def new_model_distribution(v): # The x values have been updated so get a new distribution. # If the x value has not been changed and C is set to None # and N = I then the mean of the distribution will be the # v.V_model A = generate_m_proj(v) N = np.diag(np.repeat(v.obs_variance, 2)) C = np.diag(np.repeat(v.obs_variance, 2)) d = split_re_im(v.V_obs) V = split_re_im(v.get_calibrated_visibilities()) if C is None: inv_C = np.zeros((v.nvis*2, v.nvis*2)) else: inv_C = inv(C) term1 = np.dot(A.T, np.dot(inv(N), A)) dist_covariance = inv(term1+inv_C) term2 = np.dot(A.T, np.dot(inv(N), d)+np.dot(inv_C, V)) dist_mean = np.dot(dist_covariance, term2) sigma_1 = inv(np.dot(A.T, np.dot(inv(N), A))) mu_1 = np.dot(inv(A), d) sigma_2 = C mu_2 = V dist_mean = np.dot(np.dot(sigma_2, inv(sigma_1+sigma_2)), mu_1)+np.dot(np.dot(sigma_1, inv(sigma_1+sigma_2)), mu_2) dist_covariance = np.dot(sigma_1, np.dot(inv(sigma_1+sigma_2), sigma_2)) return dist_mean, dist_covariance
def sample_x(): v = VisSim(4, random_seed=99) # Replace the simulated x with GLS solved x which conforms to # the degrees of freedom. One maginary value will be 0. # The sampling will also conform to this. v.x = gls_solve(v) # Setup sampling matrices S = None # np.eye(v.nant*2-1) A, where = reduce_dof(generate_proj(v.g_bar, v.V_model), True) N = np.diag(np.repeat(v.obs_variance, 2)) d = split_re_im(v.get_reduced_observed()) num = 10000 all_x = np.zeros((num, v.nant*2-1)) # Take num samples for i in range(num): all_x[i] = sample(S, N, A, d) # Provide information about the set of samples s, var = expected_sample_stats(S, N, A, d) print("Expected sampler solution for x", unsplit_re_im(restore_x(s, where))) print("Original x", v.x) print("Best x vectors") hist, bin_edges = np.histogramdd(all_x, bins=10) ind = np.where(hist == hist.max()) for j in range(len(ind[0])): x = np.zeros(len(ind)) for i in range(len(ind)): x[i] = (bin_edges[i][ind[i][j]+1]+bin_edges[i][ind[i][j]])/2 print(x) print("Sample stats") sample_stats("x", all_x) # Make corner plot of samples plt.clf() labels = [ r"$re(x_"+str(i)+")$" for i in range(all_x.shape[1]) ] figure = corner.corner(all_x, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2) # Overplot GLS solution # Extract the axes axes = np.array(figure.axes).reshape((v.nant*2-1, v.nant*2-1)) # Loop over the diagonal orig_x = split_re_im(v.x) for i in range(v.nant*2-1): ax = axes[i, i] ax.axvline(orig_x[i], color="r", linewidth=0.5) plt.tight_layout() plt.savefig("x_dist.png")
def sample_model(): v = VisSim(4, random_seed=99) # Setup sampling matrices A = np.diag(split_re_im(v.get_baseline_gains())) N = np.diag(np.repeat(v.obs_variance, 2)) C = np.diag(np.repeat(v.obs_variance, 2)) d = split_re_im(v.V_obs) V = split_re_im(v.V_model) term1 = np.dot(A.T, np.dot(inv(N), A)) dist_covariance = inv(term1+inv(C)) term2 = np.dot(A.T, np.dot(inv(N), d))+np.dot(inv(C), V) dist_mean = np.dot(dist_covariance, term2) num = 10000 all_v = np.random.multivariate_normal(dist_mean, dist_covariance, num) # Provide information about the set of samples print("Expected sampler solution for model", dist_mean) print("Original model", v.V_model) """ print("Best model vectors") hist, bin_edges = np.histogramdd(all_v, bins=10) ind = np.where(hist == hist.max()) for j in range(len(ind[0])): v = np.zeros(len(ind)) for i in range(len(ind)): v[i] = (bin_edges[i][ind[i][j]+1]+bin_edges[i][ind[i][j]])/2 print(v) """ print("Sample stats") sample_stats("model", all_v) plt.clf() # Make corner plot of samples labels = [ r"$re(V_model_{"+str(i)+"})$" for i in range(all_v.shape[1]) ] figure = corner.corner(all_v, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2) # Overplot GLS solution # Extract the axes axes = np.array(figure.axes).reshape((v.nvis*2, v.nvis*2)) # Loop over the diagonal orig_v = split_re_im(v.V_model) for i in range(v.nvis*2): ax = axes[i, i] ax.axvline(orig_v[i], color="r", linewidth=0.5) plt.tight_layout() plt.savefig("model_dist.png")
def estimate_variance(perturb_percent, perturb_this, level): """ Find the variance between the observed visibilities, and visibilities obtained from model times gains, after something has been perturbed. The variance used in the likelihood is important. This function simualtes lots of visivilities using the perturbation that is to be used for mcmc, gets all the differences (dy) and calculates the variance of them. This is then used when the likelihood is calculated. Parameters ---------- perturb_percent : float Perturb values by this percentage. perturb_this: str What to perturb. Currently "vis" or "gains". Returns ------- float The variance on real and imag values. """ nant = 1000 if perturb_percent == 0: return 1e2 # Generate lots of simulated visibilities from simulated # model, gains, gain offets vis_values = VisSim(nant, level=level) # Perturb values and calculate the difference between observed # visibilities and the originals. if perturb_this == "vis": vis_values = perturb_vis(vis_values, perturb_percent) else: vis_values = perturb_gains(vis_values, perturb_percent) all_dy_values = vis_values.V_obs-vis_values.get_simulated_visibilities() return np.mean(np.abs(split_re_im(all_dy_values)**2))
def new_x_distribution(v): # The model has been updated so get a new distribution. # If S is set to None and the model is never changed then # the mean of the x distribution will be the GLS solution. A, where = reduce_dof(generate_proj(v.g_bar, v.V_model), True) # depends on model N = np.diag(np.repeat(v.obs_variance, 2)) d = split_re_im(v.get_reduced_observed()) # depends on model S = np.eye(A.shape[1])*1e-8 if S is None: inv_S = np.zeros((A.shape[1], A.shape[1])) else: inv_S = inv(S) term1 = np.dot(A.T, np.dot(inv(N), A)) term2 = np.dot(A.T, np.dot(inv(N), d)) dist_mean = np.dot(inv(inv_S+term1), term2) dist_covariance = inv(inv_S+term1) return dist_mean, dist_covariance, where
def gls_solve(vis): """ Calculate the generalised least-squares solution for the gain fluctuations. The model is V_ij ~ \bar{g}_i \bar{g}_j V_ij^model (1 + x_i + x_j). The x values will be updated in vis. Parameters ---------- vis: VisSim, VisCal, or VisTrue object """ if vis.level != "approx": raise RuntimeError( "GLS can only operate on offsets used approximately") # Convert the variances into an expanded diagonal array Ninv = inv(np.diag(np.repeat(vis.obs_variance, 2))) # Generate projection matrix proj = generate_proj(vis.g_bar, vis.V_model) # Now remove a column from the proj matrix, which will reduce the # number of x values found by 1. This removes a degree of freedom # so a solution can be found. proj, where_cut = reduce_dof(proj, True) inv_cov = gls_inv_covariance(proj, Ninv) #print("RMS", rms(np.dot(Ninv, data)), rms(np.dot(proj.T, np.dot(Ninv, data)))) # Calculate GLS solution xhat = np.dot( inv_cov, np.dot(proj.T, np.dot(Ninv, split_re_im(vis.get_reduced_observed())))) # Restore the missing x value and set it to zero. Form complex numbers and update vis. return unsplit_re_im(restore_x(xhat, where_cut))
def gibbs(): def new_x_distribution(v): # The model has been updated so get a new distribution. # If S is set to None and the model is never changed then # the mean of the x distribution will be the GLS solution. A, where = reduce_dof(generate_proj(v.g_bar, v.V_model), True) # depends on model N = np.diag(np.repeat(v.obs_variance, 2)) d = split_re_im(v.get_reduced_observed()) # depends on model S = np.eye(A.shape[1])*1e-8 if S is None: inv_S = np.zeros((A.shape[1], A.shape[1])) else: inv_S = inv(S) term1 = np.dot(A.T, np.dot(inv(N), A)) term2 = np.dot(A.T, np.dot(inv(N), d)) dist_mean = np.dot(inv(inv_S+term1), term2) dist_covariance = inv(inv_S+term1) return dist_mean, dist_covariance, where def separate_terms(gi, gj, xi, xj): """ Form 2-D matrix that can be multiplied by re(model), im(model) to give re(V_obs), im(V_obs) """ a = gi.real b = gi.imag c = gj.real d = gj.imag e = xi.real f = xi.imag g = xj.real h = xj.imag return a*c+b*d + a*c*e+b*d*e-b*c*f+a*d*f + a*c*g+b*d*g+b*c*h-a*d*h, \ -(b*c-a*d) -(a*c*f+b*d*f+b*c*e-a*d*e) - (-a*c*h-b*d*h+b*c*g-a*d*g) def generate_m_proj(vis): proj = np.zeros((vis.nvis*2, vis.nvis*2)) k = 0 for i in range(vis.nant): for j in range(i+1, vis.nant): term1, term2 = separate_terms(vis.g_bar[i], vis.g_bar[j], vis.x[i], vis.x[j]) # Put them in the right place in the bigger matrix proj[k*2, k*2] = term1 proj[k*2, k*2+1] = term2 proj[k*2+1, k*2] = -term2 proj[k*2+1, k*2+1] = term1 k += 1 # Test - first line equal to second #print(split_re_im(vis.get_simulated_visibilities())) #print(np.dot(proj, split_re_im(vis.V_model))); exit() return proj def new_model_distribution(v): # The x values have been updated so get a new distribution. # If the x value has not been changed and C is set to None # and N = I then the mean of the distribution will be the # v.V_model A = generate_m_proj(v) N = np.diag(np.repeat(v.obs_variance, 2)) C = np.diag(np.repeat(v.obs_variance, 2)) d = split_re_im(v.V_obs) V = split_re_im(v.get_calibrated_visibilities()) if C is None: inv_C = np.zeros((v.nvis*2, v.nvis*2)) else: inv_C = inv(C) term1 = np.dot(A.T, np.dot(inv(N), A)) dist_covariance = inv(term1+inv_C) term2 = np.dot(A.T, np.dot(inv(N), d)+np.dot(inv_C, V)) dist_mean = np.dot(dist_covariance, term2) sigma_1 = inv(np.dot(A.T, np.dot(inv(N), A))) mu_1 = np.dot(inv(A), d) sigma_2 = C mu_2 = V dist_mean = np.dot(np.dot(sigma_2, inv(sigma_1+sigma_2)), mu_1)+np.dot(np.dot(sigma_1, inv(sigma_1+sigma_2)), mu_2) dist_covariance = np.dot(sigma_1, np.dot(inv(sigma_1+sigma_2), sigma_2)) return dist_mean, dist_covariance v = VisSim(4, random_seed=99) # Replace the simulated x with GLS solved x which conforms to # the degrees of freedom. One imaginary value will be 0. # The sampling will also conform to this. v.x = gls_solve(v) print("orig x", v.x) print("orig model", v.V_model) orig_v = copy.deepcopy(v) num = 40000 all_x = np.zeros((num, v.nant*2-1)) # -1 because there'll be a missing imaginary value all_model = np.zeros((num, v.nvis*2)) v_x_sampling = copy.deepcopy(v) v_model_sampling = copy.deepcopy(v) new_x_sample = v_model_sampling.x # Initialize # Take num samples for i in range(num): # Use the sampled x to change the model sampling distribution, and take a sample v_model_sampling.x = new_x_sample v_dist_mean, v_dist_covariance = new_model_distribution(v_model_sampling) all_model[i] = np.random.multivariate_normal(v_dist_mean, v_dist_covariance, 1) new_model_sample = unsplit_re_im(all_model[i]) # Use the sampled model to change the x sampling distribution, and take a sample v_x_sampling.V_model = new_model_sample x_dist_mean, x_dist_covariance, which_im_cut = new_x_distribution(v_x_sampling) all_x[i] = np.random.multivariate_normal(x_dist_mean, x_dist_covariance, 1) new_x_sample = unsplit_re_im(restore_x(all_x[i], which_im_cut)) all_data = np.hstack((all_x, all_model)) # Get log poseterior evaluate interpret similar to chi2 # Plot compare plot of true vis vs sampled vis (V_obs) print("Sample stats") sample_stats("x", all_x) sample_stats("model", all_model) print("N:", np.repeat(v.obs_variance, 2)) print("C:", np.repeat(v.obs_variance, 2)) print("S:", 1e-4) # Plot traces for i in range(all_model.shape[1]): plt.clf() plt.plot(all_model[:, i], linewidth=0.06) plt.savefig("model_trace_"+str(i)+".png") # Plot x dist plt.clf() labels = [ r"$re(x_"+str(i)+")$" for i in range(all_x.shape[1]) ] figure = corner.corner(all_x, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2) # Overplot GLS solution # Extract the axes axes = np.array(figure.axes).reshape((v.nant*2-1, v.nant*2-1)) # Loop over the diagonal orig_x = split_re_im(orig_v.x) for i in range(v.nant*2-1): ax = axes[i, i] ax.axvline(orig_x[i], color="r", linewidth=0.5) plt.tight_layout() plt.savefig("gibbs_x_dist.png") # Plot model dist plt.clf() # Make corner plot of samples labels = [ r"$re(V_{model}{"+str(i)+"})$" for i in range(all_model.shape[1]) ] figure = corner.corner(all_model, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2) # Overplot GLS solution # Extract the axes axes = np.array(figure.axes).reshape((v.nvis*2, v.nvis*2)) # Loop over the diagonal orig_model = split_re_im(orig_v.V_model) for i in range(v.nvis*2): ax = axes[i, i] ax.axvline(orig_model[i], color="r", linewidth=0.5) plt.tight_layout() plt.savefig("gibbs_model_dist.png") # Plot all dist plt.clf() labels = [ r"$re(x_"+str(i)+")$" for i in range(all_x.shape[1]) ] + [ r"$re(V_{model}{"+str(i)+"})$" for i in range(all_model.shape[1]) ] figure = corner.corner(all_data, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2) # Overplot GLS solution # Extract the axes #axes = np.array(figure.axes).reshape((v.nant*2-1, v.nant*2-1)) # Loop over the diagonal #orig_x = split_re_im(orig_v.x) #for i in range(v.nant*2-1): # ax = axes[i, i] # ax.axvline(orig_x[i], color="r", linewidth=0.5) plt.tight_layout() plt.savefig("gibbs_all_dist.png") f = open("gibbs_sampling.html", "w") f.write("<h2>Sampling</h2><p>\n") f.write("Gibbs sampled x<p><img src=gibbs_x_dist.png width=1200>\n<p>") f.write("Gibbs sampled model<p><img src=gibbs_model_dist.png width=1200>\n") f.write("Gibbs sampled all<p><img src=gibbs_all_dist.png width=1200>\n<p>") for i in range(all_model.shape[1]): f.write("Trace<p><img src=model_trace_"+str(i)+".png width=1200>\n") f.close() plt.clf() plt.hist(all_x[:, 0]) plt.savefig("hist") # Get the peak of the distributions and see if it creates V_obs best_x = np.array([ np.mean(all_x[:, i]) for i in range(all_x.shape[1]) ]) best_model = np.array([ np.mean(all_model[:, i]) for i in range(all_model.shape[1]) ]) print(best_x, orig_v.x) print(best_model, orig_v.V_model) v.x = unsplit_re_im(restore_x(best_x, 0)) v.V_model = unsplit_re_im(best_model) print(orig_v.V_obs) print(v.get_simulated_visibilities()) print(np.abs(orig_v.V_obs)) print(np.abs(v.get_simulated_visibilities()))