def new_model_distribution(v):
# The x values have been updated so get a new distribution.
# If the x value has not been changed and C is set to None
# and N = I then the mean of the distribution will be the
# v.V_model
A = generate_m_proj(v)
N = np.diag(np.repeat(v.obs_variance, 2))
C = np.diag(np.repeat(v.obs_variance, 2))
d = split_re_im(v.V_obs)
V = split_re_im(v.get_calibrated_visibilities())
if C is None:
inv_C = np.zeros((v.nvis*2, v.nvis*2))
else:
inv_C = inv(C)
term1 = np.dot(A.T, np.dot(inv(N), A))
dist_covariance = inv(term1+inv_C)
term2 = np.dot(A.T, np.dot(inv(N), d)+np.dot(inv_C, V))
dist_mean = np.dot(dist_covariance, term2)
sigma_1 = inv(np.dot(A.T, np.dot(inv(N), A)))
mu_1 = np.dot(inv(A), d)
sigma_2 = C
mu_2 = V
dist_mean = np.dot(np.dot(sigma_2, inv(sigma_1+sigma_2)), mu_1)+np.dot(np.dot(sigma_1, inv(sigma_1+sigma_2)), mu_2)
dist_covariance = np.dot(sigma_1, np.dot(inv(sigma_1+sigma_2), sigma_2))
return dist_mean, dist_covariance
def sample_x():
v = VisSim(4, random_seed=99)
# Replace the simulated x with GLS solved x which conforms to
# the degrees of freedom. One maginary value will be 0.
# The sampling will also conform to this.
v.x = gls_solve(v)
# Setup sampling matrices
S = None # np.eye(v.nant*2-1)
A, where = reduce_dof(generate_proj(v.g_bar, v.V_model), True)
N = np.diag(np.repeat(v.obs_variance, 2))
d = split_re_im(v.get_reduced_observed())
num = 10000
all_x = np.zeros((num, v.nant*2-1))
# Take num samples
for i in range(num):
all_x[i] = sample(S, N, A, d)
# Provide information about the set of samples
s, var = expected_sample_stats(S, N, A, d)
print("Expected sampler solution for x", unsplit_re_im(restore_x(s, where)))
print("Original x", v.x)
print("Best x vectors")
hist, bin_edges = np.histogramdd(all_x, bins=10)
ind = np.where(hist == hist.max())
for j in range(len(ind[0])):
x = np.zeros(len(ind))
for i in range(len(ind)):
x[i] = (bin_edges[i][ind[i][j]+1]+bin_edges[i][ind[i][j]])/2
print(x)
print("Sample stats")
sample_stats("x", all_x)
# Make corner plot of samples
plt.clf()
labels = [ r"$re(x_"+str(i)+")$" for i in range(all_x.shape[1]) ]
figure = corner.corner(all_x, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2)
# Overplot GLS solution
# Extract the axes
axes = np.array(figure.axes).reshape((v.nant*2-1, v.nant*2-1))
# Loop over the diagonal
orig_x = split_re_im(v.x)
for i in range(v.nant*2-1):
ax = axes[i, i]
ax.axvline(orig_x[i], color="r", linewidth=0.5)
plt.tight_layout()
plt.savefig("x_dist.png")
def sample_model():
v = VisSim(4, random_seed=99)
# Setup sampling matrices
A = np.diag(split_re_im(v.get_baseline_gains()))
N = np.diag(np.repeat(v.obs_variance, 2))
C = np.diag(np.repeat(v.obs_variance, 2))
d = split_re_im(v.V_obs)
V = split_re_im(v.V_model)
term1 = np.dot(A.T, np.dot(inv(N), A))
dist_covariance = inv(term1+inv(C))
term2 = np.dot(A.T, np.dot(inv(N), d))+np.dot(inv(C), V)
dist_mean = np.dot(dist_covariance, term2)
num = 10000
all_v = np.random.multivariate_normal(dist_mean, dist_covariance, num)
# Provide information about the set of samples
print("Expected sampler solution for model", dist_mean)
print("Original model", v.V_model)
"""
print("Best model vectors")
hist, bin_edges = np.histogramdd(all_v, bins=10)
ind = np.where(hist == hist.max())
for j in range(len(ind[0])):
v = np.zeros(len(ind))
for i in range(len(ind)):
v[i] = (bin_edges[i][ind[i][j]+1]+bin_edges[i][ind[i][j]])/2
print(v)
"""
print("Sample stats")
sample_stats("model", all_v)
plt.clf()
# Make corner plot of samples
labels = [ r"$re(V_model_{"+str(i)+"})$" for i in range(all_v.shape[1]) ]
figure = corner.corner(all_v, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2)
# Overplot GLS solution
# Extract the axes
axes = np.array(figure.axes).reshape((v.nvis*2, v.nvis*2))
# Loop over the diagonal
orig_v = split_re_im(v.V_model)
for i in range(v.nvis*2):
ax = axes[i, i]
ax.axvline(orig_v[i], color="r", linewidth=0.5)
plt.tight_layout()
plt.savefig("model_dist.png")
def estimate_variance(perturb_percent, perturb_this, level):
"""
Find the variance between the observed visibilities, and visibilities
obtained from model times gains, after something has been perturbed.
The variance used in the likelihood is important. This function
simualtes lots of visivilities using the perturbation that is to be
used for mcmc, gets all the differences (dy) and calculates the variance
of them. This is then used when the likelihood is calculated.
Parameters
----------
perturb_percent : float
Perturb values by this percentage.
perturb_this: str
What to perturb. Currently "vis" or "gains".
Returns
-------
float
The variance on real and imag values.
"""
nant = 1000
if perturb_percent == 0: return 1e2
# Generate lots of simulated visibilities from simulated
# model, gains, gain offets
vis_values = VisSim(nant, level=level)
# Perturb values and calculate the difference between observed
# visibilities and the originals.
if perturb_this == "vis":
vis_values = perturb_vis(vis_values, perturb_percent)
else:
vis_values = perturb_gains(vis_values, perturb_percent)
all_dy_values = vis_values.V_obs-vis_values.get_simulated_visibilities()
return np.mean(np.abs(split_re_im(all_dy_values)**2))
def new_x_distribution(v):
# The model has been updated so get a new distribution.
# If S is set to None and the model is never changed then
# the mean of the x distribution will be the GLS solution.
A, where = reduce_dof(generate_proj(v.g_bar, v.V_model), True) # depends on model
N = np.diag(np.repeat(v.obs_variance, 2))
d = split_re_im(v.get_reduced_observed()) # depends on model
S = np.eye(A.shape[1])*1e-8
if S is None:
inv_S = np.zeros((A.shape[1], A.shape[1]))
else:
inv_S = inv(S)
term1 = np.dot(A.T, np.dot(inv(N), A))
term2 = np.dot(A.T, np.dot(inv(N), d))
dist_mean = np.dot(inv(inv_S+term1), term2)
dist_covariance = inv(inv_S+term1)
return dist_mean, dist_covariance, where
def gls_solve(vis):
"""
Calculate the generalised least-squares solution for the gain fluctuations.
The model is V_ij ~ \bar{g}_i \bar{g}_j V_ij^model (1 + x_i + x_j).
The x values will be updated in vis.
Parameters
----------
vis: VisSim, VisCal, or VisTrue object
"""
if vis.level != "approx":
raise RuntimeError(
"GLS can only operate on offsets used approximately")
# Convert the variances into an expanded diagonal array
Ninv = inv(np.diag(np.repeat(vis.obs_variance, 2)))
# Generate projection matrix
proj = generate_proj(vis.g_bar, vis.V_model)
# Now remove a column from the proj matrix, which will reduce the
# number of x values found by 1. This removes a degree of freedom
# so a solution can be found.
proj, where_cut = reduce_dof(proj, True)
inv_cov = gls_inv_covariance(proj, Ninv)
#print("RMS", rms(np.dot(Ninv, data)), rms(np.dot(proj.T, np.dot(Ninv, data))))
# Calculate GLS solution
xhat = np.dot(
inv_cov,
np.dot(proj.T, np.dot(Ninv, split_re_im(vis.get_reduced_observed()))))
# Restore the missing x value and set it to zero. Form complex numbers and update vis.
return unsplit_re_im(restore_x(xhat, where_cut))
def gibbs():
def new_x_distribution(v):
# The model has been updated so get a new distribution.
# If S is set to None and the model is never changed then
# the mean of the x distribution will be the GLS solution.
A, where = reduce_dof(generate_proj(v.g_bar, v.V_model), True) # depends on model
N = np.diag(np.repeat(v.obs_variance, 2))
d = split_re_im(v.get_reduced_observed()) # depends on model
S = np.eye(A.shape[1])*1e-8
if S is None:
inv_S = np.zeros((A.shape[1], A.shape[1]))
else:
inv_S = inv(S)
term1 = np.dot(A.T, np.dot(inv(N), A))
term2 = np.dot(A.T, np.dot(inv(N), d))
dist_mean = np.dot(inv(inv_S+term1), term2)
dist_covariance = inv(inv_S+term1)
return dist_mean, dist_covariance, where
def separate_terms(gi, gj, xi, xj):
"""
Form 2-D matrix that can be multiplied by re(model), im(model)
to give re(V_obs), im(V_obs)
"""
a = gi.real
b = gi.imag
c = gj.real
d = gj.imag
e = xi.real
f = xi.imag
g = xj.real
h = xj.imag
return a*c+b*d + a*c*e+b*d*e-b*c*f+a*d*f + a*c*g+b*d*g+b*c*h-a*d*h, \
-(b*c-a*d) -(a*c*f+b*d*f+b*c*e-a*d*e) - (-a*c*h-b*d*h+b*c*g-a*d*g)
def generate_m_proj(vis):
proj = np.zeros((vis.nvis*2, vis.nvis*2))
k = 0
for i in range(vis.nant):
for j in range(i+1, vis.nant):
term1, term2 = separate_terms(vis.g_bar[i], vis.g_bar[j], vis.x[i], vis.x[j])
# Put them in the right place in the bigger matrix
proj[k*2, k*2] = term1
proj[k*2, k*2+1] = term2
proj[k*2+1, k*2] = -term2
proj[k*2+1, k*2+1] = term1
k += 1
# Test - first line equal to second
#print(split_re_im(vis.get_simulated_visibilities()))
#print(np.dot(proj, split_re_im(vis.V_model))); exit()
return proj
def new_model_distribution(v):
# The x values have been updated so get a new distribution.
# If the x value has not been changed and C is set to None
# and N = I then the mean of the distribution will be the
# v.V_model
A = generate_m_proj(v)
N = np.diag(np.repeat(v.obs_variance, 2))
C = np.diag(np.repeat(v.obs_variance, 2))
d = split_re_im(v.V_obs)
V = split_re_im(v.get_calibrated_visibilities())
if C is None:
inv_C = np.zeros((v.nvis*2, v.nvis*2))
else:
inv_C = inv(C)
term1 = np.dot(A.T, np.dot(inv(N), A))
dist_covariance = inv(term1+inv_C)
term2 = np.dot(A.T, np.dot(inv(N), d)+np.dot(inv_C, V))
dist_mean = np.dot(dist_covariance, term2)
sigma_1 = inv(np.dot(A.T, np.dot(inv(N), A)))
mu_1 = np.dot(inv(A), d)
sigma_2 = C
mu_2 = V
dist_mean = np.dot(np.dot(sigma_2, inv(sigma_1+sigma_2)), mu_1)+np.dot(np.dot(sigma_1, inv(sigma_1+sigma_2)), mu_2)
dist_covariance = np.dot(sigma_1, np.dot(inv(sigma_1+sigma_2), sigma_2))
return dist_mean, dist_covariance
v = VisSim(4, random_seed=99)
# Replace the simulated x with GLS solved x which conforms to
# the degrees of freedom. One imaginary value will be 0.
# The sampling will also conform to this.
v.x = gls_solve(v)
print("orig x", v.x)
print("orig model", v.V_model)
orig_v = copy.deepcopy(v)
num = 40000
all_x = np.zeros((num, v.nant*2-1)) # -1 because there'll be a missing imaginary value
all_model = np.zeros((num, v.nvis*2))
v_x_sampling = copy.deepcopy(v)
v_model_sampling = copy.deepcopy(v)
new_x_sample = v_model_sampling.x # Initialize
# Take num samples
for i in range(num):
# Use the sampled x to change the model sampling distribution, and take a sample
v_model_sampling.x = new_x_sample
v_dist_mean, v_dist_covariance = new_model_distribution(v_model_sampling)
all_model[i] = np.random.multivariate_normal(v_dist_mean, v_dist_covariance, 1)
new_model_sample = unsplit_re_im(all_model[i])
# Use the sampled model to change the x sampling distribution, and take a sample
v_x_sampling.V_model = new_model_sample
x_dist_mean, x_dist_covariance, which_im_cut = new_x_distribution(v_x_sampling)
all_x[i] = np.random.multivariate_normal(x_dist_mean, x_dist_covariance, 1)
new_x_sample = unsplit_re_im(restore_x(all_x[i], which_im_cut))
all_data = np.hstack((all_x, all_model))
# Get log poseterior evaluate interpret similar to chi2
# Plot compare plot of true vis vs sampled vis (V_obs)
print("Sample stats")
sample_stats("x", all_x)
sample_stats("model", all_model)
print("N:", np.repeat(v.obs_variance, 2))
print("C:", np.repeat(v.obs_variance, 2))
print("S:", 1e-4)
# Plot traces
for i in range(all_model.shape[1]):
plt.clf()
plt.plot(all_model[:, i], linewidth=0.06)
plt.savefig("model_trace_"+str(i)+".png")
# Plot x dist
plt.clf()
labels = [ r"$re(x_"+str(i)+")$" for i in range(all_x.shape[1]) ]
figure = corner.corner(all_x, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2)
# Overplot GLS solution
# Extract the axes
axes = np.array(figure.axes).reshape((v.nant*2-1, v.nant*2-1))
# Loop over the diagonal
orig_x = split_re_im(orig_v.x)
for i in range(v.nant*2-1):
ax = axes[i, i]
ax.axvline(orig_x[i], color="r", linewidth=0.5)
plt.tight_layout()
plt.savefig("gibbs_x_dist.png")
# Plot model dist
plt.clf()
# Make corner plot of samples
labels = [ r"$re(V_{model}{"+str(i)+"})$" for i in range(all_model.shape[1]) ]
figure = corner.corner(all_model, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2)
# Overplot GLS solution
# Extract the axes
axes = np.array(figure.axes).reshape((v.nvis*2, v.nvis*2))
# Loop over the diagonal
orig_model = split_re_im(orig_v.V_model)
for i in range(v.nvis*2):
ax = axes[i, i]
ax.axvline(orig_model[i], color="r", linewidth=0.5)
plt.tight_layout()
plt.savefig("gibbs_model_dist.png")
# Plot all dist
plt.clf()
labels = [ r"$re(x_"+str(i)+")$" for i in range(all_x.shape[1]) ] + [ r"$re(V_{model}{"+str(i)+"})$" for i in range(all_model.shape[1]) ]
figure = corner.corner(all_data, labels=labels, show_titles=True, use_math_text=True, labelpad=0.2)
# Overplot GLS solution
# Extract the axes
#axes = np.array(figure.axes).reshape((v.nant*2-1, v.nant*2-1))
# Loop over the diagonal
#orig_x = split_re_im(orig_v.x)
#for i in range(v.nant*2-1):
# ax = axes[i, i]
# ax.axvline(orig_x[i], color="r", linewidth=0.5)
plt.tight_layout()
plt.savefig("gibbs_all_dist.png")
f = open("gibbs_sampling.html", "w")
f.write("<h2>Sampling</h2><p>\n")
f.write("Gibbs sampled x<p><img src=gibbs_x_dist.png width=1200>\n<p>")
f.write("Gibbs sampled model<p><img src=gibbs_model_dist.png width=1200>\n")
f.write("Gibbs sampled all<p><img src=gibbs_all_dist.png width=1200>\n<p>")
for i in range(all_model.shape[1]):
f.write("Trace<p><img src=model_trace_"+str(i)+".png width=1200>\n")
f.close()
plt.clf()
plt.hist(all_x[:, 0])
plt.savefig("hist")
# Get the peak of the distributions and see if it creates V_obs
best_x = np.array([ np.mean(all_x[:, i]) for i in range(all_x.shape[1]) ])
best_model = np.array([ np.mean(all_model[:, i]) for i in range(all_model.shape[1]) ])
print(best_x, orig_v.x)
print(best_model, orig_v.V_model)
v.x = unsplit_re_im(restore_x(best_x, 0))
v.V_model = unsplit_re_im(best_model)
print(orig_v.V_obs)
print(v.get_simulated_visibilities())
print(np.abs(orig_v.V_obs))
print(np.abs(v.get_simulated_visibilities()))