def test_phase_unwrapp1d():
peak = 30
N = 100
assert peak/(N-1) < np.pi, "choose a valid test case"
phase_true = np.linspace(0,peak,N) + np.random.uniform(size=N)*(np.pi - peak/(N-1))
phase_wrap = np.angle(np.exp(1j*phase_true))
phase_unwrap = phase_unwrapp1d(phase_wrap)
assert np.allclose(phase_true,phase_unwrap)
phase_true = np.multiply.outer(phase_true,[1,0.9,0.5])
phase_wrap = np.angle(np.exp(1j*phase_true))
phase_unwrap = phase_unwrapp1d(phase_wrap,axis=0)
assert np.allclose(phase_true,phase_unwrap)
def robust_l2(obs_phase, freqs, solve_cs=True):
'''Solve the tec and cs for multiple datasets.
`obs_phase` : `numpy.ndarray`
the measured phase with shape (num_freqs, )
`freqs` : `numpy.ndarray`
the frequencies at the datapoints (num_freqs,)
`solve_cs` : (optional) bool
Whether to solve cs (True)
'''
obs_phase = phase_unwrapp1d(obs_phase)
if solve_cs:
def residuals(m, freqs, obs_phase):
tec,cs = m[0],m[1]
return calc_phase(tec,freqs,cs=cs) - obs_phase
else:
def residuals(m, freqs, obs_phase):
tec,cs = m[0],m[1]
return calc_phase(tec,freqs,cs=0.) - obs_phase
nan_mask = np.bitwise_not(np.isnan(obs_phase))
obs_phase_ = obs_phase[nan_mask]
freqs_ = freqs[nan_mask]
m0 = [0.0, 0.]
m = least_squares(residuals,m0,loss='soft_l1',f_scale=90.*np.pi/180.,args=(freqs_,obs_phase_))
if solve_cs:
return m.x[0], m.x[1]
else:
return m.x[0], 0.
def fit_datapack(datapack,
template_datapack,
ant_idx=-1,
time_idx=-1,
dir_idx=-1,
freq_idx=-1):
"""Fit a datapack to a template datapack using Bayesian optimization
of given kernel. Conjugation occurs at 350km."""
antennas, antenna_labels = datapack.get_antennas(ant_idx)
directions, patch_names = datapack.get_directions(dir_idx)
times, timestamps = datapack.get_times(time_idx)
freqs = datapack.get_freqs(freq_idx)
phase = datapack.get_phase(ant_idx=ant_idx,
dir_idx=dir_idx,
time_idx=time_idx,
freq_idx=freq_idx)
Na = len(antennas)
Nt = len(times)
Nd = len(directions)
Nf = len(freqs)
for i in range(Na):
for k in range(Nd):
for l in range(Nf):
phase[i, :, k, l] = phase_unwrapp1d(phase[i, :, k, l])
phase -= np.mean(phase.flatten())
X = np.zeros((Na, Nt, Nd, Nf, 8), dtype=float)
for i, t in enumerate(times):
print("Working on {}".format(t.isot))
phase_center = datapack.get_center_direction()
fixtime = times[0]
obstime = t
center = datapack.radio_array.get_center()
uvw = Pointing(location=center,
phase=phase_center,
fixtime=fixtime,
obstime=t)
ants_uvw = antennas.transform_to(uvw)
dirs_uvw = directions.transform_to(uvw)
#Na,1,Nd,Nf,8
X[:, i:i + 1, :, :, :] = np.stack([
_replicate((), (1, Nd, Nf),
ants_uvw.u.to(au.km).value),
_replicate((), (1, Nd, Nf),
ants_uvw.v.to(au.km).value),
_replicate((), (1, Nd, Nf),
ants_uvw.w.to(au.km).value),
_replicate((Na, ), (Nd, Nf), times[i:i + 1].gps),
_replicate((Na, 1), (Nf, ), dirs_uvw.u.value),
_replicate((Na, 1), (Nf, ), dirs_uvw.v.value),
_replicate((Na, 1), (Nf, ), dirs_uvw.w.value),
_replicate((Na, 1, Nd), (), freqs)
],
axis=-1)
X = X.reshape((-1, 8))
template_datapack = template_datapack.clone()
antennas_, antenna_labels_ = template_datapack.get_antennas(ant_idx)
directions_, patch_names_ = template_datapack.get_directions(dir_idx)
times_, timestamps_ = template_datapack.get_times(time_idx)
freqs_ = template_datapack.get_freqs(freq_idx)
Na_ = len(antennas_)
Nt_ = len(times_)
Nd_ = len(directions_)
Nf_ = len(freqs_)
Xstar = np.zeros((Na_, Nt_, Nd_, Nf_, 8), dtype=float)
for i, t in enumerate(times_):
print("Working on {}".format(t.isot))
phase_center = template_datapack.get_center_direction()
fixtime = times_[0]
obstime = t
center = template_datapack.radio_array.get_center()
uvw = Pointing(location=center,
phase=phase_center,
fixtime=fixtime,
obstime=t)
ants_uvw = antennas_.transform_to(uvw)
dirs_uvw = directions_.transform_to(uvw)
Xstar[:, i:i + 1, :, :, :] = np.stack([
_replicate((), (1, Nd_, Nf_),
ants_uvw.u.to(au.km).value),
_replicate((), (1, Nd_, Nf_),
ants_uvw.v.to(au.km).value),
_replicate((), (1, Nd_, Nf_),
ants_uvw.w.to(au.km).value),
_replicate((Na_, ), (Nd_, Nf_), times_[i:i + 1].gps),
_replicate((Na_, 1), (Nf_, ), dirs_uvw.u.value),
_replicate((Na_, 1), (Nf_, ), dirs_uvw.v.value),
_replicate((Na_, 1), (Nf_, ), dirs_uvw.w.value),
_replicate((Na_, 1, Nd_), (), freqs_)
],
axis=-1)
Xstar = Xstar.reshape((-1, 8))
y = phase.flatten()
sigma_y = np.ones_like(y) * 0.01
# plot_datapack(datapack,ant_idx=ant_idx,time_idx=time_idx, dir_idx=-1,freq_idx=freq_idx,figname=None,vmin=None,vmax=None,mode='perantenna',observable='phase')
# plot_datapack(datapack,ant_idx=ant_idx,time_idx=time_idx, dir_idx=-1,freq_idx=freq_idx,figname=None,vmin=None,vmax=None,mode='perantenna',observable='variance')
K = PhaseScreen()
#K = SquaredExponential()
#K.set_hyperparams_bounds('l',[1e-5,10])
P = Pipeline(1, X.shape[0], K, multi_dataset=False, share_x=True)
neg_log_mar_like = P.level2_optimize(X,
y,
sigma_y,
delta=0.001,
patience=5,
epochs=1000)
win_idx = np.argmin(neg_log_mar_like)
ystar, cov, lml = P.level1_predict(X,
y,
sigma_y,
Xstar=Xstar,
smooth=False,
batch_idx=win_idx)
ystar = ystar[0, :].reshape([Na_, Nt_, Nd_, Nf_])
cov = cov[0, :, :]
var = np.diag(cov).reshape([Na_, Nt_, Nd_, Nf_])
template_datapack.set_phase(ystar,
ant_idx=ant_idx,
time_idx=time_idx,
dir_idx=-1,
freq_idx=freq_idx,
ref_ant=datapack.ref_ant)
template_datapack.set_variance(var,
ant_idx=ant_idx,
time_idx=time_idx,
dir_idx=-1,
freq_idx=freq_idx)
plot_datapack(template_datapack,
ant_idx=ant_idx,
time_idx=time_idx,
dir_idx=-1,
freq_idx=freq_idx,
figname=None,
vmin=None,
vmax=None,
mode='perantenna',
observable='phase')
plot_datapack(template_datapack,
ant_idx=ant_idx,
time_idx=time_idx,
dir_idx=-1,
freq_idx=freq_idx,
figname=None,
vmin=None,
vmax=None,
mode='perantenna',
observable='variance')
return template_datapack
def l1_lpsolver(obs_phase, freqs, sigma_max = np.pi, fout=0.5, solve_cs=True, problem_name="l1_tec_solver"):
'''Formulate the linear problem:
Minimize 1'.(z1 + z2) s.t.
phase = (z1 - z2) + K/nu*TEC
|z1 + z2| < max_allowed
min_tec < TEC < max_tec
assumes obs_phase and freqs are for a single timestamp.
'''
nan_mask = np.isnan(obs_phase)
obs_phase = obs_phase[np.bitwise_not(nan_mask)]
freqs = freqs[np.bitwise_not(nan_mask)]
if (len(freqs)<10):
return 0.,0.,0.
obs_phase_unwrap = phase_unwrapp1d(obs_phase)
K = 8.44797256e-7*TECU
#z+, z-, a+, a-, asigma, a, sigma, tec, cs
N = len(freqs)
ncols = N*6 + 3
A_eq, b_eq = [],[]
A_lt, b_lt = [],[]
A_gt, b_gt = [],[]
c_obj = np.zeros(ncols,dtype=np.double)
for i in range(N):
idx_p = i
idx_m = N + i
idx_ap = 2*N + i
idx_am = 3*N + i
idx_as = 4*N + i
idx_a = 5*N + i
idx_s = 6*N
idx_tec = 6*N + 1
idx_cs = 6*N + 2
# 0<= a+ <= asigma
row = np.zeros(ncols,dtype=np.double)
row[[idx_ap,idx_as]] = 1., -1.
A_lt.append(row)
b_lt.append(0.)
# 0 <= z+ - a+ <= sigma - asigma
row = np.zeros(ncols,dtype=np.double)
row[[idx_p,idx_ap, idx_s, idx_as]] = 1., -1., -1., 1.
A_lt.append(row)
b_lt.append(0.)
row = np.zeros(ncols,dtype=np.double)
row[[idx_p,idx_ap]] = 1., -1.
A_gt.append(row)
b_gt.append(0.)
#same for a-
row = np.zeros(ncols,dtype=np.double)
row[[idx_am,idx_as]] = 1., -1.
A_lt.append(row)
b_lt.append(0.)
row = np.zeros(ncols,dtype=np.double)
row[[idx_m,idx_am, idx_s, idx_as]] = 1., -1., -1., 1.
A_lt.append(row)
b_lt.append(0.)
row = np.zeros(ncols,dtype=np.double)
row[[idx_m,idx_am]] = 1., -1.
A_gt.append(row)
b_gt.append(0.)
# 0 <= asigma <= a*sigma_max
row = np.zeros(ncols,dtype=np.double)
row[[idx_s,idx_a]] = 1., -sigma_max
A_lt.append(row)
b_lt.append(0.)
# 0 <= sigma - asigma <= sigma_max - a*sigma_max
row = np.zeros(ncols,dtype=np.double)
row[[idx_s,idx_as, idx_a]] = 1., -1., sigma_max
A_lt.append(row)
b_lt.append(sigma_max)
row = np.zeros(ncols,dtype=np.double)
row[[idx_s,idx_as]] = 1., -1.
A_gt.append(row)
b_gt.append(0.)
# a+ + a- >= asigma
row = np.zeros(ncols,dtype=np.double)
row[[idx_ap,idx_am, idx_as]] = 1., -1., -1.
A_gt.append(row)
b_gt.append(0.)
# z+ + z- - a+ - a- <= sigma - asigma
row = np.zeros(ncols,dtype=np.double)
row[[idx_p, idx_m, idx_ap, idx_am, idx_s, idx_as]] = 1., 1., -1., -1., -1., 1.
A_lt.append(row)
b_lt.append(0.)
# z+ - z- + K/nu*tec + cs = phase
row = np.zeros(ncols,dtype=np.double)
if solve_cs:
row[[idx_p, idx_m, idx_tec, idx_cs]] = 1., -1., K/freqs[i], 1.
else:
row[[idx_p, idx_m, idx_tec, idx_cs]] = 1., -1., K/freqs[i], 0.
A_eq.append(row)
b_eq.append(obs_phase_unwrap[i])
# minimize z+ + z- - a+ - a- + Nsigma_max
c_obj[[idx_p, idx_m, idx_ap, idx_am,idx_s]] = 1., 1., -1., -1.,N
row = np.zeros(ncols,dtype=np.double)
for i in range(N):
idx_a = 5*N + i
# sum a < fout * N
row[idx_a] = 1.
A_lt.append(row)
b_lt.append(fout*N)
A_eq, b_eq = np.array(A_eq), np.array(b_eq)
A_lt, b_lt = np.array(A_lt), np.array(b_lt)
A_gt, b_gt = np.array(A_gt), np.array(b_gt)
from mippy.lpsolver import LPSolver
lp = LPSolver(c_obj,A_eq=A_eq, b_eq=b_eq, A_lt=A_lt, b_lt=b_lt, A_gt=A_gt, b_gt=b_gt, maximize=False,problem_name=problem_name, solver_type='SIMP')
for i in range(len(freqs)):
idx_p = i
idx_m = N + i
idx_ap = 2*N + i
idx_am = 3*N + i
idx_as = 4*N + i
idx_a = 5*N + i
idx_s = 6*N
idx_tec = 6*N + 1
idx_cs = 6*N + 2
lp.set_variable_type(idx_p,'c',('>',0.))
lp.set_variable_type(idx_m,'c',('>',0.))
lp.set_variable_type(idx_ap,'c',('>',0.))
lp.set_variable_type(idx_am,'c',('>',0.))
lp.set_variable_type(idx_as,'c',('>',0.))
lp.set_variable_type(idx_a,'i',('<>',0., 1.))
lp.set_variable_type(idx_s,'c',('<>',0., sigma_max))
lp.set_variable_type(idx_tec,'c',('*',))
lp.set_variable_type(idx_cs,'c',('*',))
lp.compile()
res = lp.submit_problem()
for i in range(len(freqs)):
idx_p = i
idx_m = N + i
idx_ap = 2*N + i
idx_am = 3*N + i
idx_as = 4*N + i
idx_a = 5*N + i
idx_s = 6*N
idx_tec = 6*N + 1
idx_cs = 6*N + 2
assert np.isclose(res[idx_p]*res[idx_m], 0.) , "infeasible solution, {},{}".format(res[idx_p],res[idx_m])
return res[[6*N, 6*N+1, 6*N+2]]
def l1data_l2model_solve(obs_phase,freqs,Cd_error,Ct_ratio=0.01,m0=None, CS_solve=False):
'''Solves for the terms phase(CS,TEC) = CS + e^2/(4pi ep0 me c) * TEC/nu
Delay is taken out.
If CS is optionally solved for with option `CS_solve`.
`obs_phase` : `numpy.ndarray`
The phase as observable in radians. The shape is assumed to be (len(freqs), num_timestamps)
`freqs` : `numpy.ndarray`
The frequencies in Hz at midpoints of observables.
`Cd_error` : `float` or `numpy.ndarray`
The uncertainty of the measured `obs_phase` in degrees.
If not a float then must be of shape `obs_phase.shape`
`Ct_ratio` : `float` (optional)
The systematic uncertainty in fraction of absolute phase.
Ct will be calculated as Ct_ratio*abs(obs_phase)
`m0` : `numpy.ndarray` (optional)
The initial guess of the model. If not given then set to zeros.
Shape must be (num_timestamps, 2) [even if CS is not solved for]
m0[:,0] = tec0, m0[:,1] = CS0
`CS_solve` : bool (optional)
Whether or not to solve for CS or set it to 0.
Returns model of shape (num_timestamps, 2) where,
m[:,0] = tec, m[:,1] = CS
'''
obs_phase = phase_unwrapp1d(obs_phase,axis=0)
alpha = 0.
if CS_solve:
alpha = 1.
#whethre or not to use a priori information (regularization) (soft problem makes little difference)
beta = 0.
def calc_phase(m, freqs):
tec = m[:,0]
cs = m[:,1]
phase = 8.44797256e-7*TECU * np.multiply.outer(1./freqs,tec) + alpha*cs
return phase
def neglogL(obs_phase,m,CdCt_phase,m0,cov_m,freqs):
'''Return per timestep'''
K = 8.44797256e-7*TECU
nu = np.multiply.outer(1./freqs,np.ones(obs_phase.shape[1]))
tec = m[:,0]
cs = m[:,1]
tec_p = m0[:,0]
cs_p = m0[:,1]
sigma_tec2 = cov_m[0]
sigma_cs2 = cov_m[1]
sigma_phase = np.sqrt(CdCt_phase)
phase = obs_phase
#return np.nansum(np.abs(K*np.multiply.outer(1./freqs,tec) - phase)/sigma_phase,axis=0)
return beta*((tec - tec_p)**2/sigma_tec2 + (cs - cs_p)**2/sigma_cs2)/2 + np.nansum(np.abs(K*np.multiply.outer(1./freqs,tec) + alpha*cs - phase)/sigma_phase,axis=0)
def calc_grad(obs_phase,m,CdCt_phase,m0,cov_m,freqs):
K = 8.44797256e-7*TECU
nu = np.multiply.outer(1./freqs,np.ones(obs_phase.shape[1]))
tec = m[:,0]
cs = m[:,1]
tec_p = m0[:,0]
cs_p = m0[:,1]
sigma_tec2 = cov_m[0]
sigma_cs2 = cov_m[1]
sigma_phase = np.sqrt(CdCt_phase)
phase = obs_phase
x0 = sigma_tec2
x1 = K/nu
x1_ = K*np.multiply.outer(1./freqs,tec)
x2 = np.sign(alpha*cs - phase + x1_)/sigma_phase
x3 = sigma_cs2
grad = np.zeros([obs_phase.shape[1],2])
grad[:,0] = x0*(beta*(tec - tec_p)/x0 + np.nansum((x1*x2),axis=0))
grad[:,1] = x3 * (beta*(cs - cs_p)/x3 + np.nansum(alpha*x2,axis=0))
return grad
def calc_epsilon_n(dir,m,freqs,CdCt,obs_phase,step=1e-3):
"""Approximate stepsize"""
g0 = calc_phase(m, freqs)
gp = calc_phase(m + step*dir, freqs)
Gm = (gp - g0)/step
dd = obs_phase - g0
epsilon_n = (np.nansum(Gm*dd/CdCt,axis=0)/np.nansum(Gm/CdCt*Gm,axis=0))
return epsilon_n
if m0 is None:
m0 = np.zeros([obs_phase.shape[1],2],dtype=np.double)
m = m0.copy()
cov_m = np.array([1e-4,1e-4])
#print( calc_phase(m,freqs) - obs_phase)
Ct = (Ct_ratio*np.abs(obs_phase))**2
Cd = (Cd_error*np.pi/180.)**2
CdCt = Cd+Ct
#print(np.sqrt(CdCt))
#print( np.nansum(np.abs(calc_phase(m,freqs) - obs_phase)/np.sqrt(CdCt),axis=0))
S = neglogL(obs_phase,m,CdCt,m0,cov_m,freqs)
#print("Initial neglogL: {}".format(S))
iter = 0
Nmax = 2#one is enough
while iter < Nmax:
grad = calc_grad(obs_phase,m,CdCt,m0,cov_m,freqs)
dir = grad
epsilon_n = calc_epsilon_n(dir,m,freqs,CdCt,obs_phase,step=1e-3)
#print("epsilon_n: {}".format(epsilon_n))
m += dir*epsilon_n
S = neglogL(obs_phase,m,CdCt,m0,cov_m,freqs)
#print("Model: {}".format(m))
#print("iter {}: neglogL: {}, log|dm/m|: {}, |grad|: {}".format(iter, S, np.mean(np.log(np.abs(np.einsum("i,ij->ij",epsilon_n,dir)/m))),np.nansum(np.abs(grad))))
iter += 1
#print("Final neglogL: {}".format(S))
return m
def import_data(dd_file, di_file, slow_gain, datapack_file, clobber=False,dd_only = False,chan_per_block = 20):
"""Create a datapack from the direction (de)independent files.
dd_file : str
path to dd solutions made by NDPPP
di_file : str
path to di solutions made with losoto
datapack_file : str
path to store resulting DataPack
Note: assumes dtypes in hdf5 files are float
"""
f_dd = h5py.File(dd_file,"r",libver="earliest")
f_di = h5py.File(di_file,"r")
f_sg = h5py.File(slow_gain,"r")
assert (os.path.isfile(datapack_file) and clobber) or (not os.path.isfile(datapack_file)), "datapack file {} already exists and clobber is False".format(datapack_file)
antenna_labels = []
antenna_positions = []
for row in f_dd['/sol000/antenna']:
antenna_labels.append(str_(row[0]))
antenna_positions.append(row[1])
antenna_labels = np.array(antenna_labels)
antenna_positions = np.array(antenna_positions)
# antenna_positions = ac.SkyCoord(antenna_positions[:,0]*au.m,
# antenna_positions[:,1]*au.m,
# antenna_positions[:,2]*au.m,frame='itrs')
Na = len(antenna_labels)
radio_array = RadioArray(array_file = RadioArray.lofar_array)
ant_idx = []#location in radio_array (we don't go by file)
for n, lab in enumerate(antenna_labels):
i = radio_array.get_antenna_idx(lab)
assert i is not None, "{} doesn't exist in radio array, please add entry: {} {}".format(lab,lab,antenna_positions[n])
ant_idx.append(i)
patch_names = []
directions = []
for row in f_dd['/sol000/source']:
patch_names.append(str_(row[0]))
directions.append(row[1])
patch_names = np.array(patch_names)
directions = np.array(directions)
directions = ac.SkyCoord(directions[:,0]*au.rad, directions[:,1]*au.rad,frame='icrs')
Nd = len(patch_names)
times = at.Time(f_dd['/sol000/tec000/time'][...]/86400., format='mjd',scale='tai')
timestamps = times.isot
Nt = len(times)
#freqs = f_di['/sol000/phasesoffset000/freq'][...]#Hz
freqs = f_sg['/sol000/phase000/freq'][...]
df = (freqs[1] - freqs[0])/2.
freqs = np.sort(np.concatenate([np.linspace(f-df,f+df,int(chan_per_block)) for f in freqs]))
Nf = len(freqs)
data_dict = {"radio_array":radio_array,"times":times, "timestamps": timestamps, "directions": directions,
"patch_names": patch_names, "antennas": radio_array.get_antenna_locs(),
"antenna_labels": radio_array.get_antenna_labels(), 'freqs': freqs}
#now construct data
#Direction1: scalarphase000(dir1) + tec000(dir1) + clock000+ tec000+ phaseoffset000
phase = np.zeros([Na,Nt,Nd,Nf],dtype=float)
freqs_inv = 1./freqs
# for i in range(Na):
# for j in range(Nt):
# for k in range(Nd):
#di part
print("getting phases")
if not dd_only:
#phaseoffset_di
phase += np.einsum('ij,l,k->ijkl',
f_di['/sol000/phasesoffset000/val'][0,0,:,0,:],
np.ones(Nf,dtype=float),
np.ones(Nd,dtype=float))
#tec_di
phase += tec_conversion*np.einsum('ji,k,l->ijkl',
f_di['/sol000/tec000/val'][:,:],
np.ones(Nd,dtype=float),
freqs_inv)
#clock_di
phase += (2*np.pi)*np.einsum('ji,k,l->ijkl',
f_di['/sol000/clock000/val'][:,:],
np.ones(Nd,dtype=float),
freqs)
clock = f_di['/sol000/clock000/val'][:,:].T#seconds
#dd part
#scalarphase_dd
phase += np.einsum("jik,l->ijkl",
f_dd['/sol000/scalarphase000/val'][:,:,:,0],
np.ones(Nf,dtype=float))
#tec_dd
phase += tec_conversion*np.einsum("jik,l->ijkl",
f_dd['/sol000/tec000/val'][:,:,:,0],
freqs_inv)
#Na
const = np.mean(np.mean(np.expand_dims(f_di['/sol000/phasesoffset000/val'][0,0,:,0,:], -1) + np.transpose(f_dd['/sol000/scalarphase000/val'][:,:,:,0],[1,0,2]),axis=2),axis=1)#radians
#uncertainty estimates
print("getting uncertainty")
try:
uncert = np.einsum("jik,l->ijkl",
f_dd['/sol000/error000/val'][:,:,:,0],
np.ones(Nf))
except:
#30,500,62,42
slow_gain_phases = np.mean(f_sg['/sol000/phase000/val'][:,:,:,:,:],axis=4)
t_sg = (np.arange(slow_gain_phases.shape[0]) + 0.5)*(times[-1].gps - times[0].gps) + times[0].gps
t_interp = times.gps
v0 = np.var(phase_unwrapp1d(slow_gain_phases,axis=0),axis=0,keepdims=True)
v1 = np.var(np.transpose(phase_unwrapp1d(np.transpose(slow_gain_phases,axes=[1,0,2,3]),axis=0),axes=[1,0,2,3]),axis=1,keepdims=True)
std = np.transpose(np.sqrt((v0 + v1)),axes=[2,0,3,1])
uncert = np.zeros([Na,Nt,Nd,Nf])
i = np.searchsorted(t_sg,t_interp) - 1
start = i
end = i + 1
uncert = (std[:,start,:,:] * ((t_interp - t_sg[start])[None,:,None,None]) + std[:,end,:,:] * ((t_sg[end] - t_interp)[None,:,None,None]))/((t_sg[end] - t_sg[start])[None,:,None,None])
# for j in range(Nt):
# start = i[j]
# end = i[j] + 1
# uncert[:,j,:,:] = (std[:,start,:,:] * (t_interp[j] - t_sg[start]) + std[:,end,:,:] * (t_sg[end] - t_interp[j]))/(t_sg[end] - t_sg[start])
# uncert[:,j*120:min((j+1)*120,Nt),:,:] = std[:,j:j+1,:,:]
#ijkl
#phase = phaseoffset_di + tec_di + clock_di + scalarphase_dd + tec_dd
#fill out some of the values
variance = uncert**2
#prop = tec_dd#+ scalarphase_dd#tec_di + tec_dd#radians
f_dd.close()
f_di.close()
f_sg.close()
data_dict.update({'phase':phase, 'clock':clock, 'const':const, 'variance':variance})
print("creating datapack")
datapack = DataPack(data_dict)
datapack.set_reference_antenna(antenna_labels[0])
print("saving")
datapack.save(datapack_file)
print("closing files")
return datapack
def error_map(phase):
v0 = np.expand_dims(np.var(phase_unwrapp1d(phase,axis=0),axis=0),0)
v1 = np.expand_dims(np.var(phase_unwrapp1d(phase.T,axis=0).T,axis=1),-1)
std = np.sqrt((v0 + v1))
return std
fixtime = times[0] fixfreq = freqs[Nf>>1] phase_center = datapack.get_center_direction() array_center = datapack.radio_array.get_center() uvw = [Pointing(location = array_center.earth_location,obstime = times[j],fixtime=fixtime, phase = phase_center) for j in range(1)] ants_uvw = [antennas.transform_to(uvw[j]) for j in range(1)] dirs_uvw = [directions.transform_to(uvw[j]) for j in range(1)] # In[13]: phase=np.angle(np.exp(1j*datapack.get_phase(ant_idx=ant_idx,time_idx=time_idx,dir_idx=dir_idx,freq_idx=freq_idx))) #plt.plot(phase[0,:,:,0]) from rathings.phase_unwrap import phase_unwrapp1d phase = np.transpose(phase_unwrapp1d(np.transpose(phase,axes=[1,0,2,3]),axis=0),axes=[1,0,2,3]) phase = np.transpose(phase_unwrapp1d(np.transpose(phase,axes=[3,1,2,0]),axis=0),axes=[3,1,2,0]) plt.plot(phase[51,:,:,0]) plt.show() phi = phase[:,:,:,:] phi_wrap=phi X = np.array([dirs_uvw[0].u.value,dirs_uvw[0].v.value]).T # In[ ]: # In[21]:
times, timestamps = datapack.get_times(time_idx=time_idx)
directions, patch_names = datapack.get_directions(dir_idx=-1)
freqs = datapack.get_freqs(freq_idx=freq_idx)
phase = datapack.get_phase(ant_idx=-1,
time_idx=time_idx,
dir_idx=-1,
freq_idx=freq_idx)
Na = len(antennas)
Nt = len(times)
Nd = len(directions)
Nf = len(freqs)
for i in range(Na):
for k in range(Nd):
for l in range(Nf):
phase[i, :, k, l] = phase_unwrapp1d(phase[i, :, k, l])
print("Size: {} GB".format(Na * Nt * Nd * Nf * 8 * 8 / 1024 / 1024))
output = h5py.File("datapack_vanWeeren_partial_v1.hdf5", 'w')
output['/data/rays_uvw'] = np.zeros((Na, Nt, Nd, Nf, 8), dtype=float)
output['/data/phase'] = phase
dt = h5py.special_dtype(vlen=str)
antenna_labels_ = output.create_dataset("/data/antenna_labels", (Na, ),
dtype=dt)
timestamps_ = output.create_dataset("/data/timestamps", (Nt, ), dtype=dt)
patch_names_ = output.create_dataset("/data/patch_names", (Nd, ), dtype=dt)
antenna_labels_[:] = antenna_labels
timestamps_[:] = timestamps