def test_mock_data(self):
# load into pyuvdata object
data_file = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA")
data, flgs, ap, a, f, t, l, p = hc.io.load_vis(data_file,
return_meta=True)
wgts = odict()
for k in flgs.keys():
wgts[k] = (~flgs[k]).astype(np.float)
wgts = hc.datacontainer.DataContainer(wgts)
# make mock data
dly_slope = np.array([-1e-9, 2e-9, 0])
model = odict()
for i, k in enumerate(data.keys()):
bl = np.around(ap[k[0]] - ap[k[1]], 0)
model[k] = data[k] * np.exp(2j * np.pi * f * np.dot(dly_slope, bl))
model = DataContainer(model)
# setup AbsCal
AC = hc.abscal.AbsCal(model, data, antpos=ap, wgts=wgts, freqs=f)
# run delay_slope_cal
AC.delay_slope_lincal(time_avg=True, verbose=False)
# test recovery: accuracy only checked at 10% level
nt.assert_almost_equal(AC.dly_slope_arr[0, 0, 0, 0, 0],
1e-9,
delta=1e-10)
nt.assert_almost_equal(AC.dly_slope_arr[0, 1, 0, 0, 0],
-2e-9,
delta=1e-10)
# make mock data
abs_gain = 0.02
TT_phi = np.array([1e-3, -1e-3, 0])
model = odict()
for i, k in enumerate(data.keys()):
bl = np.around(ap[k[0]] - ap[k[1]], 0)
model[k] = data[k] * np.exp(abs_gain + 1j * np.dot(TT_phi, bl))
model = DataContainer(model)
# setup AbsCal
AC = hc.abscal.AbsCal(model, data, antpos=ap, wgts=wgts, freqs=f)
# run abs_amp cal
AC.abs_amp_logcal(verbose=False)
# run TT_phs_logcal
AC.TT_phs_logcal(verbose=False)
nt.assert_almost_equal(np.median(
AC.abs_eta_arr[0, :, :,
0][AC.wgts[(24, 25, 'xx')].astype(np.bool)]),
-0.01,
delta=1e-3)
nt.assert_almost_equal(np.median(
AC.TT_Phi_arr[0, 0, :, :,
0][AC.wgts[(24, 25, 'xx')].astype(np.bool)]),
-1e-3,
delta=1e-4)
nt.assert_almost_equal(np.median(
AC.TT_Phi_arr[0, 1, :, :,
0][AC.wgts[(24, 25, 'xx')].astype(np.bool)]),
1e-3,
delta=1e-4)
def fourier_interp(data,
flags,
kernel_width=10,
kernel='tophat',
axis=1,
stop_tol=1e-2,
maxiter=5,
copy_vis=True):
"""
Fourier filtering + interpolation for flagged data
Parameters:
-----------
data : type=DataContainer, holding complex visibility data
flags : type=DataContainer, holding boolean flag arrays for data
Keyword parameters passed to fourier_filter()
Output: (interp_data, model)
-------
interp_data : type=DataContainer, holding complex visibility data with
flags filled in with model
model : type=DataContainer, holding smoothed visibility model
"""
# setup output data structures
interp_data = odict()
model = odict()
# iterate over keys
for k in data.keys():
# perform fourier fit
d_int, mdl = fourier_filter(data[k],
flags[k],
kernel_width=kernel_width,
kernel=kernel,
axis=axis,
stop_tol=stop_tol,
maxiter=maxiter,
copy_vis=copy_vis)
# insert into structures
interp_data[k] = d_int
model[k] = mdl
interp_data = DataContainer(interp_data)
model = DataContainer(model)
return interp_data, model
def load_npz_flags(npzfile):
'''Load flags from a npz file (like those produced by hera_qm.xrfi) and converts
them into a DataContainer. More than one spectral window is not supported. Assumes
every baseline has the same times present and that the times are in order.
Arguments:
npzfile: path to .npz file containing flags and array metadata
Returns:
flags: Dictionary of boolean flags as a function of time and
frequency with keys in the (1,'x') format
'''
npz = np.load(npzfile)
pols = [polnum2str[p] for p in npz['polarization_array']]
nTimes = len(np.unique(npz['time_array']))
nAntpairs = len(npz['antpairs'])
nFreqs = npz['flag_array'].shape[2]
assert npz['flag_array'].shape[0] == nAntpairs * nTimes, \
'flag_array must have flags for all baselines for all times.'
flags = {}
for p, pol in enumerate(pols):
flag_array = np.reshape(npz['flag_array'][:, 0, :, p],
(nTimes, nAntpairs, nFreqs))
for n, (i, j) in enumerate(npz['antpairs']):
flags[i, j, pol] = flag_array[:, n, :]
return DataContainer(flags)
def test_match_red_baselines(self):
model = copy.deepcopy(self.data)
model = DataContainer(
odict([((k[0] + 1, k[1] + 1, k[2]), model[k])
for i, k in enumerate(model.keys())]))
del model[(25, 54, 'xx')]
model_antpos = odict([(k + 1, self.antpos[k])
for i, k in enumerate(self.antpos.keys())])
new_model = hc.abscal.match_red_baselines(model,
model_antpos,
self.data,
self.antpos,
tol=2.0,
verbose=False)
nt.assert_equal(len(new_model.keys()), 8)
nt.assert_true((24, 37, 'xx') in new_model)
nt.assert_false((24, 53, 'xx') in new_model)
def compute_ubls(self, data, gain_sols):
"""Given a set of guess gain solutions, return a dictionary of calibrated visbilities
averged over a redundant group. Not strictly necessary for typical operation."""
dc = DataContainer(data)
ubl_sols = {}
for ubl, blgrp in enumerate(self.reds):
d_gp = [dc[bl] for bl in blgrp]
ubl_sols[blgrp[0]] = np.average(
d_gp, axis=0) # XXX add option for median here?
return ubl_sols
def test_lstbin(self):
dlst = 0.0007830490163484
# test basic execution
output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=self.flgs_list, dlst=None,
median=True, lst_low=0, lst_hi=np.pi, verbose=False)
output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=None, dlst=0.01,
verbose=False)
output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=self.flgs_list, dlst=dlst,
verbose=False)
# check shape and dtype
nt.assert_equal(output[1][(24,25,'xx')].dtype, np.complex)
nt.assert_equal(output[1][(24,25,'xx')].shape, (224, 64))
# check number of points in each bin
nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[0,30], 1)
nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[30,30], 2)
nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[100,30], 3)
nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[190,30], 2)
nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[220,30], 1)
# check with large spacing lst_grid
output = hc.lstbin.lst_bin(self.data_list, self.lst_list, dlst=.01, verbose=False)
nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[10,30], 39)
# check flgs are propagated
flgs1 = copy.deepcopy(self.flgs1)
flgs1[(24, 25, 'xx')][:, 32] = True
flgs2 = copy.deepcopy(self.flgs2)
flgs2[(24, 25, 'xx')][:, 32] = True
flgs3 = copy.deepcopy(self.flgs3)
flgs_list = [flgs1, flgs2, flgs3]
output = hc.lstbin.lst_bin(self.data_list, self.lst_list, dlst=dlst, flags_list=flgs_list)
nt.assert_almost_equal(output[2][(24, 25, 'xx')][0, 32], True)
nt.assert_almost_equal(output[2][(24, 25, 'xx')][180, 32], False)
nt.assert_almost_equal(output[2][(24, 25, 'xx')][210, 32], False)
# test return no avg
output = hc.lstbin.lst_bin(self.data_list, self.lst_list, dlst=dlst, flags_list=self.flgs_list, return_no_avg=True)
nt.assert_equal(len(output[2][output[2].keys()[0]][100]), 3)
nt.assert_equal(len(output[2][output[2].keys()[0]][100][0]), 64)
# test switch bl
conj_data3 = DataContainer(odict(map(lambda k: (hc.lstbin.switch_bl(k), np.conj(self.data3[k])), self.data3.keys())))
data_list = [self.data1, self.data2, conj_data3]
output = hc.lstbin.lst_bin(data_list, self.lst_list, dlst=dlst)
nt.assert_equal(output[1][(24,25,'xx')].shape, (224, 64))
# test sigma clip
output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=None, dlst=0.01,
verbose=False, sig_clip=True, min_N=5, sigma=2)
# test wrapping
lst_list = map(lambda l: (copy.deepcopy(l) + 6) % (2*np.pi), self.lst_list)
output = hc.lstbin.lst_bin(self.data_list, lst_list, dlst=0.001, lst_start=np.pi)
nt.assert_true(output[0][0] > output[0][-1])
nt.assert_equal(len(output[0]), 175)
# test appropriate data_count
output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=None, dlst=dlst, lst_low=0.25, lst_hi=0.3,
verbose=False)
nt.assert_true(np.isclose(output[4][(24, 25, 'xx')], 3.0).all())
def _solver(self,
solver,
data,
wgts={},
detrend_phs=False,
sparse=False,
**kwargs):
"""Instantiates a linsolve solver for performing redcal.
Args:
solver: linsolve solver (e.g. linsolve.LogProductSolver or linsolve.LinProductSolver)
data: visibility data in the dictionary format {(ant1,ant2,pol): np.array}
wgts: dictionary of linear weights in the same format as data. Defaults to equal wgts.
detrend_phs: takes out average phase, useful for logcal
sparse: represent the A matrix (visibilities to parameters) sparsely in linsolve
**kwargs: other keyword arguments passed into the solver for use by linsolve
Returns:
solver: instantiated solver with redcal equations and weights
"""
dc = DataContainer(data)
eqs = self.build_eqs(dc.keys())
self.phs_avg = {
} # detrend phases within redundant group, used for logcal to avoid phase wraps
if detrend_phs:
for blgrp in self.reds:
self.phs_avg[blgrp[0]] = np.exp(-1j * np.median(np.unwrap(
[np.log(dc[bl]).imag for bl in blgrp], axis=0),
axis=0))
for bl in blgrp:
self.phs_avg[bl] = self.phs_avg[blgrp[0]]
d_ls, w_ls = {}, {}
for eq, key in eqs.items():
d_ls[eq] = dc[key] * self.phs_avg.get(key, 1)
if len(wgts) > 0:
wc = DataContainer(wgts)
for eq, key in eqs.items():
w_ls[eq] = wc[key]
return solver(data=d_ls, wgts=w_ls, sparse=sparse, **kwargs)
def test_mirror_data_to_red_bls(self):
# make fake data
reds = hc.redcal.get_reds(self.antpos, pols=['xx'])
data = DataContainer(
odict(map(lambda k: (k[0], self.data[k[0]]), reds[:5])))
# test execuation
d = hc.abscal.mirror_data_to_red_bls(data, self.antpos)
nt.assert_equal(len(d.keys()), 16)
nt.assert_true((24, 25, 'xx') in d)
# test correct value is propagated
nt.assert_almost_equal(data[(24, 25, 'xx')][30, 30],
d[(38, 39, 'xx')][30, 30])
# test reweighting
w = hc.abscal.mirror_data_to_red_bls(self.wgts,
self.antpos,
weights=True)
nt.assert_equal(w[(24, 25, 'xx')].dtype, np.float)
nt.assert_almost_equal(w[(24, 25, 'xx')].max(), 16.0)
def lst_bin(data_list,
lst_list,
flags_list=None,
dlst=None,
lst_start=None,
lst_low=None,
lst_hi=None,
flag_thresh=0.7,
atol=1e-10,
median=False,
truncate_empty=True,
sig_clip=False,
sigma=4.0,
min_N=4,
return_no_avg=False,
antpos=None,
rephase=False,
freq_array=None,
lat=-30.72152,
verbose=True):
"""
Bin data in Local Sidereal Time (LST) onto an LST grid. An LST grid
is defined as an array of points increasing in Local Sidereal Time, with each point marking
the center of the LST bin.
Parameters:
-----------
data_list : type=list, list of DataContainer dictionaries holding complex visibility data
lst_list : type=list, list of ndarrays holding LST bin centers of each data dictionary in data_list.
These LST arrays must be monotonically increasing, except for a possible wrap at 2pi.
flags_list : type=list, list of DataContainer dictionaries holding flags for each data dict
in data_list. Flagged data do not contribute to the average of an LST bin.
dlst : type=float, delta-LST spacing for lst_grid. If None, will use the delta-LST of the first
array in lst_list.
lst_start : type=float, starting LST for making the lst_grid, extending from
[lst_start, lst_start+2pi). Default is lst_start = 0 radians.
lst_low : type=float, lower bound on LST bin centers used for contructing LST grid
lst_hi : type=float, upper bound on LST bin centers used for contructing LST grid
flag_thresh : type=float, minimum fraction of flagged points in an LST bin needed to
flag the entire bin.
atol : type=float, absolute tolerance for comparing LST bin center floats
median : type=boolean, if True use median for LST binning. Warning: this is slower.
truncate_empty : type=boolean, if True, truncate output time integrations that have no data
in them.
sig_clip : type=boolean, if True, perform a sigma clipping algorithm of the LST bins on the
real and imag components separately. Warning: This is considerably slow.
sigma : type=float, input sigma threshold to use for sigma clipping algorithm.
min_N : type=int, minimum number of points in an LST bin to perform sigma clipping
return_no_avg : type=boolean, if True, return binned but un-averaged data and flags.
rephase : type=bool, if True, phase data to center of LST bin before binning.
Note that this produces a copy of the data.
antpos : type=dictionary, holds antenna position vectors in ENU frame in meters with
antenna integers as keys and 3D ndarrays as values. See io.load_vis(). Needed for rephase.
freq_array : type=ndarray, 1D array of unique data frequencies channels in Hz. Needed for rephase.
lat : type=float, latitude of array in degrees North. Needed for rephase.
verbose : type=bool, if True report feedback to stdout
Output: (lst_bins, data_avg, flags_min, data_std, data_count)
-------
lst_bins : ndarray containing final lst grid of data (marks bin centers)
data_avg : dictionary of data having averaged in each LST bin
flags_min : dictionary of minimum of data flags in each LST bin
data_std : dictionary of data with real component holding LST bin std along real axis
and imag component holding std along imag axis
data_count : dictionary containing the number count of data points averaged in each LST bin.
if return_no_avg:
Output: (lst_bins, data_bin, flags_min)
data_bin : dictionary with (ant1,ant2,pol) as keys and ndarrays holding
un-averaged complex visibilities in each LST bin as values.
flags_min : dictionary with data flags
"""
# get visibility shape
Ntimes, Nfreqs = data_list[0][data_list[0].keys()[0]].shape
# get dlst if not provided
if dlst is None:
dlst = np.median(np.diff(lst_list[0]))
# construct lst_grid
lst_grid = make_lst_grid(dlst, lst_start=lst_start, verbose=verbose)
dlst = np.median(np.diff(lst_grid))
# test for special case of lst grid restriction
if lst_low is not None and lst_hi is not None and lst_hi < lst_low:
lst_grid = lst_grid[(lst_grid > (lst_low - atol)) | (lst_grid <
(lst_hi + atol))]
else:
# restrict lst_grid based on lst_low and lst_high
if lst_low is not None:
lst_grid = lst_grid[lst_grid > (lst_low - atol)]
if lst_hi is not None:
lst_grid = lst_grid[lst_grid < (lst_hi + atol)]
# Raise Exception if lst_grid is empty
if len(lst_grid) == 0:
raise ValueError(
"len(lst_grid) == 0; consider changing lst_low and/or lst_hi.")
# move lst_grid centers to the left
lst_grid_left = lst_grid - dlst / 2
# form new dictionaries
# data is a dictionary that will hold other dictionaries as values, which will
# themselves hold lists of ndarrays
data = odict()
flags = odict()
all_lst_indices = set()
# iterate over data_list
for i, d in enumerate(data_list):
# get lst array
l = copy.copy(lst_list[i])
# ensure l isn't wrapped relative to lst_grid
l[l < lst_grid_left.min() - atol] += 2 * np.pi
# digitize data lst array "l"
grid_indices = np.digitize(l, lst_grid_left[1:], right=True)
# make data_in_bin boolean array, and set to False data that don't fall in any bin
data_in_bin = np.ones_like(l, np.bool)
data_in_bin[(l < lst_grid_left.min() - atol)] = False
data_in_bin[(l > lst_grid_left.max() + dlst + atol)] = False
# update all_lst_indices
all_lst_indices.update(set(grid_indices[data_in_bin]))
if rephase:
# rephase each integration in d to nearest LST bin
if freq_array is None or antpos is None:
raise ValueError("freq_array and antpos is needed for rephase")
# form baseline dictionary
bls = odict(
map(lambda k: (k, antpos[k[0]] - antpos[k[1]]), d.keys()))
# get appropriate lst_shift for each integration, then rephase
lst_shift = lst_grid[grid_indices] - l
# this makes a copy of the data in d
d = utils.lst_rephase(d,
bls,
freq_array,
lst_shift,
lat=lat,
inplace=False)
# iterate over keys in d
for j, key in enumerate(d.keys()):
# data[key] will be an odict. if data[key] doesn't exist
# create data[key] as an empty odict. if data[key] already
# exists, then pass
if key in data:
pass
elif switch_bl(key) in data:
# check to see if conj(key) exists in data
key = switch_bl(key)
d[key] = np.conj(d[switch_bl(key)])
if flags_list is not None:
flags_list[i][key] = flags_list[i][switch_bl(key)]
else:
# if key or conj(key) not in data, insert key into data
data[key] = odict()
flags[key] = odict()
# data[key] is an odict, with keys as grid index integers and
# values as lists holding the LST bin data: ndarrays of shape (Nfreqs)
# iterate over grid_indices, and append to data if data_in_bin is True
for k, ind in enumerate(grid_indices):
# ensure data_in_bin is True for this grid index
if data_in_bin[k]:
# if index not in data[key], insert it as empty list
if ind not in data[key]:
data[key][ind] = []
flags[key][ind] = []
# append data ndarray to LST bin
data[key][ind].append(d[key][k])
# also insert flags if fed
if flags_list is None:
flags[key][ind].append(
np.zeros_like(d[key][k], np.bool))
else:
flags[key][ind].append(flags_list[i][key][k])
# get final lst_bin array
if truncate_empty:
# use only lst_grid bins that have data in them
lst_bins = lst_grid[sorted(all_lst_indices)]
else:
# keep all lst_grid bins and fill empty ones with unity data and mark as flagged
for index in range(len(lst_grid)):
if index in all_lst_indices:
# skip if index already in data
continue
for key in data.keys():
# fill data with blank data
data[key][index] = [np.ones(Nfreqs, np.complex)]
flags[key][index] = [np.ones(Nfreqs, np.bool)]
# use all LST bins
lst_bins = lst_grid
# wrap lst_bins if needed
lst_bins = lst_bins % (2 * np.pi)
# make final dictionaries
flags_min = odict()
data_avg = odict()
data_count = odict()
data_std = odict()
# return un-averaged data if desired
if return_no_avg:
# return all binned data instead of just bin average
data_bin = odict(
map(
lambda k: (k,
np.array(
odict(
map(lambda k2: (k2, data[k][k2]),
sorted(data[k].keys()))).values())),
sorted(data.keys())))
flags_bin = odict(
map(
lambda k: (k,
np.array(
odict(
map(lambda k2: (k2, flags[k][k2]),
sorted(flags[k].keys()))).values())),
sorted(flags.keys())))
return lst_bins, data_bin, flags_bin
# iterate over data keys (baselines) and get statistics
for i, key in enumerate(data.keys()):
# create empty lists
real_avg = []
imag_avg = []
f_min = []
real_std = []
imag_std = []
bin_count = []
# iterate over sorted LST grid indices in data[key]
for j, ind in enumerate(sorted(data[key].keys())):
# make data and flag arrays from lists
d = np.array(data[key][ind]) # shape = (Ndays x Nfreqs)
f = np.array(flags[key][ind])
f[np.isnan(f)] = True
# replace flagged data with nan
d[f] *= np.nan
# sigma clip if desired
if sig_clip:
# clip real
real_f = sigma_clip(d.real, sigma=sigma, min_N=min_N, axis=0)
# clip imag
imag_f = sigma_clip(d.imag, sigma=sigma, min_N=min_N, axis=0)
# get real + imag flags
clip_flags = real_f + imag_f
# flag data
d[clip_flags] *= np.nan
# merge clip flags
f += (real_f + imag_f)
# check flag thresholds
if len(f) == 1:
flag_bin = np.zeros(f.shape[1], np.bool)
else:
flag_bin = np.sum(f, axis=0).astype(
np.float) / len(f) > flag_thresh
d[:, flag_bin] *= np.nan
f[:, flag_bin] = True
# take bin average
if median:
real_avg.append(np.nanmedian(d.real, axis=0))
imag_avg.append(np.nanmedian(d.imag, axis=0))
else:
real_avg.append(np.nanmean(d.real, axis=0))
imag_avg.append(np.nanmean(d.imag, axis=0))
# get minimum bin flag
f_min.append(np.nanmin(f, axis=0))
# get other stats
real_std.append(np.nanstd(d.real, axis=0))
imag_std.append(np.nanstd(d.imag, axis=0))
bin_count.append(np.nansum(~np.isnan(d), axis=0))
# get final statistics
d_avg = np.array(real_avg) + 1j * np.array(imag_avg)
f_min = np.array(f_min)
d_std = np.array(real_std) + 1j * np.array(imag_std)
d_num = np.array(bin_count).astype(np.float)
# fill nans
d_nan = np.isnan(d_avg)
d_avg[d_nan] = 1.0
f_min[d_nan] = True
d_std[d_nan] = 1.0
d_num[d_nan] = 0.0
# insert into dictionaries
data_avg[key] = d_avg
flags_min[key] = f_min
data_std[key] = d_std
data_count[key] = d_num
# turn into DataContainer objects
data_avg = DataContainer(data_avg)
flags_min = DataContainer(flags_min)
data_std = DataContainer(data_std)
data_count = DataContainer(data_count)
return lst_bins, data_avg, flags_min, data_std, data_count
def run_filter(self,
to_filter=[],
weight_dict=None,
standoff=15.,
horizon=1.,
min_dly=0.0,
tol=1e-9,
window='blackman-harris',
skip_wgt=0.1,
maxiter=100,
verbose=False,
flag_nchan_low=0,
flag_nchan_high=0,
gain=0.1,
**win_kwargs):
'''Performs uvtools.dspec.Delay_Filter on (a subset of) the data stored in the object.
Uses stored flags unless explicitly overridden with weight_dict.
Arguments:
to_filter: list of visibilities to filter in the (i,j,pol) format.
If [] (the default), all visibilities are filtered.
weight_dict: dictionary or DataContainer with all the same keys as self.data.
Linear multiplicative weights to use for the delay filter. Default, use np.logical_not
of self.flags. uvtools.dspec.delay_filter will renormalize to compensate
standoff: fixed additional delay beyond the horizon (in ns)
horizon: proportionality constant for bl_len where 1 is the horizon (full light travel time)
min_dly: minimum delay used for cleaning [ns]: if bl_len * horizon + standoff < min_dly, use min_dly.
tol: CLEAN algorithm convergence tolerance (see aipy.deconv.clean)
window: window function for filtering applied to the filtered axis.
See aipy.dsp.gen_window for options.
skip_wgt: skips filtering rows with very low total weight (unflagged fraction ~< skip_wgt).
Model is left as 0s, residual is left as data, and info is {'skipped': True} for that
time. Skipped channels are then flagged in self.flags.
Only works properly when all weights are all between 0 and 1.
maxiter: Maximum number of iterations for aipy.deconv.clean to converge.
verbose: If True print feedback to stdout
flag_nchan_low: Integer number of channels to flag on lower band edge before filtering
flag_nchan_low: Integer number of channels to flag on upper band edge before filtering
gain: The fraction of a residual used in each iteration. If this is too low, clean takes
unnecessarily long. If it is too high, clean does a poor job of deconvolving.
win_kwargs : keyword arguments to feed aipy.dsp.gen_window()
Results are stored in:
self.filtered_residuals: DataContainer formatted like self.data with only high-delay components
self.CLEAN_models: DataContainer formatted like self.data with only low-delay components
self.info: Dictionary of info from uvtools.dspec.delay_filter with the same keys as self.data
'''
self.filtered_residuals = deepcopy(self.data)
self.CLEAN_models = DataContainer({
k: np.zeros_like(self.data.values()[0])
for k in self.data.keys()
})
self.info = odict()
if to_filter == []:
to_filter = self.data.keys()
for k in to_filter:
if verbose:
print "\nStarting filter on {} at {}".format(
k, str(datetime.datetime.now()))
bl_len = np.linalg.norm(self.antpos[k[0]] - self.antpos[k[1]]
) / constants.c * 1e9 #in ns
sdf = np.median(np.diff(self.freqs)) / 1e9 #in GHz
if weight_dict is not None:
wgts = weight_dict[k]
else:
wgts = np.logical_not(self.flags[k])
# edge flag
if flag_nchan_low > 0:
self.flags[k][:, :flag_nchan_low] = True
wgts[:, :flag_nchan_low] = 0.0
if flag_nchan_high > 0:
self.flags[k][:, -flag_nchan_high:] = True
wgts[:, -flag_nchan_high:] = 0.0
d_mdl, d_res, info = delay_filter(self.data[k],
wgts,
bl_len,
sdf,
standoff=standoff,
horizon=horizon,
min_dly=min_dly,
tol=tol,
window=window,
skip_wgt=skip_wgt,
maxiter=maxiter,
gain=gain,
**win_kwargs)
self.filtered_residuals[k] = d_res
self.CLEAN_models[k] = d_mdl
self.info[k] = info
# Flag all channels for any time when skip_wgt gets triggered
for i, info_dict in enumerate(info):
if info_dict.get('skipped', False):
self.flags[k][i, :] = np.ones_like(self.flags[k][i, :])
def recalibrate_in_place(data,
data_flags,
new_gains,
cal_flags,
old_gains=None,
gain_convention='divide'):
'''Update data and data_flags in place, taking out old calibration solutions, putting in
new calibration solutions, and updating flags from those calibration solutions. Previously
flagged data is modified, but left flagged. Missing antennas from either the new gains or (if it's not None),
the old gains are automatically flagged in the data's visibilities that involves those antennas.
Arguments:
data: DataContainer containing baseline-pol complex visibility data. This is modified in place.
data_flags: DataContainer containing data flags. This is modified in place. Can also be fed as a
data weights dictionary with float dtype. In this case, wgts of 0 are treated as flagged
data and non-zero wgts are unflagged data.
new_gains: Dictionary of complex calibration gains to apply with keys like (1,'x')
cal_flags: Dictionary with keys like (1,'x') of per-antenna boolean flags to update data_flags
if either antenna in a visibility is flagged.
old_gains: Dictionary of complex calibration gains to take out with keys like (1,'x').
Default of None implies that the data is raw (i.e. uncalibrated).
gain_convention: str, either 'divide' or 'multiply'. 'divide' means V_obs = gi gj* V_true,
'multiply' means V_true = gi gj* V_obs. Assumed to be the same for new_gains and old_gains.
'''
# get datatype of data_flags to determine if flags or wgts
if np.all([(df.dtype == np.bool) for df in data_flags.values()]):
bool_flags = True
elif np.all([(df.dtype == np.float) for df in data_flags.values()]):
bool_flags = False
wgts = data_flags
data_flags = DataContainer(
dict(map(lambda k: (k, ~wgts[k].astype(np.bool)), wgts.keys())))
else:
raise ValueError("didn't recognize dtype of data_flags")
# loop over keys
for (i, j, pol) in data.keys():
# Check to see that all necessary antennas are present in the gains
if new_gains.has_key((i, pol[0])) and new_gains.has_key(
(j, pol[1])) and (old_gains == None or (old_gains.has_key(
(i, pol[0])) and old_gains.has_key((j, pol[1])))):
gigj_new = new_gains[(i, pol[0])] * np.conj(new_gains[(j, pol[1])])
if old_gains is not None:
gigj_old = old_gains[(i, pol[0])] * np.conj(
old_gains[(j, pol[1])])
else:
gigj_old = np.ones_like(gigj_new)
# update all the data, even if it was flagged
if gain_convention == 'divide':
data[(i, j, pol)] *= (gigj_old / gigj_new)
elif gain_convention == 'multiply':
data[(i, j, pol)] *= (gigj_new / gigj_old)
else:
raise KeyError(
"gain_convention must be either 'divide' or 'multiply'.")
# update data flags
if bool_flags:
# treat as flags
data_flags[(i, j, pol)][cal_flags[(i, pol[0])]] = True
data_flags[(i, j, pol)][cal_flags[(j, pol[1])]] = True
else:
# treat as data weights
wgts[(i, j, pol)][cal_flags[(i, pol[0])]] = 0.0
wgts[(i, j, pol)][cal_flags[(j, pol[1])]] = 0.0
else:
# If any antenna is missing from the gains, the data is flagged
if bool_flags:
data_flags[(i, j, pol)] = np.ones_like(data_flags[(i, j, pol)],
dtype=np.bool)
else:
wgts[(i, j, pol)] = np.zeros_like(wgts[(i, j, pol)],
dtype=np.float)
def load_vis(input_data,
return_meta=False,
filetype='miriad',
pop_autos=False,
pick_data_ants=True,
nested_dict=False):
'''Load miriad or uvfits files or UVData objects into DataContainers, optionally returning
the most useful metadata. More than one spectral window is not supported. Assumes every baseline
has the same times present and that the times are in order.
Arguments:
input_data: data file path, or UVData instance, or list of either strings of data file paths
or list of UVData instances to concatenate into a single dictionary
return_meta: boolean, if True: also return antpos, ants, freqs, times, lsts, and pols
filetype: either 'miriad' or 'uvfits', can be ignored if input_data is UVData objects
pop_autos: boolean, if True: remove autocorrelations
pick_data_ants: boolean, if True and return_meta=True, return only antennas in data
nested_dict: boolean, if True replace DataContainers with the legacy nested dictionary filetype
where visibilities and flags are accessed as data[(0,1)]['xx']
Returns:
if return_meta is True:
(data, flags, antpos, ants, freqs, times, lsts, pols)
else:
(data, flags)
data: DataContainer containing baseline-pol complex visibility data with keys
like (0,1,'xx') and with shape=(Ntimes,Nfreqs)
flags: DataContainer containing data flags
antpos: dictionary containing antennas numbers as keys and position vectors
ants: ndarray containing unique antenna indices
freqs: ndarray containing frequency channels (Hz)
times: ndarray containing julian date bins of data
lsts: ndarray containing LST bins of data (radians)
pol: ndarray containing list of polarization strings
'''
uvd = UVData()
if isinstance(input_data, (tuple, list, np.ndarray)): #List loading
if np.all([isinstance(id, str)
for id in input_data]): #List of visibility data paths
if filetype == 'miriad':
uvd.read_miriad(list(input_data))
elif filetype == 'uvfits':
#TODO: implement this
raise NotImplementedError(
'This function has not been implemented yet.')
else:
raise NotImplementedError(
"Data filetype must be either 'miriad' or 'uvfits'.")
elif np.all([isinstance(id, UVData)
for id in input_data]): #List of uvdata objects
uvd = reduce(operator.add, input_data)
else:
raise TypeError(
'If input is a list, it must be only strings or only UVData objects.'
)
elif isinstance(input_data, str): #single visibility data path
if filetype == 'miriad':
uvd.read_miriad(input_data)
elif filetype == 'uvfits':
#TODO: implement this
raise NotImplementedError(
'This function has not been implemented yet.')
else:
raise NotImplementedError(
"Data filetype must be either 'miriad' or 'uvfits'.")
elif isinstance(input_data, UVData): #single UVData object
uvd = input_data
else:
raise TypeError(
'Input must be a UVData object, a string, or a list of either.')
data, flags = odict(), odict()
# create nested dictionaries of visibilities in the data[bl][pol] filetype, removing autos if desired
for nbl, (i, j) in enumerate(uvd.get_antpairs()):
if (not pop_autos) or (i != j):
if (i, j) not in data:
data[i, j], flags[i, j] = odict(), odict()
for ip, pol in enumerate(uvd.polarization_array):
pol = polnum2str[pol]
data[(i, j)][pol] = copy.deepcopy(uvd.get_data((i, j, pol)))
flags[(i, j)][pol] = copy.deepcopy(uvd.get_flags((i, j, pol)))
# If we don't want nested dicts, convert to DataContainer
if not nested_dict:
data, flags = DataContainer(data), DataContainer(flags)
# get meta
if return_meta:
freqs = np.unique(uvd.freq_array)
times = np.unique(uvd.time_array)
lsts = []
for l in uvd.lst_array.ravel():
if l not in lsts:
lsts.append(l)
lsts = np.array(lsts)
antpos, ants = uvd.get_ENU_antpos(center=True,
pick_data_ants=pick_data_ants)
antpos = odict(zip(ants, antpos))
pols = np.array(
[polnum2str[polnum] for polnum in uvd.polarization_array])
del uvd
garbage_collector.collect()
return data, flags, antpos, ants, freqs, times, lsts, pols
else:
del uvd
garbage_collector.collect()
return data, flags
def test_recalibrate_in_place(self):
np.random.seed(21)
vis = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10)
dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)})
f = np.random.randn(10, 10) > 0
flags = DataContainer({(0, 1, 'xx'): deepcopy(f)})
g0_new = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10)
g1_new = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10)
g_new = {(0, 'x'): g0_new, (1, 'x'): g1_new}
g0_old = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10)
g1_old = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10)
g_old = {(0, 'x'): g0_old, (1, 'x'): g1_old}
cal_flags = {
(0, 'x'): np.random.randn(10, 10) > 0,
(1, 'x'): np.random.randn(10, 10) > 0
}
# test standard operation
ac.recalibrate_in_place(dc,
flags,
g_new,
cal_flags,
old_gains=g_old,
gain_convention='divide')
for i in range(10):
for j in range(10):
self.assertAlmostEqual(
dc[(0, 1, 'xx')][i, j],
vis[i, j] * g0_old[i, j] * np.conj(g1_old[i, j]) /
g0_new[i, j] / np.conj(g1_new[i, j]))
if f[i, j] or cal_flags[(0, 'x')][i,
j] or cal_flags[(1, 'x')][i,
j]:
self.assertTrue(flags[(0, 1, 'xx')][i, j])
else:
self.assertFalse(flags[(0, 1, 'xx')][i, j])
# test without old cal
dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)})
flags = DataContainer({(0, 1, 'xx'): deepcopy(f)})
ac.recalibrate_in_place(dc,
flags,
g_new,
cal_flags,
gain_convention='divide')
for i in range(10):
for j in range(10):
self.assertAlmostEqual(
dc[(0, 1, 'xx')][i, j],
vis[i, j] / g0_new[i, j] / np.conj(g1_new[i, j]))
# test multiply
dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)})
flags = DataContainer({(0, 1, 'xx'): deepcopy(f)})
ac.recalibrate_in_place(dc,
flags,
g_new,
cal_flags,
old_gains=g_old,
gain_convention='multiply')
for i in range(10):
for j in range(10):
self.assertAlmostEqual(
dc[(0, 1, 'xx')][i, j],
vis[i, j] / g0_old[i, j] / np.conj(g1_old[i, j]) *
g0_new[i, j] * np.conj(g1_new[i, j]))
# test flag propagation when missing antennas in gains
dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)})
flags = DataContainer({(0, 1, 'xx'): deepcopy(f)})
ac.recalibrate_in_place(dc,
flags, {},
cal_flags,
gain_convention='divide')
np.testing.assert_array_equal(flags[(0, 1, 'xx')], True)
dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)})
flags = DataContainer({(0, 1, 'xx'): deepcopy(f)})
ac.recalibrate_in_place(dc,
flags,
g_new,
cal_flags,
old_gains={},
gain_convention='divide')
np.testing.assert_array_equal(flags[(0, 1, 'xx')], True)
# test error
dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)})
flags = DataContainer({(0, 1, 'xx'): deepcopy(f)})
with self.assertRaises(KeyError):
ac.recalibrate_in_place(dc,
flags,
g_new,
cal_flags,
old_gains=g_old,
gain_convention='blah')
# test w/ data weights
dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)})
flags = DataContainer({(0, 1, 'xx'): deepcopy(f)})
wgts = DataContainer(
dict(map(lambda k: (k, (~flags[k]).astype(np.float)),
flags.keys())))
del g_new[(0, 'x')]
ac.recalibrate_in_place(dc,
wgts,
g_new,
cal_flags,
gain_convention='divide')
self.assertAlmostEqual(wgts[(0, 1, 'xx')].max(), 0.0)