def test_mock_data(self): # load into pyuvdata object data_file = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA") data, flgs, ap, a, f, t, l, p = hc.io.load_vis(data_file, return_meta=True) wgts = odict() for k in flgs.keys(): wgts[k] = (~flgs[k]).astype(np.float) wgts = hc.datacontainer.DataContainer(wgts) # make mock data dly_slope = np.array([-1e-9, 2e-9, 0]) model = odict() for i, k in enumerate(data.keys()): bl = np.around(ap[k[0]] - ap[k[1]], 0) model[k] = data[k] * np.exp(2j * np.pi * f * np.dot(dly_slope, bl)) model = DataContainer(model) # setup AbsCal AC = hc.abscal.AbsCal(model, data, antpos=ap, wgts=wgts, freqs=f) # run delay_slope_cal AC.delay_slope_lincal(time_avg=True, verbose=False) # test recovery: accuracy only checked at 10% level nt.assert_almost_equal(AC.dly_slope_arr[0, 0, 0, 0, 0], 1e-9, delta=1e-10) nt.assert_almost_equal(AC.dly_slope_arr[0, 1, 0, 0, 0], -2e-9, delta=1e-10) # make mock data abs_gain = 0.02 TT_phi = np.array([1e-3, -1e-3, 0]) model = odict() for i, k in enumerate(data.keys()): bl = np.around(ap[k[0]] - ap[k[1]], 0) model[k] = data[k] * np.exp(abs_gain + 1j * np.dot(TT_phi, bl)) model = DataContainer(model) # setup AbsCal AC = hc.abscal.AbsCal(model, data, antpos=ap, wgts=wgts, freqs=f) # run abs_amp cal AC.abs_amp_logcal(verbose=False) # run TT_phs_logcal AC.TT_phs_logcal(verbose=False) nt.assert_almost_equal(np.median( AC.abs_eta_arr[0, :, :, 0][AC.wgts[(24, 25, 'xx')].astype(np.bool)]), -0.01, delta=1e-3) nt.assert_almost_equal(np.median( AC.TT_Phi_arr[0, 0, :, :, 0][AC.wgts[(24, 25, 'xx')].astype(np.bool)]), -1e-3, delta=1e-4) nt.assert_almost_equal(np.median( AC.TT_Phi_arr[0, 1, :, :, 0][AC.wgts[(24, 25, 'xx')].astype(np.bool)]), 1e-3, delta=1e-4)
def fourier_interp(data, flags, kernel_width=10, kernel='tophat', axis=1, stop_tol=1e-2, maxiter=5, copy_vis=True): """ Fourier filtering + interpolation for flagged data Parameters: ----------- data : type=DataContainer, holding complex visibility data flags : type=DataContainer, holding boolean flag arrays for data Keyword parameters passed to fourier_filter() Output: (interp_data, model) ------- interp_data : type=DataContainer, holding complex visibility data with flags filled in with model model : type=DataContainer, holding smoothed visibility model """ # setup output data structures interp_data = odict() model = odict() # iterate over keys for k in data.keys(): # perform fourier fit d_int, mdl = fourier_filter(data[k], flags[k], kernel_width=kernel_width, kernel=kernel, axis=axis, stop_tol=stop_tol, maxiter=maxiter, copy_vis=copy_vis) # insert into structures interp_data[k] = d_int model[k] = mdl interp_data = DataContainer(interp_data) model = DataContainer(model) return interp_data, model
def load_npz_flags(npzfile): '''Load flags from a npz file (like those produced by hera_qm.xrfi) and converts them into a DataContainer. More than one spectral window is not supported. Assumes every baseline has the same times present and that the times are in order. Arguments: npzfile: path to .npz file containing flags and array metadata Returns: flags: Dictionary of boolean flags as a function of time and frequency with keys in the (1,'x') format ''' npz = np.load(npzfile) pols = [polnum2str[p] for p in npz['polarization_array']] nTimes = len(np.unique(npz['time_array'])) nAntpairs = len(npz['antpairs']) nFreqs = npz['flag_array'].shape[2] assert npz['flag_array'].shape[0] == nAntpairs * nTimes, \ 'flag_array must have flags for all baselines for all times.' flags = {} for p, pol in enumerate(pols): flag_array = np.reshape(npz['flag_array'][:, 0, :, p], (nTimes, nAntpairs, nFreqs)) for n, (i, j) in enumerate(npz['antpairs']): flags[i, j, pol] = flag_array[:, n, :] return DataContainer(flags)
def test_match_red_baselines(self): model = copy.deepcopy(self.data) model = DataContainer( odict([((k[0] + 1, k[1] + 1, k[2]), model[k]) for i, k in enumerate(model.keys())])) del model[(25, 54, 'xx')] model_antpos = odict([(k + 1, self.antpos[k]) for i, k in enumerate(self.antpos.keys())]) new_model = hc.abscal.match_red_baselines(model, model_antpos, self.data, self.antpos, tol=2.0, verbose=False) nt.assert_equal(len(new_model.keys()), 8) nt.assert_true((24, 37, 'xx') in new_model) nt.assert_false((24, 53, 'xx') in new_model)
def compute_ubls(self, data, gain_sols): """Given a set of guess gain solutions, return a dictionary of calibrated visbilities averged over a redundant group. Not strictly necessary for typical operation.""" dc = DataContainer(data) ubl_sols = {} for ubl, blgrp in enumerate(self.reds): d_gp = [dc[bl] for bl in blgrp] ubl_sols[blgrp[0]] = np.average( d_gp, axis=0) # XXX add option for median here? return ubl_sols
def test_lstbin(self): dlst = 0.0007830490163484 # test basic execution output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=self.flgs_list, dlst=None, median=True, lst_low=0, lst_hi=np.pi, verbose=False) output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=None, dlst=0.01, verbose=False) output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=self.flgs_list, dlst=dlst, verbose=False) # check shape and dtype nt.assert_equal(output[1][(24,25,'xx')].dtype, np.complex) nt.assert_equal(output[1][(24,25,'xx')].shape, (224, 64)) # check number of points in each bin nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[0,30], 1) nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[30,30], 2) nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[100,30], 3) nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[190,30], 2) nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[220,30], 1) # check with large spacing lst_grid output = hc.lstbin.lst_bin(self.data_list, self.lst_list, dlst=.01, verbose=False) nt.assert_almost_equal(output[-1][(24, 25, 'xx')].real[10,30], 39) # check flgs are propagated flgs1 = copy.deepcopy(self.flgs1) flgs1[(24, 25, 'xx')][:, 32] = True flgs2 = copy.deepcopy(self.flgs2) flgs2[(24, 25, 'xx')][:, 32] = True flgs3 = copy.deepcopy(self.flgs3) flgs_list = [flgs1, flgs2, flgs3] output = hc.lstbin.lst_bin(self.data_list, self.lst_list, dlst=dlst, flags_list=flgs_list) nt.assert_almost_equal(output[2][(24, 25, 'xx')][0, 32], True) nt.assert_almost_equal(output[2][(24, 25, 'xx')][180, 32], False) nt.assert_almost_equal(output[2][(24, 25, 'xx')][210, 32], False) # test return no avg output = hc.lstbin.lst_bin(self.data_list, self.lst_list, dlst=dlst, flags_list=self.flgs_list, return_no_avg=True) nt.assert_equal(len(output[2][output[2].keys()[0]][100]), 3) nt.assert_equal(len(output[2][output[2].keys()[0]][100][0]), 64) # test switch bl conj_data3 = DataContainer(odict(map(lambda k: (hc.lstbin.switch_bl(k), np.conj(self.data3[k])), self.data3.keys()))) data_list = [self.data1, self.data2, conj_data3] output = hc.lstbin.lst_bin(data_list, self.lst_list, dlst=dlst) nt.assert_equal(output[1][(24,25,'xx')].shape, (224, 64)) # test sigma clip output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=None, dlst=0.01, verbose=False, sig_clip=True, min_N=5, sigma=2) # test wrapping lst_list = map(lambda l: (copy.deepcopy(l) + 6) % (2*np.pi), self.lst_list) output = hc.lstbin.lst_bin(self.data_list, lst_list, dlst=0.001, lst_start=np.pi) nt.assert_true(output[0][0] > output[0][-1]) nt.assert_equal(len(output[0]), 175) # test appropriate data_count output = hc.lstbin.lst_bin(self.data_list, self.lst_list, flags_list=None, dlst=dlst, lst_low=0.25, lst_hi=0.3, verbose=False) nt.assert_true(np.isclose(output[4][(24, 25, 'xx')], 3.0).all())
def _solver(self, solver, data, wgts={}, detrend_phs=False, sparse=False, **kwargs): """Instantiates a linsolve solver for performing redcal. Args: solver: linsolve solver (e.g. linsolve.LogProductSolver or linsolve.LinProductSolver) data: visibility data in the dictionary format {(ant1,ant2,pol): np.array} wgts: dictionary of linear weights in the same format as data. Defaults to equal wgts. detrend_phs: takes out average phase, useful for logcal sparse: represent the A matrix (visibilities to parameters) sparsely in linsolve **kwargs: other keyword arguments passed into the solver for use by linsolve Returns: solver: instantiated solver with redcal equations and weights """ dc = DataContainer(data) eqs = self.build_eqs(dc.keys()) self.phs_avg = { } # detrend phases within redundant group, used for logcal to avoid phase wraps if detrend_phs: for blgrp in self.reds: self.phs_avg[blgrp[0]] = np.exp(-1j * np.median(np.unwrap( [np.log(dc[bl]).imag for bl in blgrp], axis=0), axis=0)) for bl in blgrp: self.phs_avg[bl] = self.phs_avg[blgrp[0]] d_ls, w_ls = {}, {} for eq, key in eqs.items(): d_ls[eq] = dc[key] * self.phs_avg.get(key, 1) if len(wgts) > 0: wc = DataContainer(wgts) for eq, key in eqs.items(): w_ls[eq] = wc[key] return solver(data=d_ls, wgts=w_ls, sparse=sparse, **kwargs)
def test_mirror_data_to_red_bls(self): # make fake data reds = hc.redcal.get_reds(self.antpos, pols=['xx']) data = DataContainer( odict(map(lambda k: (k[0], self.data[k[0]]), reds[:5]))) # test execuation d = hc.abscal.mirror_data_to_red_bls(data, self.antpos) nt.assert_equal(len(d.keys()), 16) nt.assert_true((24, 25, 'xx') in d) # test correct value is propagated nt.assert_almost_equal(data[(24, 25, 'xx')][30, 30], d[(38, 39, 'xx')][30, 30]) # test reweighting w = hc.abscal.mirror_data_to_red_bls(self.wgts, self.antpos, weights=True) nt.assert_equal(w[(24, 25, 'xx')].dtype, np.float) nt.assert_almost_equal(w[(24, 25, 'xx')].max(), 16.0)
def lst_bin(data_list, lst_list, flags_list=None, dlst=None, lst_start=None, lst_low=None, lst_hi=None, flag_thresh=0.7, atol=1e-10, median=False, truncate_empty=True, sig_clip=False, sigma=4.0, min_N=4, return_no_avg=False, antpos=None, rephase=False, freq_array=None, lat=-30.72152, verbose=True): """ Bin data in Local Sidereal Time (LST) onto an LST grid. An LST grid is defined as an array of points increasing in Local Sidereal Time, with each point marking the center of the LST bin. Parameters: ----------- data_list : type=list, list of DataContainer dictionaries holding complex visibility data lst_list : type=list, list of ndarrays holding LST bin centers of each data dictionary in data_list. These LST arrays must be monotonically increasing, except for a possible wrap at 2pi. flags_list : type=list, list of DataContainer dictionaries holding flags for each data dict in data_list. Flagged data do not contribute to the average of an LST bin. dlst : type=float, delta-LST spacing for lst_grid. If None, will use the delta-LST of the first array in lst_list. lst_start : type=float, starting LST for making the lst_grid, extending from [lst_start, lst_start+2pi). Default is lst_start = 0 radians. lst_low : type=float, lower bound on LST bin centers used for contructing LST grid lst_hi : type=float, upper bound on LST bin centers used for contructing LST grid flag_thresh : type=float, minimum fraction of flagged points in an LST bin needed to flag the entire bin. atol : type=float, absolute tolerance for comparing LST bin center floats median : type=boolean, if True use median for LST binning. Warning: this is slower. truncate_empty : type=boolean, if True, truncate output time integrations that have no data in them. sig_clip : type=boolean, if True, perform a sigma clipping algorithm of the LST bins on the real and imag components separately. Warning: This is considerably slow. sigma : type=float, input sigma threshold to use for sigma clipping algorithm. min_N : type=int, minimum number of points in an LST bin to perform sigma clipping return_no_avg : type=boolean, if True, return binned but un-averaged data and flags. rephase : type=bool, if True, phase data to center of LST bin before binning. Note that this produces a copy of the data. antpos : type=dictionary, holds antenna position vectors in ENU frame in meters with antenna integers as keys and 3D ndarrays as values. See io.load_vis(). Needed for rephase. freq_array : type=ndarray, 1D array of unique data frequencies channels in Hz. Needed for rephase. lat : type=float, latitude of array in degrees North. Needed for rephase. verbose : type=bool, if True report feedback to stdout Output: (lst_bins, data_avg, flags_min, data_std, data_count) ------- lst_bins : ndarray containing final lst grid of data (marks bin centers) data_avg : dictionary of data having averaged in each LST bin flags_min : dictionary of minimum of data flags in each LST bin data_std : dictionary of data with real component holding LST bin std along real axis and imag component holding std along imag axis data_count : dictionary containing the number count of data points averaged in each LST bin. if return_no_avg: Output: (lst_bins, data_bin, flags_min) data_bin : dictionary with (ant1,ant2,pol) as keys and ndarrays holding un-averaged complex visibilities in each LST bin as values. flags_min : dictionary with data flags """ # get visibility shape Ntimes, Nfreqs = data_list[0][data_list[0].keys()[0]].shape # get dlst if not provided if dlst is None: dlst = np.median(np.diff(lst_list[0])) # construct lst_grid lst_grid = make_lst_grid(dlst, lst_start=lst_start, verbose=verbose) dlst = np.median(np.diff(lst_grid)) # test for special case of lst grid restriction if lst_low is not None and lst_hi is not None and lst_hi < lst_low: lst_grid = lst_grid[(lst_grid > (lst_low - atol)) | (lst_grid < (lst_hi + atol))] else: # restrict lst_grid based on lst_low and lst_high if lst_low is not None: lst_grid = lst_grid[lst_grid > (lst_low - atol)] if lst_hi is not None: lst_grid = lst_grid[lst_grid < (lst_hi + atol)] # Raise Exception if lst_grid is empty if len(lst_grid) == 0: raise ValueError( "len(lst_grid) == 0; consider changing lst_low and/or lst_hi.") # move lst_grid centers to the left lst_grid_left = lst_grid - dlst / 2 # form new dictionaries # data is a dictionary that will hold other dictionaries as values, which will # themselves hold lists of ndarrays data = odict() flags = odict() all_lst_indices = set() # iterate over data_list for i, d in enumerate(data_list): # get lst array l = copy.copy(lst_list[i]) # ensure l isn't wrapped relative to lst_grid l[l < lst_grid_left.min() - atol] += 2 * np.pi # digitize data lst array "l" grid_indices = np.digitize(l, lst_grid_left[1:], right=True) # make data_in_bin boolean array, and set to False data that don't fall in any bin data_in_bin = np.ones_like(l, np.bool) data_in_bin[(l < lst_grid_left.min() - atol)] = False data_in_bin[(l > lst_grid_left.max() + dlst + atol)] = False # update all_lst_indices all_lst_indices.update(set(grid_indices[data_in_bin])) if rephase: # rephase each integration in d to nearest LST bin if freq_array is None or antpos is None: raise ValueError("freq_array and antpos is needed for rephase") # form baseline dictionary bls = odict( map(lambda k: (k, antpos[k[0]] - antpos[k[1]]), d.keys())) # get appropriate lst_shift for each integration, then rephase lst_shift = lst_grid[grid_indices] - l # this makes a copy of the data in d d = utils.lst_rephase(d, bls, freq_array, lst_shift, lat=lat, inplace=False) # iterate over keys in d for j, key in enumerate(d.keys()): # data[key] will be an odict. if data[key] doesn't exist # create data[key] as an empty odict. if data[key] already # exists, then pass if key in data: pass elif switch_bl(key) in data: # check to see if conj(key) exists in data key = switch_bl(key) d[key] = np.conj(d[switch_bl(key)]) if flags_list is not None: flags_list[i][key] = flags_list[i][switch_bl(key)] else: # if key or conj(key) not in data, insert key into data data[key] = odict() flags[key] = odict() # data[key] is an odict, with keys as grid index integers and # values as lists holding the LST bin data: ndarrays of shape (Nfreqs) # iterate over grid_indices, and append to data if data_in_bin is True for k, ind in enumerate(grid_indices): # ensure data_in_bin is True for this grid index if data_in_bin[k]: # if index not in data[key], insert it as empty list if ind not in data[key]: data[key][ind] = [] flags[key][ind] = [] # append data ndarray to LST bin data[key][ind].append(d[key][k]) # also insert flags if fed if flags_list is None: flags[key][ind].append( np.zeros_like(d[key][k], np.bool)) else: flags[key][ind].append(flags_list[i][key][k]) # get final lst_bin array if truncate_empty: # use only lst_grid bins that have data in them lst_bins = lst_grid[sorted(all_lst_indices)] else: # keep all lst_grid bins and fill empty ones with unity data and mark as flagged for index in range(len(lst_grid)): if index in all_lst_indices: # skip if index already in data continue for key in data.keys(): # fill data with blank data data[key][index] = [np.ones(Nfreqs, np.complex)] flags[key][index] = [np.ones(Nfreqs, np.bool)] # use all LST bins lst_bins = lst_grid # wrap lst_bins if needed lst_bins = lst_bins % (2 * np.pi) # make final dictionaries flags_min = odict() data_avg = odict() data_count = odict() data_std = odict() # return un-averaged data if desired if return_no_avg: # return all binned data instead of just bin average data_bin = odict( map( lambda k: (k, np.array( odict( map(lambda k2: (k2, data[k][k2]), sorted(data[k].keys()))).values())), sorted(data.keys()))) flags_bin = odict( map( lambda k: (k, np.array( odict( map(lambda k2: (k2, flags[k][k2]), sorted(flags[k].keys()))).values())), sorted(flags.keys()))) return lst_bins, data_bin, flags_bin # iterate over data keys (baselines) and get statistics for i, key in enumerate(data.keys()): # create empty lists real_avg = [] imag_avg = [] f_min = [] real_std = [] imag_std = [] bin_count = [] # iterate over sorted LST grid indices in data[key] for j, ind in enumerate(sorted(data[key].keys())): # make data and flag arrays from lists d = np.array(data[key][ind]) # shape = (Ndays x Nfreqs) f = np.array(flags[key][ind]) f[np.isnan(f)] = True # replace flagged data with nan d[f] *= np.nan # sigma clip if desired if sig_clip: # clip real real_f = sigma_clip(d.real, sigma=sigma, min_N=min_N, axis=0) # clip imag imag_f = sigma_clip(d.imag, sigma=sigma, min_N=min_N, axis=0) # get real + imag flags clip_flags = real_f + imag_f # flag data d[clip_flags] *= np.nan # merge clip flags f += (real_f + imag_f) # check flag thresholds if len(f) == 1: flag_bin = np.zeros(f.shape[1], np.bool) else: flag_bin = np.sum(f, axis=0).astype( np.float) / len(f) > flag_thresh d[:, flag_bin] *= np.nan f[:, flag_bin] = True # take bin average if median: real_avg.append(np.nanmedian(d.real, axis=0)) imag_avg.append(np.nanmedian(d.imag, axis=0)) else: real_avg.append(np.nanmean(d.real, axis=0)) imag_avg.append(np.nanmean(d.imag, axis=0)) # get minimum bin flag f_min.append(np.nanmin(f, axis=0)) # get other stats real_std.append(np.nanstd(d.real, axis=0)) imag_std.append(np.nanstd(d.imag, axis=0)) bin_count.append(np.nansum(~np.isnan(d), axis=0)) # get final statistics d_avg = np.array(real_avg) + 1j * np.array(imag_avg) f_min = np.array(f_min) d_std = np.array(real_std) + 1j * np.array(imag_std) d_num = np.array(bin_count).astype(np.float) # fill nans d_nan = np.isnan(d_avg) d_avg[d_nan] = 1.0 f_min[d_nan] = True d_std[d_nan] = 1.0 d_num[d_nan] = 0.0 # insert into dictionaries data_avg[key] = d_avg flags_min[key] = f_min data_std[key] = d_std data_count[key] = d_num # turn into DataContainer objects data_avg = DataContainer(data_avg) flags_min = DataContainer(flags_min) data_std = DataContainer(data_std) data_count = DataContainer(data_count) return lst_bins, data_avg, flags_min, data_std, data_count
def run_filter(self, to_filter=[], weight_dict=None, standoff=15., horizon=1., min_dly=0.0, tol=1e-9, window='blackman-harris', skip_wgt=0.1, maxiter=100, verbose=False, flag_nchan_low=0, flag_nchan_high=0, gain=0.1, **win_kwargs): '''Performs uvtools.dspec.Delay_Filter on (a subset of) the data stored in the object. Uses stored flags unless explicitly overridden with weight_dict. Arguments: to_filter: list of visibilities to filter in the (i,j,pol) format. If [] (the default), all visibilities are filtered. weight_dict: dictionary or DataContainer with all the same keys as self.data. Linear multiplicative weights to use for the delay filter. Default, use np.logical_not of self.flags. uvtools.dspec.delay_filter will renormalize to compensate standoff: fixed additional delay beyond the horizon (in ns) horizon: proportionality constant for bl_len where 1 is the horizon (full light travel time) min_dly: minimum delay used for cleaning [ns]: if bl_len * horizon + standoff < min_dly, use min_dly. tol: CLEAN algorithm convergence tolerance (see aipy.deconv.clean) window: window function for filtering applied to the filtered axis. See aipy.dsp.gen_window for options. skip_wgt: skips filtering rows with very low total weight (unflagged fraction ~< skip_wgt). Model is left as 0s, residual is left as data, and info is {'skipped': True} for that time. Skipped channels are then flagged in self.flags. Only works properly when all weights are all between 0 and 1. maxiter: Maximum number of iterations for aipy.deconv.clean to converge. verbose: If True print feedback to stdout flag_nchan_low: Integer number of channels to flag on lower band edge before filtering flag_nchan_low: Integer number of channels to flag on upper band edge before filtering gain: The fraction of a residual used in each iteration. If this is too low, clean takes unnecessarily long. If it is too high, clean does a poor job of deconvolving. win_kwargs : keyword arguments to feed aipy.dsp.gen_window() Results are stored in: self.filtered_residuals: DataContainer formatted like self.data with only high-delay components self.CLEAN_models: DataContainer formatted like self.data with only low-delay components self.info: Dictionary of info from uvtools.dspec.delay_filter with the same keys as self.data ''' self.filtered_residuals = deepcopy(self.data) self.CLEAN_models = DataContainer({ k: np.zeros_like(self.data.values()[0]) for k in self.data.keys() }) self.info = odict() if to_filter == []: to_filter = self.data.keys() for k in to_filter: if verbose: print "\nStarting filter on {} at {}".format( k, str(datetime.datetime.now())) bl_len = np.linalg.norm(self.antpos[k[0]] - self.antpos[k[1]] ) / constants.c * 1e9 #in ns sdf = np.median(np.diff(self.freqs)) / 1e9 #in GHz if weight_dict is not None: wgts = weight_dict[k] else: wgts = np.logical_not(self.flags[k]) # edge flag if flag_nchan_low > 0: self.flags[k][:, :flag_nchan_low] = True wgts[:, :flag_nchan_low] = 0.0 if flag_nchan_high > 0: self.flags[k][:, -flag_nchan_high:] = True wgts[:, -flag_nchan_high:] = 0.0 d_mdl, d_res, info = delay_filter(self.data[k], wgts, bl_len, sdf, standoff=standoff, horizon=horizon, min_dly=min_dly, tol=tol, window=window, skip_wgt=skip_wgt, maxiter=maxiter, gain=gain, **win_kwargs) self.filtered_residuals[k] = d_res self.CLEAN_models[k] = d_mdl self.info[k] = info # Flag all channels for any time when skip_wgt gets triggered for i, info_dict in enumerate(info): if info_dict.get('skipped', False): self.flags[k][i, :] = np.ones_like(self.flags[k][i, :])
def recalibrate_in_place(data, data_flags, new_gains, cal_flags, old_gains=None, gain_convention='divide'): '''Update data and data_flags in place, taking out old calibration solutions, putting in new calibration solutions, and updating flags from those calibration solutions. Previously flagged data is modified, but left flagged. Missing antennas from either the new gains or (if it's not None), the old gains are automatically flagged in the data's visibilities that involves those antennas. Arguments: data: DataContainer containing baseline-pol complex visibility data. This is modified in place. data_flags: DataContainer containing data flags. This is modified in place. Can also be fed as a data weights dictionary with float dtype. In this case, wgts of 0 are treated as flagged data and non-zero wgts are unflagged data. new_gains: Dictionary of complex calibration gains to apply with keys like (1,'x') cal_flags: Dictionary with keys like (1,'x') of per-antenna boolean flags to update data_flags if either antenna in a visibility is flagged. old_gains: Dictionary of complex calibration gains to take out with keys like (1,'x'). Default of None implies that the data is raw (i.e. uncalibrated). gain_convention: str, either 'divide' or 'multiply'. 'divide' means V_obs = gi gj* V_true, 'multiply' means V_true = gi gj* V_obs. Assumed to be the same for new_gains and old_gains. ''' # get datatype of data_flags to determine if flags or wgts if np.all([(df.dtype == np.bool) for df in data_flags.values()]): bool_flags = True elif np.all([(df.dtype == np.float) for df in data_flags.values()]): bool_flags = False wgts = data_flags data_flags = DataContainer( dict(map(lambda k: (k, ~wgts[k].astype(np.bool)), wgts.keys()))) else: raise ValueError("didn't recognize dtype of data_flags") # loop over keys for (i, j, pol) in data.keys(): # Check to see that all necessary antennas are present in the gains if new_gains.has_key((i, pol[0])) and new_gains.has_key( (j, pol[1])) and (old_gains == None or (old_gains.has_key( (i, pol[0])) and old_gains.has_key((j, pol[1])))): gigj_new = new_gains[(i, pol[0])] * np.conj(new_gains[(j, pol[1])]) if old_gains is not None: gigj_old = old_gains[(i, pol[0])] * np.conj( old_gains[(j, pol[1])]) else: gigj_old = np.ones_like(gigj_new) # update all the data, even if it was flagged if gain_convention == 'divide': data[(i, j, pol)] *= (gigj_old / gigj_new) elif gain_convention == 'multiply': data[(i, j, pol)] *= (gigj_new / gigj_old) else: raise KeyError( "gain_convention must be either 'divide' or 'multiply'.") # update data flags if bool_flags: # treat as flags data_flags[(i, j, pol)][cal_flags[(i, pol[0])]] = True data_flags[(i, j, pol)][cal_flags[(j, pol[1])]] = True else: # treat as data weights wgts[(i, j, pol)][cal_flags[(i, pol[0])]] = 0.0 wgts[(i, j, pol)][cal_flags[(j, pol[1])]] = 0.0 else: # If any antenna is missing from the gains, the data is flagged if bool_flags: data_flags[(i, j, pol)] = np.ones_like(data_flags[(i, j, pol)], dtype=np.bool) else: wgts[(i, j, pol)] = np.zeros_like(wgts[(i, j, pol)], dtype=np.float)
def load_vis(input_data, return_meta=False, filetype='miriad', pop_autos=False, pick_data_ants=True, nested_dict=False): '''Load miriad or uvfits files or UVData objects into DataContainers, optionally returning the most useful metadata. More than one spectral window is not supported. Assumes every baseline has the same times present and that the times are in order. Arguments: input_data: data file path, or UVData instance, or list of either strings of data file paths or list of UVData instances to concatenate into a single dictionary return_meta: boolean, if True: also return antpos, ants, freqs, times, lsts, and pols filetype: either 'miriad' or 'uvfits', can be ignored if input_data is UVData objects pop_autos: boolean, if True: remove autocorrelations pick_data_ants: boolean, if True and return_meta=True, return only antennas in data nested_dict: boolean, if True replace DataContainers with the legacy nested dictionary filetype where visibilities and flags are accessed as data[(0,1)]['xx'] Returns: if return_meta is True: (data, flags, antpos, ants, freqs, times, lsts, pols) else: (data, flags) data: DataContainer containing baseline-pol complex visibility data with keys like (0,1,'xx') and with shape=(Ntimes,Nfreqs) flags: DataContainer containing data flags antpos: dictionary containing antennas numbers as keys and position vectors ants: ndarray containing unique antenna indices freqs: ndarray containing frequency channels (Hz) times: ndarray containing julian date bins of data lsts: ndarray containing LST bins of data (radians) pol: ndarray containing list of polarization strings ''' uvd = UVData() if isinstance(input_data, (tuple, list, np.ndarray)): #List loading if np.all([isinstance(id, str) for id in input_data]): #List of visibility data paths if filetype == 'miriad': uvd.read_miriad(list(input_data)) elif filetype == 'uvfits': #TODO: implement this raise NotImplementedError( 'This function has not been implemented yet.') else: raise NotImplementedError( "Data filetype must be either 'miriad' or 'uvfits'.") elif np.all([isinstance(id, UVData) for id in input_data]): #List of uvdata objects uvd = reduce(operator.add, input_data) else: raise TypeError( 'If input is a list, it must be only strings or only UVData objects.' ) elif isinstance(input_data, str): #single visibility data path if filetype == 'miriad': uvd.read_miriad(input_data) elif filetype == 'uvfits': #TODO: implement this raise NotImplementedError( 'This function has not been implemented yet.') else: raise NotImplementedError( "Data filetype must be either 'miriad' or 'uvfits'.") elif isinstance(input_data, UVData): #single UVData object uvd = input_data else: raise TypeError( 'Input must be a UVData object, a string, or a list of either.') data, flags = odict(), odict() # create nested dictionaries of visibilities in the data[bl][pol] filetype, removing autos if desired for nbl, (i, j) in enumerate(uvd.get_antpairs()): if (not pop_autos) or (i != j): if (i, j) not in data: data[i, j], flags[i, j] = odict(), odict() for ip, pol in enumerate(uvd.polarization_array): pol = polnum2str[pol] data[(i, j)][pol] = copy.deepcopy(uvd.get_data((i, j, pol))) flags[(i, j)][pol] = copy.deepcopy(uvd.get_flags((i, j, pol))) # If we don't want nested dicts, convert to DataContainer if not nested_dict: data, flags = DataContainer(data), DataContainer(flags) # get meta if return_meta: freqs = np.unique(uvd.freq_array) times = np.unique(uvd.time_array) lsts = [] for l in uvd.lst_array.ravel(): if l not in lsts: lsts.append(l) lsts = np.array(lsts) antpos, ants = uvd.get_ENU_antpos(center=True, pick_data_ants=pick_data_ants) antpos = odict(zip(ants, antpos)) pols = np.array( [polnum2str[polnum] for polnum in uvd.polarization_array]) del uvd garbage_collector.collect() return data, flags, antpos, ants, freqs, times, lsts, pols else: del uvd garbage_collector.collect() return data, flags
def test_recalibrate_in_place(self): np.random.seed(21) vis = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10) dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)}) f = np.random.randn(10, 10) > 0 flags = DataContainer({(0, 1, 'xx'): deepcopy(f)}) g0_new = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10) g1_new = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10) g_new = {(0, 'x'): g0_new, (1, 'x'): g1_new} g0_old = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10) g1_old = np.random.randn(10, 10) + 1.0j * np.random.randn(10, 10) g_old = {(0, 'x'): g0_old, (1, 'x'): g1_old} cal_flags = { (0, 'x'): np.random.randn(10, 10) > 0, (1, 'x'): np.random.randn(10, 10) > 0 } # test standard operation ac.recalibrate_in_place(dc, flags, g_new, cal_flags, old_gains=g_old, gain_convention='divide') for i in range(10): for j in range(10): self.assertAlmostEqual( dc[(0, 1, 'xx')][i, j], vis[i, j] * g0_old[i, j] * np.conj(g1_old[i, j]) / g0_new[i, j] / np.conj(g1_new[i, j])) if f[i, j] or cal_flags[(0, 'x')][i, j] or cal_flags[(1, 'x')][i, j]: self.assertTrue(flags[(0, 1, 'xx')][i, j]) else: self.assertFalse(flags[(0, 1, 'xx')][i, j]) # test without old cal dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)}) flags = DataContainer({(0, 1, 'xx'): deepcopy(f)}) ac.recalibrate_in_place(dc, flags, g_new, cal_flags, gain_convention='divide') for i in range(10): for j in range(10): self.assertAlmostEqual( dc[(0, 1, 'xx')][i, j], vis[i, j] / g0_new[i, j] / np.conj(g1_new[i, j])) # test multiply dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)}) flags = DataContainer({(0, 1, 'xx'): deepcopy(f)}) ac.recalibrate_in_place(dc, flags, g_new, cal_flags, old_gains=g_old, gain_convention='multiply') for i in range(10): for j in range(10): self.assertAlmostEqual( dc[(0, 1, 'xx')][i, j], vis[i, j] / g0_old[i, j] / np.conj(g1_old[i, j]) * g0_new[i, j] * np.conj(g1_new[i, j])) # test flag propagation when missing antennas in gains dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)}) flags = DataContainer({(0, 1, 'xx'): deepcopy(f)}) ac.recalibrate_in_place(dc, flags, {}, cal_flags, gain_convention='divide') np.testing.assert_array_equal(flags[(0, 1, 'xx')], True) dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)}) flags = DataContainer({(0, 1, 'xx'): deepcopy(f)}) ac.recalibrate_in_place(dc, flags, g_new, cal_flags, old_gains={}, gain_convention='divide') np.testing.assert_array_equal(flags[(0, 1, 'xx')], True) # test error dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)}) flags = DataContainer({(0, 1, 'xx'): deepcopy(f)}) with self.assertRaises(KeyError): ac.recalibrate_in_place(dc, flags, g_new, cal_flags, old_gains=g_old, gain_convention='blah') # test w/ data weights dc = DataContainer({(0, 1, 'xx'): deepcopy(vis)}) flags = DataContainer({(0, 1, 'xx'): deepcopy(f)}) wgts = DataContainer( dict(map(lambda k: (k, (~flags[k]).astype(np.float)), flags.keys()))) del g_new[(0, 'x')] ac.recalibrate_in_place(dc, wgts, g_new, cal_flags, gain_convention='divide') self.assertAlmostEqual(wgts[(0, 1, 'xx')].max(), 0.0)