def test_load_vis_nested(self): #duplicated testing from firstcal.UVData_to_dict str2pol = io.polstr2num filename1 = os.path.join(DATA_PATH, 'zen.2458043.12552.xx.HH.uvORA') filename2 = os.path.join(DATA_PATH, 'zen.2458043.13298.xx.HH.uvORA') uvd1 = UVData() uvd1.read_miriad(filename1) uvd2 = UVData() uvd2.read_miriad(filename2) if uvd1.phase_type != 'drift': uvd1.unphase_to_drift() if uvd2.phase_type != 'drift': uvd2.unphase_to_drift() uvd = uvd1 + uvd2 d, f = io.load_vis([uvd1, uvd2], nested_dict=True) for i, j in d: for pol in d[i, j]: uvpol = list(uvd1.polarization_array).index(str2pol[pol]) uvmask = np.all(np.array(zip(uvd.ant_1_array, uvd.ant_2_array)) == [i, j], axis=1) np.testing.assert_equal( d[i, j][pol], np.resize(uvd.data_array[uvmask][:, 0, :, uvpol], d[i, j][pol].shape)) np.testing.assert_equal( f[i, j][pol], np.resize(uvd.flag_array[uvmask][:, 0, :, uvpol], f[i, j][pol].shape)) d, f = io.load_vis([filename1, filename2], nested_dict=True) for i, j in d: for pol in d[i, j]: uvpol = list(uvd.polarization_array).index(str2pol[pol]) uvmask = np.all(np.array(zip(uvd.ant_1_array, uvd.ant_2_array)) == [i, j], axis=1) np.testing.assert_equal( d[i, j][pol], np.resize(uvd.data_array[uvmask][:, 0, :, uvpol], d[i, j][pol].shape)) np.testing.assert_equal( f[i, j][pol], np.resize(uvd.flag_array[uvmask][:, 0, :, uvpol], f[i, j][pol].shape)) uvd = UVData() uvd.read_miriad(filename1) self.assertEqual(len(io.load_vis([uvd], nested_dict=True)[0]), uvd.Nbls) # reorder baseline array uvd.baseline_array = uvd.baseline_array[np.argsort(uvd.baseline_array)] self.assertEqual(len(io.load_vis(filename1, nested_dict=True)[0]), uvd.Nbls)
def test_mul(self): test_file = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA") d, f = io.load_vis(test_file, pop_autos=True) f[(24, 25, 'xx')][:, 0] = False f2 = f * f self.assertFalse(f2[(24, 25, 'xx')][0, 0]) # test exception d2, f2 = io.load_vis(test_file, pop_autos=True) d2[d2.keys()[0]] = d2[d2.keys()[0]][:, :10] self.assertRaises(ValueError, d.__mul__, d2) d2[d2.keys()[0]] = d2[d2.keys()[0]][:10, :] self.assertRaises(ValueError, d.__mul__, d2)
def test_adder(self): test_file = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA") d, f = io.load_vis(test_file, pop_autos=True) d2 = d + d self.assertAlmostEqual(d2[(24, 25, 'xx')][30, 30], d[(24, 25, 'xx')][30, 30] * 2) # test exception d2, f2 = io.load_vis(test_file, pop_autos=True) d2[d2.keys()[0]] = d2[d2.keys()[0]][:, :10] self.assertRaises(ValueError, d.__add__, d2) d2[d2.keys()[0]] = d2[d2.keys()[0]][:10, :] self.assertRaises(ValueError, d.__add__, d2)
def test_concatenate(self): test_file = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA") d, f = io.load_vis(test_file, pop_autos=True) d2 = d.concatenate(d) self.assertEqual(d2[(24, 25, 'xx')].shape[0], d[(24, 25, 'xx')].shape[0] * 2) d2 = d.concatenate(d, axis=1) self.assertEqual(d2[(24, 25, 'xx')].shape[1], d[(24, 25, 'xx')].shape[1] * 2) d2 = d.concatenate([d, d], axis=0) self.assertEqual(d2[(24, 25, 'xx')].shape[0], d[(24, 25, 'xx')].shape[0] * 3) # test exceptions d2, f2 = io.load_vis(test_file, pop_autos=True) d2[d2.keys()[0]] = d2[d2.keys()[0]][:10, :10] self.assertRaises(ValueError, d.concatenate, d2, axis=0) self.assertRaises(ValueError, d.concatenate, d2, axis=1)
def test_lst_rephase(): # load point source sim w/ array at latitude = 0 fname = os.path.join(DATA_PATH, "PAPER_point_source_sim.uv") data, flags, antpos, ants, freqs, times, lsts, pols = io.load_vis( fname, return_meta=True) data_drift = copy.deepcopy(data) transit_integration = 50 # get integration time in LST, baseline dict dlst = np.median(np.diff(lsts)) bls = odict(map(lambda k: (k, antpos[k[0]] - antpos[k[1]]), data.keys())) # basic test: single dlst for all integrations hc.utils.lst_rephase(data, bls, freqs, dlst, lat=0.0) # get phase error on shortest EW baseline k = (0, 1, 'xx') # check error at transit phs_err = np.angle(data[k][transit_integration, 4] / data_drift[k][transit_integration + 1, 4]) nt.assert_true(np.isclose(phs_err, 0, atol=1e-7)) # check error across file phs_err = np.angle(data[k][:-1, 4] / data_drift[k][1:, 4]) nt.assert_true(np.abs(phs_err).max() < 1e-4) # multiple phase term test: dlst per integration dlst = np.array([np.median(np.diff(lsts))] * data[k].shape[0]) data = copy.deepcopy(data_drift) hc.utils.lst_rephase(data, bls, freqs, dlst, lat=0.0) # check error at transit phs_err = np.angle(data[k][transit_integration, 4] / data_drift[k][transit_integration + 1, 4]) nt.assert_true(np.isclose(phs_err, 0, atol=1e-7)) # check err across file phs_err = np.angle(data[k][:-1, 4] / data_drift[k][1:, 4]) nt.assert_true(np.abs(phs_err).max() < 1e-4) # phase all integrations to a single integration dlst = lsts[50] - lsts data = copy.deepcopy(data_drift) hc.utils.lst_rephase(data, bls, freqs, dlst, lat=0.0) # check error at transit phs_err = np.angle(data[k][transit_integration, 4] / data_drift[k][transit_integration, 4]) nt.assert_true(np.isclose(phs_err, 0, atol=1e-7)) # check error across file phs_err = np.angle(data[k][:, 4] / data_drift[k][50, 4]) nt.assert_true(np.abs(phs_err).max() < 1e-4) # test operation on array k = (0, 1, 'xx') d = data_drift[k].copy() d_phs = hc.utils.lst_rephase(d, bls[k], freqs, dlst, lat=0.0, array=True) nt.assert_almost_equal( np.abs(np.angle(d_phs[50] / data[k][50])).max(), 0.0)
def load_data(self, input_data, filetype='miriad'): '''Loads in and stores data for delay filtering. Arguments: input_data: data file path, or UVData instance, or list of either strings of data file paths or list of UVData instances to concatenate into a single internal DataContainer filetype: file format of data. Default 'miriad.' Ignored if input_data is UVData object(s). ''' if isinstance(input_data, (str, UVData)): self.writable = True self.input_data, self.filetype = input_data, filetype self.data, self.flags, self.antpos, _, self.freqs, self.times, _, _ = io.load_vis( input_data, return_meta=True, filetype=filetype) self.Nfreqs = len(self.freqs)
def test_update_vis(self): # load in cal fname = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA") outname = os.path.join( DATA_PATH, "test_output/zen.2458043.12552.xx.HH.modified.uvORA") uvd = UVData() uvd.read_miriad(fname) data, flags, antpos, ants, freqs, times, lsts, pols = io.load_vis( fname, return_meta=True) #make some modifications new_data = {key: (2. + 1.j) * val for key, val in data.items()} new_flags = {key: np.logical_not(val) for key, val in flags.items()} io.update_vis(fname, outname, data=new_data, flags=new_flags, add_to_history='hello world', clobber=True, telescope_name='PAPER') #test modifications data, flags, antpos, ants, freqs, times, lsts, pols = io.load_vis( outname, return_meta=True) for k in data.keys(): self.assertTrue(np.all(new_data[k] == data[k])) self.assertTrue(np.all(new_flags[k] == flags[k])) uvd2 = UVData() uvd2.read_miriad(outname) self.assertTrue( pyuvdata.utils.check_histories(uvd2.history, uvd.history + 'hello world')) self.assertEqual(uvd2.telescope_name, 'PAPER') shutil.rmtree(outname) # Coverage for errors with self.assertRaises(NotImplementedError): io.update_vis(uvd, outname, data=new_data, flags=new_flags, filetype_out='uvfits', add_to_history='hello world', clobber=True, telescope_name='PAPER') with self.assertRaises(NotImplementedError): io.update_vis(fname, outname, data=new_data, flags=new_flags, filetype_in='uvfits', add_to_history='hello world', clobber=True, telescope_name='PAPER') with self.assertRaises(TypeError): io.update_vis(uvd, outname, data=new_data, flags=new_flags, filetype_out='not_a_real_filetype', add_to_history='hello world', clobber=True, telescope_name='PAPER') with self.assertRaises(TypeError): io.update_vis(fname, outname, data=new_data, flags=new_flags, filetype_in='not_a_real_filetype', add_to_history='hello world', clobber=True, telescope_name='PAPER') # #now try the same thing but with a UVData object instead of path io.update_vis(uvd, outname, data=new_data, flags=new_flags, add_to_history='hello world', clobber=True, telescope_name='PAPER') data, flags, antpos, ants, freqs, times, lsts, pols = io.load_vis( outname, return_meta=True) for k in data.keys(): self.assertTrue(np.all(new_data[k] == data[k])) self.assertTrue(np.all(new_flags[k] == flags[k])) uvd2 = UVData() uvd2.read_miriad(outname) self.assertTrue( pyuvdata.utils.check_histories(uvd2.history, uvd.history + 'hello world')) self.assertEqual(uvd2.telescope_name, 'PAPER') shutil.rmtree(outname)
def test_load_vis(self): #duplicated testing from abscal_funcs.UVData2AbsCalDict # load into pyuvdata object self.data_file = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA") self.uvd = UVData() self.uvd.read_miriad(self.data_file) self.freq_array = np.unique(self.uvd.freq_array) self.antpos, self.ants = self.uvd.get_ENU_antpos(center=True, pick_data_ants=True) self.antpos = odict(zip(self.ants, self.antpos)) self.time_array = np.unique(self.uvd.time_array) fname = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA") data, flags = io.load_vis(fname, pop_autos=False) self.assertEqual(data[(24, 25, 'xx')].shape, (60, 64)) self.assertEqual(flags[(24, 25, 'xx')].shape, (60, 64)) self.assertEqual((24, 24, 'xx') in data, True) data, flags = io.load_vis([fname]) self.assertEqual(data[(24, 25, 'xx')].shape, (60, 64)) # test pop autos data, flags = io.load_vis(fname, pop_autos=True) self.assertEqual((24, 24, 'xx') in data, False) # test uvd object uvd = UVData() uvd.read_miriad(fname) data, flags = io.load_vis(uvd) self.assertEqual(data[(24, 25, 'xx')].shape, (60, 64)) data, flags = io.load_vis([uvd]) self.assertEqual(data[(24, 25, 'xx')].shape, (60, 64)) # test multiple fname2 = os.path.join(DATA_PATH, "zen.2458043.13298.xx.HH.uvORA") data, flags = io.load_vis([fname, fname2]) self.assertEqual(data[(24, 25, 'xx')].shape, (120, 64)) self.assertEqual(flags[(24, 25, 'xx')].shape, (120, 64)) # test w/ meta d, f, ap, a, f, t, l, p = io.load_vis([fname, fname2], return_meta=True) self.assertEqual(len(ap[24]), 3) self.assertEqual(len(f), len(self.freq_array)) #test uvfits fname = os.path.join(DATA_PATH, 'zen.2458043.12552.xx.HH.uvA.vis.uvfits') with self.assertRaises(NotImplementedError): d, f = io.load_vis(fname, filetype='uvfits') with self.assertRaises(NotImplementedError): d, f = io.load_vis([fname, fname], filetype='uvfits') #self.assertEqual(d[(0,1,'xx')].shape, (60,64)) with self.assertRaises(NotImplementedError): d, f = io.load_vis(fname, filetype='not_a_real_filetype') with self.assertRaises(NotImplementedError): d, f = io.load_vis(['str1', 'str2'], filetype='not_a_real_filetype') with self.assertRaises(TypeError): d, f = io.load_vis([1, 2], filetype='uvfits') # test w/ meta pick_data_ants fname = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA") d, f, ap, a, f, t, l, p = io.load_vis(fname, return_meta=True, pick_data_ants=False) self.assertEqual(len(ap[24]), 3) self.assertEqual(len(a), 47) self.assertEqual(len(f), len(self.freq_array)) with self.assertRaises(TypeError): d, f = io.load_vis(1.0)
def test_write_vis(self): # get data uvd = UVData() uvd.read_miriad( os.path.join(DATA_PATH, "zen.2458044.41632.xx.HH.uvXRAA")) data, flgs, ap, a, f, t, l, p = io.load_vis(uvd, return_meta=True) nsample = copy.deepcopy(data) for k in nsample.keys(): nsample[k] = np.ones_like(nsample[k], np.float) # test basic execution uvd = io.write_vis("ex.uv", data, l, f, ap, start_jd=2458044, return_uvd=True, overwrite=True, verbose=True) uvd2 = UVData() uvd2.read_miriad('ex.uv') self.assertTrue(os.path.exists('ex.uv')) self.assertEqual(uvd.data_array.shape, (1680, 1, 64, 1)) self.assertEqual(uvd2.data_array.shape, (1680, 1, 64, 1)) self.assertAlmostEqual(data[(24, 25, 'xx')][30, 32], uvd.get_data(24, 25, 'xx')[30, 32]) self.assertAlmostEqual(data[(24, 25, 'xx')][30, 32], uvd2.get_data(24, 25, 'xx')[30, 32]) # test with nsample and flags uvd = io.write_vis("ex.uv", data, l, f, ap, start_jd=2458044, flags=flgs, nsamples=nsample, return_uvd=True, overwrite=True, verbose=True) self.assertEqual(uvd.nsample_array.shape, (1680, 1, 64, 1)) self.assertEqual(uvd.flag_array.shape, (1680, 1, 64, 1)) self.assertAlmostEqual(nsample[(24, 25, 'xx')][30, 32], uvd.get_nsamples(24, 25, 'xx')[30, 32]) self.assertAlmostEqual(flgs[(24, 25, 'xx')][30, 32], uvd.get_flags(24, 25, 'xx')[30, 32]) # test exceptions self.assertRaises(AttributeError, io.write_vis, "ex.uv", data, l, f, ap) self.assertRaises(AttributeError, io.write_vis, "ex.uv", data, l, f, ap, start_jd=2458044, filetype='foo') if os.path.exists('ex.uv'): shutil.rmtree('ex.uv')
def lst_bin_files(data_files, dlst=None, verbose=True, ntimes_per_file=60, file_ext="{}.{}.{:7.5f}.uv", outdir=None, overwrite=False, history=' ', lst_start=0, atol=1e-6, sig_clip=True, sigma=5.0, min_N=5, rephase=False, output_file_select=None, **kwargs): """ LST bin a series of miriad files with identical frequency bins, but varying time bins. Output miriad file meta data (frequency bins, antennas positions, time_array) are taken from the first file in data_files. Parameters: ----------- data_files : type=list of lists: nested set of lists, with each nested list containing paths to miriad files from a particular night. These files should be sorted by ascending Julian Date. Frequency axis of each file must be identical. dlst : type=float, LST bin width. If None, will get this from the first file in data_files. lst_start : type=float, starting LST for binner as it sweeps from lst_start to lst_start + 2pi. ntimes_per_file : type=int, number of LST bins in a single output file file_ext : type=str, extension to "zen." for output miriad files. This must have three formatting placeholders, first for polarization(s), second for type of file Ex. ["LST", "STD", "NUM"] and third for starting LST bin of file. outdir : type=str, output directory overwrite : type=bool, if True overwrite output files rephase : type=bool, if True, rephase data points in LST bin to center of bin bin_kwargs : type=dictionary, keyword arguments for lst_bin. atol : type=float, absolute tolerance for LST bin float comparison output_file_select : type=int or integer list, list of integer indices of the output files to run on. Default is all files. kwargs : type=dictionary, keyword arguments to pass to io.write_vis() Result: ------- zen.{pol}.LST.{file_lst}.uv : holds LST bin avg (data_array) and bin count (nsample_array) zen.{pol}.STD.{file_lst}.uv : holds LST bin stand dev along real and imag (data_array) """ (data_times, lst_grid, dlst, file_lsts, start_lst) = config_lst_bin_files(data_files, dlst=dlst, lst_start=lst_start, ntimes_per_file=ntimes_per_file, verbose=verbose) nfiles = len(file_lsts) # select file_lsts if output_file_select is not None: if isinstance(output_file_select, (int, np.integer)): output_file_select = [output_file_select] output_file_select = [int(o) for o in output_file_select] try: file_lsts = map(lambda i: file_lsts[i], output_file_select) except IndexError: print "Warning: one or more indices in output_file_select {} caused an index error with length {} file_lsts list, exiting...".format( output_file_select, nfiles) return # create data file status: None if not opened, data objects if opened data_status = map(lambda d: map(lambda f: None, d), data_files) # get outdir if outdir is None: outdir = os.path.dirname( os.path.commonprefix(abscal.flatten(data_files))) # update miriad_kwrgs kwargs['outdir'] = outdir kwargs['overwrite'] = overwrite # get metadata from the zeroth data file d, fl, ap, a, f, t, l, p = io.load_vis(data_files[0][0], return_meta=True, pick_data_ants=False) # push time and lst arrays forward by half an integration abscal.echo("pushing time and lst arrays forward by half an integration b/c pyuvdata "\ "currently does not do this for us....", verbose=verbose) if len(t) > 1: t += np.median(np.diff(t)) / 2.0 # get frequency, time and antenna position information freq_array = copy.copy(f) antpos = copy.deepcopy(ap) start_jd = np.floor(t)[0] kwargs['start_jd'] = start_jd kwargs['integration_time'] = np.median(np.diff(t)) * 24 * 3600. del d, fl, ap, a, f, t, l, p garbage_collector.collect() # iterate over output LST files for i, f_lst in enumerate(file_lsts): abscal.echo("LST file {} / {}: {}".format(i + 1, len(file_lsts), datetime.datetime.now()), type=1, verbose=verbose) # create empty data_list and lst_list data_list = [] file_list = [] flgs_list = [] lst_list = [] # locate all data files that fall within the range of lst for this output file f_min = np.min(f_lst) f_max = np.max(f_lst) f_select = np.array( map( lambda d: map(lambda f: (f[1] > f_min) & (f[0] < f_max + atol), d), data_times)) if i == 0: old_f_select = copy.copy(f_select) # open necessary files, close ones that are no longer needed # append necessary files to data_list for j in range(len(data_files)): nightly_data_list = [] nightly_flgs_list = [] nightly_lst_list = [] for k in range(len(data_files[j])): if f_select[j][k] == True and data_status[j][k] is None: # open file d, fl, ap, a, f, t, l, p = io.load_vis(data_files[j][k], return_meta=True) d.phase_type = 'drift' # unwrap l relative to itself l[l < l[0]] += 2 * np.pi # push time and lst arrays forward by half an integration # b/c pyuvdata does not currently do this for us.... if len(l) > 1: l += np.median(np.diff(l)) / 2.0 t += np.median(np.diff(t)) / 2.0 # unwrap l relative to start_lst l[l < start_lst - atol] += 2 * np.pi # pass data references to data_status list data_status[j][k] = [d, fl, ap, a, f, t, l, p] # erase references from namespace del d, fl, ap, a, f, t, l, p elif f_select[j][k] == False and old_f_select[j][k] == True: # close file del data_status[j][k] data_status[j].insert(k, None) # if file is needed, append data references to data_list if f_select[j][k] == True: file_list.append(data_files[j][k]) nightly_data_list.append( data_status[j][k][0]) # this is data nightly_flgs_list.append( data_status[j][k][1]) # this is flgs nightly_lst_list.append( data_status[j][k][6]) # this is lsts # skip if nothing accumulated in nightly files if len(nightly_data_list) == 0: continue # extend to data lists data_list.extend(nightly_data_list) flgs_list.extend(nightly_flgs_list) lst_list.extend(nightly_lst_list) del nightly_data_list, nightly_flgs_list, nightly_lst_list # skip if data_list is empty if len(data_list) == 0: abscal.echo("data_list is empty for beginning LST {}".format( f_lst[0]), verbose=verbose) # erase data references del file_list, data_list, flgs_list, lst_list # assign old f_select old_f_select = copy.copy(f_select) continue # pass through lst-bin function (bin_lst, bin_data, flag_data, std_data, num_data) = lst_bin(data_list, lst_list, flags_list=flgs_list, dlst=dlst, lst_start=start_lst, lst_low=f_min, lst_hi=f_max, truncate_empty=False, sig_clip=sig_clip, sigma=sigma, min_N=min_N, rephase=rephase, freq_array=freq_array, antpos=antpos) # push lst_bins back to bin start rather than bin center # b/c pyuvdata does not do this for us yet... abscal.echo("pushing time and lst arrays backward by half an integration b/c pyuvdata "\ "currently does not do this for us....", verbose=verbose) # unwrap bin_lst bin_lst[bin_lst < bin_lst[0] - atol] += 2 * np.pi # push back if len(bin_lst) > 1: bin_lst -= np.median(np.diff(bin_lst)) / 2.0 # re-wrap bin_lst bin_lst = bin_lst % (2 * np.pi) # update history file_history = history + " Input files: " + "-".join( map(lambda ff: os.path.basename(ff), file_list)) kwargs['history'] = file_history # erase data references del file_list, data_list, flgs_list, lst_list garbage_collector.collect() # assign old f_select old_f_select = copy.copy(f_select) # get polarizations pols = bin_data.pols() # configure filenames bin_file = "zen.{}".format( file_ext.format('.'.join(pols), "LST", bin_lst[0])) std_file = "zen.{}".format( file_ext.format('.'.join(pols), "STD", bin_lst[0])) # check for overwrite if os.path.exists(bin_file) and overwrite is False: abscal.echo("{} exists, not overwriting".format(bin_file), verbose=verbose) continue # write to file io.write_vis(bin_file, bin_data, bin_lst, freq_array, antpos, flags=flag_data, verbose=verbose, nsamples=num_data, filetype='miriad', **kwargs) io.write_vis(std_file, std_data, bin_lst, freq_array, antpos, flags=flag_data, verbose=verbose, nsamples=num_data, filetype='miriad', **kwargs) del bin_file, std_file, bin_data, std_data, num_data, bin_lst, flag_data garbage_collector.collect()
def lst_align_files(data_files, file_ext=".L.{:7.5f}", dlst=None, overwrite=None, outdir=None, miriad_kwargs={}, align_kwargs={}, verbose=True): """ Align a series of data files with a universal LST grid. Parameters: ----------- data_files : type=list, list of paths to miriad files, or a single miriad file path file_ext : type=str, file_extension for each file in data_files when writing to disk dlst : type=float, LST grid bin interval, if None get it from first file in data_files overwrite : type=boolean, if True overwrite output files miriad_kwargs : type=dictionary, keyword arguments to feed to miriad_to_data() align_kwargs : keyword arguments to feed to lst_align() Result: ------- A series of "data_files + file_ext" miriad files written to disk. """ # check type of data_files if type(data_files) == str: data_files = [data_files] # get dlst if None if dlst is None: start, stop, int_time = utils.get_miriad_times(data_files[0]) dlst = int_time # iterate over data files for i, f in enumerate(data_files): # load data (data, flgs, apos, ants, freqs, times, lsts, pols) = io.load_vis(f, return_meta=True) # lst align interp_data, interp_flgs, interp_lsts = lst_align(data, lsts, flags=flgs, dlst=dlst, **align_kwargs) # check output output_fname = os.path.basename(f) + file_ext.format(interp_lsts[0]) # write to miriad file if overwrite is not None: miriad_kwargs['overwrite'] = overwrite if outdir is not None: miriad_kwargs['outdir'] = outdir miriad_kwargs['start_jd'] = np.floor(times[0]) io.write_vis(output_fname, interp_data, interp_lsts, freqs, apos, flags=interp_flgs, verbose=verbose, filetype='miriad', **miriad_kwargs)
def _search_and_iterate_firstcal(uv, info, option_parser): '''Searches and iterates over firstcal Iteratively run firstcal and look for rotated antennas. If rotated antennas found, fix the antenna and rerun firstcal. Args: uv (pyuvdata.UVData): UVData object info (FirstcalRedundantInfo): info object option_parser (OptionParser): option parser object Return: dict : firstcal solutions list : list of rotated antennas ''' switched_history = [] # keep track of all rotated antennas switched = [] # keep track of rotated antennas in each iteration niters = 0 while niters == 0 or len(switched) > 0: datapack, flagpack = io.load_vis([uv], nested_dict=True) datapack = _apply_pi_shift(datapack, switched) wgtpack = {k: {p: np.logical_not(flagpack[k][p]) for p in flagpack[k]} for k in flagpack} fqs = uv.freq_array[0, :] / 1e9 # gets phase solutions per frequency. fc = FirstCal(datapack, wgtpack, fqs, info) sols = fc.run(finetune=option_parser.finetune, verbose=option_parser.verbose, average=option_parser.average, window='none') # Now we need to check if antennas are flipped medians = {} cal_data = copy.deepcopy(datapack) cal_data = _apply_first_cal(cal_data, sols, fqs*1e9, info) pol = cal_data.values()[0].keys()[0][0] ratio_bls = [(bl1, bl2) for bls in info.get_reds() for b1, bl1 in enumerate(bls) for bl2 in bls[b1 + 1:] if bl1 != bl2] for i, ((a1, a2), (a3, a4)) in enumerate(ratio_bls): if (a1, a2) not in cal_data.keys(): # conjugate d1 = np.conj(cal_data[a2, a1][pol * 2]) else: d1 = cal_data[a1, a2][pol * 2] if (a3, a4) not in cal_data.keys(): # conjugate d2 = np.conj(cal_data[a4, a3][pol * 2]) else: d2 = cal_data[a3, a4][pol * 2] median_over_freqs = np.median(np.angle(d1 * np.conj(d2)), axis=1) for ai in [a1, a2, a3, a4]: antpol = (ai, pol) if antpol in medians: medians[antpol] = np.append(medians[antpol], median_over_freqs) else: medians[antpol] = median_over_freqs median_over_ant_time = {k: np.median(np.abs(m)) for k, m in medians.items()} # if the ratio is greater than np.pi/2, it is a switched antenna. switched = [k for k, m in median_over_ant_time.items() if m > np.pi / 2] switched_history += switched niters += 1 if niters >= option_parser.maxiter: break return sols, switched_history
def apply_cal(data_infilename, data_outfilename, new_calibration, old_calibration=None, flags_npz=None, flag_nchan_low=0, flag_nchan_high=0, filetype='miriad', gain_convention='divide', add_to_history='', clobber=False, **kwargs): '''Update the calibration solution and flags on the data, writing to a new file. Takes out old calibration and puts in new calibration solution, including its flags. Also enables appending to history. Arguments: data_infilename: filename (or UVData object) of the data file to be updated. data_outfilename: filename of the resultant data file with the new calibration and flags. new_calibration: filename of the calfits file (or a list of filenames) or UVCal object for the calibration to be applied, along with its new flags (if any). old_calibration: filename of the calfits file (or a list of filenames) or UVCal object for the calibration to be unapplied. Default None means that the input data is raw (i.e. uncalibrated). flags_npz: optional path to npz file containing just flags to be ORed with flags in input data flag_chan_low: integer number of channels at the low frequency end of the band to always flag (default 0) flag_chan_high: integer number of channels at the high frequency end of the band to always flag (default 0) filetype: filename for the new file, either 'miriad' or 'uvfits' gain_convention: str, either 'divide' or 'multiply'. 'divide' means V_obs = gi gj* V_true, 'multiply' means V_true = gi gj* V_obs. Assumed to be the same for new_gains and old_gains. add_to_history: appends a string to the history of the output file. This will preceed combined histories of flags_npz (if applicable), new_calibration and, old_calibration (if applicable). clobber: if True, overwrites existing file at outfilename kwargs: dictionary mapping updated attributes to their new values. See pyuvdata.UVData documentation for more info. ''' # load data, flags, and (optionally) npz flags uvd = UVData() if filetype == 'miriad': uvd.read_miriad(data_infilename) else: raise NotImplementedError( 'This function has not been implemented yet.') if flags_npz is not None: npz_flags = np.load(flags_npz) uvd.flag_array = np.logical_or(npz_flags['flag_array'], uvd.flag_array) add_to_history += ' FLAGS_NPZ_HISTORY: ' + str( npz_flags['history']) + '\n' data, data_flags = io.load_vis(uvd) # apply band edge flags for bl in data_flags.keys(): data_flags[bl][:, 0:flag_nchan_low] = True data_flags[bl][:, data_flags[bl].shape[1] - flag_nchan_high:] = True # load new calibration solution if new_calibration is None: raise ValueError('Must provide a calibration solution to apply.') if isinstance(new_calibration, UVCal): uvc = new_calibration else: uvc = UVCal() uvc.read_calfits(new_calibration) add_to_history += ' NEW_CALFITS_HISTORY: ' + uvc.history + '\n' new_gains, new_flags = io.load_cal(uvc) # load old calibration solution if old_calibration is not None: if isinstance(old_calibration, UVCal): old_uvc = old_calibration else: old_uvc = UVCal() old_uvc.read_calfits(old_calibration) add_to_history += ' OLD_CALFITS_HISTORY: ' + old_uvc.history + '\n' old_calibration, _ = io.load_cal(old_uvc) recalibrate_in_place(data, data_flags, new_gains, new_flags, old_gains=old_calibration, gain_convention=gain_convention) io.update_vis(data_infilename, data_outfilename, filetype_in=filetype, filetype_out=filetype, data=data, flags=data_flags, add_to_history=add_to_history, clobber=clobber, **kwargs)
def test_apply_cal(self): fname = os.path.join(DATA_PATH, "zen.2457698.40355.xx.HH.uvcA") outname = os.path.join( DATA_PATH, "test_output/zen.2457698.40355.xx.HH.applied.uvcA") old_cal = os.path.join( DATA_PATH, "test_input/zen.2457698.40355.HH.uvcA.omni.calfits") new_cal = os.path.join( DATA_PATH, "test_input/zen.2457698.40355.HH.uvcA.omni.calfits") flags_npz = os.path.join( DATA_PATH, "zen.2457698.40355.xx.HH.uvcA.fake_flags.npz") uvd = UVData() uvd.read_miriad(fname) uvd.flag_array = np.logical_or(uvd.flag_array, np.load(flags_npz)['flag_array']) data, data_flags = io.load_vis(uvd) new_gains, new_flags = io.load_cal(new_cal) uvc_old = UVCal() uvc_old.read_calfits(old_cal) uvc_old.gain_array *= (3.0 + 4.0j) ac.apply_cal(fname, outname, new_cal, old_calibration=uvc_old, gain_convention='divide', flags_npz=flags_npz, filetype='miriad', clobber=True, vis_units='Jy') u = UVData() u.read_miriad(outname) self.assertEqual(u.vis_units, 'Jy') new_data, new_flags = io.load_vis(outname) for k in new_data.keys(): for i in range(new_data[k].shape[0]): for j in range(new_data[k].shape[1]): if not new_flags[k][i, j]: self.assertAlmostEqual(new_data[k][i, j] / 25.0, data[k][i, j], 4) if j < 300 or j > 923: self.assertTrue(new_flags[k][i, j]) # test band edge flagging ac.apply_cal(fname, outname, new_cal, old_calibration=uvc_old, gain_convention='divide', flag_nchan_low=450, flag_nchan_high=400, filetype='miriad', clobber=True) new_data, new_flags = io.load_vis(outname) for k in new_data.keys(): for i in range(new_data[k].shape[0]): for j in range(new_data[k].shape[1]): if not new_flags[k][i, j]: self.assertAlmostEqual(new_data[k][i, j] / 25.0, data[k][i, j], 4) if j < 450 or j > 623: self.assertTrue(new_flags[k][i, j]) with self.assertRaises(ValueError): ac.apply_cal(fname, outname, None) shutil.rmtree(outname)