def test_load_vis_nested(self):
#duplicated testing from firstcal.UVData_to_dict
str2pol = io.polstr2num
filename1 = os.path.join(DATA_PATH, 'zen.2458043.12552.xx.HH.uvORA')
filename2 = os.path.join(DATA_PATH, 'zen.2458043.13298.xx.HH.uvORA')
uvd1 = UVData()
uvd1.read_miriad(filename1)
uvd2 = UVData()
uvd2.read_miriad(filename2)
if uvd1.phase_type != 'drift':
uvd1.unphase_to_drift()
if uvd2.phase_type != 'drift':
uvd2.unphase_to_drift()
uvd = uvd1 + uvd2
d, f = io.load_vis([uvd1, uvd2], nested_dict=True)
for i, j in d:
for pol in d[i, j]:
uvpol = list(uvd1.polarization_array).index(str2pol[pol])
uvmask = np.all(np.array(zip(uvd.ant_1_array,
uvd.ant_2_array)) == [i, j],
axis=1)
np.testing.assert_equal(
d[i, j][pol],
np.resize(uvd.data_array[uvmask][:, 0, :, uvpol],
d[i, j][pol].shape))
np.testing.assert_equal(
f[i, j][pol],
np.resize(uvd.flag_array[uvmask][:, 0, :, uvpol],
f[i, j][pol].shape))
d, f = io.load_vis([filename1, filename2], nested_dict=True)
for i, j in d:
for pol in d[i, j]:
uvpol = list(uvd.polarization_array).index(str2pol[pol])
uvmask = np.all(np.array(zip(uvd.ant_1_array,
uvd.ant_2_array)) == [i, j],
axis=1)
np.testing.assert_equal(
d[i, j][pol],
np.resize(uvd.data_array[uvmask][:, 0, :, uvpol],
d[i, j][pol].shape))
np.testing.assert_equal(
f[i, j][pol],
np.resize(uvd.flag_array[uvmask][:, 0, :, uvpol],
f[i, j][pol].shape))
uvd = UVData()
uvd.read_miriad(filename1)
self.assertEqual(len(io.load_vis([uvd], nested_dict=True)[0]),
uvd.Nbls)
# reorder baseline array
uvd.baseline_array = uvd.baseline_array[np.argsort(uvd.baseline_array)]
self.assertEqual(len(io.load_vis(filename1, nested_dict=True)[0]),
uvd.Nbls)
def test_mul(self):
test_file = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA")
d, f = io.load_vis(test_file, pop_autos=True)
f[(24, 25, 'xx')][:, 0] = False
f2 = f * f
self.assertFalse(f2[(24, 25, 'xx')][0, 0])
# test exception
d2, f2 = io.load_vis(test_file, pop_autos=True)
d2[d2.keys()[0]] = d2[d2.keys()[0]][:, :10]
self.assertRaises(ValueError, d.__mul__, d2)
d2[d2.keys()[0]] = d2[d2.keys()[0]][:10, :]
self.assertRaises(ValueError, d.__mul__, d2)
def test_adder(self):
test_file = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA")
d, f = io.load_vis(test_file, pop_autos=True)
d2 = d + d
self.assertAlmostEqual(d2[(24, 25, 'xx')][30, 30],
d[(24, 25, 'xx')][30, 30] * 2)
# test exception
d2, f2 = io.load_vis(test_file, pop_autos=True)
d2[d2.keys()[0]] = d2[d2.keys()[0]][:, :10]
self.assertRaises(ValueError, d.__add__, d2)
d2[d2.keys()[0]] = d2[d2.keys()[0]][:10, :]
self.assertRaises(ValueError, d.__add__, d2)
def test_concatenate(self):
test_file = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA")
d, f = io.load_vis(test_file, pop_autos=True)
d2 = d.concatenate(d)
self.assertEqual(d2[(24, 25, 'xx')].shape[0],
d[(24, 25, 'xx')].shape[0] * 2)
d2 = d.concatenate(d, axis=1)
self.assertEqual(d2[(24, 25, 'xx')].shape[1],
d[(24, 25, 'xx')].shape[1] * 2)
d2 = d.concatenate([d, d], axis=0)
self.assertEqual(d2[(24, 25, 'xx')].shape[0],
d[(24, 25, 'xx')].shape[0] * 3)
# test exceptions
d2, f2 = io.load_vis(test_file, pop_autos=True)
d2[d2.keys()[0]] = d2[d2.keys()[0]][:10, :10]
self.assertRaises(ValueError, d.concatenate, d2, axis=0)
self.assertRaises(ValueError, d.concatenate, d2, axis=1)
def test_lst_rephase():
# load point source sim w/ array at latitude = 0
fname = os.path.join(DATA_PATH, "PAPER_point_source_sim.uv")
data, flags, antpos, ants, freqs, times, lsts, pols = io.load_vis(
fname, return_meta=True)
data_drift = copy.deepcopy(data)
transit_integration = 50
# get integration time in LST, baseline dict
dlst = np.median(np.diff(lsts))
bls = odict(map(lambda k: (k, antpos[k[0]] - antpos[k[1]]), data.keys()))
# basic test: single dlst for all integrations
hc.utils.lst_rephase(data, bls, freqs, dlst, lat=0.0)
# get phase error on shortest EW baseline
k = (0, 1, 'xx')
# check error at transit
phs_err = np.angle(data[k][transit_integration, 4] /
data_drift[k][transit_integration + 1, 4])
nt.assert_true(np.isclose(phs_err, 0, atol=1e-7))
# check error across file
phs_err = np.angle(data[k][:-1, 4] / data_drift[k][1:, 4])
nt.assert_true(np.abs(phs_err).max() < 1e-4)
# multiple phase term test: dlst per integration
dlst = np.array([np.median(np.diff(lsts))] * data[k].shape[0])
data = copy.deepcopy(data_drift)
hc.utils.lst_rephase(data, bls, freqs, dlst, lat=0.0)
# check error at transit
phs_err = np.angle(data[k][transit_integration, 4] /
data_drift[k][transit_integration + 1, 4])
nt.assert_true(np.isclose(phs_err, 0, atol=1e-7))
# check err across file
phs_err = np.angle(data[k][:-1, 4] / data_drift[k][1:, 4])
nt.assert_true(np.abs(phs_err).max() < 1e-4)
# phase all integrations to a single integration
dlst = lsts[50] - lsts
data = copy.deepcopy(data_drift)
hc.utils.lst_rephase(data, bls, freqs, dlst, lat=0.0)
# check error at transit
phs_err = np.angle(data[k][transit_integration, 4] /
data_drift[k][transit_integration, 4])
nt.assert_true(np.isclose(phs_err, 0, atol=1e-7))
# check error across file
phs_err = np.angle(data[k][:, 4] / data_drift[k][50, 4])
nt.assert_true(np.abs(phs_err).max() < 1e-4)
# test operation on array
k = (0, 1, 'xx')
d = data_drift[k].copy()
d_phs = hc.utils.lst_rephase(d, bls[k], freqs, dlst, lat=0.0, array=True)
nt.assert_almost_equal(
np.abs(np.angle(d_phs[50] / data[k][50])).max(), 0.0)
def load_data(self, input_data, filetype='miriad'):
'''Loads in and stores data for delay filtering.
Arguments:
input_data: data file path, or UVData instance, or list of either strings of data file paths
or list of UVData instances to concatenate into a single internal DataContainer
filetype: file format of data. Default 'miriad.' Ignored if input_data is UVData object(s).
'''
if isinstance(input_data, (str, UVData)):
self.writable = True
self.input_data, self.filetype = input_data, filetype
self.data, self.flags, self.antpos, _, self.freqs, self.times, _, _ = io.load_vis(
input_data, return_meta=True, filetype=filetype)
self.Nfreqs = len(self.freqs)
def test_update_vis(self):
# load in cal
fname = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA")
outname = os.path.join(
DATA_PATH, "test_output/zen.2458043.12552.xx.HH.modified.uvORA")
uvd = UVData()
uvd.read_miriad(fname)
data, flags, antpos, ants, freqs, times, lsts, pols = io.load_vis(
fname, return_meta=True)
#make some modifications
new_data = {key: (2. + 1.j) * val for key, val in data.items()}
new_flags = {key: np.logical_not(val) for key, val in flags.items()}
io.update_vis(fname,
outname,
data=new_data,
flags=new_flags,
add_to_history='hello world',
clobber=True,
telescope_name='PAPER')
#test modifications
data, flags, antpos, ants, freqs, times, lsts, pols = io.load_vis(
outname, return_meta=True)
for k in data.keys():
self.assertTrue(np.all(new_data[k] == data[k]))
self.assertTrue(np.all(new_flags[k] == flags[k]))
uvd2 = UVData()
uvd2.read_miriad(outname)
self.assertTrue(
pyuvdata.utils.check_histories(uvd2.history,
uvd.history + 'hello world'))
self.assertEqual(uvd2.telescope_name, 'PAPER')
shutil.rmtree(outname)
# Coverage for errors
with self.assertRaises(NotImplementedError):
io.update_vis(uvd,
outname,
data=new_data,
flags=new_flags,
filetype_out='uvfits',
add_to_history='hello world',
clobber=True,
telescope_name='PAPER')
with self.assertRaises(NotImplementedError):
io.update_vis(fname,
outname,
data=new_data,
flags=new_flags,
filetype_in='uvfits',
add_to_history='hello world',
clobber=True,
telescope_name='PAPER')
with self.assertRaises(TypeError):
io.update_vis(uvd,
outname,
data=new_data,
flags=new_flags,
filetype_out='not_a_real_filetype',
add_to_history='hello world',
clobber=True,
telescope_name='PAPER')
with self.assertRaises(TypeError):
io.update_vis(fname,
outname,
data=new_data,
flags=new_flags,
filetype_in='not_a_real_filetype',
add_to_history='hello world',
clobber=True,
telescope_name='PAPER')
# #now try the same thing but with a UVData object instead of path
io.update_vis(uvd,
outname,
data=new_data,
flags=new_flags,
add_to_history='hello world',
clobber=True,
telescope_name='PAPER')
data, flags, antpos, ants, freqs, times, lsts, pols = io.load_vis(
outname, return_meta=True)
for k in data.keys():
self.assertTrue(np.all(new_data[k] == data[k]))
self.assertTrue(np.all(new_flags[k] == flags[k]))
uvd2 = UVData()
uvd2.read_miriad(outname)
self.assertTrue(
pyuvdata.utils.check_histories(uvd2.history,
uvd.history + 'hello world'))
self.assertEqual(uvd2.telescope_name, 'PAPER')
shutil.rmtree(outname)
def test_load_vis(self):
#duplicated testing from abscal_funcs.UVData2AbsCalDict
# load into pyuvdata object
self.data_file = os.path.join(DATA_PATH,
"zen.2458043.12552.xx.HH.uvORA")
self.uvd = UVData()
self.uvd.read_miriad(self.data_file)
self.freq_array = np.unique(self.uvd.freq_array)
self.antpos, self.ants = self.uvd.get_ENU_antpos(center=True,
pick_data_ants=True)
self.antpos = odict(zip(self.ants, self.antpos))
self.time_array = np.unique(self.uvd.time_array)
fname = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA")
data, flags = io.load_vis(fname, pop_autos=False)
self.assertEqual(data[(24, 25, 'xx')].shape, (60, 64))
self.assertEqual(flags[(24, 25, 'xx')].shape, (60, 64))
self.assertEqual((24, 24, 'xx') in data, True)
data, flags = io.load_vis([fname])
self.assertEqual(data[(24, 25, 'xx')].shape, (60, 64))
# test pop autos
data, flags = io.load_vis(fname, pop_autos=True)
self.assertEqual((24, 24, 'xx') in data, False)
# test uvd object
uvd = UVData()
uvd.read_miriad(fname)
data, flags = io.load_vis(uvd)
self.assertEqual(data[(24, 25, 'xx')].shape, (60, 64))
data, flags = io.load_vis([uvd])
self.assertEqual(data[(24, 25, 'xx')].shape, (60, 64))
# test multiple
fname2 = os.path.join(DATA_PATH, "zen.2458043.13298.xx.HH.uvORA")
data, flags = io.load_vis([fname, fname2])
self.assertEqual(data[(24, 25, 'xx')].shape, (120, 64))
self.assertEqual(flags[(24, 25, 'xx')].shape, (120, 64))
# test w/ meta
d, f, ap, a, f, t, l, p = io.load_vis([fname, fname2],
return_meta=True)
self.assertEqual(len(ap[24]), 3)
self.assertEqual(len(f), len(self.freq_array))
#test uvfits
fname = os.path.join(DATA_PATH,
'zen.2458043.12552.xx.HH.uvA.vis.uvfits')
with self.assertRaises(NotImplementedError):
d, f = io.load_vis(fname, filetype='uvfits')
with self.assertRaises(NotImplementedError):
d, f = io.load_vis([fname, fname], filetype='uvfits')
#self.assertEqual(d[(0,1,'xx')].shape, (60,64))
with self.assertRaises(NotImplementedError):
d, f = io.load_vis(fname, filetype='not_a_real_filetype')
with self.assertRaises(NotImplementedError):
d, f = io.load_vis(['str1', 'str2'],
filetype='not_a_real_filetype')
with self.assertRaises(TypeError):
d, f = io.load_vis([1, 2], filetype='uvfits')
# test w/ meta pick_data_ants
fname = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA")
d, f, ap, a, f, t, l, p = io.load_vis(fname,
return_meta=True,
pick_data_ants=False)
self.assertEqual(len(ap[24]), 3)
self.assertEqual(len(a), 47)
self.assertEqual(len(f), len(self.freq_array))
with self.assertRaises(TypeError):
d, f = io.load_vis(1.0)
def test_write_vis(self):
# get data
uvd = UVData()
uvd.read_miriad(
os.path.join(DATA_PATH, "zen.2458044.41632.xx.HH.uvXRAA"))
data, flgs, ap, a, f, t, l, p = io.load_vis(uvd, return_meta=True)
nsample = copy.deepcopy(data)
for k in nsample.keys():
nsample[k] = np.ones_like(nsample[k], np.float)
# test basic execution
uvd = io.write_vis("ex.uv",
data,
l,
f,
ap,
start_jd=2458044,
return_uvd=True,
overwrite=True,
verbose=True)
uvd2 = UVData()
uvd2.read_miriad('ex.uv')
self.assertTrue(os.path.exists('ex.uv'))
self.assertEqual(uvd.data_array.shape, (1680, 1, 64, 1))
self.assertEqual(uvd2.data_array.shape, (1680, 1, 64, 1))
self.assertAlmostEqual(data[(24, 25, 'xx')][30, 32],
uvd.get_data(24, 25, 'xx')[30, 32])
self.assertAlmostEqual(data[(24, 25, 'xx')][30, 32],
uvd2.get_data(24, 25, 'xx')[30, 32])
# test with nsample and flags
uvd = io.write_vis("ex.uv",
data,
l,
f,
ap,
start_jd=2458044,
flags=flgs,
nsamples=nsample,
return_uvd=True,
overwrite=True,
verbose=True)
self.assertEqual(uvd.nsample_array.shape, (1680, 1, 64, 1))
self.assertEqual(uvd.flag_array.shape, (1680, 1, 64, 1))
self.assertAlmostEqual(nsample[(24, 25, 'xx')][30, 32],
uvd.get_nsamples(24, 25, 'xx')[30, 32])
self.assertAlmostEqual(flgs[(24, 25, 'xx')][30, 32],
uvd.get_flags(24, 25, 'xx')[30, 32])
# test exceptions
self.assertRaises(AttributeError, io.write_vis, "ex.uv", data, l, f,
ap)
self.assertRaises(AttributeError,
io.write_vis,
"ex.uv",
data,
l,
f,
ap,
start_jd=2458044,
filetype='foo')
if os.path.exists('ex.uv'):
shutil.rmtree('ex.uv')
def lst_bin_files(data_files,
dlst=None,
verbose=True,
ntimes_per_file=60,
file_ext="{}.{}.{:7.5f}.uv",
outdir=None,
overwrite=False,
history=' ',
lst_start=0,
atol=1e-6,
sig_clip=True,
sigma=5.0,
min_N=5,
rephase=False,
output_file_select=None,
**kwargs):
"""
LST bin a series of miriad files with identical frequency bins, but varying
time bins. Output miriad file meta data (frequency bins, antennas positions, time_array)
are taken from the first file in data_files.
Parameters:
-----------
data_files : type=list of lists: nested set of lists, with each nested list containing
paths to miriad files from a particular night. These files should be sorted
by ascending Julian Date. Frequency axis of each file must be identical.
dlst : type=float, LST bin width. If None, will get this from the first file in data_files.
lst_start : type=float, starting LST for binner as it sweeps from lst_start to lst_start + 2pi.
ntimes_per_file : type=int, number of LST bins in a single output file
file_ext : type=str, extension to "zen." for output miriad files. This must have three
formatting placeholders, first for polarization(s), second for type of file
Ex. ["LST", "STD", "NUM"] and third for starting LST bin of file.
outdir : type=str, output directory
overwrite : type=bool, if True overwrite output files
rephase : type=bool, if True, rephase data points in LST bin to center of bin
bin_kwargs : type=dictionary, keyword arguments for lst_bin.
atol : type=float, absolute tolerance for LST bin float comparison
output_file_select : type=int or integer list, list of integer indices of the output files to run on.
Default is all files.
kwargs : type=dictionary, keyword arguments to pass to io.write_vis()
Result:
-------
zen.{pol}.LST.{file_lst}.uv : holds LST bin avg (data_array) and bin count (nsample_array)
zen.{pol}.STD.{file_lst}.uv : holds LST bin stand dev along real and imag (data_array)
"""
(data_times, lst_grid, dlst, file_lsts,
start_lst) = config_lst_bin_files(data_files,
dlst=dlst,
lst_start=lst_start,
ntimes_per_file=ntimes_per_file,
verbose=verbose)
nfiles = len(file_lsts)
# select file_lsts
if output_file_select is not None:
if isinstance(output_file_select, (int, np.integer)):
output_file_select = [output_file_select]
output_file_select = [int(o) for o in output_file_select]
try:
file_lsts = map(lambda i: file_lsts[i], output_file_select)
except IndexError:
print "Warning: one or more indices in output_file_select {} caused an index error with length {} file_lsts list, exiting...".format(
output_file_select, nfiles)
return
# create data file status: None if not opened, data objects if opened
data_status = map(lambda d: map(lambda f: None, d), data_files)
# get outdir
if outdir is None:
outdir = os.path.dirname(
os.path.commonprefix(abscal.flatten(data_files)))
# update miriad_kwrgs
kwargs['outdir'] = outdir
kwargs['overwrite'] = overwrite
# get metadata from the zeroth data file
d, fl, ap, a, f, t, l, p = io.load_vis(data_files[0][0],
return_meta=True,
pick_data_ants=False)
# push time and lst arrays forward by half an integration
abscal.echo("pushing time and lst arrays forward by half an integration b/c pyuvdata "\
"currently does not do this for us....", verbose=verbose)
if len(t) > 1:
t += np.median(np.diff(t)) / 2.0
# get frequency, time and antenna position information
freq_array = copy.copy(f)
antpos = copy.deepcopy(ap)
start_jd = np.floor(t)[0]
kwargs['start_jd'] = start_jd
kwargs['integration_time'] = np.median(np.diff(t)) * 24 * 3600.
del d, fl, ap, a, f, t, l, p
garbage_collector.collect()
# iterate over output LST files
for i, f_lst in enumerate(file_lsts):
abscal.echo("LST file {} / {}: {}".format(i + 1, len(file_lsts),
datetime.datetime.now()),
type=1,
verbose=verbose)
# create empty data_list and lst_list
data_list = []
file_list = []
flgs_list = []
lst_list = []
# locate all data files that fall within the range of lst for this output file
f_min = np.min(f_lst)
f_max = np.max(f_lst)
f_select = np.array(
map(
lambda d: map(lambda f:
(f[1] > f_min) & (f[0] < f_max + atol), d),
data_times))
if i == 0:
old_f_select = copy.copy(f_select)
# open necessary files, close ones that are no longer needed
# append necessary files to data_list
for j in range(len(data_files)):
nightly_data_list = []
nightly_flgs_list = []
nightly_lst_list = []
for k in range(len(data_files[j])):
if f_select[j][k] == True and data_status[j][k] is None:
# open file
d, fl, ap, a, f, t, l, p = io.load_vis(data_files[j][k],
return_meta=True)
d.phase_type = 'drift'
# unwrap l relative to itself
l[l < l[0]] += 2 * np.pi
# push time and lst arrays forward by half an integration
# b/c pyuvdata does not currently do this for us....
if len(l) > 1:
l += np.median(np.diff(l)) / 2.0
t += np.median(np.diff(t)) / 2.0
# unwrap l relative to start_lst
l[l < start_lst - atol] += 2 * np.pi
# pass data references to data_status list
data_status[j][k] = [d, fl, ap, a, f, t, l, p]
# erase references from namespace
del d, fl, ap, a, f, t, l, p
elif f_select[j][k] == False and old_f_select[j][k] == True:
# close file
del data_status[j][k]
data_status[j].insert(k, None)
# if file is needed, append data references to data_list
if f_select[j][k] == True:
file_list.append(data_files[j][k])
nightly_data_list.append(
data_status[j][k][0]) # this is data
nightly_flgs_list.append(
data_status[j][k][1]) # this is flgs
nightly_lst_list.append(
data_status[j][k][6]) # this is lsts
# skip if nothing accumulated in nightly files
if len(nightly_data_list) == 0:
continue
# extend to data lists
data_list.extend(nightly_data_list)
flgs_list.extend(nightly_flgs_list)
lst_list.extend(nightly_lst_list)
del nightly_data_list, nightly_flgs_list, nightly_lst_list
# skip if data_list is empty
if len(data_list) == 0:
abscal.echo("data_list is empty for beginning LST {}".format(
f_lst[0]),
verbose=verbose)
# erase data references
del file_list, data_list, flgs_list, lst_list
# assign old f_select
old_f_select = copy.copy(f_select)
continue
# pass through lst-bin function
(bin_lst, bin_data, flag_data, std_data,
num_data) = lst_bin(data_list,
lst_list,
flags_list=flgs_list,
dlst=dlst,
lst_start=start_lst,
lst_low=f_min,
lst_hi=f_max,
truncate_empty=False,
sig_clip=sig_clip,
sigma=sigma,
min_N=min_N,
rephase=rephase,
freq_array=freq_array,
antpos=antpos)
# push lst_bins back to bin start rather than bin center
# b/c pyuvdata does not do this for us yet...
abscal.echo("pushing time and lst arrays backward by half an integration b/c pyuvdata "\
"currently does not do this for us....", verbose=verbose)
# unwrap bin_lst
bin_lst[bin_lst < bin_lst[0] - atol] += 2 * np.pi
# push back
if len(bin_lst) > 1:
bin_lst -= np.median(np.diff(bin_lst)) / 2.0
# re-wrap bin_lst
bin_lst = bin_lst % (2 * np.pi)
# update history
file_history = history + " Input files: " + "-".join(
map(lambda ff: os.path.basename(ff), file_list))
kwargs['history'] = file_history
# erase data references
del file_list, data_list, flgs_list, lst_list
garbage_collector.collect()
# assign old f_select
old_f_select = copy.copy(f_select)
# get polarizations
pols = bin_data.pols()
# configure filenames
bin_file = "zen.{}".format(
file_ext.format('.'.join(pols), "LST", bin_lst[0]))
std_file = "zen.{}".format(
file_ext.format('.'.join(pols), "STD", bin_lst[0]))
# check for overwrite
if os.path.exists(bin_file) and overwrite is False:
abscal.echo("{} exists, not overwriting".format(bin_file),
verbose=verbose)
continue
# write to file
io.write_vis(bin_file,
bin_data,
bin_lst,
freq_array,
antpos,
flags=flag_data,
verbose=verbose,
nsamples=num_data,
filetype='miriad',
**kwargs)
io.write_vis(std_file,
std_data,
bin_lst,
freq_array,
antpos,
flags=flag_data,
verbose=verbose,
nsamples=num_data,
filetype='miriad',
**kwargs)
del bin_file, std_file, bin_data, std_data, num_data, bin_lst, flag_data
garbage_collector.collect()
def lst_align_files(data_files,
file_ext=".L.{:7.5f}",
dlst=None,
overwrite=None,
outdir=None,
miriad_kwargs={},
align_kwargs={},
verbose=True):
"""
Align a series of data files with a universal LST grid.
Parameters:
-----------
data_files : type=list, list of paths to miriad files, or a single miriad file path
file_ext : type=str, file_extension for each file in data_files when writing to disk
dlst : type=float, LST grid bin interval, if None get it from first file in data_files
overwrite : type=boolean, if True overwrite output files
miriad_kwargs : type=dictionary, keyword arguments to feed to miriad_to_data()
align_kwargs : keyword arguments to feed to lst_align()
Result:
-------
A series of "data_files + file_ext" miriad files written to disk.
"""
# check type of data_files
if type(data_files) == str:
data_files = [data_files]
# get dlst if None
if dlst is None:
start, stop, int_time = utils.get_miriad_times(data_files[0])
dlst = int_time
# iterate over data files
for i, f in enumerate(data_files):
# load data
(data, flgs, apos, ants, freqs, times, lsts,
pols) = io.load_vis(f, return_meta=True)
# lst align
interp_data, interp_flgs, interp_lsts = lst_align(data,
lsts,
flags=flgs,
dlst=dlst,
**align_kwargs)
# check output
output_fname = os.path.basename(f) + file_ext.format(interp_lsts[0])
# write to miriad file
if overwrite is not None:
miriad_kwargs['overwrite'] = overwrite
if outdir is not None:
miriad_kwargs['outdir'] = outdir
miriad_kwargs['start_jd'] = np.floor(times[0])
io.write_vis(output_fname,
interp_data,
interp_lsts,
freqs,
apos,
flags=interp_flgs,
verbose=verbose,
filetype='miriad',
**miriad_kwargs)
def _search_and_iterate_firstcal(uv, info, option_parser):
'''Searches and iterates over firstcal
Iteratively run firstcal and look for rotated antennas.
If rotated antennas found, fix the antenna and rerun firstcal.
Args:
uv (pyuvdata.UVData): UVData object
info (FirstcalRedundantInfo): info object
option_parser (OptionParser): option parser object
Return:
dict : firstcal solutions
list : list of rotated antennas
'''
switched_history = [] # keep track of all rotated antennas
switched = [] # keep track of rotated antennas in each iteration
niters = 0
while niters == 0 or len(switched) > 0:
datapack, flagpack = io.load_vis([uv], nested_dict=True)
datapack = _apply_pi_shift(datapack, switched)
wgtpack = {k: {p: np.logical_not(flagpack[k][p]) for p in flagpack[k]} for k in flagpack}
fqs = uv.freq_array[0, :] / 1e9
# gets phase solutions per frequency.
fc = FirstCal(datapack, wgtpack, fqs, info)
sols = fc.run(finetune=option_parser.finetune,
verbose=option_parser.verbose,
average=option_parser.average,
window='none')
# Now we need to check if antennas are flipped
medians = {}
cal_data = copy.deepcopy(datapack)
cal_data = _apply_first_cal(cal_data, sols, fqs*1e9, info)
pol = cal_data.values()[0].keys()[0][0]
ratio_bls = [(bl1, bl2) for bls in info.get_reds() for b1, bl1 in enumerate(bls) for bl2 in bls[b1 + 1:] if bl1 != bl2]
for i, ((a1, a2), (a3, a4)) in enumerate(ratio_bls):
if (a1, a2) not in cal_data.keys():
# conjugate
d1 = np.conj(cal_data[a2, a1][pol * 2])
else:
d1 = cal_data[a1, a2][pol * 2]
if (a3, a4) not in cal_data.keys():
# conjugate
d2 = np.conj(cal_data[a4, a3][pol * 2])
else:
d2 = cal_data[a3, a4][pol * 2]
median_over_freqs = np.median(np.angle(d1 * np.conj(d2)), axis=1)
for ai in [a1, a2, a3, a4]:
antpol = (ai, pol)
if antpol in medians:
medians[antpol] = np.append(medians[antpol], median_over_freqs)
else:
medians[antpol] = median_over_freqs
median_over_ant_time = {k: np.median(np.abs(m)) for k, m in medians.items()}
# if the ratio is greater than np.pi/2, it is a switched antenna.
switched = [k for k, m in median_over_ant_time.items() if m > np.pi / 2]
switched_history += switched
niters += 1
if niters >= option_parser.maxiter:
break
return sols, switched_history
def apply_cal(data_infilename,
data_outfilename,
new_calibration,
old_calibration=None,
flags_npz=None,
flag_nchan_low=0,
flag_nchan_high=0,
filetype='miriad',
gain_convention='divide',
add_to_history='',
clobber=False,
**kwargs):
'''Update the calibration solution and flags on the data, writing to a new file. Takes out old calibration
and puts in new calibration solution, including its flags. Also enables appending to history.
Arguments:
data_infilename: filename (or UVData object) of the data file to be updated.
data_outfilename: filename of the resultant data file with the new calibration and flags.
new_calibration: filename of the calfits file (or a list of filenames) or UVCal object for the calibration
to be applied, along with its new flags (if any).
old_calibration: filename of the calfits file (or a list of filenames) or UVCal object for the calibration
to be unapplied. Default None means that the input data is raw (i.e. uncalibrated).
flags_npz: optional path to npz file containing just flags to be ORed with flags in input data
flag_chan_low: integer number of channels at the low frequency end of the band to always flag (default 0)
flag_chan_high: integer number of channels at the high frequency end of the band to always flag (default 0)
filetype: filename for the new file, either 'miriad' or 'uvfits'
gain_convention: str, either 'divide' or 'multiply'. 'divide' means V_obs = gi gj* V_true,
'multiply' means V_true = gi gj* V_obs. Assumed to be the same for new_gains and old_gains.
add_to_history: appends a string to the history of the output file. This will preceed combined histories
of flags_npz (if applicable), new_calibration and, old_calibration (if applicable).
clobber: if True, overwrites existing file at outfilename
kwargs: dictionary mapping updated attributes to their new values.
See pyuvdata.UVData documentation for more info.
'''
# load data, flags, and (optionally) npz flags
uvd = UVData()
if filetype == 'miriad':
uvd.read_miriad(data_infilename)
else:
raise NotImplementedError(
'This function has not been implemented yet.')
if flags_npz is not None:
npz_flags = np.load(flags_npz)
uvd.flag_array = np.logical_or(npz_flags['flag_array'], uvd.flag_array)
add_to_history += ' FLAGS_NPZ_HISTORY: ' + str(
npz_flags['history']) + '\n'
data, data_flags = io.load_vis(uvd)
# apply band edge flags
for bl in data_flags.keys():
data_flags[bl][:, 0:flag_nchan_low] = True
data_flags[bl][:, data_flags[bl].shape[1] - flag_nchan_high:] = True
# load new calibration solution
if new_calibration is None:
raise ValueError('Must provide a calibration solution to apply.')
if isinstance(new_calibration, UVCal):
uvc = new_calibration
else:
uvc = UVCal()
uvc.read_calfits(new_calibration)
add_to_history += ' NEW_CALFITS_HISTORY: ' + uvc.history + '\n'
new_gains, new_flags = io.load_cal(uvc)
# load old calibration solution
if old_calibration is not None:
if isinstance(old_calibration, UVCal):
old_uvc = old_calibration
else:
old_uvc = UVCal()
old_uvc.read_calfits(old_calibration)
add_to_history += ' OLD_CALFITS_HISTORY: ' + old_uvc.history + '\n'
old_calibration, _ = io.load_cal(old_uvc)
recalibrate_in_place(data,
data_flags,
new_gains,
new_flags,
old_gains=old_calibration,
gain_convention=gain_convention)
io.update_vis(data_infilename,
data_outfilename,
filetype_in=filetype,
filetype_out=filetype,
data=data,
flags=data_flags,
add_to_history=add_to_history,
clobber=clobber,
**kwargs)
def test_apply_cal(self):
fname = os.path.join(DATA_PATH, "zen.2457698.40355.xx.HH.uvcA")
outname = os.path.join(
DATA_PATH, "test_output/zen.2457698.40355.xx.HH.applied.uvcA")
old_cal = os.path.join(
DATA_PATH, "test_input/zen.2457698.40355.HH.uvcA.omni.calfits")
new_cal = os.path.join(
DATA_PATH, "test_input/zen.2457698.40355.HH.uvcA.omni.calfits")
flags_npz = os.path.join(
DATA_PATH, "zen.2457698.40355.xx.HH.uvcA.fake_flags.npz")
uvd = UVData()
uvd.read_miriad(fname)
uvd.flag_array = np.logical_or(uvd.flag_array,
np.load(flags_npz)['flag_array'])
data, data_flags = io.load_vis(uvd)
new_gains, new_flags = io.load_cal(new_cal)
uvc_old = UVCal()
uvc_old.read_calfits(old_cal)
uvc_old.gain_array *= (3.0 + 4.0j)
ac.apply_cal(fname,
outname,
new_cal,
old_calibration=uvc_old,
gain_convention='divide',
flags_npz=flags_npz,
filetype='miriad',
clobber=True,
vis_units='Jy')
u = UVData()
u.read_miriad(outname)
self.assertEqual(u.vis_units, 'Jy')
new_data, new_flags = io.load_vis(outname)
for k in new_data.keys():
for i in range(new_data[k].shape[0]):
for j in range(new_data[k].shape[1]):
if not new_flags[k][i, j]:
self.assertAlmostEqual(new_data[k][i, j] / 25.0,
data[k][i, j], 4)
if j < 300 or j > 923:
self.assertTrue(new_flags[k][i, j])
# test band edge flagging
ac.apply_cal(fname,
outname,
new_cal,
old_calibration=uvc_old,
gain_convention='divide',
flag_nchan_low=450,
flag_nchan_high=400,
filetype='miriad',
clobber=True)
new_data, new_flags = io.load_vis(outname)
for k in new_data.keys():
for i in range(new_data[k].shape[0]):
for j in range(new_data[k].shape[1]):
if not new_flags[k][i, j]:
self.assertAlmostEqual(new_data[k][i, j] / 25.0,
data[k][i, j], 4)
if j < 450 or j > 623:
self.assertTrue(new_flags[k][i, j])
with self.assertRaises(ValueError):
ac.apply_cal(fname, outname, None)
shutil.rmtree(outname)
