def test_jones_delay(tmp_path):
"""
Test when data file has more than one element in Jones matrix.
Currently we do not have a testfile, so we will artifically create one
and check for internal consistency.
"""
cal_in = UVCal()
cal_out = UVCal()
testfile = os.path.join(DATA_PATH, "zen.2457698.40355.xx.delay.calfits")
write_file = str(tmp_path / "outtest_jones.fits")
cal_in.read_calfits(testfile)
# Create filler jones info
cal_in.jones_array = np.array([-5, -6, -7, -8])
cal_in.Njones = 4
cal_in.flag_array = np.zeros(cal_in._flag_array.expected_shape(cal_in),
dtype=bool)
cal_in.delay_array = np.ones(cal_in._delay_array.expected_shape(cal_in),
dtype=np.float64)
cal_in.quality_array = np.zeros(
cal_in._quality_array.expected_shape(cal_in))
cal_in.write_calfits(write_file, clobber=True)
cal_out.read_calfits(write_file)
assert cal_in == cal_out
def test_fits_header_errors_delay(tmp_path, header_dict, error_msg):
# change values for various axes in flag and total quality hdus to not
# match primary hdu
cal_in = UVCal()
cal_out = UVCal()
testfile = os.path.join(DATA_PATH, "zen.2457698.40355.xx.delay.calfits")
write_file = str(tmp_path / "outtest_firstcal.fits")
write_file2 = str(tmp_path / "outtest_firstcal2.fits")
cal_in.read_calfits(testfile)
# Create filler jones info
cal_in.jones_array = np.array([-5, -6, -7, -8])
cal_in.Njones = 4
cal_in.flag_array = np.zeros(cal_in._flag_array.expected_shape(cal_in),
dtype=bool)
cal_in.delay_array = np.ones(cal_in._delay_array.expected_shape(cal_in),
dtype=np.float64)
cal_in.quality_array = np.zeros(
cal_in._quality_array.expected_shape(cal_in))
# add total_quality_array so that can be tested as well
cal_in.total_quality_array = np.zeros(
cal_in._total_quality_array.expected_shape(cal_in))
# write file
cal_in.write_calfits(write_file, clobber=True)
unit = list(header_dict.keys())[0]
keyword = header_dict[unit]
fname = fits.open(write_file)
data = fname[0].data
primary_hdr = fname[0].header
hdunames = uvutils._fits_indexhdus(fname)
ant_hdu = fname[hdunames["ANTENNAS"]]
flag_hdu = fname[hdunames["FLAGS"]]
flag_hdr = flag_hdu.header
totqualhdu = fname[hdunames["TOTQLTY"]]
totqualhdr = totqualhdu.header
if unit == "flag":
flag_hdr[keyword] *= 2
elif unit == "totqual":
totqualhdr[keyword] *= 2
prihdu = fits.PrimaryHDU(data=data, header=primary_hdr)
hdulist = fits.HDUList([prihdu, ant_hdu])
flag_hdu = fits.ImageHDU(data=flag_hdu.data, header=flag_hdr)
hdulist.append(flag_hdu)
totqualhdu = fits.ImageHDU(data=totqualhdu.data, header=totqualhdr)
hdulist.append(totqualhdu)
hdulist.writeto(write_file2, overwrite=True)
hdulist.close()
with pytest.raises(ValueError, match=error_msg):
cal_out.read_calfits(write_file2)
return
def combine_calfits(files, fname, outdir=None, overwrite=False, broadcast_flags=True, verbose=True):
"""
multiply together multiple calfits gain solutions (overlapping in time and frequency)
Parameters:
-----------
files : type=list, dtype=str, list of files to multiply together
fname : type=str, path to output filename
outdir : type=str, path to output directory
overwrite : type=bool, overwrite output file
broadcast_flags : type=bool, if True, broadcast flags from each calfits to final solution
"""
# get io params
if outdir is None:
outdir = "./"
output_fname = os.path.join(outdir, fname)
if os.path.exists(fname) and overwrite is False:
raise IOError("{} exists, not overwriting".format(output_fname))
# iterate over files
for i, f in enumerate(files):
if i == 0:
echo("...loading {}".format(f), verbose=verbose)
uvc = UVCal()
uvc.read_calfits(f)
f1 = copy.copy(f)
# set flagged data to unity
uvc.gain_array[uvc.flag_array] /= uvc.gain_array[uvc.flag_array]
else:
uvc2 = UVCal()
uvc2.read_calfits(f)
# set flagged data to unity
gain_array = uvc2.gain_array
gain_array[uvc2.flag_array] /= gain_array[uvc2.flag_array]
# multiply gain solutions in
uvc.gain_array *= uvc2.gain_array
# pass flags
if broadcast_flags:
uvc.flag_array += uvc2.flag_array
else:
uvc.flag_array = uvc.flag_array * uvc2.flag_array
# write to file
echo("...saving {}".format(output_fname), verbose=verbose)
uvc.write_calfits(output_fname, clobber=True)
def blank_uvcal_from_uvdata(uvdata):
"""initialize UVCal object with same times, antennas, and frequencies as uvdata.
Parameters
----------
uvdata: UVData object
UVData object that you wish to generate a blanck UVCal object from.
Returns
-------
uvcal: UVCal object
UVCal object with all flags set to False
and all gains set to unity with same antennas, freqs, jones, and times
as uvdata.
"""
uvcal = UVCal()
uvcal.Nfreqs = uvdata.Nfreqs
uvcal.Njones = uvdata.Npols
uvcal.Ntimes = uvdata.Ntimes
uvcal.Nspws = uvdata.Nspws
uvcal.history = ""
uvcal.Nspws = uvdata.Nspws
uvcal.telescope_name = uvdata.telescope_name
uvcal.telescope_location = uvdata.telescope_location
uvcal.Nants_data = uvdata.Nants_data
uvcal.Nants_telescope = uvdata.Nants_telescope
uvcal.ant_array = np.asarray(list(set(uvdata.ant_1_array).union(set(uvdata.ant_2_array))))
uvcal.antenna_names = uvdata.antenna_names
uvcal.antenna_numbers = uvdata.antenna_numbers
uvcal.antenna_positions = uvdata.antenna_positions
uvcal.spw_array = uvdata.spw_array
uvcal.freq_array = uvdata.freq_array
uvcal.jones_array = uvdata.polarization_array
uvcal.time_array = np.unique(uvdata.time_array)
uvcal.integration_time = np.mean(uvdata.integration_time)
uvcal.lst_array = np.unique(uvdata.lst_array)
uvcal.gain_convention = "divide" # always use divide for this package.
uvcal.flag_array = np.zeros(
(uvcal.Nants_data, uvcal.Nspws, uvcal.Nfreqs, uvcal.Ntimes, uvcal.Njones),
dtype=np.bool,
)
uvcal.quality_array = np.zeros_like(uvcal.flag_array, dtype=np.float64)
uvcal.x_orientation = uvdata.x_orientation
uvcal.gain_array = np.ones_like(uvcal.flag_array, dtype=np.complex128)
uvcal.cal_style = "redundant"
uvcal.cal_type = "gain"
uvcal.time_range = (
uvcal.time_array.min() - uvcal.integration_time / 2.0,
uvcal.time_array.max() + uvcal.integration_time / 2.0,
)
uvcal.channel_width = np.median(np.diff(uvcal.freq_array))
return uvcal
def uvcal_from_data(self):
"""
Generate an empty uvcal object from visibility parameters
"""
uvc = UVCal()
uvc.Njones = self.measured_vis.Npols
uvc.Nfreqs = self.measured_vis.Nfreqs
uvc.Ntimes = self.measured_vis.Ntimes
uvc.Nspws = self.measured_vis.Nspws
uvc.time_range = (self.measured_vis.time_array.min(),
self.measured_vis.time_array.max())
uvc.telescope_name = self.measured_vis.telescope_name
uvc.Nants_data = self.measured_vis.Nants_data
uvc.Nants_telescope = self.measured_vis.Nants_telescope
uvc.ant_array = np.unique(self.measured_vis.ant_1_array)
uvc.antenna_names = self.measured_vis.antenna_names
uvc.antenna_numbers = self.measured_vis.antenna_numbers
uvc.freq_array = self.measured_vis.freq_array
uvc.channel_width = self.measured_vis.channel_width
uvc.jones_array = self.measured_vis.polarization_array
uvc.time_array = np.unique(self.measured_vis.time_array)
uvc.integration_time = self.measured_vis.integration_time
uvc.x_orientation = 'east' #always
uvc.cal_type = 'gain'
uvc.quality_array = np.zeros(
(self.measured_vis.Nants_data, 1, self.measured_vis.Nfreqs,
self.measured_vis.Ntimes, self.measured_vis.Npols))
uvc.git_origin_cal='calibrated with stefcal_uvdata version %s with run id %s'\
%(self.meta_params.stefcal_version_str,self.meta_params.id)
uvc.gain_array = np.ones(
(self.meta_params.Nants_data, 1, self.model_vis.Nfreqs,
self.model_vis.Ntimes, self.model_vis.Npols),
dtype=complex)
uvc.flag_array = np.empty(
(self.meta_params.Nants_data, 1, self.model_vis.Nfreqs,
self.model_vis.Ntimes, self.model_vis.Npols),
dtype=bool)
uvc.flag_array[:] = False
return uvc
def test_flags2waterfall():
uv = UVData()
uv.read_uvfits(
os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvc.vis.uvfits'))
np.random.seed(0)
uv.flag_array = np.random.randint(0,
2,
size=uv.flag_array.shape,
dtype=bool)
wf = utils.flags2waterfall(uv)
nt.assert_almost_equal(np.mean(wf), np.mean(uv.flag_array))
nt.assert_equal(wf.shape, (uv.Ntimes, uv.Nfreqs))
wf = utils.flags2waterfall(uv, keep_pol=True)
nt.assert_equal(wf.shape, (uv.Ntimes, uv.Nfreqs, uv.Npols))
# Test external flag_array
uv.flag_array = np.zeros_like(uv.flag_array)
f = np.random.randint(0, 2, size=uv.flag_array.shape, dtype=bool)
wf = utils.flags2waterfall(uv, flag_array=f)
nt.assert_almost_equal(np.mean(wf), np.mean(f))
nt.assert_equal(wf.shape, (uv.Ntimes, uv.Nfreqs))
# UVCal version
uvc = UVCal()
uvc.read_calfits(
os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA.omni.calfits'))
uvc.flag_array = np.random.randint(0,
2,
size=uvc.flag_array.shape,
dtype=bool)
wf = utils.flags2waterfall(uvc)
nt.assert_almost_equal(np.mean(wf), np.mean(uvc.flag_array))
nt.assert_equal(wf.shape, (uvc.Ntimes, uvc.Nfreqs))
wf = utils.flags2waterfall(uvc, keep_pol=True)
nt.assert_equal(wf.shape, (uvc.Ntimes, uvc.Nfreqs, uvc.Njones))
def test_flags2waterfall():
uv = UVData()
uv.read_miriad(test_d_file)
np.random.seed(0)
uv.flag_array = np.random.randint(0,
2,
size=uv.flag_array.shape,
dtype=bool)
wf = flags2waterfall(uv)
assert np.allclose(np.mean(wf), np.mean(uv.flag_array))
assert wf.shape == (uv.Ntimes, uv.Nfreqs)
wf = flags2waterfall(uv, keep_pol=True)
assert wf.shape == (uv.Ntimes, uv.Nfreqs, uv.Npols)
# Test external flag_array
uv.flag_array = np.zeros_like(uv.flag_array)
f = np.random.randint(0, 2, size=uv.flag_array.shape, dtype=bool)
wf = flags2waterfall(uv, flag_array=f)
assert np.allclose(np.mean(wf), np.mean(f))
assert wf.shape == (uv.Ntimes, uv.Nfreqs)
# UVCal version
uvc = UVCal()
uvc.read_calfits(test_c_file)
uvc.flag_array = np.random.randint(0,
2,
size=uvc.flag_array.shape,
dtype=bool)
wf = flags2waterfall(uvc)
assert np.allclose(np.mean(wf), np.mean(uvc.flag_array))
assert wf.shape == (uvc.Ntimes, uvc.Nfreqs)
wf = flags2waterfall(uvc, keep_pol=True)
assert wf.shape == (uvc.Ntimes, uvc.Nfreqs, uvc.Njones)
def test_errors():
"""
Test for various errors.
"""
cal_in = UVCal()
cal_out = UVCal()
testfile = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.delay.calfits')
write_file = os.path.join(DATA_PATH, 'test/outtest_firstcal.fits')
cal_in.read_calfits(testfile)
cal_in.set_unknown_cal_type()
pytest.raises(ValueError,
cal_in.write_calfits,
write_file,
run_check=False,
clobber=True)
# change values for various axes in flag and total quality hdus to not match primary hdu
cal_in.read_calfits(testfile)
# Create filler jones info
cal_in.jones_array = np.array([-5, -6, -7, -8])
cal_in.Njones = 4
cal_in.flag_array = np.zeros(cal_in._flag_array.expected_shape(cal_in),
dtype=bool)
cal_in.delay_array = np.ones(cal_in._delay_array.expected_shape(cal_in),
dtype=np.float64)
cal_in.quality_array = np.zeros(
cal_in._quality_array.expected_shape(cal_in))
# add total_quality_array so that can be tested as well
cal_in.total_quality_array = np.zeros(
cal_in._total_quality_array.expected_shape(cal_in))
header_vals_to_double = [{
'flag': 'CDELT2'
}, {
'flag': 'CDELT3'
}, {
'flag': 'CRVAL5'
}, {
'totqual': 'CDELT1'
}, {
'totqual': 'CDELT2'
}, {
'totqual': 'CRVAL4'
}]
for i, hdr_dict in enumerate(header_vals_to_double):
cal_in.write_calfits(write_file, clobber=True)
unit = list(hdr_dict.keys())[0]
keyword = hdr_dict[unit]
F = fits.open(write_file)
data = F[0].data
primary_hdr = F[0].header
hdunames = uvutils._fits_indexhdus(F)
ant_hdu = F[hdunames['ANTENNAS']]
flag_hdu = F[hdunames['FLAGS']]
flag_hdr = flag_hdu.header
totqualhdu = F[hdunames['TOTQLTY']]
totqualhdr = totqualhdu.header
if unit == 'flag':
flag_hdr[keyword] *= 2
elif unit == 'totqual':
totqualhdr[keyword] *= 2
prihdu = fits.PrimaryHDU(data=data, header=primary_hdr)
hdulist = fits.HDUList([prihdu, ant_hdu])
flag_hdu = fits.ImageHDU(data=flag_hdu.data, header=flag_hdr)
hdulist.append(flag_hdu)
totqualhdu = fits.ImageHDU(data=totqualhdu.data, header=totqualhdr)
hdulist.append(totqualhdu)
hdulist.writeto(write_file, overwrite=True)
pytest.raises(ValueError,
cal_out.read_calfits,
write_file,
strict_fits=True)
# repeat for gain type file
testfile = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.gain.calfits')
write_file = os.path.join(DATA_PATH, 'test/outtest_omnical.fits')
cal_in.read_calfits(testfile)
# Create filler jones info
cal_in.jones_array = np.array([-5, -6, -7, -8])
cal_in.Njones = 4
cal_in.flag_array = np.zeros(cal_in._flag_array.expected_shape(cal_in),
dtype=bool)
cal_in.gain_array = np.ones(cal_in._gain_array.expected_shape(cal_in),
dtype=np.complex64)
cal_in.quality_array = np.zeros(
cal_in._quality_array.expected_shape(cal_in))
# add total_quality_array so that can be tested as well
cal_in.total_quality_array = np.zeros(
cal_in._total_quality_array.expected_shape(cal_in))
header_vals_to_double = [{
'totqual': 'CDELT1'
}, {
'totqual': 'CDELT2'
}, {
'totqual': 'CDELT3'
}, {
'totqual': 'CRVAL4'
}]
for i, hdr_dict in enumerate(header_vals_to_double):
cal_in.write_calfits(write_file, clobber=True)
unit = list(hdr_dict.keys())[0]
keyword = hdr_dict[unit]
F = fits.open(write_file)
data = F[0].data
primary_hdr = F[0].header
hdunames = uvutils._fits_indexhdus(F)
ant_hdu = F[hdunames['ANTENNAS']]
totqualhdu = F[hdunames['TOTQLTY']]
totqualhdr = totqualhdu.header
if unit == 'totqual':
totqualhdr[keyword] *= 2
prihdu = fits.PrimaryHDU(data=data, header=primary_hdr)
hdulist = fits.HDUList([prihdu, ant_hdu])
totqualhdu = fits.ImageHDU(data=totqualhdu.data, header=totqualhdr)
hdulist.append(totqualhdu)
hdulist.writeto(write_file, overwrite=True)
pytest.raises(ValueError,
cal_out.read_calfits,
write_file,
strict_fits=True)
'HEAD']).strip().decode('UTF-8')
uvc.history = 'Created using mkgains.py\nCommand run: mkgains.py %s\nmkgains.py Git Hash: %s' % (
command_given, git_hash)
uvc.Njones = uvd.Npols
uvc.jones_array = uvd.polarization_array
uvc.ant_array = np.arange(uvc.Nants_data)
uvc.time_range = [
uvc.time_array[0] - (uvc.integration_time / 172800.),
uvc.time_array[-1] + (uvc.integration_time / 172800.)
]
uvc.quality_array = np.zeros(
(uvc.Nants_data, uvc.Nspws, uvc.Nfreqs, uvc.Ntimes, uvc.Njones),
dtype='float64')
uvc.flag_array = np.zeros(
(uvc.Nants_data, uvc.Nspws, uvc.Nfreqs, uvc.Ntimes, uvc.Njones),
dtype='bool')
uvc.time_array = np.unique(uvd.time_array)
if uvc.x_orientation == None:
uvc.x_orientation = 'East'
# Create random gains
gain_shape = (uvc.Nants_data, uvc.Nspws, uvc.Nfreqs, uvc.Ntimes, uvc.Njones)
gains = np.random.normal(1.0, args.ampstdev, gain_shape) * \
np.exp(np.random.normal(0.0, args.phasestdev, gain_shape) * 1j)
# Preparing variables for transformations
timesize = int(args.time / uvc.integration_time)
if timesize == 0:
raise ValueError('--time is too small')
cal.integration_time = 1800. # cal.gain_convention = 'divide' # Use this operation to apply gain solution. cal.x_orientation = 'east' # orientation of 1st jones parameter. # # JD's this can applied to. Below is Season 1 # cal.time_range = [2456528., 2456626.] # cal.telescope_name = 'MWA' cal.Nants_data = Nants_data cal.Nants_telescope = Nants_data # have solutions for all antennas in array. cal.ant_array = ant_array cal.antenna_names = antenna_names cal.antenna_numbers = ant_array cal.flag_array = flags cal.gain_array = gains cal.quality_array = chisq # # Put your name in as creator # cal.observer = '<YOUR NAME HERE>' # # Put in the git url of the code that generated the cals # cal.git_origin_cal = 'https://github.com/EoRImaging/FHD' # # And if you know the git hash, put that in as well # #cal.git_hash_cal =
