def test_UVH5OptionalParameters(): """ Test reading and writing optional parameters not in sample files """ uv_in = UVData() uv_out = UVData() uvfits_file = os.path.join(DATA_PATH, 'day2_TDEM0003_10s_norx_1src_1spw.uvfits') uvtest.checkWarnings(uv_in.read_uvfits, [uvfits_file], message='Telescope EVLA is not') testfile = os.path.join(DATA_PATH, 'test', 'outtest_uvfits.uvh5') # set optional parameters uv_in.x_orientation = 'east' uv_in.antenna_diameters = np.ones_like(uv_in.antenna_numbers) * 1. uv_in.uvplane_reference_time = 0 # write out and read back in uv_in.write_uvh5(testfile, clobber=True) uv_out.read(testfile) nt.assert_equal(uv_in, uv_out) # clean up os.remove(testfile) return
def test_readwriteread(): """ CASA tutorial uvfits loopback test. Read in uvfits file, write out new uvfits file, read back in and check for object equality. """ uv_in = UVData() uv_out = UVData() testfile = os.path.join(DATA_PATH, 'day2_TDEM0003_10s_norx_1src_1spw.uvfits') write_file = os.path.join(DATA_PATH, 'test/outtest_casa.uvfits') uvtest.checkWarnings(uv_in.read_uvfits, [testfile], message='Telescope EVLA is not') uv_in.write_uvfits(write_file) uvtest.checkWarnings(uv_out.read_uvfits, [write_file], message='Telescope EVLA is not') nt.assert_equal(uv_in, uv_out) # check that if x_orientation is set, it's read back out properly uv_in.x_orientation = 'east' uv_in.write_uvfits(write_file) uvtest.checkWarnings(uv_out.read_uvfits, [write_file], message='Telescope EVLA is not') nt.assert_equal(uv_in, uv_out) # check that if antenna_diameters is set, it's read back out properly uvtest.checkWarnings(uv_in.read_uvfits, [testfile], message='Telescope EVLA is not') uv_in.antenna_diameters = np.zeros( (uv_in.Nants_telescope, ), dtype=np.float) + 14.0 uv_in.write_uvfits(write_file) uvtest.checkWarnings(uv_out.read_uvfits, [write_file], message='Telescope EVLA is not') nt.assert_equal(uv_in, uv_out) # check error if timesys is 'IAT' uv_in.timesys = 'IAT' nt.assert_raises(ValueError, uv_in.write_uvfits, write_file) uv_in.timesys = 'UTC' # check that unflagged data with nsample = 0 will cause warnings uv_in.nsample_array[range(11, 22)] = 0 uv_in.flag_array[range(11, 22)] = False uvtest.checkWarnings(uv_in.write_uvfits, [write_file], message='Some unflagged data has nsample = 0') del (uv_in) del (uv_out)
def test_read_ms_read_uvfits(): """ Test that a uvdata object instantiated from an ms file created with CASA's importuvfits is equal to a uvdata object instantiated from the original uvfits file (tests equivalence with importuvfits in uvdata). Since the histories are different, this test sets both uvdata histories to identical empty strings before comparing them. """ ms_uv = UVData() uvfits_uv = UVData() ms_file = os.path.join(DATA_PATH, "day2_TDEM0003_10s_norx_1src_1spw.ms") uvfits_file = os.path.join(DATA_PATH, "day2_TDEM0003_10s_norx_1src_1spw.uvfits") uvtest.checkWarnings(uvfits_uv.read, [uvfits_file], message="Telescope EVLA is not") ms_uv.read(ms_file) # set histories to identical blank strings since we do not expect # them to be the same anyways. ms_uv.history = "" uvfits_uv.history = "" # the objects won't be equal because uvfits adds some optional parameters # and the ms sets default antenna diameters even though the uvfits file # doesn't have them assert uvfits_uv != ms_uv # they are equal if only required parameters are checked: assert uvfits_uv.__eq__(ms_uv, check_extra=False) # set those parameters to none to check that the rest of the objects match ms_uv.antenna_diameters = None for p in uvfits_uv.extra(): fits_param = getattr(uvfits_uv, p) ms_param = getattr(ms_uv, p) if fits_param.name in UVFITS.uvfits_required_extra and ms_param.value is None: fits_param.value = None setattr(uvfits_uv, p, fits_param) # extra keywords are also different, set both to empty dicts uvfits_uv.extra_keywords = {} ms_uv.extra_keywords = {} assert uvfits_uv == ms_uv del ms_uv del uvfits_uv
def test_multi_files(casa_uvfits, axis): """ Reading multiple files at once. """ uv_full = casa_uvfits.copy() uv_multi = UVData() testfile1 = os.path.join(DATA_PATH, "multi_1.ms") testfile2 = os.path.join(DATA_PATH, "multi_2.ms") filesread = [testfile1, testfile2] # test once as list and once as an array if axis is None: filesread = np.array(filesread) uv_multi.read(filesread, axis=axis) # Casa scrambles the history parameter. Replace for now. uv_multi.history = uv_full.history # the objects won't be equal because uvfits adds some optional parameters # and the ms sets default antenna diameters even though the uvfits file # doesn't have them assert uv_multi != uv_full # they are equal if only required parameters are checked: assert uv_multi.__eq__(uv_full, check_extra=False) # set those parameters to none to check that the rest of the objects match uv_multi.antenna_diameters = None for p in uv_full.extra(): fits_param = getattr(uv_full, p) ms_param = getattr(uv_multi, p) if fits_param.name in UVFITS.uvfits_required_extra and ms_param.value is None: fits_param.value = None setattr(uv_full, p, fits_param) # extra keywords are also different, set both to empty dicts uv_full.extra_keywords = {} uv_multi.extra_keywords = {} assert uv_multi == uv_full del uv_full del uv_multi
def test_multi_files(): """ Reading multiple files at once. """ uv_full = UVData() uv_multi = UVData() uvfits_file = os.path.join(DATA_PATH, "day2_TDEM0003_10s_norx_1src_1spw.uvfits") uvtest.checkWarnings(uv_full.read, [uvfits_file], message="Telescope EVLA is not") testfile1 = os.path.join(DATA_PATH, "multi_1.ms") testfile2 = os.path.join(DATA_PATH, "multi_2.ms") uv_multi.read(np.array([testfile1, testfile2])) # Casa scrambles the history parameter. Replace for now. uv_multi.history = uv_full.history # the objects won't be equal because uvfits adds some optional parameters # and the ms sets default antenna diameters even though the uvfits file # doesn't have them assert uv_multi != uv_full # they are equal if only required parameters are checked: assert uv_multi.__eq__(uv_full, check_extra=False) # set those parameters to none to check that the rest of the objects match uv_multi.antenna_diameters = None for p in uv_full.extra(): fits_param = getattr(uv_full, p) ms_param = getattr(uv_multi, p) if fits_param.name in UVFITS.uvfits_required_extra and ms_param.value is None: fits_param.value = None setattr(uv_full, p, fits_param) # extra keywords are also different, set both to empty dicts uv_full.extra_keywords = {} uv_multi.extra_keywords = {} assert uv_multi == uv_full del uv_full del uv_multi
def test_readWriteReadMiriad(): """ PAPER file Miriad loopback test. Read in Miriad PAPER file, write out as new Miriad file, read back in and check for object equality. """ uv_in = UVData() uv_out = UVData() testfile = os.path.join(DATA_PATH, 'zen.2456865.60537.xy.uvcRREAA') write_file = os.path.join(DATA_PATH, 'test/outtest_miriad.uv') uvtest.checkWarnings(uv_in.read_miriad, [testfile], known_warning='miriad') uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that trying to overwrite without clobber raises an error nt.assert_raises(ValueError, uv_in.write_miriad, write_file) # check that if x_orientation is set, it's read back out properly uv_in.x_orientation = 'east' uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that if antenna_diameters is set, it's read back out properly uvtest.checkWarnings(uv_in.read_miriad, [testfile], known_warning='miriad') uv_in.antenna_diameters = np.zeros( (uv_in.Nants_telescope, ), dtype=np.float) + 14.0 uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that antenna diameters get written if not exactly float uv_in.antenna_diameters = np.zeros( (uv_in.Nants_telescope, ), dtype=np.float32) + 14.0 uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that trying to write a file with unknown phasing raises an error uv_in.set_unknown_phase_type() nt.assert_raises(ValueError, uv_in.write_miriad, write_file, clobber=True) # check for backwards compatibility with old keyword 'diameter' for antenna diameters testfile_diameters = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcA') uv_in.read_miriad(testfile_diameters) uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that variables 'ischan' and 'nschan' were written to new file # need to use aipy, since pyuvdata is not currently capturing these variables uv_in.read_miriad(write_file) uv_aipy = amiriad.UV( write_file ) # on enterprise, this line makes it so you cant delete the file nfreqs = uv_in.Nfreqs nschan = uv_aipy['nschan'] ischan = uv_aipy['ischan'] nt.assert_equal(nschan, nfreqs) nt.assert_equal(ischan, 1) del (uv_aipy) # close the file so it can be used later # check partial IO selections full = UVData() uvtest.checkWarnings(full.read_miriad, [testfile], known_warning='miriad') full.write_miriad(write_file, clobber=True) uv_in = UVData() # test only specified bls were read, and that flipped antpair is loaded too uv_in.read_miriad(write_file, ant_pairs_nums=[(0, 0), (0, 1), (4, 2)]) nt.assert_equal(uv_in.get_antpairs(), [(0, 0), (0, 1), (2, 4)]) exp_uv = full.select(ant_pairs_nums=[(0, 0), (0, 1), (4, 2)], inplace=False) nt.assert_equal(uv_in, exp_uv) # test all bls w/ 0 are loaded uv_in.read_miriad(write_file, antenna_nums=[0]) diff = set(full.get_antpairs()) - set(uv_in.get_antpairs()) nt.assert_true(0 not in np.unique(diff)) exp_uv = full.select(antenna_nums=[0], inplace=False) nt.assert_true(np.max(exp_uv.ant_1_array) == 0) nt.assert_true(np.max(exp_uv.ant_2_array) == 0) nt.assert_equal(uv_in, exp_uv) uv_in.read_miriad(write_file, antenna_nums=[0], ant_pairs_nums=[(2, 4)]) nt.assert_true( np.array([bl in uv_in.get_antpairs() for bl in [(0, 0), (2, 4)]]).all()) exp_uv = full.select(antenna_nums=[0], ant_pairs_nums=[(2, 4)], inplace=False) nt.assert_equal(uv_in, exp_uv) # test time loading uv_in.read_miriad(write_file, time_range=[2456865.607, 2456865.609]) full_times = np.unique(full.time_array[(full.time_array > 2456865.607) & (full.time_array < 2456865.609)]) nt.assert_true(np.isclose(np.unique(uv_in.time_array), full_times).all()) exp_uv = full.select(times=full_times, inplace=False) nt.assert_equal(uv_in, exp_uv) # test polarization loading uv_in.read_miriad(write_file, polarizations=['xy']) nt.assert_equal(full.polarization_array, uv_in.polarization_array) exp_uv = full.select(polarizations=['xy'], inplace=False) nt.assert_equal(uv_in, exp_uv) uv_in.read_miriad(write_file, polarizations=[-7]) nt.assert_equal(full.polarization_array, uv_in.polarization_array) exp_uv = full.select(polarizations=[-7], inplace=False) nt.assert_equal(uv_in, exp_uv) # test ant_str uv_in.read_miriad(write_file, ant_str='auto') nt.assert_true( np.array([blp[0] == blp[1] for blp in uv_in.get_antpairs()]).all()) exp_uv = full.select(ant_str='auto', inplace=False) nt.assert_equal(uv_in, exp_uv) uv_in.read_miriad(write_file, ant_str='cross') nt.assert_true( np.array([blp[0] != blp[1] for blp in uv_in.get_antpairs()]).all()) exp_uv = full.select(ant_str='cross', inplace=False) nt.assert_equal(uv_in, exp_uv) uv_in.read_miriad(write_file, ant_str='all') nt.assert_equal(uv_in, full) nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, ant_str='auto', antenna_nums=[0, 1]) # assert exceptions nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, ant_pairs_nums='foo') nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, ant_pairs_nums=[[0, 1]]) nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, ant_pairs_nums=[('foo', )]) nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, antenna_nums=np.array([(0, 10)])) nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, polarizations='xx') nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, polarizations=[1.0]) nt.assert_raises(ValueError, uv_in.read_miriad, write_file, polarizations=['yy']) nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, time_range='foo') nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, time_range=[1, 2, 3]) nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, time_range=['foo', 'bar']) nt.assert_raises(ValueError, uv_in.read_miriad, write_file, time_range=[10.1, 10.2]) nt.assert_raises(AssertionError, uv_in.read_miriad, write_file, ant_str=0) # assert partial-read and select are same uv_in.read_miriad(write_file, polarizations=[-7], ant_pairs_nums=[(4, 4)]) exp_uv = full.select(polarizations=[-7], ant_pairs_nums=[(4, 4)], inplace=False) nt.assert_equal(uv_in, exp_uv) # assert partial-read and select are same t = np.unique(full.time_array) uv_in.read_miriad(write_file, antenna_nums=[0], time_range=[2456865.607, 2456865.609]) exp_uv = full.select(antenna_nums=[0], times=t[((t > 2456865.607) & (t < 2456865.609))], inplace=False) nt.assert_equal(uv_in, exp_uv) # assert partial-read and select are same t = np.unique(full.time_array) uv_in.read_miriad(write_file, polarizations=[-7], time_range=[2456865.607, 2456865.609]) exp_uv = full.select(polarizations=[-7], times=t[((t > 2456865.607) & (t < 2456865.609))], inplace=False) nt.assert_equal(uv_in, exp_uv) del (uv_in) del (uv_out) del (full)
def test_readWriteReadMiriad(): """ PAPER file Miriad loopback test. Read in Miriad PAPER file, write out as new Miriad file, read back in and check for object equality. """ uv_in = UVData() uv_out = UVData() testfile = os.path.join(DATA_PATH, 'zen.2456865.60537.xy.uvcRREAA') write_file = os.path.join(DATA_PATH, 'test/outtest_miriad.uv') uvtest.checkWarnings(uv_in.read_miriad, [testfile], known_warning='miriad') uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that trying to overwrite without clobber raises an error nt.assert_raises(ValueError, uv_in.write_miriad, write_file) # check that if x_orientation is set, it's read back out properly uv_in.x_orientation = 'east' uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that if antenna_diameters is set, it's read back out properly uvtest.checkWarnings(uv_in.read_miriad, [testfile], known_warning='miriad') uv_in.antenna_diameters = np.zeros( (uv_in.Nants_telescope, ), dtype=np.float) + 14.0 uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that antenna diameters get written if not exactly float uv_in.antenna_diameters = np.zeros( (uv_in.Nants_telescope, ), dtype=np.float32) + 14.0 uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that trying to write a file with unknown phasing raises an error uv_in.set_unknown_phase_type() nt.assert_raises(ValueError, uv_in.write_miriad, write_file, clobber=True) # check for backwards compatibility with old keyword 'diameter' for antenna diameters testfile_diameters = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcA') uv_in.read_miriad(testfile_diameters) uv_in.write_miriad(write_file, clobber=True) uv_out.read_miriad(write_file) nt.assert_equal(uv_in, uv_out) # check that variables 'ischan' and 'nschan' were written to new file # need to use aipy, since pyuvdata is not currently capturing these variables uv_in.read_miriad(write_file) uv_aipy = amiriad.UV(write_file) nfreqs = uv_in.Nfreqs nschan = uv_aipy['nschan'] ischan = uv_aipy['ischan'] nt.assert_equal(nschan, nfreqs) nt.assert_equal(ischan, 1) del (uv_in) del (uv_out) del (uv_aipy)
def apply_bda( uv, max_decorr, pre_fs_int_time, corr_fov_angle, max_time, corr_int_time=None ): """Apply baseline dependent averaging to a UVData object. For each baseline in the UVData object, the expected decorrelation from averaging in time is computed. Baselines are averaged together in powers- of-two until the specified level of decorrelation is reached (rounded down). Parameters ---------- uv : UVData object The UVData object to apply BDA to. No changes are made to this object, and instead a copy is returned. max_decorr : float The maximum decorrelation fraction desired in the output object. Must be between 0 and 1. pre_fs_int_time : astropy Quantity The pre-finge-stopping integration time inside of the correlator. The quantity should be compatible with units of time. corr_fov_angle : astropy Angle The opening angle at which the maximum decorrelation is to be calculated. Because a priori it is not known in which direction the decorrelation will be largest, the expected decorrelation is computed in all 4 cardinal directions at `corr_fov_angle` degrees off of zenith, and the largest one is used. This is a "worst case scenario" decorrelation. max_time : astropy Quantity The maximum amount of time that spectra from different times should be combined for. The ultimate integration time for a given baseline will be for max_time or the integration time that is smaller than the specified decorrelation level, whichever is smaller. The quantity should be compatible with units of time. corr_int_time : astropy Quantity, optional The output time of the correlator. If not specified, the smallest integration_time in the UVData object is used. If specified, the quantity should be compatible with units of time. Returns ------- uv2 : UVData object The UVData object with BDA applied. Raises ------ ValueError This is raised if the input parameters are not the appropriate type or in the appropriate range. It is also raised if the input UVData object is not in drift mode (the BDA code does rephasing within an averaged set of baselines). AssertionError This is raised if the baselines of the UVData object are not time- ordered. """ if not isinstance(uv, UVData): raise ValueError( "apply_bda must be passed a UVData object as its first argument" ) if not isinstance(corr_fov_angle, Angle): raise ValueError( "corr_fov_angle must be an Angle object from astropy.coordinates" ) if not isinstance(pre_fs_int_time, units.Quantity): raise ValueError("pre_fs_int_time must be an astropy.units.Quantity") try: pre_fs_int_time.to(units.s) except UnitConversionError: raise ValueError("pre_fs_int_time must be a Quantity with units of time") if ( corr_fov_angle.to(units.deg).value < 0 or corr_fov_angle.to(units.deg).value > 90 ): raise ValueError("corr_fov_angle must be between 0 and 90 degrees") if max_decorr < 0 or max_decorr > 1: raise ValueError("max_decorr must be between 0 and 1") if not isinstance(max_time, units.Quantity): raise ValueError("max_time must be an astropy.units.Quantity") try: max_time.to(units.s) except UnitConversionError: raise ValueError("max_time must be a Quantity with units of time") if corr_int_time is None: # assume the correlator integration time is the smallest int_time of the # UVData object corr_int_time = np.unique(uv.integration_time)[0] * units.s else: if not isinstance(corr_int_time, units.Quantity): raise ValueError("corr_int_time must be an astropy.units.Quantity") try: corr_int_time.to(units.s) except UnitConversionError: raise ValueError("corr_int_time must be a Quantity with units of time") if uv.phase_type != "drift": raise ValueError("UVData object must be in drift mode to apply BDA") # get relevant bits of metadata freq = np.amax(uv.freq_array[0, :]) * units.Hz chan_width = uv.channel_width * units.Hz antpos_enu, ants = uv.get_ENU_antpos() lat, lon, alt = uv.telescope_location_lat_lon_alt antpos_ecef = uvutils.ECEF_from_ENU(antpos_enu, lat, lon, alt) telescope_location = EarthLocation.from_geocentric( uv.telescope_location[0], uv.telescope_location[1], uv.telescope_location[2], unit="m", ) # make a new UVData object to put BDA baselines in uv2 = UVData() # copy over metadata uv2.Nbls = uv.Nbls uv2.Nfreqs = uv.Nfreqs uv2.Npols = uv.Npols uv2.vis_units = uv.vis_units uv2.Nspws = uv.Nspws uv2.spw_array = uv.spw_array uv2.freq_array = uv.freq_array uv2.polarization_array = uv.polarization_array uv2.channel_width = uv.channel_width uv2.object_name = uv.object_name uv2.telescope_name = uv.telescope_name uv2.instrument = uv.instrument uv2.telescope_location = uv.telescope_location history = uv.history + " Baseline dependent averaging applied." uv2.history = history uv2.Nants_data = uv.Nants_data uv2.Nants_telescope = uv.Nants_telescope uv2.antenna_names = uv.antenna_names uv2.antenna_numbers = uv.antenna_numbers uv2.x_orientation = uv.x_orientation uv2.extra_keywords = uv.extra_keywords uv2.antenna_positions = uv.antenna_positions uv2.antenna_diameters = uv.antenna_diameters uv2.gst0 = uv.gst0 uv2.rdate = uv.rdate uv2.earth_omega = uv.earth_omega uv2.dut1 = uv.dut1 uv2.timesys = uv.timesys uv2.uvplane_reference_time = uv.uvplane_reference_time # initialize place-keeping variables and Nblt-sized metadata start_index = 0 uv2.Nblts = 0 uv2.uvw_array = np.zeros_like(uv.uvw_array) uv2.time_array = np.zeros_like(uv.time_array) uv2.lst_array = np.zeros_like(uv.lst_array) uv2.ant_1_array = np.zeros_like(uv.ant_1_array) uv2.ant_2_array = np.zeros_like(uv.ant_2_array) uv2.baseline_array = np.zeros_like(uv.baseline_array) uv2.integration_time = np.zeros_like(uv.integration_time) uv2.data_array = np.zeros_like(uv.data_array) uv2.flag_array = np.zeros_like(uv.flag_array) uv2.nsample_array = np.zeros_like(uv.nsample_array) # iterate over baselines for key in uv.get_antpairs(): print("averaging baseline ", key) ind1, ind2, indp = uv._key2inds(key) if len(ind2) != 0: raise AssertionError( "ind2 from _key2inds() is not 0--exiting. This should not happen, " "please contact the package maintainers." ) data = uv._smart_slicing( uv.data_array, ind1, ind2, indp, squeeze="none", force_copy=True ) flags = uv._smart_slicing( uv.flag_array, ind1, ind2, indp, squeeze="none", force_copy=True ) nsamples = uv._smart_slicing( uv.nsample_array, ind1, ind2, indp, squeeze="none", force_copy=True ) # get lx and ly for baseline ant1 = np.where(ants == key[0])[0][0] ant2 = np.where(ants == key[1])[0][0] x1, y1, z1 = antpos_ecef[ant1, :] x2, y2, z2 = antpos_ecef[ant2, :] lx = np.abs(x2 - x1) * units.m ly = np.abs(y2 - y1) * units.m # figure out how many time samples we can combine together if key[0] == key[1]: # autocorrelation--don't average n_two_foldings = 0 else: n_two_foldings = dc.bda_compression_factor( max_decorr, freq, lx, ly, corr_fov_angle, chan_width, pre_fs_int_time, corr_int_time, ) # convert from max_time to max_samples max_samples = (max_time / corr_int_time).to(units.dimensionless_unscaled) max_two_foldings = int(np.floor(np.log2(max_samples))) n_two_foldings = min(n_two_foldings, max_two_foldings) n_int = 2 ** (n_two_foldings) print("averaging {:d} time samples...".format(n_int)) # figure out how many output samples we're going to have n_in = len(ind1) n_out = n_in // n_int + min(1, n_in % n_int) # get relevant metdata uvw_array = uv.uvw_array[ind1, :] times = uv.time_array[ind1] if not np.all(times == np.sort(times)): raise AssertionError( "times of uvdata object are not monotonically increasing; " "throwing our hands up" ) lsts = uv.lst_array[ind1] int_time = uv.integration_time[ind1] # do the averaging input_shape = data.shape assert input_shape == (n_in, 1, uv.Nfreqs, uv.Npols) output_shape = (n_out, 1, uv.Nfreqs, uv.Npols) data_out = np.empty(output_shape, dtype=np.complex128) flags_out = np.empty(output_shape, dtype=np.bool_) nsamples_out = np.empty(output_shape, dtype=np.float32) uvws_out = np.empty((n_out, 3), dtype=np.float64) times_out = np.empty((n_out,), dtype=np.float64) lst_out = np.empty((n_out,), dtype=np.float64) int_time_out = np.empty((n_out,), dtype=np.float64) if n_out == n_in: # we don't need to average current_index = start_index + n_out uv2.data_array[start_index:current_index, :, :, :] = data uv2.flag_array[start_index:current_index, :, :, :] = flags uv2.nsample_array[start_index:current_index, :, :, :] = nsamples uv2.uvw_array[start_index:current_index, :] = uvw_array uv2.time_array[start_index:current_index] = times uv2.lst_array[start_index:current_index] = lsts uv2.integration_time[start_index:current_index] = int_time uv2.ant_1_array[start_index:current_index] = key[0] uv2.ant_2_array[start_index:current_index] = key[1] uv2.baseline_array[start_index:current_index] = uvutils.antnums_to_baseline( ant1, ant2, None ) start_index = current_index else: # rats, we actually have to do work... # phase up the data along each chunk of times for i in range(n_out): # compute zenith of the desired output time i1 = i * n_int i2 = min((i + 1) * n_int, n_in) assert i2 - i1 > 0 t0 = Time((times[i1] + times[i2 - 1]) / 2, scale="utc", format="jd") zenith_coord = SkyCoord( alt=Angle(90 * units.deg), az=Angle(0 * units.deg), obstime=t0, frame="altaz", location=telescope_location, ) obs_zenith_coord = zenith_coord.transform_to("icrs") zenith_ra = obs_zenith_coord.ra zenith_dec = obs_zenith_coord.dec # get data, flags, and nsamples of slices data_chunk = data[i1:i2, :, :, :] flags_chunk = flags[i1:i2, :, :, :] nsamples_chunk = nsamples[i1:i2, :, :, :] # actually phase now # compute new uvw coordinates icrs_coord = SkyCoord( ra=zenith_ra, dec=zenith_dec, unit="radian", frame="icrs" ) uvws = np.float64(uvw_array[i1:i2, :]) itrs_telescope_location = SkyCoord( x=uv.telescope_location[0] * units.m, y=uv.telescope_location[1] * units.m, z=uv.telescope_location[2] * units.m, representation_type="cartesian", frame="itrs", obstime=t0, ) itrs_lat_lon_alt = uv.telescope_location_lat_lon_alt frame_telescope_location = itrs_telescope_location.transform_to("icrs") frame_telescope_location.representation_type = "cartesian" uvw_ecef = uvutils.ECEF_from_ENU(uvws, *itrs_lat_lon_alt) itrs_uvw_coord = SkyCoord( x=uvw_ecef[:, 0] * units.m, y=uvw_ecef[:, 1] * units.m, z=uvw_ecef[:, 2] * units.m, representation_type="cartesian", frame="itrs", obstime=t0, ) frame_uvw_coord = itrs_uvw_coord.transform_to("icrs") frame_rel_uvw = ( frame_uvw_coord.cartesian.get_xyz().value.T - frame_telescope_location.cartesian.get_xyz().value ) new_uvws = uvutils.phase_uvw( icrs_coord.ra.rad, icrs_coord.dec.rad, frame_rel_uvw ) # average these uvws together to get the "average" position in # the uv-plane avg_uvws = np.average(new_uvws, axis=0) # calculate and apply phasor w_lambda = ( new_uvws[:, 2].reshape((i2 - i1), 1) / const.c.to("m/s").value * uv.freq_array.reshape(1, uv.Nfreqs) ) phs = np.exp(-1j * 2 * np.pi * w_lambda[:, None, :, None]) data_chunk *= phs # sum data, propagate flag array, and adjusting nsample accordingly data_slice = np.sum(data_chunk, axis=0) flag_slice = np.sum(flags_chunk, axis=0) nsamples_slice = np.sum(nsamples_chunk, axis=0) / (i2 - i1) data_out[i, :, :, :] = data_slice flags_out[i, :, :, :] = flag_slice nsamples_out[i, :, :, :] = nsamples_slice # update metadata uvws_out[i, :] = avg_uvws times_out[i] = (times[i1] + times[i2 - 1]) / 2 lst_out[i] = (lsts[i1] + lsts[i2 - 1]) / 2 int_time_out[i] = np.average(int_time[i1:i2]) * (i2 - i1) # update data and metadata when we're done with this baseline current_index = start_index + n_out uv2.data_array[start_index:current_index, :, :, :] = data_out uv2.flag_array[start_index:current_index, :, :, :] = flags_out uv2.nsample_array[start_index:current_index, :, :, :] = nsamples_out uv2.uvw_array[start_index:current_index, :] = uvws_out uv2.time_array[start_index:current_index] = times_out uv2.lst_array[start_index:current_index] = lst_out uv2.integration_time[start_index:current_index] = int_time_out uv2.ant_1_array[start_index:current_index] = key[0] uv2.ant_2_array[start_index:current_index] = key[1] uv2.baseline_array[start_index:current_index] = uvutils.antnums_to_baseline( ant1, ant2, None ) start_index = current_index # clean up -- shorten all arrays to actually be size nblts nblts = start_index uv2.Nblts = nblts uv2.data_array = uv2.data_array[:nblts, :, :, :] uv2.flag_array = uv2.flag_array[:nblts, :, :, :] uv2.nsample_array = uv2.nsample_array[:nblts, :, :, :] uv2.uvw_array = uv2.uvw_array[:nblts, :] uv2.time_array = uv2.time_array[:nblts] uv2.lst_array = uv2.lst_array[:nblts] uv2.integration_time = uv2.integration_time[:nblts] uv2.ant_1_array = uv2.ant_1_array[:nblts] uv2.ant_2_array = uv2.ant_2_array[:nblts] uv2.baseline_array = uv2.baseline_array[:nblts] uv2.Ntimes = len(np.unique(uv2.time_array)) # set phasing info uv2.phase_type = "phased" uv2.phase_center_ra = zenith_ra.rad uv2.phase_center_dec = zenith_dec.rad uv2.phase_center_frame = 2000.0 # fix up to correct old phasing method uv2.phase(zenith_ra.rad, zenith_dec.rad, epoch="J2000", fix_old_proj=True) # run a check uv2.check() return uv2
def test_readwriteread(): """ CASA tutorial uvfits loopback test. Read in uvfits file, write out new uvfits file, read back in and check for object equality. """ uv_in = UVData() uv_out = UVData() testfile = os.path.join(DATA_PATH, 'day2_TDEM0003_10s_norx_1src_1spw.uvfits') write_file = os.path.join(DATA_PATH, 'test/outtest_casa.uvfits') uvtest.checkWarnings(uv_in.read, [testfile], message='Telescope EVLA is not') uv_in.write_uvfits(write_file) uvtest.checkWarnings(uv_out.read, [write_file], message='Telescope EVLA is not') assert uv_in == uv_out # test that it works with write_lst = False uv_in.write_uvfits(write_file, write_lst=False) uvtest.checkWarnings(uv_out.read, [write_file], message='Telescope EVLA is not') assert uv_in == uv_out # check that if x_orientation is set, it's read back out properly uv_in.x_orientation = 'east' uv_in.write_uvfits(write_file) uvtest.checkWarnings(uv_out.read, [write_file], message='Telescope EVLA is not') assert uv_in == uv_out # check that if antenna_diameters is set, it's read back out properly uvtest.checkWarnings(uv_in.read, [testfile], message='Telescope EVLA is not') uv_in.antenna_diameters = np.zeros( (uv_in.Nants_telescope, ), dtype=np.float) + 14.0 uv_in.write_uvfits(write_file) uvtest.checkWarnings(uv_out.read, [write_file], message='Telescope EVLA is not') assert uv_in == uv_out # check that if antenna_numbers are > 256 everything works uvtest.checkWarnings(uv_in.read, [testfile], message='Telescope EVLA is not') uv_in.antenna_numbers = uv_in.antenna_numbers + 256 uv_in.ant_1_array = uv_in.ant_1_array + 256 uv_in.ant_2_array = uv_in.ant_2_array + 256 uv_in.baseline_array = uv_in.antnums_to_baseline(uv_in.ant_1_array, uv_in.ant_2_array) uvtest.checkWarnings( uv_in.write_uvfits, [write_file], message='antnums_to_baseline: found > 256 antennas, using 2048 baseline' ) uvtest.checkWarnings(uv_out.read, [write_file], message='Telescope EVLA is not') assert uv_in == uv_out # check missing telescope_name, timesys vs timsys spelling, xyz_telescope_frame=???? with fits.open(write_file, memmap=True) as hdu_list: hdunames = uvutils._fits_indexhdus(hdu_list) vis_hdu = hdu_list[0] vis_hdr = vis_hdu.header.copy() vis_hdr.pop('TELESCOP') vis_hdu.header = vis_hdr ant_hdu = hdu_list[hdunames['AIPS AN']] ant_hdr = ant_hdu.header.copy() time_sys = ant_hdr.pop('TIMSYS') ant_hdr['TIMESYS'] = time_sys ant_hdr['FRAME'] = '????' ant_hdu.header = ant_hdr hdulist = fits.HDUList(hdus=[vis_hdu, ant_hdu]) hdulist.writeto(write_file, overwrite=True) uvtest.checkWarnings(uv_out.read, [write_file], message='Telescope EVLA is not') assert uv_out.telescope_name == 'EVLA' assert uv_out.timesys == time_sys # check error if timesys is 'IAT' uvtest.checkWarnings(uv_in.read, [testfile], message='Telescope EVLA is not') uv_in.timesys = 'IAT' pytest.raises(ValueError, uv_in.write_uvfits, write_file) uv_in.timesys = 'UTC' # check error if one time & no inttime specified uv_singlet = uv_in.select(times=uv_in.time_array[0], inplace=False) uv_singlet.write_uvfits(write_file) with fits.open(write_file, memmap=True) as hdu_list: hdunames = uvutils._fits_indexhdus(hdu_list) vis_hdu = hdu_list[0] vis_hdr = vis_hdu.header.copy() raw_data_array = vis_hdu.data.data par_names = np.array(vis_hdu.data.parnames) pars_use = np.where(par_names != 'INTTIM')[0] par_names = par_names[pars_use].tolist() group_parameter_list = [vis_hdu.data.par(name) for name in par_names] vis_hdu = fits.GroupData(raw_data_array, parnames=par_names, pardata=group_parameter_list, bitpix=-32) vis_hdu = fits.GroupsHDU(vis_hdu) vis_hdu.header = vis_hdr ant_hdu = hdu_list[hdunames['AIPS AN']] hdulist = fits.HDUList(hdus=[vis_hdu, ant_hdu]) hdulist.writeto(write_file, overwrite=True) uvtest.checkWarnings( pytest.raises, [ValueError, uv_out.read, write_file], message=[ 'Telescope EVLA is not', 'ERFA function "utcut1" yielded 1 of "dubious year (Note 3)"', 'ERFA function "utctai" yielded 1 of "dubious year (Note 3)"', 'LST values stored in this file are not self-consistent' ], nwarnings=4, category=[ UserWarning, astropy._erfa.core.ErfaWarning, astropy._erfa.core.ErfaWarning, UserWarning ]) # check that unflagged data with nsample = 0 will cause warnings uv_in.nsample_array[list(range(11, 22))] = 0 uv_in.flag_array[list(range(11, 22))] = False uvtest.checkWarnings(uv_in.write_uvfits, [write_file], message='Some unflagged data has nsample = 0') del (uv_in) del (uv_out)
def uvc2uv(uvcfile, calfile, outdir=None, overwrite=False, name_prefix="HH"): """ """ # check output if outdir is None: outdir = os.path.dirname(uvcfile) uvfile = os.path.splitext(os.path.basename(uvcfile)) uvfile = uvfile[0] + ".uv" output_fname = os.path.join(outdir, uvfile) if os.path.exists(output_fname) and overwrite is False: raise IOError("...{} exists, not overwriting") # load file uvc = UVData() uvc.read_miriad(uvcfile) # get antenna numbers from data uvc_ants = np.unique(np.concatenate([uvc.ant_1_array, uvc.ant_2_array])) # reorder according to data antenna_numbers = copy.copy(uvc.antenna_numbers) uvc_antcut = [True if a in uvc_ants else False for a in antenna_numbers] antenna_numbers = antenna_numbers[uvc_antcut] uvc_antsort = [] for a in uvc_ants: if a in antenna_numbers: uvc_antsort.append(antenna_numbers.tolist().index(a)) uvc_ants = uvc_ants[uvc_antsort] Nants_data = len(uvc_ants) if uvc.antenna_diameters is not None: uvc_ant_diameters = uvc.antenna_diameters[uvc_antcut][uvc_antsort] else: uvc_ant_diameters = np.ones_like(uvc_antcut, np.float) # get antenna positions from calfile aa = aipy.cal.get_aa(calfile, np.array([0.15])) info = hc.omni.aa_to_info(aa, pols=['y'], tol=5.0) cf_ants = info.subsetant cf_antpos = info.antloc # check ants in uvc are at least in cf assert len(set(uvc_ants) - set(cf_ants)) == 0, "ants {} found in data but not calfile".format(set(uvc_ants) - set(cf_ants)) # reorder cf_ants to match uvc_ants cf_antsort = [cf_ants.tolist().index(a) for a in uvc_ants] cf_ants = cf_ants[cf_antsort] cf_antpos = cf_antpos[cf_antsort] # convert antpos from ENU meters to ITRF nanosec ECEF_antpos = uvutils.ECEF_from_ENU(cf_antpos.T, *uvc.telescope_location_lat_lon_alt).T - uvc.telescope_location # reset antenna position and number information uvc.Nants_data = Nants_data uvc.Nants_telescope = Nants_data uvc.antenna_numbers = cf_ants uvc.antenna_names = map(lambda a: "{}{}".format(name_prefix, a), cf_ants) uvc.antenna_positions = ECEF_antpos uvc.antenna_diameters = uvc_ant_diameters # write to file print("..saving {}".format(output_fname)) uvc.write_miriad(output_fname, clobber=True)
def test_readwriteread(tmp_path): """ CASA tutorial uvfits loopback test. Read in uvfits file, write out new uvfits file, read back in and check for object equality. """ uv_in = UVData() uv_out = UVData() testfile = os.path.join(DATA_PATH, "day2_TDEM0003_10s_norx_1src_1spw.uvfits") write_file = str(tmp_path / "outtest_casa.uvfits") uv_in.read(testfile) uv_in.write_uvfits(write_file) uv_out.read(write_file) assert uv_in == uv_out # test that it works with write_lst = False uv_in.write_uvfits(write_file, write_lst=False) uv_out.read(write_file) assert uv_in == uv_out # check that if x_orientation is set, it's read back out properly uv_in.x_orientation = "east" uv_in.write_uvfits(write_file) uv_out.read(write_file) assert uv_in == uv_out # check that if antenna_diameters is set, it's read back out properly uv_in.read(testfile) uv_in.antenna_diameters = np.zeros( (uv_in.Nants_telescope, ), dtype=np.float) + 14.0 uv_in.write_uvfits(write_file) uv_out.read(write_file) assert uv_in == uv_out # check that if antenna_numbers are > 256 everything works uv_in.read(testfile) uv_in.antenna_numbers = uv_in.antenna_numbers + 256 uv_in.ant_1_array = uv_in.ant_1_array + 256 uv_in.ant_2_array = uv_in.ant_2_array + 256 uv_in.baseline_array = uv_in.antnums_to_baseline(uv_in.ant_1_array, uv_in.ant_2_array) uvtest.checkWarnings( uv_in.write_uvfits, [write_file], message= "antnums_to_baseline: found > 256 antennas, using 2048 baseline", ) uv_out.read(write_file) assert uv_in == uv_out # check missing telescope_name, timesys vs timsys spelling, xyz_telescope_frame=???? with fits.open(write_file, memmap=True) as hdu_list: hdunames = uvutils._fits_indexhdus(hdu_list) vis_hdu = hdu_list[0] vis_hdr = vis_hdu.header.copy() vis_hdr.pop("TELESCOP") vis_hdu.header = vis_hdr ant_hdu = hdu_list[hdunames["AIPS AN"]] ant_hdr = ant_hdu.header.copy() time_sys = ant_hdr.pop("TIMSYS") ant_hdr["TIMESYS"] = time_sys ant_hdr["FRAME"] = "????" ant_hdu.header = ant_hdr hdulist = fits.HDUList(hdus=[vis_hdu, ant_hdu]) hdulist.writeto(write_file, overwrite=True) uv_out.read(write_file) assert uv_out.telescope_name == "EVLA" assert uv_out.timesys == time_sys # check error if timesys is 'IAT' uv_in.read(testfile) uv_in.timesys = "IAT" with pytest.raises(ValueError) as cm: uv_in.write_uvfits(write_file) assert str(cm.value).startswith( "This file has a time system IAT. " 'Only "UTC" time system files are supported') uv_in.timesys = "UTC" # check error if one time & no inttime specified uv_singlet = uv_in.select(times=uv_in.time_array[0], inplace=False) uv_singlet.write_uvfits(write_file) with fits.open(write_file, memmap=True) as hdu_list: hdunames = uvutils._fits_indexhdus(hdu_list) vis_hdu = hdu_list[0] vis_hdr = vis_hdu.header.copy() raw_data_array = vis_hdu.data.data par_names = np.array(vis_hdu.data.parnames) pars_use = np.where(par_names != "INTTIM")[0] par_names = par_names[pars_use].tolist() group_parameter_list = [vis_hdu.data.par(name) for name in par_names] vis_hdu = fits.GroupData(raw_data_array, parnames=par_names, pardata=group_parameter_list, bitpix=-32) vis_hdu = fits.GroupsHDU(vis_hdu) vis_hdu.header = vis_hdr ant_hdu = hdu_list[hdunames["AIPS AN"]] hdulist = fits.HDUList(hdus=[vis_hdu, ant_hdu]) hdulist.writeto(write_file, overwrite=True) with pytest.raises(ValueError) as cm: uvtest.checkWarnings( uv_out.read, func_args=[write_file], message=[ "Telescope EVLA is not", 'ERFA function "utcut1" yielded 1 of "dubious year (Note 3)"', 'ERFA function "utctai" yielded 1 of "dubious year (Note 3)"', "LST values stored in this file are not self-consistent", ], nwarnings=4, category=[ UserWarning, astropy._erfa.core.ErfaWarning, astropy._erfa.core.ErfaWarning, UserWarning, ], ) assert str(cm.value).startswith( "integration time not specified and only one time present") # check that unflagged data with nsample = 0 will cause warnings uv_in.nsample_array[list(range(11, 22))] = 0 uv_in.flag_array[list(range(11, 22))] = False uvtest.checkWarnings(uv_in.write_uvfits, [write_file], message="Some unflagged data has nsample = 0")
def test_readWriteReadMiriad(): """ PAPER file Miriad loopback test. Read in Miriad PAPER file, write out as new Miriad file, read back in and check for object equality. """ uv_in = UVData() uv_out = UVData() testfile = os.path.join(DATA_PATH, 'zen.2456865.60537.xy.uvcRREAA') write_file = os.path.join(DATA_PATH, 'test/outtest_miriad.uv') write_file2 = os.path.join(DATA_PATH, 'test/outtest_miriad2.uv') uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad') uv_in.write_miriad(write_file, clobber=True) uv_out.read(write_file) nt.assert_equal(uv_in, uv_out) # check that we can read & write phased data uv_in2 = copy.deepcopy(uv_in) uv_in2.phase_to_time(Time(np.mean(uv_in2.time_array), format='jd')) uv_in2.write_miriad(write_file, clobber=True) uv_out.read(write_file) nt.assert_equal(uv_in2, uv_out) # check that trying to overwrite without clobber raises an error nt.assert_raises(ValueError, uv_in.write_miriad, write_file) # check that if x_orientation is set, it's read back out properly uv_in.x_orientation = 'east' uv_in.write_miriad(write_file, clobber=True) uv_out.read(write_file) nt.assert_equal(uv_in, uv_out) # check that if antenna_diameters is set, it's read back out properly uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad') uv_in.antenna_diameters = np.zeros( (uv_in.Nants_telescope, ), dtype=np.float) + 14.0 uv_in.write_miriad(write_file, clobber=True) uv_out.read(write_file) nt.assert_equal(uv_in, uv_out) # check that antenna diameters get written if not exactly float uv_in.antenna_diameters = np.zeros( (uv_in.Nants_telescope, ), dtype=np.float32) + 14.0 uv_in.write_miriad(write_file, clobber=True) uv_out.read(write_file) nt.assert_equal(uv_in, uv_out) # check that trying to write a file with unknown phasing raises an error uv_in.set_unknown_phase_type() nt.assert_raises(ValueError, uv_in.write_miriad, write_file, clobber=True) # check for backwards compatibility with old keyword 'diameter' for antenna diameters testfile_diameters = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcA') uv_in.read(testfile_diameters) uv_in.write_miriad(write_file, clobber=True) uv_out.read(write_file) nt.assert_equal(uv_in, uv_out) # check that variables 'ischan' and 'nschan' were written to new file # need to use aipy, since pyuvdata is not currently capturing these variables uv_in.read(write_file) uv_aipy = aipy_extracts.UV( write_file ) # on enterprise, this line makes it so you cant delete the file nfreqs = uv_in.Nfreqs nschan = uv_aipy['nschan'] ischan = uv_aipy['ischan'] nt.assert_equal(nschan, nfreqs) nt.assert_equal(ischan, 1) del (uv_aipy) # close the file so it can be used later # check partial IO selections full = UVData() uvtest.checkWarnings(full.read, [testfile], known_warning='miriad') full.write_miriad(write_file, clobber=True) uv_in = UVData() # test only specified bls were read, and that flipped antpair is loaded too uv_in.read(write_file, bls=[(0, 0), (0, 1), (4, 2)]) nt.assert_equal(uv_in.get_antpairs(), [(0, 0), (0, 1), (2, 4)]) exp_uv = full.select(bls=[(0, 0), (0, 1), (4, 2)], inplace=False) nt.assert_equal(uv_in, exp_uv) # test all bls w/ 0 are loaded uv_in.read(write_file, antenna_nums=[0]) diff = set(full.get_antpairs()) - set(uv_in.get_antpairs()) nt.assert_true(0 not in np.unique(diff)) exp_uv = full.select(antenna_nums=[0], inplace=False) nt.assert_true(np.max(exp_uv.ant_1_array) == 0) nt.assert_true(np.max(exp_uv.ant_2_array) == 0) nt.assert_equal(uv_in, exp_uv) uv_in.read(write_file, antenna_nums=[0], bls=[(2, 4)]) nt.assert_true( np.array([bl in uv_in.get_antpairs() for bl in [(0, 0), (2, 4)]]).all()) exp_uv = full.select(antenna_nums=[0], bls=[(2, 4)], inplace=False) nt.assert_equal(uv_in, exp_uv) uv_in.read(write_file, bls=[(2, 4, 'xy')]) nt.assert_true( np.array([bl in uv_in.get_antpairs() for bl in [(2, 4)]]).all()) exp_uv = full.select(bls=[(2, 4, 'xy')], inplace=False) nt.assert_equal(uv_in, exp_uv) uv_in.read(write_file, bls=[(4, 2, 'yx')]) nt.assert_true( np.array([bl in uv_in.get_antpairs() for bl in [(2, 4)]]).all()) exp_uv = full.select(bls=[(4, 2, 'yx')], inplace=False) nt.assert_equal(uv_in, exp_uv) uv_in.read(write_file, bls=(4, 2, 'yx')) nt.assert_true( np.array([bl in uv_in.get_antpairs() for bl in [(2, 4)]]).all()) exp_uv = full.select(bls=[(4, 2, 'yx')], inplace=False) nt.assert_equal(uv_in, exp_uv) # test time loading uv_in.read(write_file, time_range=[2456865.607, 2456865.609]) full_times = np.unique(full.time_array[(full.time_array > 2456865.607) & (full.time_array < 2456865.609)]) nt.assert_true(np.isclose(np.unique(uv_in.time_array), full_times).all()) exp_uv = full.select(times=full_times, inplace=False) nt.assert_equal(uv_in, exp_uv) # test polarization loading uv_in.read(write_file, polarizations=['xy']) nt.assert_equal(full.polarization_array, uv_in.polarization_array) exp_uv = full.select(polarizations=['xy'], inplace=False) nt.assert_equal(uv_in, exp_uv) uv_in.read(write_file, polarizations=[-7]) nt.assert_equal(full.polarization_array, uv_in.polarization_array) exp_uv = full.select(polarizations=[-7], inplace=False) nt.assert_equal(uv_in, exp_uv) # test ant_str uv_in.read(write_file, ant_str='auto') nt.assert_true( np.array([blp[0] == blp[1] for blp in uv_in.get_antpairs()]).all()) exp_uv = full.select(ant_str='auto', inplace=False) nt.assert_equal(uv_in, exp_uv) uv_in.read(write_file, ant_str='cross') nt.assert_true( np.array([blp[0] != blp[1] for blp in uv_in.get_antpairs()]).all()) exp_uv = full.select(ant_str='cross', inplace=False) nt.assert_equal(uv_in, exp_uv) uv_in.read(write_file, ant_str='all') nt.assert_equal(uv_in, full) nt.assert_raises(AssertionError, uv_in.read, write_file, ant_str='auto', antenna_nums=[0, 1]) # assert exceptions nt.assert_raises(ValueError, uv_in.read, write_file, bls='foo') nt.assert_raises(ValueError, uv_in.read, write_file, bls=[[0, 1]]) nt.assert_raises(ValueError, uv_in.read, write_file, bls=[('foo', 'bar')]) nt.assert_raises(ValueError, uv_in.read, write_file, bls=[('foo', )]) nt.assert_raises(ValueError, uv_in.read, write_file, bls=[(1, 2), (2, 3, 'xx')]) nt.assert_raises(ValueError, uv_in.read, write_file, bls=[(2, 4, 0)]) nt.assert_raises(ValueError, uv_in.read, write_file, bls=[(2, 4, 'xy')], polarizations=['xy']) nt.assert_raises(AssertionError, uv_in.read, write_file, antenna_nums=np.array([(0, 10)])) nt.assert_raises(AssertionError, uv_in.read, write_file, polarizations='xx') nt.assert_raises((AssertionError, ValueError), uv_in.read, write_file, polarizations=[1.0]) nt.assert_raises(ValueError, uv_in.read, write_file, polarizations=['yy']) nt.assert_raises(AssertionError, uv_in.read, write_file, time_range='foo') nt.assert_raises(AssertionError, uv_in.read, write_file, time_range=[1, 2, 3]) nt.assert_raises(AssertionError, uv_in.read, write_file, time_range=['foo', 'bar']) nt.assert_raises(ValueError, uv_in.read, write_file, time_range=[10.1, 10.2]) nt.assert_raises(AssertionError, uv_in.read, write_file, ant_str=0) # assert partial-read and select are same uv_in.read(write_file, polarizations=[-7], bls=[(4, 4)]) exp_uv = full.select(polarizations=[-7], bls=[(4, 4)], inplace=False) nt.assert_equal(uv_in, exp_uv) # assert partial-read and select are same uv_in.read(write_file, bls=[(4, 4, 'xy')]) exp_uv = full.select(bls=[(4, 4, 'xy')], inplace=False) nt.assert_equal(uv_in, exp_uv) # assert partial-read and select are same unique_times = np.unique(full.time_array) time_range = [2456865.607, 2456865.609] times_to_keep = unique_times[((unique_times > 2456865.607) & (unique_times < 2456865.609))] uv_in.read(write_file, antenna_nums=[0], time_range=time_range) exp_uv = full.select(antenna_nums=[0], times=times_to_keep, inplace=False) nt.assert_equal(uv_in, exp_uv) # assert partial-read and select are same uv_in.read(write_file, polarizations=[-7], time_range=time_range) exp_uv = full.select(polarizations=[-7], times=times_to_keep, inplace=False) nt.assert_equal(uv_in, exp_uv) # check handling for generic read selections unsupported by read_miriad uvtest.checkWarnings(uv_in.read, [write_file], {'times': times_to_keep}, message=['Warning: a select on read keyword is set']) exp_uv = full.select(times=times_to_keep, inplace=False) nt.assert_equal(uv_in, exp_uv) # check handling for generic read selections unsupported by read_miriad blts_select = np.where(full.time_array == unique_times[0])[0] ants_keep = [0, 2, 4] uvtest.checkWarnings( uv_in.read, [write_file], { 'blt_inds': blts_select, 'antenna_nums': ants_keep }, message=['Warning: blt_inds is set along with select on read']) exp_uv = full.select(blt_inds=blts_select, antenna_nums=ants_keep, inplace=False) nt.assert_not_equal(uv_in, exp_uv) del (uv_in) del (uv_out) del (full) # try metadata only read uv_in = UVData() uvtest.checkWarnings(uv_in.read, [testfile], {'read_data': False}, known_warning='miriad') nt.assert_equal(uv_in.time_array, None) nt.assert_equal(uv_in.data_array, None) nt.assert_equal(uv_in.integration_time, None) metadata = [ 'antenna_positions', 'antenna_names', 'antenna_positions', 'channel_width', 'history', 'vis_units', 'telescope_location' ] for m in metadata: nt.assert_true(getattr(uv_in, m) is not None) # test exceptions # multiple file read-in uv_in = UVData() uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad') new_uv = uv_in.select(freq_chans=np.arange(5), inplace=False) new_uv.write_miriad(write_file, clobber=True) new_uv = uv_in.select(freq_chans=np.arange(5) + 5, inplace=False) new_uv.write_miriad(write_file2, clobber=True) nt.assert_raises(ValueError, uv_in.read, [write_file, write_file2], read_data=False) # read-in when data already exists uv_in = UVData() uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad') nt.assert_raises(ValueError, uv_in.read, testfile, read_data=False) # test load_telescope_coords w/ blank Miriad uv_in = Miriad() uv = aipy_extracts.UV(testfile) uvtest.checkWarnings(uv_in._load_telescope_coords, [uv], known_warning='miriad') nt.assert_true(uv_in.telescope_location_lat_lon_alt is not None) # test load_antpos w/ blank Miriad uv_in = Miriad() uv = aipy_extracts.UV(testfile) uvtest.checkWarnings(uv_in._load_antpos, [uv], known_warning='miriad') nt.assert_true(uv_in.antenna_positions is not None) # test that changing precision of integraiton_time is okay # tolerance of integration_time (1e-3) is larger than floating point type conversions uv_in = UVData() uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad') uv_in.integration_time = uv_in.integration_time.astype(np.float32) uv_in.write_miriad(write_file, clobber=True) new_uv = UVData() new_uv.read(write_file) nt.assert_equal(uv_in, new_uv)
print("skipping {}...".format(fn_in)) sys.exit(0) print("scanning {}...".format(fn_in)) uvd.read_uvh5(fn_in, read_data=False, run_check=False) # Figure out which antennas have valid data and perform select-on-read. # Here, "valid data" means data that comes from actual SNAP inputs # (instead of dummy placeholder data). The way that the correlator # denotes valid SNAP input is to change the antenna number in the # object's ant_[1,2]_array to a value less than 350 (the maximum number # of valid antennas for HERA). We find all such antenna numbers # corresponding to valid input, and downselect to keep only those. ant_nums = np.unique(np.concatenate((uvd.ant_1_array, uvd.ant_2_array))) inds = np.where(ant_nums < 350) data_ants = ant_nums[inds] print("reading {}...".format(fn_in)) uvd.read_uvh5(fn_in, antenna_nums=data_ants) # fix up the metadata to reflect the antennas in the dataset uvd.Nants_telescope = len(data_ants) uvd.antenna_names = [uvd.antenna_names[ind] for ind in data_ants] uvd.antenna_numbers = uvd.antenna_numbers[data_ants] uvd.antenna_positions = uvd.antenna_positions[data_ants, :] uvd.antenna_diameters = uvd.antenna_diameters[data_ants] print("writing {}...".format(fn_out)) uvd.write_uvh5(fn_out, data_write_dtype=_hera_corr_dtype, flags_compression='lzf', nsample_compression='lzf', clobber=True)