def test_write():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
outfile = os.path.join(DATA_PATH, 'test_write.uvfits')
ss = SS()
ss.read(testfile)
custom = np.zeros_like(ss.data_array.mask)
custom[:ss.Nbls] = 1
# Flags the first time and no others
ss.apply_flags(flag_choice='custom', custom=custom)
# Write this out without combining flags
ss.write(outfile, 'uvfits', filename_in=testfile, combine=False)
# Check if the flags propagated correctly
UV = UVData()
UV.read(outfile)
assert np.all(
UV.flag_array[:2 * UV.Nbls]), "Not all expected flags were propagated"
assert not np.any(
UV.flag_array[2 * UV.Nbls:]), "More flags were made than expected"
os.remove(outfile)
def test_write():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, f'{obs}.uvfits')
file_type = 'uvfits'
outfile = os.path.join(DATA_PATH, 'test_write.uvfits')
ss = SS()
ss.read(testfile, diff=True)
blt_inds = np.where(ss.time_array == np.unique(ss.time_array)[10])
custom = np.zeros_like(ss.data_array.mask)
custom[blt_inds, :, 64:128, :] = 1
# Flags the first time and no others
ss.apply_flags(flag_choice='custom', custom=custom)
# Write this out without combining flags, will issue a warning
with pytest.warns(
UserWarning,
match=
"Some nsamples are 0, which will result in failure to propagate flags. Setting nsample to default values where 0."
):
ss.write(outfile, 'uvfits', filename_in=testfile, combine=False)
# Check if the flags propagated correctly
UV = UVData()
UV.read(outfile)
blt_inds = np.isin(UV.time_array, np.unique(UV.time_array)[10:12])
assert np.all(
UV.flag_array[blt_inds, :,
64:128, :]), "Not all expected flags were propagated"
new_blt_inds = np.logical_not(
np.isin(UV.time_array,
np.unique(UV.time_array)[10:12]))
assert not np.any(
UV.flag_array[new_blt_inds, :,
64:128, :]), "More flags were made than expected"
os.remove(outfile)
# Test bad read.
bad_uv_filepath = os.path.join(DATA_PATH, '1061312640_mix.uvfits')
bad_uv = UVData()
bad_uv.read(bad_uv_filepath)
with pytest.raises(
ValueError,
match="UVData and SS objects were found to be incompatible."):
ss.write(outfile, 'uvfits', bad_uv)
def test_newmask():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
ss = SS()
ss.read(testfile, diff=False)
assert not isinstance(ss.data_array, np.ma.MaskedArray)
ss.apply_flags()
assert ss.flag_choice is None
assert isinstance(ss.data_array, np.ma.MaskedArray)
def test_mixture_prob():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
ss = SS()
ss.read(testfile)
ss.apply_flags('original')
# Generate the mixture probabilities
mixture_prob = ss.mixture_prob(bins='auto')
# Check that they sum to close to 1
assert np.isclose(np.sum(mixture_prob),
1), "Probabilities did not add up to close to 1"
def test_mixture_prob():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
ss = SS()
ss.read(testfile, diff=True)
ss.apply_flags('original')
# Generate the mixture probabilities
mixture_prob = ss.mixture_prob(bins='auto')
# Check that they sum to close to 1
assert np.isclose(np.sum(mixture_prob),
1), "Probabilities did not add up to close to 1"
# Do a new read, but don't diff. Run and check mask.
ss = SS()
ss.read(testfile, diff=False)
mixture_prob = ss.mixture_prob(bins='auto')
assert ss.flag_choice is None
settings_file = f"{args.cal_soln_dir}/{args.obsid}_settings.txt"
files_missing = []
for file in [cal_file, obs_file, settings_file]:
if not os.path.exists(file):
files_missing.append(file)
if len(files_missing) > 0:
raise ValueError(f"The following calibration files were not found: {files_missing}")
# Should already be pre-processed, so no frills on read.
ss = SS()
ss.read(args.uvd, diff=False)
# Manually unflag so that cal flags can be used without original SSINS flags, otherwise RFI flux cannot be gathered
ss.flag_array[:] = False
uvc = UVCal()
uvc.read_fhd_cal(cal_file, obs_file, settings_file=settings_file, raw=False)
utils.uvcalibrate(ss, uvc, prop_flags=True, inplace=True, ant_check=True)
ss.diff()
ss.apply_flags(flag_choice="original")
ins = INS(ss)
prefix = f"{args.outdir}/{args.obsid}_cal"
ins.write(prefix)
Catalog_Plot.INS_plot(ins, prefix, data_cmap=cm.plasma, ms_vmin=-5, ms_vmax=5,
title=args.obsid, xlabel='Frequency (Mhz)',
ylabel='Time (UTC)')
# print(version.version)
from SSINS import INS
import numpy as np
import matplotlib as plt
import os
# Create an SS object and read in uvfits file
ss = SS()
uvfits_filename = str(input("Enter the .uvfits file you would like to use: "))
uvfits_file = '/Users/imaniware/Desktop/CHAMP/hera_uvfits/' + uvfits_filename
ss.read(uvfits_file, ant_str='cross')
#apply flags to SS data
ss.apply_flags(flag_choice='INS')
# Make a VDH plot
# import Catalog_Plot from SSINS
from SSINS import Catalog_Plot as cp
# set a file prefix for plot
prefix = 'tutorial_outputs/IW_plot_1.1_' + uvfits_filename
# Pre-flagged and Post-flagged
cp.VDH_plot(ss,
prefix,
file_ext='pdf',
xlabel='Visibility Differences',
xscale='log',
yscale='log',
args = parser.parse_args()
sys.stdout.write("Reading in observation at %s" % time.strftime("%H:%M:%S"))
ss = SS(obs=args.obs, inpath=args.inpath, outpath=args.outpath)
sys.stdout.write("Finished reading in observation at %s" %
time.strftime("%H:%M:%S"))
sys.stdout.write("Beginning flagging at %s" % time.strftime("%H:%M:%S"))
auto_bls = ss.UV.ant_1_array[:ss.UV.Nbls] == ss.UV.ant_2_array[:ss.UV.Nbls]
custom = np.copy(ss.UV.flag_array)
custom[:, auto_bls] = 1
custom[-4:-1] = 1
ss.apply_flags(choice='custom', custom=custom)
ss.MF_prepare(sig_thresh=5,
shape_dict={
'TV6': [1.74e8, 1.81e8],
'TV7': [1.81e8, 1.88e8],
'TV8': [1.88e8, 1.95e8],
'TV7_broad': [1.79e8, 1.9e8]
})
ss.MF.apply_match_test()
ss.apply_flags(choice='INS', INS=ss.INS)
sys.stdout.write("Done with flagging at %s" % time.strftime("%H:%M:%S"))
sys.stdout.write("Beginning writing at %s" % time.strftime("%H:%M:%S"))
def test_apply_flags():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)
ss = SS()
ss.read(testfile)
# Make sure no flags are applied to start with
assert not np.any(
ss.data_array.mask), "There are some flags to start with."
# Apply flags, test equality, test attribute change
ss.apply_flags(flag_choice='original')
assert np.all(
ss.flag_array == ss.data_array.mask), "Flag arrays are not equal"
assert ss.flag_choice is 'original', "Flag choice attribute was not changed"
# Revert flags back, test equality, test attribute change
ss.apply_flags(flag_choice=None)
assert not np.any(ss.data_array.mask), "Flags did not revert back back"
assert ss.flag_choice is None, "Flag choice attribute did not revert back"
# Make a custom flag array where everything is flagged, check application
custom = np.ones_like(ss.flag_array)
ss.apply_flags(flag_choice='custom', custom=custom)
assert np.all(ss.data_array.mask), "The custom flag array was not applied"
assert ss.flag_choice is 'custom', "The flag choice attribute was not changed"
# Read an INS in (no flags by default) and flag a channel, test if it applies correctly
ins = INS(insfile)
ins.metric_array.mask[:, 0] = True
ss.apply_flags(flag_choice='INS', INS=ins)
assert np.all(
ss.data_array.mask[:, 0, 0]), "Not all of the 0th channel was flagged."
assert not np.any(ss.data_array.mask[:, 0, 1:]
), "Some of the channels other than the 0th were flagged"
assert ss.flag_choice is 'INS'
# Make flag_choice custom but do not provide array - should keep old flags
ss.apply_flags(flag_choice='custom', custom=None)
assert not np.any(ss.data_array.mask
), "Some of the channels other than the 0th were flagged"
assert ss.flag_choice is None
with pytest.raises(ValueError):
ss.apply_flags(flag_choice='bad_choice')
UV.read(args.inpath, file_type='uvfits')
JD = np.unique(UV.time_array)[-2]
where = np.where(UV.time_array == JD)
UV.flag_array[where] = 1
auto_bl = np.where(UV.ant_1_array == UV.ant_2_array)
UV.flag_array[auto_bl] = 1
return (UV)
UV = UV_construct(args)
shape_dict = {
'TV6': [1.74e8, 1.81e8],
'TV7': [1.81e8, 1.88e8],
'TV8': [1.88e8, 1.95e8],
'broad6': [1.72e8, 1.83e8],
'broad7': [1.79e8, 1.9e8],
'broad8': [1.86e8, 1.97e8]
}
sky_sub = SS(obs=args.obs, UV=UV, outpath=args.outpath, flag_choice='original')
sky_sub.INS_prepare()
sky_sub.MF_prepare(sig_thresh=5, shape_dict=shape_dict, N_thresh=15)
sky_sub.MF.apply_match_test(apply_N_thresh=True)
sky_sub.apply_flags(choice='INS', INS=sky_sub.INS)
sky_sub.write('%s/%s.uvfits' % (args.outpath, args.obs),
'uvfits',
UV=UV_construct(args))
from SSINS import SS
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('obs')
parser.add_argument('inpath')
parser.add_argument('outpath')
args = parser.parse_args()
sky_sub = SS(obs=args.obs,
outpath=args.outpath,
inpath=args.inpath,
bad_time_indices=[0, -1, -2, -3],
read_kwargs={'ant_str': 'cross'})
sky_sub.INS_prepare()
sky_sub.save_data()
sky_sub.apply_flags(choice='original')
sky_sub.INS_prepare()
sky_sub.save_data()
indir = args.infile[:args.infile.rfind('/')]
if indir == args.outdir:
raise ValueError("indir and outdir are the same")
ss = SS()
ss.read(args.infile, flag_choice='original')
ins = INS(ss)
ins.metric_array[-5:] = np.ma.masked
ins.metric_ms = ins.mean_subtract()
shape_dict = {
'TV4': [1.74e8, 1.81e8],
'TV5': [1.81e8, 1.88e8],
'TV6': [1.88e8, 1.95e8],
'broad6': [1.72e8, 1.83e8],
'broad7': [1.79e8, 1.9e8],
'broad8': [1.86e8, 1.98e8]
}
mf = MF(ins.freq_array, 5, shape_dict=shape_dict, N_samp_thresh=25)
mf.apply_match_test(ins, apply_samp_thresh=True)
cp.INS_plot(ins, '%s/%s' % (args.outdir, args.obsid))
ss.apply_flags(flag_choice='INS', INS=ins)
ss.write('%s/%s.uvfits' % (args.outdir, args.obsid),
'uvfits',
nsample_default=16)
def test_apply_flags():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)
ss = SS()
ss.read(testfile, diff=True)
# Make sure no flags are applied to start with
assert not np.any(
ss.data_array.mask), "There are some flags to start with."
# Apply flags, test equality, test attribute change
ss.apply_flags(flag_choice='original')
assert np.all(
ss.flag_array == ss.data_array.mask), "Flag arrays are not equal"
assert ss.flag_choice is 'original', "Flag choice attribute was not changed"
# Revert flags back, test equality, test attribute change
ss.apply_flags(flag_choice=None)
assert not np.any(ss.data_array.mask), "Flags did not revert back back"
assert ss.flag_choice is None, "Flag choice attribute did not revert back"
# Make a custom flag array where everything is flagged, check application
custom = np.ones_like(ss.flag_array)
ss.apply_flags(flag_choice='custom', custom=custom)
assert np.all(ss.data_array.mask), "The custom flag array was not applied"
assert ss.flag_choice is 'custom', "The flag choice attribute was not changed"
# Read an INS in (no flags by default) and flag a channel for two times stuff, see if applied correctly
ins = INS(insfile)
ins.metric_array.mask[[2, 4], 1, :] = True
ss.apply_flags(flag_choice='INS', INS=ins)
assert np.all(ss.data_array.mask[2::ss.Ntimes, :,
1, :]), "The 2nd time was not flagged."
assert np.all(ss.data_array.mask[4::ss.Ntimes, :,
1, :]), "The 4th time was not flagged."
assert not np.any(ss.data_array.mask[:, :,
[0] + list(range(2, ss.Nfreqs)), :]
), "Channels were flagged that should not have been."
assert ss.flag_choice is 'INS'
# Make a bad time array to test an error
ins.time_array = ins.time_array + 1
with pytest.raises(ValueError):
ss.apply_flags(flag_choice='INS', INS=ins)
# Make flag_choice custom but do not provide array - should unflag everything and issue a warning
with pytest.warns(
UserWarning,
match=
"Custom flags were chosen, but custom flags were None type. Setting flag_choice to None and unmasking data."
):
ss.apply_flags(flag_choice='custom', custom=None)
assert not np.any(
ss.data_array.mask), "Some of the channels were still flagged"
assert ss.flag_choice is None
with pytest.raises(ValueError):
ss.apply_flags(flag_choice='bad_choice')
