class TestMethods(unittest.TestCase):
def setUp(self):
self.data = UVData()
filename = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA')
self.data.read_miriad(filename)
# massage the object to make it work with check_noise_variance
self.data.select(antenna_nums=self.data.get_ants()[0:10])
self.data.select(freq_chans=range(100))
# Data file only has three times... need more.
while self.data.Ntimes < 90:
d2 = copy.deepcopy(self.data)
d2.time_array += d2.time_array.max() + d2.integration_time / (24 *
3600)
self.data += d2
ntimes = self.data.Ntimes
nchan = self.data.Nfreqs
self.data1 = qmtest.noise(size=(ntimes, nchan))
self.data2 = qmtest.noise(size=(ntimes, nchan))
ant_dat = {}
for i in self.data.get_ants():
ant_dat[i] = qmtest.noise(size=(ntimes, nchan)) + 0.1 * self.data1
for key in self.data.get_antpairpols():
ind = self.data._key2inds(key)[0]
self.data.data_array[ind, 0, :,
0] = ant_dat[key[0]] * ant_dat[key[1]].conj()
def test_check_noise_variance(self):
nos = vis_metrics.check_noise_variance(self.data)
for bl in self.data.get_antpairs():
inds = self.data.antpair2ind(*bl)
n = nos[bl + (uvutils.parse_polstr('xx'), )]
self.assertEqual(n.shape, (self.data.Nfreqs - 1, ))
nsamp = self.data.channel_width * self.data.integration_time[inds][
0]
np.testing.assert_almost_equal(n,
np.ones_like(n) * nsamp,
-np.log10(nsamp))
def test_check_noise_variance_inttime_error(self):
self.data.integration_time = (
self.data.integration_time *
np.arange(self.data.integration_time.size))
self.assertRaises(NotImplementedError,
vis_metrics.check_noise_variance, self.data)
def vismetrics_data():
data = UVData()
filename = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA')
data.read_miriad(filename)
# massage the object to make it work with check_noise_variance
data.select(antenna_nums=data.get_ants()[0:10])
data.select(freq_chans=range(100))
# Data file only has three times... need more.
while data.Ntimes < 90:
d2 = copy.deepcopy(data)
d2.time_array += d2.time_array.max() + d2.integration_time / (24 *
3600)
data += d2
ntimes = data.Ntimes
nchan = data.Nfreqs
data1 = qmtest.noise(size=(ntimes, nchan))
data2 = qmtest.noise(size=(ntimes, nchan))
ant_dat = {}
for i in data.get_ants():
ant_dat[i] = qmtest.noise(size=(ntimes, nchan)) + 0.1 * data1
for key in data.get_antpairpols():
ind = data._key2inds(key)[0]
data.data_array[ind, 0, :,
0] = ant_dat[key[0]] * ant_dat[key[1]].conj()
class DataHolder(object):
def __init__(self, data, data1, data2):
self.data = data
self.data1 = data1
self.data2 = data2
vismetrics_data = DataHolder(data, data1, data2)
# yield lets us return the data and then continue with clean up after
yield vismetrics_data
# post test clean up
del (vismetrics_data)
return
class Eris(object):
def __init__(self, Zeus, pol):
self.Zeus = Zeus
self.pol = pol
self.npz_name = str()
self.caldata = dict()
self.uvd = None
self.uvd2 = None
self.aa = None
self.info = dict()
self.data = dict()
self.flags = dict()
self.vis_sq_bl = None
self.vis_sq_slope = None
self.vis_sq_antpair = None
self.bl_slices = list()
self.sl_slices = list()
self.ant_slices = list()
def name_npz(self):
zen = 'zen'
HH = 'HH'
extension = self.Zeus.ext
pol_naming = self.Zeus.catalog[extension].keys()[0]
data_file0 = self.Zeus.catalog[extension][pol_naming][0]
data_filef = self.Zeus.catalog[extension][pol_naming][-1]
file0 = data_file0['file']
filef = data_filef['file']
JD = int(data_file0['JD'])
JDT0 = data_file0['JD'] - JD
JDTf = data_filef['JD'] - JD
JD = str(JD)
JDT0 = str(JDT0)[2:7]
JDTf = str(JDTf)[2:7]
num_files = np.unique(
self.Zeus.catalog[extension][pol_naming]['file']).shape[0]
JDT = '{JDT0}_{num_files}_{JDTf}'.format(JDT0=JDT0,
num_files=num_files,
JDTf=JDTf)
pol = self.pol
if self.Zeus.exants:
exants = [str(ant) for ant in self.Zeus.exants]
exants = '_'.join(exants)
else:
exants = 'None'
freqrange = '{start}_{end}'.format(start=self.Zeus.freqrange[0],
end=self.Zeus.freqrange[1])
npz_name = [zen, JD, JDT, pol, exants, freqrange, HH, extension, 'npz']
npz_name = '.'.join(npz_name)
self.npz_name = os.path.join(self.Zeus.path, npz_name)
print(self.npz_name)
def load_MIRIAD(self):
"""Formats data and flags array for a specific polarization, also apply flags."""
# Check what type of polarization was specified
if self.Zeus.pol_type == 'standard':
self.uvd = UVData()
self.uvd.read_miriad(self.Zeus.files[self.pol])
elif self.pol == 'I' or self.pol == 'Q':
self.uvd = UVData()
self.uvd.read_miriad(self.Zeus.files['xx'])
self.uvd2 = UVData()
self.uvd2.read_miriad(self.Zeus.files['yy'])
elif self.pol == 'U' or self.pol == 'V':
self.uvd = UVData()
self.uvd.read_miriad(self.Zeus.files['xy'])
self.uvd2 = UVData()
self.uvd2.read_miriad(self.Zeus.files['yx'])
self.calculate_caldata()
# Get metadata and convert freqs array from Hz -> GHz
self.info['freqs'] = self.uvd.freq_array[0, :] / 1e9
self.info['times'] = np.unique(self.uvd.time_array)
# Extract and format data and flags arrays
for key, d in self.uvd.antpairpol_iter():
# Ignore auto-correlation
if key[0] == key[1]:
continue
ind1, ind2, ipol = self.uvd._key2inds(key)
for ind in [ind1, ind2]:
if len(ind) == 0:
continue
antkey = key[:2]
if self.Zeus.pol_type == 'standard':
f = self.uvd.flag_array[ind, 0, :, ipol]
polkey = key[2].lower()
self.data[antkey] = {polkey: d}
self.flags[antkey] = {polkey: f}
elif self.Zeus.pol_type == 'stokes':
f = self.uvd.flag_array[ind, 0, :, 0]
d2 = self.uvd2.data_array[ind, 0, :, 0]
f2 = self.uvd2.flag_array[ind, 0, :, 0]
if self.pol == 'I':
self.data[antkey] = {'I': d + d2}
self.flags[antkey] = {'I': f + f2}
elif self.pol == 'Q':
self.data[antkey] = {'Q': d - d2}
self.flags[antkey] = {'Q': f + f2}
elif self.pol == 'U':
self.data[antkey] = {'U': d + d2}
self.flags[antkey] = {'U': f + f2}
elif self.pol == 'V':
self.data[antkey] = {'V': -1j * d + 1j * d2}
self.flags[antkey] = {'V': f + f2}
del self.uvd2
# Apply flags to data by looking for where the flags array is true, and zeroing the corresponding data elements.
for pair in self.caldata['pairs']:
self.data[pair][self.pol] = np.where(self.flags[pair][self.pol],
0. + 0. * 1j,
self.data[pair][self.pol])
for i in self.flags:
for j in self.flags[i]:
self.flags[i][j] = np.logical_not(self.flags[i][j]).astype(int)
def calculate_caldata(self):
"""Returns a dictionary of baseline lengths and the corresponding pairs.
The data is based on a calfile. ex_ants is a list of integers that
specify antennae to be exlcuded from calculation.
Requires cal file to be in PYTHONPATH."""
if self.Zeus.calfile:
try:
print('Reading calfile: %s...' % self.Zeus.calfile)
exec('import %s as cal' % self.Zeus.calfile, globals(),
locals())
antennae = cal.prms['antpos_ideal']
except ImportError:
raise Exception('Unable to import: %s' % self.Zeus.calfile)
elif self.uvd:
# Build antenna positions from data file itself.
print('Generating calibration information from MIRIAD file.')
antennae = {}
lat, lon, alt = self.uvd.telescope_location_lat_lon_alt
for i, antnum in enumerate(self.uvd.antenna_numbers):
pos = self.uvd.antenna_positions[
i, :] + self.uvd.telescope_location
xyz = uvutils.ENU_from_ECEF(pos,
latitude=lat,
longitude=lon,
altitude=alt)
antennae[antnum] = {
'top_x': xyz[0],
'top_y': xyz[1],
'top_z': xyz[2]
}
else:
raise Exception(
'UVData object does not exist. Try supplying a calfile.')
# Check that self.Zeus.exants contain only antennae that exist
if not set(self.Zeus.exants).issubset(set(antennae.keys())):
raise Exception('You provided invalid antenna(e) to exclude.')
# Remove all placeholder antennae from consideration
# Remove all antennae from exants from consideration
ants = []
for ant in antennae.keys():
if (not antennae[ant]['top_z']
== -1) and (ant not in self.Zeus.exants):
ants.append(ant)
ants = np.array(ants)
# Set the bins for binning the baselines
bins_baseline = np.arange(0, 750, self.Zeus.BIN_WIDTH)
# Initialize baselines, slopes, pairs arrays
baselines = np.array([], dtype=np.float128)
slopes = np.array([], dtype=np.float128)
pairs = {}
# Cycle through antennae to create pairs
for ant_i in ants:
for ant_j in ants:
if ant_i >= ant_j:
continue
pair = (ant_i, ant_j)
# Find the baseline length of the pair, bin it, format it, and add it to the baselines array
dx = antennae[pair[1]]['top_x'] - antennae[pair[0]]['top_x']
dy = antennae[pair[1]]['top_y'] - antennae[pair[0]]['top_y']
baseline = np.sqrt(np.power(dx, 2) + np.power(dy, 2))
baseline = np.round(baseline, decimals=2)
baseline = np.digitize(baseline,
bins_baseline) * self.Zeus.BIN_WIDTH
baselines = np.append(baselines, baseline)
# Find the slope of the pair, format it, and add it to the slopes array
dy = antennae[pair[1]]['top_y'] - antennae[pair[0]]['top_y']
dx = antennae[pair[1]]['top_x'] - antennae[pair[0]]['top_x']
if dx != 0:
slope = float(np.round(dy / dx, decimals=2))
else:
slope = np.inf
slopes = np.append(slopes, slope)
# Add the pair and its slope and baseline to the pairs dictionary: {(ant_1, ant_2): (baseline, slope), ...}
pairs[pair] = (np.float(np.round(baseline, 1)),
np.float(np.round(slope, 2)))
# Sort and remove duplicates from baselines and slopes
baselines = np.unique(np.sort(baselines))
slopes = np.unique(np.sort(slopes))
# Sort pairs into antdict and slopedict
antdict = {}
slopedict = {}
for pair, (baseline, slope) in pairs.items():
if baseline in antdict:
antdict[baseline].append(pair)
else:
antdict[baseline] = [pair]
if baseline in slopedict:
if slope in slopedict[baseline]:
slopedict[baseline][slope].append(pair)
else:
slopedict[baseline][slope] = [pair]
else:
slopedict[baseline] = {slope: []}
slopedict[baseline][slope].append(pair)
self.caldata = {
'antdict': antdict,
'slopedict': slopedict,
'pairs': pairs,
'baselines': baselines,
'slopes': slopes
}
def pitchfork(self):
self.info['freqs'] = self.info['freqs'][self.Zeus.freqrange[0]:self.
Zeus.freqrange[1]]
for antpair in self.data.keys():
self.data[antpair][self.pol] = self.data[antpair][
self.pol][:, self.Zeus.freqrange[0]:self.Zeus.freqrange[1]]
self.flags[antpair][self.pol] = self.flags[antpair][
self.pol][:, self.Zeus.freqrange[0]:self.Zeus.freqrange[1]]
ntimes = len(self.info['times'])
nchan = len(self.info['freqs'])
if self.Zeus.calfile:
self.aa = hera_cal.utils.get_aa_from_calfile(
self.info['freqs'], self.Zeus.calfile)
else:
self.aa = hera_cal.utils.get_aa_from_uv(self.uvd,
self.info['freqs'])
self.aa.set_active_pol(self.pol)
for baseline in sorted(self.caldata['slopedict']):
self.vis_sq_bl = np.zeros((ntimes // 2, nchan),
dtype=np.complex128)
for slope in sorted(self.caldata['slopedict'][baseline]):
self.vis_sq_slope = np.zeros((ntimes // 2, nchan),
dtype=np.complex128)
for antpair in sorted(
self.caldata['slopedict'][baseline][slope]):
self.vis_sq_antpair = np.zeros((ntimes // 2, nchan),
dtype=np.complex128)
self.fft_clean_phase(antpair, ntimes, self.Zeus.CLEAN)
self.ant_slices.append(self.vis_sq_antpair)
self.vis_sq_slope += self.vis_sq_antpair
self.vis_sq_bl += self.vis_sq_antpair
self.vis_sq_slope /= len(
self.caldata['slopedict'][baseline][slope])
self.sl_slices.append(self.vis_sq_slope)
self.vis_sq_bl /= len(self.caldata['antdict'][baseline])
self.bl_slices.append(self.vis_sq_bl)
def fft_clean_phase(self, antpair, ntimes, CLEAN):
# FFT
w = aipy.dsp.gen_window(self.data[antpair][self.pol].shape[-1],
window='blackman-harris')
_dw = np.fft.ifft(self.data[antpair][self.pol] * w)
# CLEAN
_ker = np.fft.ifft(self.flags[antpair][self.pol] * w)
gain = aipy.img.beam_gain(_ker)
for time in range(_dw.shape[0]):
_dw[time, :], info = aipy.deconv.clean(_dw[time, :],
_ker[time, :],
tol=CLEAN)
_dw[time, :] += info['res'] / gain
fft_2Ddata = np.ma.array(_dw)
# Phase
for i in range(ntimes):
if i != 0:
old_zenith = zenith
time = self.info['times'][i]
self.aa.set_jultime(time)
lst = self.aa.sidereal_time()
zenith = aipy.phs.RadioFixedBody(lst, self.aa.lat)
zenith.compute(self.aa)
if i % 2:
v1 = fft_2Ddata[i - 1, :]
phase_correction = np.conj(
self.aa.gen_phs(zenith, antpair[0],
antpair[1])) * self.aa.gen_phs(
old_zenith, antpair[0], antpair[1])
v2 = fft_2Ddata[i, :] * phase_correction
self.vis_sq_antpair[i // 2, :] = np.conj(v1) * v2
def save(self):
channel_width = np.diff(self.info['freqs'])[0]
num_bins = self.info['freqs'].shape[0]
delays = np.fft.fftshift(np.fft.fftfreq(num_bins, channel_width))
np.savez(self.npz_name,
Zeus=self.Zeus,
bl=self.bl_slices,
sl=self.sl_slices,
ant=self.ant_slices,
caldata=self.caldata,
pol=self.pol,
freqs=self.info['freqs'],
delays=delays)
if len(file_glob) == 0:
raise FileNotFoundError("Something went wrong--no files were found.")
# load in the data, downselect to autos and linear pols
use_pols = [polstr2num(pol) for pol in ('xx', 'yy')]
uvd = UVData()
uvd.read(file_glob, ant_str='auto', polarizations=use_pols)
# just do everything in this script; first isolate the rfi
rfi_data = np.zeros_like(uvd.data_array, dtype=np.float)
normalized_rfi_data = np.zeros_like(rfi_data, dtype=np.float)
rfi_flags = np.zeros_like(rfi_data, dtype=np.bool)
for antpairpol in uvd.get_antpairpols():
# get indices for properly slicing through data array
blt_inds, conj_blt_inds, pol_inds = uvd._key2inds(antpairpol)
this_slice = slice(blt_inds, 0, None, pol_inds[0])
# approximately remove all non-rfi signal
this_data = uvd.get_data(antpairpol).real
filt_data = xrfi.medminfilt(this_data)
this_rfi = this_data - filt_data
this_rfi[this_rfi <= 0] = 1
this_ratio = this_data / filt_data
# detrend the original data to find where the stations are
detrended_data = xrfi.detrend_medfilt(data)
station_flags = np.where(detrended_data > 100, True, False)
# update flags with watershed algorithm
station_flags = xrfi._ws_flag_waterfall(detrended_data,
