class TestMethods(unittest.TestCase): def setUp(self): self.data = UVData() filename = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA') self.data.read_miriad(filename) # massage the object to make it work with check_noise_variance self.data.select(antenna_nums=self.data.get_ants()[0:10]) self.data.select(freq_chans=range(100)) # Data file only has three times... need more. while self.data.Ntimes < 90: d2 = copy.deepcopy(self.data) d2.time_array += d2.time_array.max() + d2.integration_time / (24 * 3600) self.data += d2 ntimes = self.data.Ntimes nchan = self.data.Nfreqs self.data1 = qmtest.noise(size=(ntimes, nchan)) self.data2 = qmtest.noise(size=(ntimes, nchan)) ant_dat = {} for i in self.data.get_ants(): ant_dat[i] = qmtest.noise(size=(ntimes, nchan)) + 0.1 * self.data1 for key in self.data.get_antpairpols(): ind = self.data._key2inds(key)[0] self.data.data_array[ind, 0, :, 0] = ant_dat[key[0]] * ant_dat[key[1]].conj() def test_check_noise_variance(self): nos = vis_metrics.check_noise_variance(self.data) for bl in self.data.get_antpairs(): inds = self.data.antpair2ind(*bl) n = nos[bl + (uvutils.parse_polstr('xx'), )] self.assertEqual(n.shape, (self.data.Nfreqs - 1, )) nsamp = self.data.channel_width * self.data.integration_time[inds][ 0] np.testing.assert_almost_equal(n, np.ones_like(n) * nsamp, -np.log10(nsamp)) def test_check_noise_variance_inttime_error(self): self.data.integration_time = ( self.data.integration_time * np.arange(self.data.integration_time.size)) self.assertRaises(NotImplementedError, vis_metrics.check_noise_variance, self.data)
def vismetrics_data(): data = UVData() filename = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA') data.read_miriad(filename) # massage the object to make it work with check_noise_variance data.select(antenna_nums=data.get_ants()[0:10]) data.select(freq_chans=range(100)) # Data file only has three times... need more. while data.Ntimes < 90: d2 = copy.deepcopy(data) d2.time_array += d2.time_array.max() + d2.integration_time / (24 * 3600) data += d2 ntimes = data.Ntimes nchan = data.Nfreqs data1 = qmtest.noise(size=(ntimes, nchan)) data2 = qmtest.noise(size=(ntimes, nchan)) ant_dat = {} for i in data.get_ants(): ant_dat[i] = qmtest.noise(size=(ntimes, nchan)) + 0.1 * data1 for key in data.get_antpairpols(): ind = data._key2inds(key)[0] data.data_array[ind, 0, :, 0] = ant_dat[key[0]] * ant_dat[key[1]].conj() class DataHolder(object): def __init__(self, data, data1, data2): self.data = data self.data1 = data1 self.data2 = data2 vismetrics_data = DataHolder(data, data1, data2) # yield lets us return the data and then continue with clean up after yield vismetrics_data # post test clean up del (vismetrics_data) return
class Eris(object): def __init__(self, Zeus, pol): self.Zeus = Zeus self.pol = pol self.npz_name = str() self.caldata = dict() self.uvd = None self.uvd2 = None self.aa = None self.info = dict() self.data = dict() self.flags = dict() self.vis_sq_bl = None self.vis_sq_slope = None self.vis_sq_antpair = None self.bl_slices = list() self.sl_slices = list() self.ant_slices = list() def name_npz(self): zen = 'zen' HH = 'HH' extension = self.Zeus.ext pol_naming = self.Zeus.catalog[extension].keys()[0] data_file0 = self.Zeus.catalog[extension][pol_naming][0] data_filef = self.Zeus.catalog[extension][pol_naming][-1] file0 = data_file0['file'] filef = data_filef['file'] JD = int(data_file0['JD']) JDT0 = data_file0['JD'] - JD JDTf = data_filef['JD'] - JD JD = str(JD) JDT0 = str(JDT0)[2:7] JDTf = str(JDTf)[2:7] num_files = np.unique( self.Zeus.catalog[extension][pol_naming]['file']).shape[0] JDT = '{JDT0}_{num_files}_{JDTf}'.format(JDT0=JDT0, num_files=num_files, JDTf=JDTf) pol = self.pol if self.Zeus.exants: exants = [str(ant) for ant in self.Zeus.exants] exants = '_'.join(exants) else: exants = 'None' freqrange = '{start}_{end}'.format(start=self.Zeus.freqrange[0], end=self.Zeus.freqrange[1]) npz_name = [zen, JD, JDT, pol, exants, freqrange, HH, extension, 'npz'] npz_name = '.'.join(npz_name) self.npz_name = os.path.join(self.Zeus.path, npz_name) print(self.npz_name) def load_MIRIAD(self): """Formats data and flags array for a specific polarization, also apply flags.""" # Check what type of polarization was specified if self.Zeus.pol_type == 'standard': self.uvd = UVData() self.uvd.read_miriad(self.Zeus.files[self.pol]) elif self.pol == 'I' or self.pol == 'Q': self.uvd = UVData() self.uvd.read_miriad(self.Zeus.files['xx']) self.uvd2 = UVData() self.uvd2.read_miriad(self.Zeus.files['yy']) elif self.pol == 'U' or self.pol == 'V': self.uvd = UVData() self.uvd.read_miriad(self.Zeus.files['xy']) self.uvd2 = UVData() self.uvd2.read_miriad(self.Zeus.files['yx']) self.calculate_caldata() # Get metadata and convert freqs array from Hz -> GHz self.info['freqs'] = self.uvd.freq_array[0, :] / 1e9 self.info['times'] = np.unique(self.uvd.time_array) # Extract and format data and flags arrays for key, d in self.uvd.antpairpol_iter(): # Ignore auto-correlation if key[0] == key[1]: continue ind1, ind2, ipol = self.uvd._key2inds(key) for ind in [ind1, ind2]: if len(ind) == 0: continue antkey = key[:2] if self.Zeus.pol_type == 'standard': f = self.uvd.flag_array[ind, 0, :, ipol] polkey = key[2].lower() self.data[antkey] = {polkey: d} self.flags[antkey] = {polkey: f} elif self.Zeus.pol_type == 'stokes': f = self.uvd.flag_array[ind, 0, :, 0] d2 = self.uvd2.data_array[ind, 0, :, 0] f2 = self.uvd2.flag_array[ind, 0, :, 0] if self.pol == 'I': self.data[antkey] = {'I': d + d2} self.flags[antkey] = {'I': f + f2} elif self.pol == 'Q': self.data[antkey] = {'Q': d - d2} self.flags[antkey] = {'Q': f + f2} elif self.pol == 'U': self.data[antkey] = {'U': d + d2} self.flags[antkey] = {'U': f + f2} elif self.pol == 'V': self.data[antkey] = {'V': -1j * d + 1j * d2} self.flags[antkey] = {'V': f + f2} del self.uvd2 # Apply flags to data by looking for where the flags array is true, and zeroing the corresponding data elements. for pair in self.caldata['pairs']: self.data[pair][self.pol] = np.where(self.flags[pair][self.pol], 0. + 0. * 1j, self.data[pair][self.pol]) for i in self.flags: for j in self.flags[i]: self.flags[i][j] = np.logical_not(self.flags[i][j]).astype(int) def calculate_caldata(self): """Returns a dictionary of baseline lengths and the corresponding pairs. The data is based on a calfile. ex_ants is a list of integers that specify antennae to be exlcuded from calculation. Requires cal file to be in PYTHONPATH.""" if self.Zeus.calfile: try: print('Reading calfile: %s...' % self.Zeus.calfile) exec('import %s as cal' % self.Zeus.calfile, globals(), locals()) antennae = cal.prms['antpos_ideal'] except ImportError: raise Exception('Unable to import: %s' % self.Zeus.calfile) elif self.uvd: # Build antenna positions from data file itself. print('Generating calibration information from MIRIAD file.') antennae = {} lat, lon, alt = self.uvd.telescope_location_lat_lon_alt for i, antnum in enumerate(self.uvd.antenna_numbers): pos = self.uvd.antenna_positions[ i, :] + self.uvd.telescope_location xyz = uvutils.ENU_from_ECEF(pos, latitude=lat, longitude=lon, altitude=alt) antennae[antnum] = { 'top_x': xyz[0], 'top_y': xyz[1], 'top_z': xyz[2] } else: raise Exception( 'UVData object does not exist. Try supplying a calfile.') # Check that self.Zeus.exants contain only antennae that exist if not set(self.Zeus.exants).issubset(set(antennae.keys())): raise Exception('You provided invalid antenna(e) to exclude.') # Remove all placeholder antennae from consideration # Remove all antennae from exants from consideration ants = [] for ant in antennae.keys(): if (not antennae[ant]['top_z'] == -1) and (ant not in self.Zeus.exants): ants.append(ant) ants = np.array(ants) # Set the bins for binning the baselines bins_baseline = np.arange(0, 750, self.Zeus.BIN_WIDTH) # Initialize baselines, slopes, pairs arrays baselines = np.array([], dtype=np.float128) slopes = np.array([], dtype=np.float128) pairs = {} # Cycle through antennae to create pairs for ant_i in ants: for ant_j in ants: if ant_i >= ant_j: continue pair = (ant_i, ant_j) # Find the baseline length of the pair, bin it, format it, and add it to the baselines array dx = antennae[pair[1]]['top_x'] - antennae[pair[0]]['top_x'] dy = antennae[pair[1]]['top_y'] - antennae[pair[0]]['top_y'] baseline = np.sqrt(np.power(dx, 2) + np.power(dy, 2)) baseline = np.round(baseline, decimals=2) baseline = np.digitize(baseline, bins_baseline) * self.Zeus.BIN_WIDTH baselines = np.append(baselines, baseline) # Find the slope of the pair, format it, and add it to the slopes array dy = antennae[pair[1]]['top_y'] - antennae[pair[0]]['top_y'] dx = antennae[pair[1]]['top_x'] - antennae[pair[0]]['top_x'] if dx != 0: slope = float(np.round(dy / dx, decimals=2)) else: slope = np.inf slopes = np.append(slopes, slope) # Add the pair and its slope and baseline to the pairs dictionary: {(ant_1, ant_2): (baseline, slope), ...} pairs[pair] = (np.float(np.round(baseline, 1)), np.float(np.round(slope, 2))) # Sort and remove duplicates from baselines and slopes baselines = np.unique(np.sort(baselines)) slopes = np.unique(np.sort(slopes)) # Sort pairs into antdict and slopedict antdict = {} slopedict = {} for pair, (baseline, slope) in pairs.items(): if baseline in antdict: antdict[baseline].append(pair) else: antdict[baseline] = [pair] if baseline in slopedict: if slope in slopedict[baseline]: slopedict[baseline][slope].append(pair) else: slopedict[baseline][slope] = [pair] else: slopedict[baseline] = {slope: []} slopedict[baseline][slope].append(pair) self.caldata = { 'antdict': antdict, 'slopedict': slopedict, 'pairs': pairs, 'baselines': baselines, 'slopes': slopes } def pitchfork(self): self.info['freqs'] = self.info['freqs'][self.Zeus.freqrange[0]:self. Zeus.freqrange[1]] for antpair in self.data.keys(): self.data[antpair][self.pol] = self.data[antpair][ self.pol][:, self.Zeus.freqrange[0]:self.Zeus.freqrange[1]] self.flags[antpair][self.pol] = self.flags[antpair][ self.pol][:, self.Zeus.freqrange[0]:self.Zeus.freqrange[1]] ntimes = len(self.info['times']) nchan = len(self.info['freqs']) if self.Zeus.calfile: self.aa = hera_cal.utils.get_aa_from_calfile( self.info['freqs'], self.Zeus.calfile) else: self.aa = hera_cal.utils.get_aa_from_uv(self.uvd, self.info['freqs']) self.aa.set_active_pol(self.pol) for baseline in sorted(self.caldata['slopedict']): self.vis_sq_bl = np.zeros((ntimes // 2, nchan), dtype=np.complex128) for slope in sorted(self.caldata['slopedict'][baseline]): self.vis_sq_slope = np.zeros((ntimes // 2, nchan), dtype=np.complex128) for antpair in sorted( self.caldata['slopedict'][baseline][slope]): self.vis_sq_antpair = np.zeros((ntimes // 2, nchan), dtype=np.complex128) self.fft_clean_phase(antpair, ntimes, self.Zeus.CLEAN) self.ant_slices.append(self.vis_sq_antpair) self.vis_sq_slope += self.vis_sq_antpair self.vis_sq_bl += self.vis_sq_antpair self.vis_sq_slope /= len( self.caldata['slopedict'][baseline][slope]) self.sl_slices.append(self.vis_sq_slope) self.vis_sq_bl /= len(self.caldata['antdict'][baseline]) self.bl_slices.append(self.vis_sq_bl) def fft_clean_phase(self, antpair, ntimes, CLEAN): # FFT w = aipy.dsp.gen_window(self.data[antpair][self.pol].shape[-1], window='blackman-harris') _dw = np.fft.ifft(self.data[antpair][self.pol] * w) # CLEAN _ker = np.fft.ifft(self.flags[antpair][self.pol] * w) gain = aipy.img.beam_gain(_ker) for time in range(_dw.shape[0]): _dw[time, :], info = aipy.deconv.clean(_dw[time, :], _ker[time, :], tol=CLEAN) _dw[time, :] += info['res'] / gain fft_2Ddata = np.ma.array(_dw) # Phase for i in range(ntimes): if i != 0: old_zenith = zenith time = self.info['times'][i] self.aa.set_jultime(time) lst = self.aa.sidereal_time() zenith = aipy.phs.RadioFixedBody(lst, self.aa.lat) zenith.compute(self.aa) if i % 2: v1 = fft_2Ddata[i - 1, :] phase_correction = np.conj( self.aa.gen_phs(zenith, antpair[0], antpair[1])) * self.aa.gen_phs( old_zenith, antpair[0], antpair[1]) v2 = fft_2Ddata[i, :] * phase_correction self.vis_sq_antpair[i // 2, :] = np.conj(v1) * v2 def save(self): channel_width = np.diff(self.info['freqs'])[0] num_bins = self.info['freqs'].shape[0] delays = np.fft.fftshift(np.fft.fftfreq(num_bins, channel_width)) np.savez(self.npz_name, Zeus=self.Zeus, bl=self.bl_slices, sl=self.sl_slices, ant=self.ant_slices, caldata=self.caldata, pol=self.pol, freqs=self.info['freqs'], delays=delays)
if len(file_glob) == 0: raise FileNotFoundError("Something went wrong--no files were found.") # load in the data, downselect to autos and linear pols use_pols = [polstr2num(pol) for pol in ('xx', 'yy')] uvd = UVData() uvd.read(file_glob, ant_str='auto', polarizations=use_pols) # just do everything in this script; first isolate the rfi rfi_data = np.zeros_like(uvd.data_array, dtype=np.float) normalized_rfi_data = np.zeros_like(rfi_data, dtype=np.float) rfi_flags = np.zeros_like(rfi_data, dtype=np.bool) for antpairpol in uvd.get_antpairpols(): # get indices for properly slicing through data array blt_inds, conj_blt_inds, pol_inds = uvd._key2inds(antpairpol) this_slice = slice(blt_inds, 0, None, pol_inds[0]) # approximately remove all non-rfi signal this_data = uvd.get_data(antpairpol).real filt_data = xrfi.medminfilt(this_data) this_rfi = this_data - filt_data this_rfi[this_rfi <= 0] = 1 this_ratio = this_data / filt_data # detrend the original data to find where the stations are detrended_data = xrfi.detrend_medfilt(data) station_flags = np.where(detrended_data > 100, True, False) # update flags with watershed algorithm station_flags = xrfi._ws_flag_waterfall(detrended_data,