def test_antenna(self): """Test the AIPS IDI ANTENNA table.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-AN.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Aips(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) an = hdulist['ANTENNA'].data # Correct number of stands self.assertEqual(len(data['antennas']), len(an.field('ANTENNA_NO'))) # Correct FREQIDs for freqid in an.field('FREQID'): self.assertEqual(freqid, 1) hdulist.close()
def test_antenna(self): """Test the 'AIPS AN' table, part 2.""" testTime = time.time() - 2 * 86400.0 testFile = os.path.join(self.testPath, 'uv-test-AN.fits') # Get some data data = self._init_data() # Start the file fits = uvfits.Uv(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) an = hdulist['AIPS AN'].data # Correct number of stands self.assertEqual(len(data['antennas']), len(an.field('NOSTA'))) hdulist.close()
def test_array_geometry(self): """Test the AIPS IDI ARRAY_GEOMETRY table.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-AG.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Aips(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) ag = hdulist['ARRAY_GEOMETRY'].data # Correct number of stands self.assertEqual(len(data['antennas']), len(ag.field('NOSTA'))) # Correct stand names names = ['L%03i' % ant.stand.id for ant in data['antennas']] for name, anname in zip(names, ag.field('ANNAME')): self.assertEqual(name, anname) hdulist.close()
def test_frequency(self): """Test the AIPS IDI FREQUENCY table.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-FQ.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Aips(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) fq = hdulist['FREQUENCY'].data # Correct number of FREQIDs self.assertEqual(len(fq.field('FREQID')), 1) # Correct channel width self.assertAlmostEqual(fq.field('CH_WIDTH')[0], numpy.abs(data['freq'][1]-data['freq'][0]), 4) # Correct bandwidth self.assertAlmostEqual(fq.field('TOTAL_BANDWIDTH')[0], numpy.abs(data['freq'][-1]-data['freq'][0]).astype(numpy.float32), 4) # Correct sideband self.assertEqual(fq.field('SIDEBAND')[0], 1) hdulist.close()
def test_bandpass(self): """Test the AIPS IDI BANDPASS table.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-BP.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Aips(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) bp = hdulist['BANDPASS'].data # Correct number of entries self.assertEqual(len(data['antennas']), len(bp.field('ANTENNA_NO'))) # Correct Source ID number for src in bp.field('SOURCE_ID'): self.assertEqual(src, 0) # Correct FREQIDs for freqid in bp.field('FREQID'): self.assertEqual(freqid, 1) hdulist.close()
def test_write_tables(self): """Test if the AIPS IDI writer writes all of the tables.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-W.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Aips(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_comment('This is a comment') fits.add_history('This is history') fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) # Check that all of the extensions are there extNames = [hdu.name for hdu in hdulist] for ext in ['ARRAY_GEOMETRY', 'FREQUENCY', 'ANTENNA', 'BANDPASS', 'SOURCE', 'UV_DATA']: self.assertTrue(ext in extNames) # Check the comments and history self.assertTrue('This is a comment' in str(hdulist[0].header['COMMENT']).split('\n')) self.assertTrue('This is history' in str(hdulist[0].header['HISTORY']).split('\n')) hdulist.close()
def test_source(self): """Test the AIPS IDI SOURCE table.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-SO.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Aips(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) so = hdulist['SOURCE'].data # Correct number of entries self.assertEqual(len(so.field('SOURCE_ID')), 1) # Correct Source ID number self.assertEqual(so.field('SOURCE_ID'), 1) hdulist.close()
def test_frequency(self): """Test the 'AIPS FQ' table.""" testTime = time.time() - 2 * 86400.0 testFile = os.path.join(self.testPath, 'uv-test-FQ.fits') # Get some data data = self._init_data() # Start the file fits = uvfits.Uv(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) fq = hdulist['AIPS FQ'].data # Correct number of IFs self.assertEqual(len(fq.field('FRQSEL')), 1) # Correct channel width self.assertEqual( fq.field('CH WIDTH')[0], data['freq'][1] - data['freq'][0]) # Correct bandwidth self.assertEqual( fq.field('TOTAL BANDWIDTH')[0], numpy.abs(data['freq'][-1] - data['freq'][0]).astype( numpy.float32), 4) hdulist.close()
def test_mapper(self): """Test the AIPS IDI NOSTA_MAPPER table.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-SM.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Aips(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) extNames = [hdu.name for hdu in hdulist] maxStand = -1 for ant in data['antennas']: if ant.stand.id > maxStand: maxStand = ant.stand.id if maxStand > 99: self.assertTrue('NOSTA_MAPPER' in extNames) # Make sure the mapper makes sense mp = hdulist['NOSTA_MAPPER'].data ag = hdulist['ARRAY_GEOMETRY'].data mNoSta = mp.field('NOSTA') aNoSta = ag.field('NOSTA') mNoAct = mp.field('NOACT') aAnNam = ag.field('ANNAME') for msta, mact, asta, anam in zip(mNoSta, mNoAct, aNoSta, aAnNam): self.assertEqual(msta, asta) self.assertEqual(mact, int(anam[1:])) hdulist.close()
def test_writer_errors(self): """Test that common FITS IDI error conditions are caught.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-ERR.fits') # Get some data data = self._init_data() for i in range(4): # Start the file fits = fitsidi.Idi(testFile, ref_time=testTime, overwrite=True) if i != 0: fits.set_stokes(['xx']) if i != 1: fits.set_frequency(data['freq']) if i != 2: fits.set_geometry(data['site'], data['antennas']) if i != 3: fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) self.assertRaises(RuntimeError, fits.write) fits.close()
def test_write_tables(self): """Test if the FITS IDI writer writes all of the tables.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-W.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Idi(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.set_observer('Dowell, Jayce', 'LD009', 'Test') fits.add_header_keyword('EXAMPLE', 'example keyword') fits.add_comment('This is a comment') fits.add_history('This is history') fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) # Check that all of the extensions are there extNames = [hdu.name for hdu in hdulist] for ext in ['ARRAY_GEOMETRY', 'FREQUENCY', 'ANTENNA', 'BANDPASS', 'SOURCE', 'UV_DATA']: with self.subTest(table=ext): self.assertTrue(ext in extNames) # Check header values that we set self.assertEqual('Dowell, Jayce', hdulist[0].header['OBSERVER']) self.assertEqual('LD009', hdulist[0].header['PROJECT']) self.assertEqual('Test', hdulist[0].header['LWATYPE']) self.assertEqual('example keyword', hdulist[0].header['EXAMPLE']) # Check the comments and history self.assertTrue('This is a comment' in str(hdulist[0].header['COMMENT']).split('\n')) self.assertTrue('This is history' in str(hdulist[0].header['HISTORY']).split('\n')) hdulist.close()
def test_bandpass(self): """Test the 'AIPS BP' table.""" testTime = time.time() - 2 * 86400.0 testFile = os.path.join(self.testPath, 'uv-test-BP.fits') # Get some data data = self._init_data() # Start the file fits = uvfits.Uv(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) su = hdulist['AIPS BP'].data hdulist.close()
def test_write_tables(self): """Test if the MeasurementSet writer writes all of the tables.""" testTime = time.time() testFile = os.path.join(self.testPath, 'ms-test-W.ms') # Get some data data = self._init_data() # Start the table tbl = measurementset.Ms(testFile, ref_time=testTime) tbl.set_stokes(['xx']) tbl.set_frequency(data['freq']) tbl.set_geometry(data['site'], data['antennas']) tbl.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) tbl.write() # Make sure everyone is there self.assertTrue(os.path.exists(testFile)) for tbl in ('ANTENNA', 'DATA_DESCRIPTION', 'FEED', 'FIELD', 'FLAG_CMD', 'HISTORY', 'OBSERVATION', 'POINTING', 'POLARIZATION', 'PROCESSOR', 'SOURCE', 'SPECTRAL_WINDOW', 'STATE'): self.assertTrue(os.path.exists(os.path.join(testFile, tbl)))
def process_chunk(idf, site, good, filename, int_time=5.0, pols=[ 'xx', ], chunk_size=100): """ Given a lsl.reader.ldp.TBNFile instances and various parameters for the cross-correlation, write cross-correlate the data and save it to a file. """ # Get antennas antennas = site.antennas # Get the metadata sample_rate = idf.get_info('sample_rate') freq = idf.get_info('freq1') # Create the list of good digitizers and a digitizer to Antenna instance mapping. # These are: # toKeep -> mapping of digitizer number to array location # mapper -> mapping of Antenna instance to array location toKeep = [antennas[i].digitizer - 1 for i in good] mapper = [antennas[i] for i in good] # Create a list of unqiue stands to know what style of IDI file to create stands = set([antennas[i].stand.id for i in good]) # Main loop over the input file to read in the data and organize it. Several control # variables are defined for this: # ref_time -> time (in seconds since the UNIX epoch) for the first data set # setTime -> time (in seconds since the UNIX epoch) for the current data set ref_time = 0.0 setTime = 0.0 wallTime = time.time() for s in range(chunk_size): try: readT, t, data = idf.read(int_time) except Exception as e: print("Error: %s" % str(e)) continue ## Prune out what we don't want data = data[toKeep, :, :] ## Split the polarizations antennasX, antennasY = [ a for i, a in enumerate(antennas) if a.pol == 0 and i in toKeep ], [a for i, a in enumerate(antennas) if a.pol == 1 and i in toKeep] dataX, dataY = data[0::2, :, :], data[1::2, :, :] validX = numpy.ones((dataX.shape[0], dataX.shape[2]), dtype=numpy.uint8) validY = numpy.ones((dataY.shape[0], dataY.shape[2]), dtype=numpy.uint8) ## Apply the cable delays as phase rotations for i in range(dataX.shape[0]): gain = numpy.sqrt(antennasX[i].cable.gain(freq)) phaseRot = numpy.exp(2j*numpy.pi*freq*(antennasX[i].cable.delay(freq) \ -antennasX[i].stand.z/speedOfLight)) for j in range(dataX.shape[2]): dataX[i, :, j] *= phaseRot / gain for i in range(dataY.shape[0]): gain = numpy.sqrt(antennasY[i].cable.gain(freq)) phaseRot = numpy.exp(2j*numpy.pi*freq*(antennasY[i].cable.delay(freq)\ -antennasY[i].stand.z/speedOfLight)) for j in range(dataY.shape[2]): dataY[i, :, j] *= phaseRot / gain setTime = t if s == 0: ref_time = setTime # Setup the set time as a python datetime instance so that it can be easily printed setDT = setTime.datetime print("Working on set #%i (%.3f seconds after set #1 = %s)" % ((s + 1), (setTime - ref_time), setDT.strftime("%Y/%m/%d %H:%M:%S.%f"))) # Loop over polarization products for pol in pols: print("-> %s" % pol) if pol[0] == 'x': a1, d1, v1 = antennasX, dataX, validX else: a1, d1, v1 = antennasY, dataY, validY if pol[1] == 'x': a2, d2, v2 = antennasX, dataX, validX else: a2, d2, v2 = antennasY, dataY, validY ## Get the baselines baselines = uvutils.get_baselines(a1, antennas2=a2, include_auto=True, indicies=True) blList = [] for bl in range(len(baselines)): blList.append((a1[baselines[bl][0]], a2[baselines[bl][1]])) ## Run the cross multiply and accumulate vis = XEngine2(d1, d2, v1, v2) # Select the right range of channels to save toUse = numpy.where((freq > 5.0e6) & (freq < 93.0e6)) toUse = toUse[0] # If we are in the first polarazation product of the first iteration, setup # the FITS IDI file. if s == 0 and pol == pols[0]: pol1, pol2 = fxc.pol_to_pols(pol) if len(stands) > 255: fits = fitsidi.ExtendedIdi(filename, ref_time=ref_time) else: fits = fitsidi.Idi(filename, ref_time=ref_time) fits.set_stokes(pols) fits.set_frequency(freq[toUse]) fits.set_geometry(site, [a for a in mapper if a.pol == pol1]) # Convert the setTime to a MJD and save the visibilities to the FITS IDI file obsTime = astro.unix_to_taimjd(setTime) fits.add_data_set(obsTime, readT, blList, vis[:, toUse], pol=pol) print("-> Cummulative Wall Time: %.3f s (%.3f s per integration)" % ((time.time() - wallTime), (time.time() - wallTime) / (s + 1))) # Cleanup after everything is done fits.write() fits.close() del (fits) del (data) del (vis) return True
def process_chunk(idf, site, good, filename, int_time=5.0, LFFT=64, overlap=1, pfb=False, pols=[ 'xx', ], chunk_size=100): """ Given a lsl.reader.ldp.TBNFile instances and various parameters for the cross-correlation, write cross-correlate the data and save it to a file. """ # Get antennas antennas = [] for a in site.antennas: if a.digitizer != 0: antennas.append(a) # Get the metadata sample_rate = idf.get_info('sample_rate') central_freq = idf.get_info('freq1') # Create the list of good digitizers and a digitizer to Antenna instance mapping. # These are: # toKeep -> mapping of digitizer number to array location # mapper -> mapping of Antenna instance to array location toKeep = [antennas[i].digitizer - 1 for i in good] mapper = [antennas[i] for i in good] # Create a list of unqiue stands to know what style of IDI file to create stands = set([antennas[i].stand.id for i in good]) # Main loop over the input file to read in the data and organize it. Several control # variables are defined for this: # ref_time -> time (in seconds since the UNIX epoch) for the first data set # setTime -> time (in seconds since the UNIX epoch) for the current data set ref_time = 0.0 setTime = 0.0 wallTime = time.time() for s in range(chunk_size): try: readT, t, data = idf.read(int_time) except Exception as e: print("Error: %s" % str(e)) continue ## Prune out what we don't want data = data[toKeep, :] setTime = t if s == 0: ref_time = setTime # Setup the set time as a python datetime instance so that it can be easily printed setDT = datetime.utcfromtimestamp(setTime) setDT.replace(tzinfo=UTC()) print("Working on set #%i (%.3f seconds after set #1 = %s)" % ((s + 1), (setTime - ref_time), setDT.strftime("%Y/%m/%d %H:%M:%S.%f"))) # Loop over polarization products for pol in pols: print("-> %s" % pol) blList, freq, vis = fxc.FXMaster(data, mapper, LFFT=LFFT, overlap=overlap, pfb=pfb, include_auto=True, verbose=False, sample_rate=sample_rate, central_freq=central_freq, pol=pol, return_baselines=True, gain_correct=True) # Select the right range of channels to save toUse = numpy.where((freq > 5.0e6) & (freq < 93.0e6)) toUse = toUse[0] # If we are in the first polarazation product of the first iteration, setup # the FITS IDI file. if s == 0 and pol == pols[0]: pol1, pol2 = fxc.pol_to_pols(pol) if len(stands) > 255: fits = fitsidi.ExtendedIdi(filename, ref_time=ref_time) else: fits = fitsidi.Idi(filename, ref_time=ref_time) fits.set_stokes(pols) fits.set_frequency(freq[toUse]) fits.set_geometry(site, [a for a in mapper if a.pol == pol1]) # Convert the setTime to a MJD and save the visibilities to the FITS IDI file obsTime = astro.unix_to_taimjd(setTime) fits.add_data_set(obsTime, readT, blList, vis[:, toUse], pol=pol) print("-> Cummulative Wall Time: %.3f s (%.3f s per integration)" % ((time.time() - wallTime), (time.time() - wallTime) / (s + 1))) # Cleanup after everything is done fits.write() fits.close() del (fits) del (data) del (vis) return True
def main(args): # Parse the command line filenames = args.filename if args.regex: filenames = [] for regex in args.filename: filenames.extend(glob.glob(regex)) try: filenames.sort(cmp=cmpNPZ) except TypeError: filenames.sort(key=cmp_to_key(cmpNPZ)) if args.limit != -1: filenames = filenames[:args.limit] # Build up the station site = stations.lwa1 observer = site.get_observer() # Load in the file file to figure out what to do dataDict = numpy.load(filenames[0]) tStart = dataDict['tStart'].item() tInt = dataDict['tInt'] freq = dataDict['freq1'] config, refSrc, junk1, junk2, junk3, junk4, antennas = read_correlator_configuration( dataDict) try: if config['basis'] == 'linear': args.linear = True args.circular = False args.stokes = False elif config['basis'] == 'circular': args.linear = False args.circular = True args.stokes = False elif config['basis'] == 'stokes': args.linear = False args.circular = False args.stokes = True print( "NOTE: Set output polarization basis to '%s' per user defined configuration" % config['basis']) except (TypeError, KeyError): pass visXX = dataDict['vis1XX'].astype(numpy.complex64) visXY = dataDict['vis1XY'].astype(numpy.complex64) visYX = dataDict['vis1YX'].astype(numpy.complex64) visYY = dataDict['vis1YY'].astype(numpy.complex64) dataDict.close() # Build up the master list of antennas and report master_antennas = antennas obs_groups = [ os.path.basename(filenames[0]).split('-vis2', 1)[0], ] for filename in filenames: group = os.path.basename(filename).split('-vis2', 1)[0] if group not in obs_groups: dataDict = numpy.load(filename) config, refSrc, junk1, junk2, junk3, junk4, antennas = read_correlator_configuration( dataDict) del dataDict for ant in antennas: ## The FITS IDI writer only cares about the stand ID if (ant.stand.id, ant.pol) not in [(ma.stand.id, ma.pol) for ma in master_antennas]: master_antennas.append(ant) obs_groups.append(group) master_antennas.sort(key=lambda x: (x.stand.id, x.pol)) for i in range(len(master_antennas)): master_antennas[i].id = i + 1 print("Antennas:") for ant in master_antennas: print(" Antenna %i: Stand %i, Pol. %i" % (ant.id, ant.stand.id, ant.pol)) nchan = visXX.shape[1] master_blList = uvutils.get_baselines( [ant for ant in master_antennas if ant.pol == 0], include_auto=True) if args.decimate > 1: to_trim = (freq.size / args.decimate) * args.decimate to_drop = freq.size - to_trim if to_drop != 0: print("Warning: Dropping %i channels (%.1f%%; %.3f kHz)" % (to_drop, 100.0 * to_drop / freq.size, to_drop * (freq[1] - freq[0]) / 1e3)) nchan //= args.decimate if to_drop != 0: freq = freq[:to_trim] freq.shape = (freq.size // args.decimate, args.decimate) freq = freq.mean(axis=1) # Figure out the visibility conjugation problem in LSL, pre-1.1.4 conjugateVis = False if float(fitsidi.__version__) < 0.9: print("Warning: Applying conjugate to visibility data") conjugateVis = True # Figure out our revision try: repo = git.Repo(os.path.dirname(os.path.abspath(__file__))) try: branch = repo.active_branch.name hexsha = repo.active_branch.commit.hexsha except TypeError: branch = '<detached>' hexsha = repo.head.commit.hexsha shortsha = hexsha[-7:] dirty = ' (dirty)' if repo.is_dirty() else '' except git.exc.GitError: branch = 'unknown' hexsha = 'unknown' shortsha = 'unknown' dirty = '' # Fill in the data for i, filename in enumerate(filenames): ## Load in the integration group = os.path.basename(filename).split('-vis2', 1)[0] dataDict = numpy.load(filename) junk0, refSrc, junk1, junk2, junk3, junk4, antennas = read_correlator_configuration( dataDict) try: refSrc.name = refSrc.name.upper() # For AIPS if refSrc.name[:12] == 'ELWA_SESSION': ## Convert ELWA_SESSION names to "real" source names refSrc.name = getSourceName(refSrc).replace(' ', '').upper() except AttributeError: ## Moving sources cannot have their names changed pass blList = uvutils.get_baselines( [ant for ant in antennas if ant.pol == 0], include_auto=True) ## Make sure the frequencies are compatible cFreq = dataDict['freq1'] if cFreq.size != freq.size: error_msg = "Incompatible frequencies at %s: %i != %i" % ( group, cFreq.size, freq.size) if args.ignore_incompatible: warnings.warn(error_msg, RuntimeWarning) continue else: raise RuntimeError(error_msg) elif cFreq[0] != freq[0] or cFreq[-1] != freq[-1]: error_msg = "Incompatible frequencies at %s: %.3f MHz != %.3f MHz or %.3f MHz != %.3f MHz" % ( group, cFreq[0] / 1e6, freq[0] / 1e6, cFreq[-1] / 1e6, freq[-1] / 1e6) if args.ignore_incompatible: warnings.warn(error_msg, RuntimeWarning) continue else: raise RuntimeError(error_msg) tStart = dataDict['tStart'].item() tInt = dataDict['tInt'].item() visXX = dataDict['vis1XX'].astype(numpy.complex64) visXY = dataDict['vis1XY'].astype(numpy.complex64) visYX = dataDict['vis1YX'].astype(numpy.complex64) visYY = dataDict['vis1YY'].astype(numpy.complex64) dataDict.close() if args.decimate > 1: if to_drop != 0: visXX = visXX[:, :to_trim] visXY = visXY[:, :to_trim] visYX = visYX[:, :to_trim] visYY = visYY[:, :to_trim] visXX.shape = (visXX.shape[0], visXX.shape[1] // args.decimate, args.decimate) visXX = visXX.mean(axis=2) visXY.shape = (visXY.shape[0], visXY.shape[1] // args.decimate, args.decimate) visXY = visXY.mean(axis=2) visYX.shape = (visYX.shape[0], visYX.shape[1] // args.decimate, args.decimate) visYX = visYX.mean(axis=2) visYY.shape = (visYY.shape[0], visYY.shape[1] // args.decimate, args.decimate) visYY = visYY.mean(axis=2) if conjugateVis: visXX = visXX.conj() visXY = visXY.conj() visYX = visYX.conj() visYY = visYY.conj() if args.circular or args.stokes: visI = visXX + visYY visQ = visXX - visYY visU = visXY + visYX visV = (visXY - visYX) / 1.0j if args.circular: visRR = visI + visV visRL = visQ + 1j * visU visLR = visQ - 1j * visU visLL = visI - visV if i % args.split == 0: ## Clean up the previous file try: fits.write() fits.close() except NameError: pass ## Create the FITS-IDI file as needed ### What to call it if args.tag is None: outname = 'buildIDI.FITS_%i' % (i // args.split + 1, ) else: outname = 'buildIDI_%s.FITS_%i' % ( args.tag, i // args.split + 1, ) ### Does it already exist or not if os.path.exists(outname): if not args.force: yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname) else: yn = 'y' if yn not in ('n', 'N'): os.unlink(outname) else: raise RuntimeError("Output file '%s' already exists" % outname) ### Create the file fits = fitsidi.Idi(outname, ref_time=tStart) if args.circular: fits.set_stokes(['RR', 'RL', 'LR', 'LL']) elif args.stokes: fits.set_stokes(['I', 'Q', 'U', 'V']) else: fits.set_stokes(['XX', 'XY', 'YX', 'YY']) fits.set_frequency(freq) fits.set_geometry(stations.lwa1, [a for a in master_antennas if a.pol == 0]) if config['context'] is not None: mode = 'LSBI' if min([ma.stand.id for ma in master_antennas]) < 50 \ and max([ma.stand.id for ma in master_antennas]) > 50: mode = 'ELWA' elif min([ma.stand.id for ma in master_antennas]) < 50: mode = 'VLA' fits.set_observer(config['context']['observer'], config['context']['project'], 'eLWA') if config['context']['session'] is not None: fits.add_header_keyword('session', config['context']['session']) if config['context']['vlaref'] is not None: fits.add_header_keyword('vlaref', config['context']['vlaref']) fits.add_header_keyword('instrume', mode) fits.add_history( 'Created with %s, revision %s.%s%s' % (os.path.basename(__file__), branch, shortsha, dirty)) print("Opening %s for writing" % outname) if i % 10 == 0: print(i) ## Update the observation observer.date = datetime.utcfromtimestamp(tStart).strftime( '%Y/%m/%d %H:%M:%S.%f') refSrc.compute(observer) ## Convert the setTime to a MJD and save the visibilities to the FITS IDI file obsTime = astro.unix_to_taimjd(tStart) if args.circular: fits.add_data_set(obsTime, tInt, blList, visRR, pol='RR', source=refSrc) fits.add_data_set(obsTime, tInt, blList, visRL, pol='RL', source=refSrc) fits.add_data_set(obsTime, tInt, blList, visLR, pol='LR', source=refSrc) fits.add_data_set(obsTime, tInt, blList, visLL, pol='LL', source=refSrc) elif args.stokes: fits.add_data_set(obsTime, tInt, blList, visI, pol='I', source=refSrc) fits.add_data_set(obsTime, tInt, blList, visQ, pol='Q', source=refSrc) fits.add_data_set(obsTime, tInt, blList, visU, pol='U', source=refSrc) fits.add_data_set(obsTime, tInt, blList, visV, pol='V', source=refSrc) else: fits.add_data_set(obsTime, tInt, blList, visXX, pol='XX', source=refSrc) fits.add_data_set(obsTime, tInt, blList, visXY, pol='XY', source=refSrc) fits.add_data_set(obsTime, tInt, blList, visYX, pol='YX', source=refSrc) fits.add_data_set(obsTime, tInt, blList, visYY, pol='YY', source=refSrc) # Cleanup the last file fits.write() fits.close()
def test_main_table(self): """Test the primary data table.""" testTime = time.time() testFile = os.path.join(self.testPath, 'ms-test-UV.ms') # Get some data data = self._init_data() # Start the file fits = measurementset.Ms(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() # Open the table and examine ms = casacore.tables.table(testFile, ack=False) uvw = ms.getcol('UVW') ant1 = ms.getcol('ANTENNA1') ant2 = ms.getcol('ANTENNA2') vis = ms.getcol('DATA') ms2 = casacore.tables.table(os.path.join(testFile, 'ANTENNA'), ack=False) mapper = ms2.getcol('NAME') mapper = [int(m[3:], 10) for m in mapper] # Correct number of visibilities self.assertEqual(uvw.shape[0], data['vis'].shape[0]) self.assertEqual(vis.shape[0], data['vis'].shape[0]) # Correct number of uvw coordinates self.assertEqual(uvw.shape[1], 3) # Correct number of frequencies self.assertEqual(vis.shape[1], data['freq'].size) # Correct values for row in range(uvw.shape[0]): stand1 = ant1[row] stand2 = ant2[row] visData = vis[row, :, 0] # Find out which visibility set in the random data corresponds to the # current visibility i = 0 for a1, a2 in data['bl']: if a1.stand.id == mapper[stand1] and a2.stand.id == mapper[ stand2]: break else: i = i + 1 # Run the comparison for vd, sd in zip(visData, data['vis'][i, :]): self.assertAlmostEqual(vd, sd, 8) i = i + 1 ms.close() ms2.close()
def test_multi_if(self): """writing more than one spectral window to a MeasurementSet.""" testTime = time.time() testFile = os.path.join(self.testPath, 'ms-test-MultiIF.ms') # Get some data data = self._init_data() # Start the file fits = measurementset.Ms(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_frequency(data['freq'] + 10e6) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set( unix_to_taimjd(testTime), 6.0, data['bl'], numpy.concatenate([data['vis'], 10 * data['vis']], axis=1)) fits.write() # Open the table and examine ms = casacore.tables.table(testFile, ack=False) uvw = ms.getcol('UVW') ant1 = ms.getcol('ANTENNA1') ant2 = ms.getcol('ANTENNA2') ddsc = ms.getcol('DATA_DESC_ID') vis = ms.getcol('DATA') ms2 = casacore.tables.table(os.path.join(testFile, 'ANTENNA'), ack=False) mapper = ms2.getcol('NAME') mapper = [int(m[3:], 10) for m in mapper] ms3 = casacore.tables.table(os.path.join(testFile, 'DATA_DESCRIPTION'), ack=False) spw = [i for i in ms3.getcol('SPECTRAL_WINDOW_ID')] # Correct number of visibilities self.assertEqual(uvw.shape[0], 2 * data['vis'].shape[0]) self.assertEqual(vis.shape[0], 2 * data['vis'].shape[0]) # Correct number of uvw coordinates self.assertEqual(uvw.shape[1], 3) # Correct number of frequencies self.assertEqual(vis.shape[1], data['freq'].size) # Correct values for row in range(uvw.shape[0]): stand1 = ant1[row] stand2 = ant2[row] descid = ddsc[row] visData = vis[row, :, 0] # Find out which visibility set in the random data corresponds to the # current visibility i = 0 for a1, a2 in data['bl']: if a1.stand.id == mapper[stand1] and a2.stand.id == mapper[ stand2]: break else: i = i + 1 # Find out which spectral window this corresponds to if spw[descid] == 0: compData = data['vis'] else: compData = 10 * data['vis'] # Run the comparison for vd, sd in zip(visData, compData[i, :]): self.assertAlmostEqual(vd, sd, 8) ms.close() ms2.close() ms3.close()
def main(args): # Setup the site information station = stations.lwasv ants = station.antennas nAnt = len([a for a in ants if a.pol == 0]) phase_freq_range = [0, 0] for filename in args.filename: # Open the file and get ready to go idf = CORFile(filename) nBL = idf.get_info('nbaseline') nchan = idf.get_info('nchan') tInt = idf.get_info('tint') nFpO = nBL * nchan / 72 nInts = idf.get_info('nframe') / nFpO jd = astro.unix_to_utcjd(idf.get_info('start_time')) date = idf.get_info('start_time').datetime central_freq = idf.get_info('freq1') central_freq = central_freq[len(central_freq)/2] print("Data type: %s" % type(idf)) print("Samples per observations: %i" % (nFpO,)) print("Integration Time: %.3f s" % tInt) print("Tuning frequency: %.3f Hz" % central_freq) print("Captures in file: %i (%.1f s)" % (nInts, nInts*tInt)) print("==") print("Station: %s" % station.name) print("Date observed: %s" % date) print("Julian day: %.5f" % jd) print(" ") # Offset into the file offset = idf.offset(args.skip) if offset != 0.0: print("Skipped %.3f s into the file" % offset) # Open the file and go nFiles = int(args.duration / tInt) if nFiles == 0: nFiles = numpy.inf fileCount = 0 while fileCount < nFiles: try: tInt, tStart, data = idf.read(tInt) except Exception as e: print("ERROR: %s" % str(e)) break freqs = idf.get_info('freq1') beginJD = astro.unix_to_utcjd( tStart ) beginTime = datetime.utcfromtimestamp( tStart ) if freqs[0] != phase_freq_range[0] or freqs[-1] != phase_freq_range[1]: print("Updating phasing for %.3f to %.3f MHz" % (freqs[0]/1e6, freqs[-1]/1e6)) phase_freq_range[0] = freqs[ 0] phase_freq_range[1] = freqs[-1] k = 0 phase = numpy.zeros((nBL, nchan, 2, 2), dtype=numpy.complex64) gaix = [a.cable.gain(freqs) for a in ants if a.pol == 0] gaiy = [a.cable.gain(freqs) for a in ants if a.pol == 1] dlyx = [a.cable.delay(freqs) - a.stand.z / speedOfLight for a in ants if a.pol == 0] dlyy = [a.cable.delay(freqs) - a.stand.z / speedOfLight for a in ants if a.pol == 1] for i in xrange(nAnt): for j in xrange(i, nAnt): phase[k,:,0,0] = numpy.exp(2j*numpy.pi*freqs*(dlyx[i] - dlyx[j])) \ / numpy.sqrt(gaix[i]*gaix[j]) phase[k,:,0,1] = numpy.exp(2j*numpy.pi*freqs*(dlyx[i] - dlyy[j])) \ / numpy.sqrt(gaix[i]*gaiy[j]) phase[k,:,1,0] = numpy.exp(2j*numpy.pi*freqs*(dlyy[i] - dlyx[j])) \ / numpy.sqrt(gaiy[i]*gaix[j]) phase[k,:,1,1] = numpy.exp(2j*numpy.pi*freqs*(dlyy[i] - dlyy[j])) \ / numpy.sqrt(gaiy[i]*gaiy[j]) k += 1 for i in xrange(data.shape[-1]): data[...,i] *= phase # Convert to a dataDict try: blList # pylint: disable=used-before-assignment except NameError: blList = uvutils.get_baselines(ants[0::2], include_auto=True) if args.output is None: outname = os.path.basename(filename) outname = os.path.splitext(outname)[0] outname = "%s_%i_%s.ms" % (outname, int(beginJD-astro.MJD_OFFSET), beginTime.strftime("%H_%M_%S")) else: base, ext = os.path.splitext(args.output) if ext == '': ext = '.ms' outname = "%s_%i_%s%s" % (base, int(beginJD-astro.MJD_OFFSET), beginTime.strftime("%H_%M_%S"), ext) fits = measurementset.Ms(outname, ref_time=tStart) fits.set_stokes(['xx', 'xy', 'yx', 'yy']) fits.set_frequency(freqs) fits.set_geometry(station, ants[0::2]) obsTime = astro.unix_to_taimjd(tStart) fits.add_data_set(obsTime, tInt, blList, data[:,:,0,0,0], pol='xx') fits.add_data_set(obsTime, tInt, blList, data[:,:,0,1,0], pol='xy') fits.add_data_set(obsTime, tInt, blList, data[:,:,1,0,0], pol='yx') fits.add_data_set(obsTime, tInt, blList, data[:,:,1,1,0], pol='yy') fits.write() fits.close() fileCount += 1 idf.close()
def test_uvdata(self): """Test the AIPS IDI UV_DATA table.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-UV.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Aips(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) uv = hdulist['UV_DATA'].data # Load the mapper try: mp = hdulist['NOSTA_MAPPER'].data nosta = mp.field('NOSTA') noact = mp.field('NOACT') except KeyError: ag = hdulist['ARRAY_GEOMETRY'].data nosta = ag.field('NOSTA') noact = ag.field('NOSTA') mapper = {} for s,a in zip(nosta, noact): mapper[s] = a # Correct number of visibilities self.assertEqual(len(uv.field('FLUX')), data['vis'].shape[0]) # Correct number of frequencies for vis in uv.field('FLUX'): self.assertEqual(len(vis), 2*len(data['freq'])) # Correct values for bl, vis in zip(uv.field('BASELINE'), uv.field('FLUX')): # Convert mapped stands to real stands stand1 = mapper[(bl >> 8) & 255] stand2 = mapper[bl & 255] # Find out which visibility set in the random data corresponds to the # current visibility i = 0 for ant1,ant2 in data['bl']: if ant1.stand.id == stand1 and ant2.stand.id == stand2: break else: i = i + 1 # Extract the data and run the comparison visData = numpy.zeros(len(data['freq']), dtype=numpy.complex64) visData.real = vis[0::2] visData.imag = vis[1::2] numpy.testing.assert_allclose(visData, data['vis'][i,:]) i = i + 1 hdulist.close()
def process_chunk(idf, site, good, filename, LFFT=64, overlap=1, pfb=False, pols=['xx', 'yy']): """ Given an lsl.reader.ldp.TBWFile instances and various parameters for the cross-correlation, write cross-correlate the data and save it to a file. """ # Get antennas antennas = site.antennas # Get the metadata sample_rate = idf.get_info('sample_rate') # Create the list of good digitizers and a digitizer to Antenna instance mapping. # These are: # toKeep -> mapping of digitizer number to array location # mapper -> mapping of Antenna instance to array location toKeep = [antennas[i].digitizer - 1 for i in good] mapper = [antennas[i] for i in good] # Create a list of unqiue stands to know what style of IDI file to create stands = set([antennas[i].stand.id for i in good]) wallTime = time.time() readT, t, data = idf.read() setTime = t ref_time = t # Setup the set time as a python datetime instance so that it can be easily printed setDT = setTime.datetime print("Working on set #1 (%.3f seconds after set #1 = %s)" % ((setTime - ref_time), setDT.strftime("%Y/%m/%d %H:%M:%S.%f"))) # In order for the TBW stuff to actaully run, we need to run in with sub- # integrations. 8 sub-integrations (61.2 ms / 8 = 7.7 ms per section) # seems to work ok with a "reasonable" number of channels. nSec = 8 secSize = data.shape[1] // nSec # Loop over polarizations (there should be only 1) for pol in pols: print("-> %s" % pol) try: tempVis *= 0 # pylint:disable=undefined-variable except NameError: pass # Set up the progress bar so we can keep up with how the sub-integrations # are progressing pb = ProgressBar(max=nSec) sys.stdout.write(pb.show() + '\r') sys.stdout.flush() # Loop over sub-integrations (set by nSec) for k in range(nSec): blList, freq, vis = fxc.FXMaster(data[toKeep, k * secSize:(k + 1) * secSize], mapper, LFFT=LFFT, overlap=overlap, pfb=pfb, include_auto=True, verbose=False, sample_rate=sample_rate, central_freq=0.0, Pol=pol, return_baselines=True, gain_correct=True) toUse = numpy.where((freq >= 5.0e6) & (freq <= 93.0e6)) toUse = toUse[0] try: tempVis += vis except: tempVis = vis pb.inc(amount=1) sys.stdout.write(pb.show() + '\r') sys.stdout.flush() # Average the sub-integrations together vis = tempVis / float(nSec) # Set up the FITS IDI file is we need to if pol == pols[0]: pol1, pol2 = fxc.pol_to_pols(pol) if len(stands) > 255: fits = fitsidi.ExtendedIdi(filename, ref_time=ref_time) else: fits = fitsidi.Idi(filename, ref_time=ref_time) fits.set_stokes(pols) fits.set_frequency(freq[toUse]) fits.set_geometry(site, [a for a in mapper if a.pol == pol1]) # Add the visibilities obsTime = astro.unix_to_taimjd(setTime) fits.add_data_set(obsTime, readT, blList, vis[:, toUse], pol=pol) sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() print("-> Cummulative Wall Time: %.3f s (%.3f s per integration)" % ((time.time() - wallTime), (time.time() - wallTime))) fits.write() fits.close() del (fits) del (data) del (vis) return True
def test_multi_if(self): """Test writing more than one IF to a FITS IDI file.""" testTime = time.time() - 2*86400.0 testFile = os.path.join(self.testPath, 'idi-test-MultiIF.fits') # Get some data data = self._init_data() # Start the file fits = fitsidi.Idi(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_frequency(data['freq']+10e6) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], numpy.concatenate([data['vis'], 10*data['vis']], axis=1)) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) # Check that all of the extensions are there extNames = [hdu.name for hdu in hdulist] for ext in ['ARRAY_GEOMETRY', 'FREQUENCY', 'ANTENNA', 'BANDPASS', 'SOURCE', 'UV_DATA']: self.assertTrue(ext in extNames) # Load the mapper try: mp = hdulist['NOSTA_MAPPER'].data nosta = mp.field('NOSTA') noact = mp.field('NOACT') except KeyError: ag = hdulist['ARRAY_GEOMETRY'].data nosta = ag.field('NOSTA') noact = ag.field('NOSTA') mapper = {} for s,a in zip(nosta, noact): mapper[s] = a # Correct number of visibilities uv = hdulist['UV_DATA'].data self.assertEqual(len(uv.field('FLUX')), data['vis'].shape[0]) # Correct number of frequencies for vis in uv.field('FLUX'): self.assertEqual(len(vis), 2*2*len(data['freq'])) # Correct values for bl, vis in zip(uv.field('BASELINE'), uv.field('FLUX')): # Convert mapped stands to real stands stand1 = mapper[(bl >> 8) & 255] stand2 = mapper[bl & 255] # Find out which visibility set in the random data corresponds to the # current visibility i = 0 for ant1,ant2 in data['bl']: if ant1.stand.id == stand1 and ant2.stand.id == stand2: break else: i = i + 1 # Extract the data and run the comparison - IF 1 visData = numpy.zeros(2*len(data['freq']), dtype=numpy.complex64) visData.real = vis[0::2] visData.imag = vis[1::2] numpy.testing.assert_allclose(visData[:len(data['freq'])], data['vis'][i,:]) # Extract the data and run the comparison - IF 2 visData = numpy.zeros(2*len(data['freq']), dtype=numpy.complex64) visData.real = vis[0::2] visData.imag = vis[1::2] numpy.testing.assert_allclose(visData[len(data['freq']):], 10*data['vis'][i,:]) hdulist.close()
def test_uvdata(self): """Test the primary data table.""" testTime = time.time() - 2 * 86400.0 testFile = os.path.join(self.testPath, 'uv-test-UV.fits') # Get some data data = self._init_data() # Start the file fits = uvfits.Uv(testFile, ref_time=testTime) fits.set_stokes(['xx']) fits.set_frequency(data['freq']) fits.set_geometry(data['site'], data['antennas']) fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis']) fits.write() fits.close() # Open the file and examine hdulist = astrofits.open(testFile) uv = hdulist[0] # Load the mapper try: mp = hdulist['NOSTA_MAPPER'].data nosta = mp.field('NOSTA') noact = mp.field('NOACT') except KeyError: ag = hdulist['AIPS AN'].data nosta = ag.field('NOSTA') noact = ag.field('NOSTA') mapper = {} for s, a in zip(nosta, noact): mapper[s] = a # Correct number of visibilities self.assertEqual(len(uv.data), data['vis'].shape[0]) # Correct number of frequencies for row in uv.data: vis = row['DATA'][0, 0, :, :, :] self.assertEqual(len(vis), len(data['freq'])) # Correct values for row in uv.data: bl = int(row['BASELINE']) vis = row['DATA'][0, 0, :, :, :] # Unpack the baseline if bl >= 65536: a1 = int((bl - 65536) / 2048) a2 = int((bl - 65536) % 2048) else: a1 = int(bl / 256) a2 = int(bl % 256) # Convert mapped stands to real stands stand1 = mapper[a1] stand2 = mapper[a2] # Find out which visibility set in the random data corresponds to the # current visibility i = 0 for ant1, ant2 in data['bl']: if ant1.stand.id == stand1 and ant2.stand.id == stand2: break else: i = i + 1 # Extract the data and run the comparison visData = numpy.zeros(len(data['freq']), dtype=numpy.complex64) visData.real = vis[:, 0, 0] visData.imag = vis[:, 0, 1] numpy.testing.assert_allclose(visData, data['vis'][i, :]) i = i + 1 hdulist.close()