def test_source(self):
"""Test the SOURCE table."""
testTime = time.time() - 2 * 86400.0
testFile = os.path.join(self.testPath, 'idi-test-SO.fits')
# Get some data
data = self.__initData()
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime)
fits.set_stokes(['xx'])
fits.set_frequency(data['freq'])
fits.set_geometry(data['site'], data['antennas'])
fits.add_data_set(testTime, 6.0, data['bl'], data['vis'])
fits.write()
# Open the file and examine
hdulist = astrofits.open(testFile)
so = hdulist['SOURCE'].data
# Correct number of entries
self.assertEqual(len(so.field('SOURCE_ID')), 1)
# Correct Source ID number
self.assertEqual(so.field('SOURCE_ID'), 1)
hdulist.close()
def test_antenna(self):
"""Test the ANTENNA table."""
testTime = time.time() - 2 * 86400.0
testFile = os.path.join(self.testPath, 'idi-test-AN.fits')
# Get some data
data = self.__initData()
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime)
fits.set_stokes(['xx'])
fits.set_frequency(data['freq'])
fits.set_geometry(data['site'], data['antennas'])
fits.add_data_set(testTime, 6.0, data['bl'], data['vis'])
fits.write()
# Open the file and examine
hdulist = astrofits.open(testFile)
an = hdulist['ANTENNA'].data
# Correct number of stands
self.assertEqual(len(data['antennas']), len(an.field('ANTENNA_NO')))
# Correct FREQIDs
for freqid in an.field('FREQID'):
self.assertEqual(freqid, 1)
hdulist.close()
def test_bandpass(self):
"""Test the BANDPASS table."""
testTime = time.time() - 2 * 86400.0
testFile = os.path.join(self.testPath, 'idi-test-BP.fits')
# Get some data
data = self.__initData()
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime)
fits.set_stokes(['xx'])
fits.set_frequency(data['freq'])
fits.set_geometry(data['site'], data['antennas'])
fits.add_data_set(testTime, 6.0, data['bl'], data['vis'])
fits.write()
# Open the file and examine
hdulist = astrofits.open(testFile)
bp = hdulist['BANDPASS'].data
# Correct number of entries
self.assertEqual(len(data['antennas']), len(bp.field('ANTENNA_NO')))
# Correct Source ID number
for src in bp.field('SOURCE_ID'):
self.assertEqual(src, 0)
# Correct FREQIDs
for freqid in bp.field('FREQID'):
self.assertEqual(freqid, 1)
hdulist.close()
def test_array_geometry(self):
"""Test the ARRAY_GEOMETRY table."""
testTime = time.time() - 2 * 86400.0
testFile = os.path.join(self.testPath, 'idi-test-AG.fits')
# Get some data
data = self.__initData()
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime)
fits.set_stokes(['xx'])
fits.set_frequency(data['freq'])
fits.set_geometry(data['site'], data['antennas'])
fits.add_data_set(testTime, 6.0, data['bl'], data['vis'])
fits.write()
# Open the file and examine
hdulist = astrofits.open(testFile)
ag = hdulist['ARRAY_GEOMETRY'].data
# Correct number of stands
self.assertEqual(len(data['antennas']), len(ag.field('NOSTA')))
# Correct stand names
names = ['LWA%03i' % ant.stand.id for ant in data['antennas']]
for name, anname in zip(names, ag.field('ANNAME')):
self.assertEqual(name, anname)
hdulist.close()
def test_frequency(self):
"""Test the FREQUENCY table."""
testTime = time.time() - 2*86400.0
testFile = os.path.join(self.testPath, 'idi-test-FQ.fits')
# Get some data
data = self._init_data()
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime)
fits.set_stokes(['xx'])
fits.set_frequency(data['freq'])
fits.set_geometry(data['site'], data['antennas'])
fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis'])
fits.write()
fits.close()
# Open the file and examine
hdulist = astrofits.open(testFile)
fq = hdulist['FREQUENCY'].data
# Correct number of FREQIDs
self.assertEqual(len(fq.field('FREQID')), 1)
# Correct channel width
self.assertAlmostEqual(fq.field('CH_WIDTH')[0], numpy.abs(data['freq'][1]-data['freq'][0]), 4)
# Correct bandwidth
self.assertAlmostEqual(fq.field('TOTAL_BANDWIDTH')[0], numpy.abs(data['freq'][-1]-data['freq'][0]).astype(numpy.float32), 4)
# Correct sideband
self.assertEqual(fq.field('SIDEBAND')[0], 1)
hdulist.close()
def test_write_tables(self):
"""Test if the FITS IDI writer writes all of the tables."""
testTime = time.time() - 2 * 86400.0
testFile = os.path.join(self.testPath, 'idi-test-W.fits')
# Get some data
data = self.__initData()
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime)
fits.set_stokes(['xx'])
fits.set_frequency(data['freq'])
fits.set_geometry(data['site'], data['antennas'])
fits.set_observer('Dowell, Jayce', 'LD009', 'Test')
fits.add_header_keyword('EXAMPLE', 'example keyword')
fits.add_comment('This is a comment')
fits.add_history('This is history')
fits.add_data_set(testTime, 6.0, data['bl'], data['vis'])
fits.write()
# Open the file and examine
hdulist = astrofits.open(testFile)
# Check that all of the extensions are there
extNames = [hdu.name for hdu in hdulist]
for ext in [
'ARRAY_GEOMETRY', 'FREQUENCY', 'ANTENNA', 'BANDPASS', 'SOURCE',
'UV_DATA'
]:
self.assertTrue(ext in extNames)
# Check header values that we set
self.assertEqual('Dowell, Jayce', hdulist[0].header['OBSERVER'])
self.assertEqual('LD009', hdulist[0].header['PROJECT'])
self.assertEqual('Test', hdulist[0].header['LWATYPE'])
self.assertEqual('example keyword', hdulist[0].header['EXAMPLE'])
# Check the comments and history
self.assertTrue('This is a comment' in str(
hdulist[0].header['COMMENT']).split('\n'))
self.assertTrue(
'This is history' in str(hdulist[0].header['HISTORY']).split('\n'))
hdulist.close()
def test_writer_errors(self):
"""Test that common FITS IDI error conditions are caught."""
testTime = time.time() - 2 * 86400.0
testFile = os.path.join(self.testPath, 'idi-test-ERR.fits')
# Get some data
data = self.__initData()
for i in range(4):
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime, overwrite=True)
if i != 0:
fits.set_stokes(['xx'])
if i != 1:
fits.set_frequency(data['freq'])
if i != 2:
fits.set_geometry(data['site'], data['antennas'])
if i != 3:
fits.add_data_set(testTime, 6.0, data['bl'], data['vis'])
self.assertRaises(RuntimeError, fits.write)
def test_mapper(self):
"""Test the NOSTA_MAPPER table."""
testTime = time.time() - 2*86400.0
testFile = os.path.join(self.testPath, 'idi-test-SM.fits')
# Get some data
data = self._init_data()
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime)
fits.set_stokes(['xx'])
fits.set_frequency(data['freq'])
fits.set_geometry(data['site'], data['antennas'])
fits.add_data_set(unix_to_taimjd(testTime), 6.0, data['bl'], data['vis'])
fits.write()
fits.close()
# Open the file and examine
hdulist = astrofits.open(testFile)
extNames = [hdu.name for hdu in hdulist]
maxStand = -1
for ant in data['antennas']:
if ant.stand.id > maxStand:
maxStand = ant.stand.id
if maxStand > 255:
self.assertTrue('NOSTA_MAPPER' in extNames)
# Make sure the mapper makes sense
mp = hdulist['NOSTA_MAPPER'].data
ag = hdulist['ARRAY_GEOMETRY'].data
mNoSta = mp.field('NOSTA')
aNoSta = ag.field('NOSTA')
mNoAct = mp.field('NOACT')
aAnNam = ag.field('ANNAME')
for msta, mact, asta, anam in zip(mNoSta, mNoAct, aNoSta, aAnNam):
self.assertEqual(msta, asta)
self.assertEqual(mact, int(anam[3:]))
hdulist.close()
def process_chunk(idf,
site,
good,
filename,
int_time=5.0,
pols=[
'xx',
],
chunk_size=100):
"""
Given a lsl.reader.ldp.TBNFile instances and various parameters for the
cross-correlation, write cross-correlate the data and save it to a file.
"""
# Get antennas
antennas = site.antennas
# Get the metadata
sample_rate = idf.get_info('sample_rate')
freq = idf.get_info('freq1')
# Create the list of good digitizers and a digitizer to Antenna instance mapping.
# These are:
# toKeep -> mapping of digitizer number to array location
# mapper -> mapping of Antenna instance to array location
toKeep = [antennas[i].digitizer - 1 for i in good]
mapper = [antennas[i] for i in good]
# Create a list of unqiue stands to know what style of IDI file to create
stands = set([antennas[i].stand.id for i in good])
# Main loop over the input file to read in the data and organize it. Several control
# variables are defined for this:
# ref_time -> time (in seconds since the UNIX epoch) for the first data set
# setTime -> time (in seconds since the UNIX epoch) for the current data set
ref_time = 0.0
setTime = 0.0
wallTime = time.time()
for s in range(chunk_size):
try:
readT, t, data = idf.read(int_time)
except Exception as e:
print("Error: %s" % str(e))
continue
## Prune out what we don't want
data = data[toKeep, :, :]
## Split the polarizations
antennasX, antennasY = [
a for i, a in enumerate(antennas) if a.pol == 0 and i in toKeep
], [a for i, a in enumerate(antennas) if a.pol == 1 and i in toKeep]
dataX, dataY = data[0::2, :, :], data[1::2, :, :]
validX = numpy.ones((dataX.shape[0], dataX.shape[2]),
dtype=numpy.uint8)
validY = numpy.ones((dataY.shape[0], dataY.shape[2]),
dtype=numpy.uint8)
## Apply the cable delays as phase rotations
for i in range(dataX.shape[0]):
gain = numpy.sqrt(antennasX[i].cable.gain(freq))
phaseRot = numpy.exp(2j*numpy.pi*freq*(antennasX[i].cable.delay(freq) \
-antennasX[i].stand.z/speedOfLight))
for j in range(dataX.shape[2]):
dataX[i, :, j] *= phaseRot / gain
for i in range(dataY.shape[0]):
gain = numpy.sqrt(antennasY[i].cable.gain(freq))
phaseRot = numpy.exp(2j*numpy.pi*freq*(antennasY[i].cable.delay(freq)\
-antennasY[i].stand.z/speedOfLight))
for j in range(dataY.shape[2]):
dataY[i, :, j] *= phaseRot / gain
setTime = t
if s == 0:
ref_time = setTime
# Setup the set time as a python datetime instance so that it can be easily printed
setDT = setTime.datetime
print("Working on set #%i (%.3f seconds after set #1 = %s)" %
((s + 1),
(setTime - ref_time), setDT.strftime("%Y/%m/%d %H:%M:%S.%f")))
# Loop over polarization products
for pol in pols:
print("-> %s" % pol)
if pol[0] == 'x':
a1, d1, v1 = antennasX, dataX, validX
else:
a1, d1, v1 = antennasY, dataY, validY
if pol[1] == 'x':
a2, d2, v2 = antennasX, dataX, validX
else:
a2, d2, v2 = antennasY, dataY, validY
## Get the baselines
baselines = uvutils.get_baselines(a1,
antennas2=a2,
include_auto=True,
indicies=True)
blList = []
for bl in range(len(baselines)):
blList.append((a1[baselines[bl][0]], a2[baselines[bl][1]]))
## Run the cross multiply and accumulate
vis = XEngine2(d1, d2, v1, v2)
# Select the right range of channels to save
toUse = numpy.where((freq > 5.0e6) & (freq < 93.0e6))
toUse = toUse[0]
# If we are in the first polarazation product of the first iteration, setup
# the FITS IDI file.
if s == 0 and pol == pols[0]:
pol1, pol2 = fxc.pol_to_pols(pol)
if len(stands) > 255:
fits = fitsidi.ExtendedIdi(filename, ref_time=ref_time)
else:
fits = fitsidi.Idi(filename, ref_time=ref_time)
fits.set_stokes(pols)
fits.set_frequency(freq[toUse])
fits.set_geometry(site, [a for a in mapper if a.pol == pol1])
# Convert the setTime to a MJD and save the visibilities to the FITS IDI file
obsTime = astro.unix_to_taimjd(setTime)
fits.add_data_set(obsTime, readT, blList, vis[:, toUse], pol=pol)
print("-> Cummulative Wall Time: %.3f s (%.3f s per integration)" %
((time.time() - wallTime), (time.time() - wallTime) / (s + 1)))
# Cleanup after everything is done
fits.write()
fits.close()
del (fits)
del (data)
del (vis)
return True
def process_chunk(idf,
site,
good,
filename,
int_time=5.0,
LFFT=64,
overlap=1,
pfb=False,
pols=[
'xx',
],
chunk_size=100):
"""
Given a lsl.reader.ldp.TBNFile instances and various parameters for the
cross-correlation, write cross-correlate the data and save it to a file.
"""
# Get antennas
antennas = []
for a in site.antennas:
if a.digitizer != 0:
antennas.append(a)
# Get the metadata
sample_rate = idf.get_info('sample_rate')
central_freq = idf.get_info('freq1')
# Create the list of good digitizers and a digitizer to Antenna instance mapping.
# These are:
# toKeep -> mapping of digitizer number to array location
# mapper -> mapping of Antenna instance to array location
toKeep = [antennas[i].digitizer - 1 for i in good]
mapper = [antennas[i] for i in good]
# Create a list of unqiue stands to know what style of IDI file to create
stands = set([antennas[i].stand.id for i in good])
# Main loop over the input file to read in the data and organize it. Several control
# variables are defined for this:
# ref_time -> time (in seconds since the UNIX epoch) for the first data set
# setTime -> time (in seconds since the UNIX epoch) for the current data set
ref_time = 0.0
setTime = 0.0
wallTime = time.time()
for s in range(chunk_size):
try:
readT, t, data = idf.read(int_time)
except Exception as e:
print("Error: %s" % str(e))
continue
## Prune out what we don't want
data = data[toKeep, :]
setTime = t
if s == 0:
ref_time = setTime
# Setup the set time as a python datetime instance so that it can be easily printed
setDT = datetime.utcfromtimestamp(setTime)
setDT.replace(tzinfo=UTC())
print("Working on set #%i (%.3f seconds after set #1 = %s)" %
((s + 1),
(setTime - ref_time), setDT.strftime("%Y/%m/%d %H:%M:%S.%f")))
# Loop over polarization products
for pol in pols:
print("-> %s" % pol)
blList, freq, vis = fxc.FXMaster(data,
mapper,
LFFT=LFFT,
overlap=overlap,
pfb=pfb,
include_auto=True,
verbose=False,
sample_rate=sample_rate,
central_freq=central_freq,
pol=pol,
return_baselines=True,
gain_correct=True)
# Select the right range of channels to save
toUse = numpy.where((freq > 5.0e6) & (freq < 93.0e6))
toUse = toUse[0]
# If we are in the first polarazation product of the first iteration, setup
# the FITS IDI file.
if s == 0 and pol == pols[0]:
pol1, pol2 = fxc.pol_to_pols(pol)
if len(stands) > 255:
fits = fitsidi.ExtendedIdi(filename, ref_time=ref_time)
else:
fits = fitsidi.Idi(filename, ref_time=ref_time)
fits.set_stokes(pols)
fits.set_frequency(freq[toUse])
fits.set_geometry(site, [a for a in mapper if a.pol == pol1])
# Convert the setTime to a MJD and save the visibilities to the FITS IDI file
obsTime = astro.unix_to_taimjd(setTime)
fits.add_data_set(obsTime, readT, blList, vis[:, toUse], pol=pol)
print("-> Cummulative Wall Time: %.3f s (%.3f s per integration)" %
((time.time() - wallTime), (time.time() - wallTime) / (s + 1)))
# Cleanup after everything is done
fits.write()
fits.close()
del (fits)
del (data)
del (vis)
return True
def test_multi_if(self):
"""Test writing more than one IF to a FITS IDI file."""
testTime = time.time() - 2 * 86400.0
testFile = os.path.join(self.testPath, 'idi-test-MultiIF.fits')
# Get some data
data = self.__initData()
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime)
fits.set_stokes(['xx'])
fits.set_frequency(data['freq'])
fits.set_frequency(data['freq'] + 10e6)
fits.set_geometry(data['site'], data['antennas'])
fits.add_data_set(
testTime, 6.0, data['bl'],
numpy.concatenate([data['vis'], 10 * data['vis']], axis=1))
fits.write()
# Open the file and examine
hdulist = astrofits.open(testFile)
# Check that all of the extensions are there
extNames = [hdu.name for hdu in hdulist]
for ext in [
'ARRAY_GEOMETRY', 'FREQUENCY', 'ANTENNA', 'BANDPASS', 'SOURCE',
'UV_DATA'
]:
self.assertTrue(ext in extNames)
# Load the mapper
try:
mp = hdulist['NOSTA_MAPPER'].data
nosta = mp.field('NOSTA')
noact = mp.field('NOACT')
except KeyError:
ag = hdulist['ARRAY_GEOMETRY'].data
nosta = ag.field('NOSTA')
noact = ag.field('NOSTA')
mapper = {}
for s, a in zip(nosta, noact):
mapper[s] = a
# Correct number of visibilities
uv = hdulist['UV_DATA'].data
self.assertEqual(len(uv.field('FLUX')), data['vis'].shape[0])
# Correct number of frequencies
for vis in uv.field('FLUX'):
self.assertEqual(len(vis), 2 * 2 * len(data['freq']))
# Correct values
for bl, vis in zip(uv.field('BASELINE'), uv.field('FLUX')):
# Convert mapped stands to real stands
stand1 = mapper[(bl >> 8) & 255]
stand2 = mapper[bl & 255]
# Find out which visibility set in the random data corresponds to the
# current visibility
i = 0
for ant1, ant2 in data['bl']:
if ant1.stand.id == stand1 and ant2.stand.id == stand2:
break
else:
i = i + 1
# Extract the data and run the comparison - IF 1
visData = numpy.zeros(2 * len(data['freq']), dtype=numpy.complex64)
visData.real = vis[0::2]
visData.imag = vis[1::2]
for vd, sd in zip(visData[:len(data['freq'])], data['vis'][i, :]):
self.assertAlmostEqual(vd, sd, 8)
# Extract the data and run the comparison - IF 2
visData = numpy.zeros(2 * len(data['freq']), dtype=numpy.complex64)
visData.real = vis[0::2]
visData.imag = vis[1::2]
for vd, sd in zip(visData[len(data['freq']):],
10 * data['vis'][i, :]):
self.assertAlmostEqual(vd, sd, 8)
hdulist.close()
def test_uvdata(self):
"""Test the UV_DATA table."""
testTime = time.time() - 2 * 86400.0
testFile = os.path.join(self.testPath, 'idi-test-UV.fits')
# Get some data
data = self.__initData()
# Start the file
fits = fitsidi.Idi(testFile, ref_time=testTime)
fits.set_stokes(['xx'])
fits.set_frequency(data['freq'])
fits.set_geometry(data['site'], data['antennas'])
fits.add_data_set(testTime, 6.0, data['bl'], data['vis'])
fits.write()
# Open the file and examine
hdulist = astrofits.open(testFile)
uv = hdulist['UV_DATA'].data
# Load the mapper
try:
mp = hdulist['NOSTA_MAPPER'].data
nosta = mp.field('NOSTA')
noact = mp.field('NOACT')
except KeyError:
ag = hdulist['ARRAY_GEOMETRY'].data
nosta = ag.field('NOSTA')
noact = ag.field('NOSTA')
mapper = {}
for s, a in zip(nosta, noact):
mapper[s] = a
# Correct number of visibilities
self.assertEqual(len(uv.field('FLUX')), data['vis'].shape[0])
# Correct number of frequencies
for vis in uv.field('FLUX'):
self.assertEqual(len(vis), 2 * len(data['freq']))
# Correct values
for bl, vis in zip(uv.field('BASELINE'), uv.field('FLUX')):
# Convert mapped stands to real stands
stand1 = mapper[(bl >> 8) & 255]
stand2 = mapper[bl & 255]
# Find out which visibility set in the random data corresponds to the
# current visibility
i = 0
for ant1, ant2 in data['bl']:
if ant1.stand.id == stand1 and ant2.stand.id == stand2:
break
else:
i = i + 1
# Extract the data and run the comparison
visData = numpy.zeros(len(data['freq']), dtype=numpy.complex64)
visData.real = vis[0::2]
visData.imag = vis[1::2]
for vd, sd in zip(visData, data['vis'][i, :]):
self.assertAlmostEqual(vd, sd, 8)
i = i + 1
hdulist.close()
def process_chunk(idf,
site,
good,
filename,
LFFT=64,
overlap=1,
pfb=False,
pols=['xx', 'yy']):
"""
Given an lsl.reader.ldp.TBWFile instances and various parameters for the
cross-correlation, write cross-correlate the data and save it to a file.
"""
# Get antennas
antennas = site.antennas
# Get the metadata
sample_rate = idf.get_info('sample_rate')
# Create the list of good digitizers and a digitizer to Antenna instance mapping.
# These are:
# toKeep -> mapping of digitizer number to array location
# mapper -> mapping of Antenna instance to array location
toKeep = [antennas[i].digitizer - 1 for i in good]
mapper = [antennas[i] for i in good]
# Create a list of unqiue stands to know what style of IDI file to create
stands = set([antennas[i].stand.id for i in good])
wallTime = time.time()
readT, t, data = idf.read()
setTime = t
ref_time = t
# Setup the set time as a python datetime instance so that it can be easily printed
setDT = setTime.datetime
print("Working on set #1 (%.3f seconds after set #1 = %s)" %
((setTime - ref_time), setDT.strftime("%Y/%m/%d %H:%M:%S.%f")))
# In order for the TBW stuff to actaully run, we need to run in with sub-
# integrations. 8 sub-integrations (61.2 ms / 8 = 7.7 ms per section)
# seems to work ok with a "reasonable" number of channels.
nSec = 8
secSize = data.shape[1] // nSec
# Loop over polarizations (there should be only 1)
for pol in pols:
print("-> %s" % pol)
try:
tempVis *= 0 # pylint:disable=undefined-variable
except NameError:
pass
# Set up the progress bar so we can keep up with how the sub-integrations
# are progressing
pb = ProgressBar(max=nSec)
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
# Loop over sub-integrations (set by nSec)
for k in range(nSec):
blList, freq, vis = fxc.FXMaster(data[toKeep, k * secSize:(k + 1) *
secSize],
mapper,
LFFT=LFFT,
overlap=overlap,
pfb=pfb,
include_auto=True,
verbose=False,
sample_rate=sample_rate,
central_freq=0.0,
Pol=pol,
return_baselines=True,
gain_correct=True)
toUse = numpy.where((freq >= 5.0e6) & (freq <= 93.0e6))
toUse = toUse[0]
try:
tempVis += vis
except:
tempVis = vis
pb.inc(amount=1)
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
# Average the sub-integrations together
vis = tempVis / float(nSec)
# Set up the FITS IDI file is we need to
if pol == pols[0]:
pol1, pol2 = fxc.pol_to_pols(pol)
if len(stands) > 255:
fits = fitsidi.ExtendedIdi(filename, ref_time=ref_time)
else:
fits = fitsidi.Idi(filename, ref_time=ref_time)
fits.set_stokes(pols)
fits.set_frequency(freq[toUse])
fits.set_geometry(site, [a for a in mapper if a.pol == pol1])
# Add the visibilities
obsTime = astro.unix_to_taimjd(setTime)
fits.add_data_set(obsTime, readT, blList, vis[:, toUse], pol=pol)
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
print("-> Cummulative Wall Time: %.3f s (%.3f s per integration)" %
((time.time() - wallTime), (time.time() - wallTime)))
fits.write()
fits.close()
del (fits)
del (data)
del (vis)
return True
