def test_tbw_read(self):
"""Test reading in a frame from a TBW file."""
fh = open(tbwFile, 'rb')
# First frame is really TBW and stores the correct stand ID
frame1 = tbw.read_frame(fh)
self.assertTrue(frame1.header.is_tbw)
self.assertEqual(frame1.id, 2)
# Second frame
frame2 = tbw.read_frame(fh)
self.assertTrue(frame2.header.is_tbw)
self.assertEqual(frame2.id, 1)
fh.close()
def test_tbw_math(self):
"""Test mathematical operations on TBW frame data via frames."""
fh = open(tbwFile, 'rb')
# Frames 1 through 3
frames = []
for i in range(1, 4):
frames.append(tbw.read_frame(fh))
fh.close()
# Multiplication
frameT = frames[0] * 2.0
numpy.testing.assert_allclose(frameT.payload.data,
2 * frames[0].payload.data)
frameT *= 2.0
numpy.testing.assert_allclose(frameT.payload.data,
4 * frames[0].payload.data)
frameT = frames[0] * frames[1]
numpy.testing.assert_allclose(
frameT.payload.data,
frames[0].payload.data * frames[1].payload.data)
# Addition
frameA = frames[0] + 2.0
numpy.testing.assert_allclose(frameA.payload.data,
2 + frames[0].payload.data)
frameA += 2.0
numpy.testing.assert_allclose(frameA.payload.data,
4 + frames[0].payload.data)
frameA = frames[0] + frames[1]
numpy.testing.assert_allclose(
frameA.payload.data,
frames[0].payload.data + frames[1].payload.data)
def test_tbw_bits(self):
"""Test getting the data bits from a TBW file."""
fh = open(tbwFile, 'rb')
# File contains 12-bit data, two ways
self.assertEqual(tbw.get_data_bits(fh), 12)
frame1 = tbw.read_frame(fh)
self.assertEqual(frame1.data_bits, 12)
fh.close()
def _get_tbw(self):
"""Private function to load in the test TBW data and get the frames."""
with open(tbwFile, 'rb') as fh:
# Frames 1 through 8
frames = []
for i in range(1, 9):
frames.append(tbw.read_frame(fh))
return frames
def __getTBW(self):
"""Private function to load in the test TBW data and get the frames."""
fh = open(tbwFile, 'rb')
# Frames 1 through 8
frames = []
for i in range(1, 9):
frames.append(tbw.read_frame(fh))
fh.close()
return frames
def test_tbw_tbn_catch(self):
"""Test that tbw will not read tbn files and vice versa."""
fh = open(tbnFile, 'rb')
frame1 = tbw.read_frame(fh)
self.assertFalse(frame1.header.is_tbw)
fh.close()
fh = open(tbwFile, 'rb')
frame1 = tbn.read_frame(fh)
self.assertFalse(frame1.header.is_tbn)
fh.close()
def test_tbw_sort(self):
"""Test sorting TBW frames by time tags."""
fh = open(tbwFile, 'rb')
# Frames 1 through 3
frames = []
for i in range(1, 4):
frames.append(tbw.read_frame(fh))
fh.close()
frames.sort()
frames = frames[::-1]
for i in range(1, len(frames)):
self.assertTrue(frames[i - 1] >= frames[i])
def test_tbw_comps(self):
"""Test the TBW frame comparison operators (>, <, etc.) for time tags."""
fh = open(tbwFile, 'rb')
# Frames 1 through 3
frames = []
for i in range(1, 4):
frames.append(tbw.read_frame(fh))
fh.close()
self.assertTrue(0 < frames[0])
self.assertFalse(0 > frames[0])
self.assertTrue(frames[-1] >= frames[0])
self.assertFalse(frames[-1] <= frames[0])
self.assertTrue(frames[0] == frames[0])
self.assertFalse(frames[0] == frames[-1])
self.assertFalse(frames[0] != frames[0])
def test_tbw_math(self):
"""Test mathematical operations on TBW frame data via frames."""
fh = open(tbwFile, 'rb')
# Frames 1 through 3
frames = []
for i in range(1, 4):
frames.append(tbw.read_frame(fh))
fh.close()
# Multiplication
frameT = frames[0] * 2.0
for i in range(800):
self.assertAlmostEqual(frameT.payload.data[i % 2, i // 2],
2 * frames[0].payload.data[i % 2, i // 2],
2)
frameT *= 2.0
for i in range(800):
self.assertAlmostEqual(frameT.payload.data[i % 2, i // 2],
4 * frames[0].payload.data[i % 2, i // 2],
2)
frameT = frames[0] * frames[1]
for i in range(800):
self.assertAlmostEqual(
frameT.payload.data[i % 2, i // 2],
frames[0].payload.data[i % 2, i // 2] *
frames[1].payload.data[i % 2, i // 2], 2)
# Addition
frameA = frames[0] + 2.0
for i in range(800):
self.assertAlmostEqual(frameA.payload.data[i % 2, i // 2],
2 + frames[0].payload.data[i % 2, i // 2],
2)
frameA += 2.0
for i in range(800):
self.assertAlmostEqual(frameA.payload.data[i % 2, i // 2],
4 + frames[0].payload.data[i % 2, i // 2],
2)
frameA = frames[0] + frames[1]
for i in range(800):
self.assertAlmostEqual(
frameA.payload.data[i % 2, i // 2],
frames[0].payload.data[i % 2, i // 2] +
frames[1].payload.data[i % 2, i // 2], 2)
def test_tbw_errors(self):
"""Test reading errors."""
fh = open(tbwFile, 'rb')
# Frames 1 through 8
for i in range(1, 9):
frame = tbw.read_frame(fh)
# Last frame should be an error (errors.EOFError)
self.assertRaises(errors.EOFError, tbw.read_frame, fh)
fh.close()
# If we offset in the file by 1 byte, we should be a
# sync error (errors.SyncError).
fh = open(tbwFile, 'rb')
fh.seek(1)
self.assertRaises(errors.SyncError, tbw.read_frame, fh)
fh.close()
def main(args):
# Set the station
if args.metadata is not None:
station = stations.parse_ssmif(args.metadata)
ssmifContents = open(args.metadata).readlines()
else:
station = stations.lwa1
ssmifContents = open(os.path.join(dataPath,
'lwa1-ssmif.txt')).readlines()
antennas = station.antennas
# Length of the FFT
LFFT = args.fft_length
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped
maxFrames = int((30000 * 260) / float(LFFT)) * LFFT
# It seems like that would be a good idea, however... TBW data comes one
# capture at a time so doing something like this actually truncates data
# from the last set of stands for the first integration. So, we really
# should stick with
maxFrames = (30000 * 260)
fh = open(args.filename, "rb")
nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
antpols = len(antennas)
nChunks = int(math.ceil(1.0 * nFrames / maxFrames))
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("Chunks: %i" % nChunks)
print("===")
nChunks = 1
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2 * tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Setup the window function to use
if args.pfb:
window = fxc.null_window
elif args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
base, ext = os.path.splitext(args.filename)
base = os.path.basename(base)
if (not os.path.exists("%s.npz" % base)) or args.force:
# Master loop over all of the file chunks
masterSpectra = numpy.zeros((nChunks, antpols, LFFT))
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
((i + 1), framesRemaining))
data = numpy.memmap('temp.mmap',
dtype=numpy.int16,
mode='w+',
shape=(antpols,
2 * 30000 * 260 * nSamples // antpols))
# If there are fewer frames than we need to fill an FFT, skip this chunk
if data.shape[1] < 2 * LFFT:
break
# Inner loop that actually reads the frames into the data array
for j in range(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
print("WARNING: Mark 5C sync error on frame #%i" %
(int(fh.tell()) // tbw.FRAME_SIZE - 1))
continue
if not cFrame.header.is_tbw:
continue
stand = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1)
if cFrame.header.frame_count % 10000 == 0 and args.verbose:
print("%3i -> %3i %6.3f %5i %i" %
(stand, aStand, cFrame.time,
cFrame.header.frame_count, cFrame.payload.timetag))
# Actually load the data. x pol goes into the even numbers, y pol into the
# odd numbers
count = cFrame.header.frame_count - 1
data[aStand, count * nSamples:(count + 1) *
nSamples] = 1 * cFrame.payload.data[0, :]
data[aStand + 1, count * nSamples:(count + 1) *
nSamples] = 1 * cFrame.payload.data[1, :]
del cFrame
# Calculate the spectra for this block of data and then weight the results by
# the total number of frames read. This is needed to keep the averages correct.
# NB: The weighting is the same for the x and y polarizations because of how
# the data are packed in TBW
for j in xrange(0, masterSpectra.shape[1], 4):
tempData = numpy.zeros((4, data.shape[1]), dtype=data.dtype)
tempData = data[j:j + 4, :]
freq, tempSpec = fxc.SpecMaster(tempData,
LFFT=LFFT,
window=window,
verbose=args.verbose,
clip_level=args.clip_level)
masterSpectra[i, j:j + 4, :] = tempSpec
# Compute the 1 ms average power and the data range within each 1 ms window
subSize = 1960
nsegments = data.shape[1] // subSize
print(
"Computing average power and data range in %i-sample intervals"
% subSize)
pb = ProgressBar(max=data.shape[0])
avgPower = numpy.zeros((antpols, nsegments), dtype=numpy.float32)
dataRange = numpy.zeros((antpols, nsegments, 3), dtype=numpy.int16)
for s in xrange(data.shape[0]):
for p in xrange(nsegments):
subData = data[s, (p * subSize):((p + 1) * subSize)]
avgPower[s, p] = numpy.mean(numpy.abs(subData))
dataRange[s, p, 0] = subData.min()
dataRange[s, p, 1] = subData.mean()
dataRange[s, p, 2] = subData.max()
### This little block here looks for likely saturation events and save
### the raw time series around them into individual NPZ files for stand
### number 14.
#if (dataRange[s,p,0] < -1000 or dataRange[s,p,0] > 1000) and antennas[s].stand.id == 14:
#subData = data[s,((p-1)*1960):((p+2)*1960)]
#satFileName = 'stand-14-pol-%i-%i.npz' % (antennas[s].pol, (p-1)*1960)
#print(satFileName)
#numpy.savez(satFileName, start=(p-1)*1960, data=subData)
pb.inc(amount=1)
if pb.amount != 0 and pb.amount % 10 == 0:
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
# We don't really need the data array anymore, so delete it
del (data)
os.unlink('temp.mmap')
# Apply the cable loss corrections, if requested
if True:
for s in xrange(masterSpectra.shape[1]):
currGain = antennas[s].cable.gain(freq)
for c in xrange(masterSpectra.shape[0]):
masterSpectra[c, s, :] /= currGain
# Now that we have read through all of the chunks, perform the final averaging by
# dividing by all of the chunks
spec = masterSpectra.mean(axis=0)
# Estimate the dipole resonance frequencies
print("Computing dipole resonance frequencies")
pb = ProgressBar(max=spec.shape[0])
resFreq = numpy.zeros(spec.shape[0])
toCompare = numpy.where((freq > 31e6) & (freq < 70e6))[0]
for i in xrange(spec.shape[0]):
bestOrder = 0
bestRMS = 1e34
for j in xrange(3, 12):
coeff = numpy.polyfit(freq[toCompare] / 1e6,
numpy.log10(spec[i, toCompare]) * 10, j)
fit = numpy.polyval(coeff, freq[toCompare] / 1e6)
rms = ((fit - numpy.log10(spec[i, toCompare]) * 10)**2).sum()
if rms < bestRMS:
bestOrder = j
bestRMS = rms
coeff = numpy.polyfit(freq[toCompare] / 1e6,
numpy.log10(spec[i, toCompare]) * 10,
bestOrder)
fit = numpy.polyval(coeff, freq[toCompare] / 1e6)
try:
resFreq[i] = freq[toCompare[numpy.where(
fit == fit.max())[0][0]]] / 1e6
except:
pass
pb.inc(amount=1)
if pb.amount != 0 and pb.amount % 10 == 0:
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
numpy.savez("%s.npz" % base,
date=str(beginDate),
freq=freq,
masterSpectra=masterSpectra,
resFreq=resFreq,
avgPower=avgPower,
dataRange=dataRange,
ssmifContents=ssmifContents)
else:
dataDict = numpy.load("%s.npz" % base)
freq = dataDict['freq']
masterSpectra = dataDict['masterSpectra']
resFreq = dataDict['resFreq']
# Now that we have read through all of the chunks, perform the final averaging by
# dividing by all of the chunks
spec = masterSpectra.mean(axis=0)
# Create a good template spectra
specTemplate = numpy.median(spec, axis=0)
specDiff = numpy.zeros(spec.shape[0])
toCompare = numpy.where((freq > 32e6) & (freq < 50e6))[0]
print(len(toCompare))
for i in xrange(spec.shape[0]):
specDiff[i] = (spec[i, toCompare] / specTemplate[toCompare]).mean()
specDiff = numpy.where(specDiff < 2, specDiff, 2)
# Get the station
standPos = numpy.array([[ant.stand.x, ant.stand.y, ant.stand.z]
for ant in antennas if ant.pol == 0])
# Plots
if args.verbose:
fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1)
ax1.scatter(standPos[:, 0],
standPos[:, 1],
c=specDiff[0::2],
s=40.0,
alpha=0.50)
## Add the fence as a dashed line
ax1.plot([-59.827, 59.771, 60.148, -59.700, -59.827],
[59.752, 59.864, -59.618, -59.948, 59.752],
linestyle='--',
color='k')
## Add the shelter
ax1.plot([55.863, 58.144, 58.062, 55.791, 55.863],
[45.946, 45.999, 51.849, 51.838, 45.946],
linestyle='-',
color='k')
## Set the limits to just zoom in on the main stations
ax1.set_xlim([-65, 65])
ax1.set_ylim([-65, 65])
ax2 = fig.add_subplot(1, 2, 2)
ax2.plot(freq / 1e6, numpy.log10(specTemplate) * 10, alpha=0.50)
print("RBW: %.1f Hz" % (freq[1] - freq[0]))
plt.show()
def main(args):
# Set the station
if args.metadata is not None:
station = stations.parse_ssmif(args.metadata)
ssmifContents = open(args.metadata).readlines()
else:
station = stations.lwa1
ssmifContents = open(os.path.join(dataPath,
'lwa1-ssmif.txt')).readlines()
antennas = station.antennas
toKeep = []
for g in (1, 10, 54, 248, 251, 258):
for i, ant in enumerate(antennas):
if ant.stand.id == g and ant.pol == 0:
toKeep.append(i)
for i, j in enumerate(toKeep):
print(i, j, antennas[j].stand.id)
# Length of the FFT
LFFT = args.fft_length
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped
maxFrames = int((30000 * 260) / float(LFFT)) * LFFT
# It seems like that would be a good idea, however... TBW data comes one
# capture at a time so doing something like this actually truncates data
# from the last set of stands for the first integration. So, we really
# should stick with
maxFrames = (30000 * 260)
fh = open(args.filename, "rb")
nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
antpols = len(antennas)
nChunks = int(math.ceil(1.0 * nFrames / maxFrames))
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginTime = junkFrame.time
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("Chunks: %i" % nChunks)
print("===")
nChunks = 1
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2 * tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Master loop over all of the file chunks
masterSpectra = numpy.zeros((nChunks, antpols, LFFT))
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
((i + 1), framesRemaining))
data = numpy.zeros((12, 12000000), dtype=numpy.int16)
# If there are fewer frames than we need to fill an FFT, skip this chunk
if data.shape[1] < 2 * LFFT:
break
# Inner loop that actually reads the frames into the data array
for j in range(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbw.FRAME_SIZE-1))
continue
if not cFrame.header.is_tbw:
continue
stand = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1)
#if cFrame.header.frame_count % 10000 == 0 and config['verbose']:
#print("%3i -> %3i %6.3f %5i %i" % (stand, aStand, cFrame.time, cFrame.header.frame_count, cFrame.payload.timetag))
# Actually load the data. x pol goes into the even numbers, y pol into the
# odd numbers
count = cFrame.header.frame_count - 1
if aStand not in toKeep:
continue
# Convert to reduced index
aStand = 2 * toKeep.index(aStand)
data[aStand, count * nSamples:(count + 1) *
nSamples] = cFrame.payload.data[0, :]
data[aStand + 1, count * nSamples:(count + 1) *
nSamples] = cFrame.payload.data[1, :]
# Time series analysis - mean, std. dev, saturation count
tsMean = data.mean(axis=1)
tsStd = data.std(axis=1)
tsSat = numpy.where((data == 2047) | (data == -2047), 1, 0).sum(axis=1)
# Time series analysis - percentiles
p = [50, 75, 90, 95, 99]
tsPct = numpy.zeros((data.shape[0], len(p)))
for i in xrange(len(p)):
for j in xrange(data.shape[0]):
tsPct[j, i] = percentile(numpy.abs(data[j, :]), p[i])
# Frequency domain analysis - spectra
freq = numpy.fft.fftfreq(2 * args.fft_length, d=1.0 / 196e6)
freq = freq[:args.fft_length]
delays = numpy.zeros((data.shape[0], freq.size))
signalsF, validF = FEngine(data,
freq,
delays,
LFFT=args.fft_length,
Overlap=1,
sample_rate=196e6,
clip_level=0)
# Cleanup to save memory
del validF, data
print(signalsF.shape)
# SK control values
skM = signalsF.shape[2]
skN = 1
# Frequency domain analysis - spectral kurtosis
k = numpy.zeros((signalsF.shape[0], signalsF.shape[1]))
for l in xrange(signalsF.shape[0]):
for m in xrange(freq.size):
k[l, m] = kurtosis.spectral_fft(signalsF[l, m, :])
kl, kh = kurtosis.get_limits(4, skM, skN)
print(kl, kh)
# Integrate the spectra for as long as we can
masterSpectra = (numpy.abs(signalsF)**2).mean(axis=2)
del signalsF
# Mask out bad values (high spectral kurtosis) for the plot
mask = numpy.where((k < kl) | (k > kh), 1, 0)
mask = expandMask(mask, radius=4, merge=True)
masterSpectra = numpy.ma.array(masterSpectra, mask=mask)
# Save the data to an HDF5 file
outname = os.path.splitext(args.filename)[0]
outname = "%s-RFI.hdf5" % outname
f = h5py.File(outname, 'w')
f.attrs['filename'] = args.filename
f.attrs['mode'] = 'TBW'
f.attrs['station'] = 'LWA-1'
f.attrs['dataBits'] = dataBits
f.attrs['startTime'] = beginTime
f.attrs['startTime_units'] = 's'
f.attrs['startTime_sys'] = 'unix'
f.attrs['sample_rate'] = 196e6
f.attrs['sample_rate_units'] = 'Hz'
f.attrs['RBW'] = freq[1] - freq[0]
f.attrs['RBW_Units'] = 'Hz'
f.attrs['SK-M'] = skM
f.attrs['SK-N'] = skN
for l in xrange(len(toKeep)):
antX = antennas[toKeep[l]]
antY = antennas[toKeep[l] + 1]
stand = f.create_group('Stand%03i' % antX.stand.id)
stand['freq'] = freq
stand['freq'].attrs['Units'] = 'Hz'
polX = stand.create_group('X')
polY = stand.create_group('Y')
polX.attrs['tsMean'] = tsMean[2 * l]
polY.attrs['tsMean'] = tsMean[2 * l + 1]
polX.attrs['tsStd'] = tsStd[2 * l]
polY.attrs['tsStd'] = tsStd[2 * l + 1]
polX.attrs['tsSat'] = tsSat[2 * l]
polY.attrs['tsSat'] = tsSat[2 * l + 1]
for i, v in enumerate(p):
polX.attrs['ts%02i' % v] = tsPct[2 * l][i]
polY.attrs['ts%02i' % v] = tsPct[2 * l + 1][i]
polX['spectrum'] = masterSpectra[2 * l, :]
polX['spectrum'].attrs['axis0'] = 'frequency'
polY['spectrum'] = masterSpectra[2 * l + 1, :]
polY['spectrum'].attrs['axis0'] = 'frequency'
polX['kurtosis'] = k[2 * l, :]
polX['kurtosis'].attrs['axis0'] = 'frequency'
polY['kurtosis'] = k[2 * l + 1, :]
polY['kurtosis'].attrs['axis0'] = 'frequency'
# The plot
fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
ax2 = fig.add_subplot(2, 1, 2)
for l in xrange(k.shape[0]):
ant = antennas[toKeep[l / 2]]
ax1.plot(freq / 1e6,
numpy.log10(masterSpectra[l, :]) * 10,
label='Stand %i, Pol %i' %
(ant.stand.id, ant.pol + l % 2))
ax2.plot(freq / 1e6,
k[l, :],
label='Stand %i, Pol %i' %
(ant.stand.id, ant.pol + l % 2))
ax2.hlines(kl,
freq[0] / 1e6,
freq[-1] / 1e6,
linestyle=':',
label='Kurtosis Limit 4$\sigma$')
ax2.hlines(kh,
freq[0] / 1e6,
freq[-1] / 1e6,
linestyle=':',
label='Kurtosis Limit 4$\sigma$')
ax1.set_xlabel('Frequency [MHz]')
ax1.set_ylabel('PSD [arb. dB/RBW]')
ax1.legend(loc=0)
ax2.set_ylim((kl / 2, kh * 2))
ax2.set_xlabel('Frequency [MHz]')
ax2.set_ylabel('Spectral Kurtosis')
ax2.legend(loc=0)
plt.show()
def main(args):
filename = args[0]
stands = [int(i) for i in args[1:]]
antennas = lwa1.antennas
fh = open(filename, "rb")
nFrames = os.path.getsize(filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
antpols = len(antennas)
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginTime = junkFrame.time
beginDate = junkFrame.time.datetime
# Figure out which digitizers to keep
toKeep = []
for a in antennas:
if a.stand.id in stands:
toKeep.append( a.digitizer )
# File summary
print("Filename: %s" % filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("===")
print("Keeping Stands:")
for a in toKeep:
print(" Stand #%3i, pol %i (digitizer %3i)" % (antennas[a-1].stand.id, antennas[a-1].pol, antennas[a-1].digitizer))
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2*tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Create the HDF5 file
outname = os.path.splitext(filename)[0]
outname = "%shdf5" % outname
f = h5py.File(outname, 'w')
f.attrs['filename'] = filename
f.attrs['mode'] = 'TBW'
f.attrs['station'] = 'LWA-1'
f.attrs['dataBits'] = dataBits
f.attrs['startTime'] = beginTime
f.attrs['startTime_units'] = 's'
f.attrs['startTime_sys'] = 'unix'
f.attrs['sample_rate'] = 196e6
f.attrs['sample_rate_units'] = 'Hz'
## Create the digitizer to dataset lookup table and the
standLookup = {}
standData = []
i = 0
for a in toKeep:
if a % 2 == 0:
continue
s = antennas[a-1].stand.id
standLookup[a] = i
temp = f.create_group('Stand%03i' % s)
### Combined status code
temp.attrs['statusCode'] = antennas[a-1].combined_status
### Antenna number
temp.attrs['antennaID'] = antennas[a-1].id
### Cable information
temp.attrs['cableID'] = antennas[a-1].cable.id
temp.attrs['cableLength'] = antennas[a-1].cable.length
temp.attrs['cableLength_units'] = 'm'
### Stand location information
temp.attrs['posX'] = antennas[a-1].stand.x
temp.attrs['posX_units'] = 'm'
temp.attrs['posY'] = antennas[a-1].stand.y
temp.attrs['posY_units'] = 'm'
temp.attrs['posZ'] = antennas[a-1].stand.z
temp.attrs['posZ_units'] = 'm'
### Time series data sets
temp.attrs['axis0'] = 'time'
xpol = temp.create_dataset('X', (12000000,), 'i2', chunks=True)
ypol = temp.create_dataset('Y', (12000000,), 'i2', chunks=True)
standData.append( (xpol, ypol, temp) )
i += 1
# Go!
while True:
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbw.FRAME_SIZE-1))
continue
if not cFrame.header.is_tbw:
continue
# Get the DP "stand" ID and the digitizer number
stand = cFrame.header.id
aStand = 2*(stand-1)
digitizer = aStand + 1
# If we don't need it, skip it
if digitizer not in toKeep:
continue
# Actually load the data.
## Frame count
count = cFrame.header.frame_count - 1
## Which data set
dataset = standLookup[digitizer]
## Load
standData[dataset][0][count*nSamples:(count+1)*nSamples] = cFrame.payload.data[0,:]
standData[dataset][1][count*nSamples:(count+1)*nSamples] = cFrame.payload.data[1,:]
fh.close()
f.close()
def main(args):
# Set the station
if args.metadata is not None:
station = stations.parse_ssmif(args.metadata)
else:
station = stations.lwa1
antennas = station.antennas
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped
maxFrames = int((30000 * 260))
# It seems like that would be a good idea, however... TBW data comes one
# capture at a time so doing something like this actually truncates data
# from the last set of stands for the first integration. So, we really
# should stick with
maxFrames = (30000 * 260)
fh = open(args.filename, "rb")
nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
antpols = len(antennas)
nChunks = int(math.ceil(1.0 * nFrames / maxFrames))
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("Chunks: %i" % nChunks)
print("===")
nChunks = 1
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2 * tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Master loop over all of the file chunks
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
((i + 1), framesRemaining))
#
# NOTE
# Major change here from tbwSpectra.py/stationMaster.py. We are only keeping
# the first 30,000 frames of the TBW file since we don't really need to read
# in all of it to find bursts
#
data = numpy.zeros((antpols, 30000 * nSamples), dtype=numpy.int16)
# Inner loop that actually reads the frames into the data array
for j in range(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
print("WARNING: Mark 5C sync error on frame #%i" %
(int(fh.tell()) // tbw.FRAME_SIZE - 1))
continue
if not cFrame.header.is_tbw:
continue
# Skip frames over 30,000 on all stands
if cFrame.header.frame_count > 30000:
continue
stand = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1)
if cFrame.header.frame_count % 5000 == 0 and args.verbose:
print("%3i -> %3i %6.3f %5i %i" %
(stand, aStand, cFrame.time, cFrame.header.frame_count,
cFrame.payload.timetag))
# Actually load the data. x pol goes into the even numbers, y pol into the
# odd numbers
count = cFrame.header.frame_count - 1
data[aStand, count * nSamples:(count + 1) *
nSamples] = cFrame.payload.data[0, :]
data[aStand + 1, count * nSamples:(count + 1) *
nSamples] = cFrame.payload.data[1, :]
# Compute the power
data = numpy.abs(data)
# We need to various time series data to be aligned so we need to do a
# correction for the cable delays. Using the various Antenna instances,
# we create a array of delays (in seconds) and do everything relative to
# the longest delay.
#
# After this, we can align the data from all of the antennas.
delays = numpy.array([a.cable.delay(30e6) for a in antennas])
delays = numpy.round(delays * fS).astype(numpy.int32)
delays = delays.max() - delays
alignedData = numpy.zeros(
(data.shape[0], data.shape[1] - delays.max()), dtype=data.dtype)
for s in xrange(data.shape[0]):
alignedData[s, :] = data[s, delays[s]:(delays[s] +
alignedData.shape[1])]
del (data)
# Using the good antennas (Antenna.combined_status == 33), we need to find the
# start of the RFI pulses by looking for "large" data values. To make sure
# that we aren't getting stuck on a first partial burst, skip the first one
# and use the second set of "large" data values found.
#
# I was using "large" as saturation/clipping (>= 2047), but the new lower
# value for the ARX gain makes me want to lower to something more like
#
inOne = False
first = 0
status = numpy.array([ant.combined_status for ant in antennas])
good = numpy.where(status == 33)[0]
while first < alignedData.shape[1]:
mv = alignedData[good, first].max()
if mv >= args.threshold:
if not inOne:
first += 5000
inOne = True
else:
break
else:
first += 1
print("Second burst at %i" % first)
# Keep only what would be interesting (200 samples before and 2,800 samples
# afterward) around the burst. This corresponds to a time range from 1
# microsecond before the start of the pulse to 14 microseconds later. Save
# the aligned data snippet to a NPZ file.
alignedData = alignedData[:, first - 200:first + 2800]
standPos = numpy.array([[ant.stand.x, ant.stand.y, ant.stand.z]
for ant in antennas])
junk, basename = os.path.split(args.filename)
shortname, ext = os.path.splitext(basename)
numpy.savez('%s-burst.npz' % shortname,
data=alignedData,
ssmif=args.metadata)
# Make the movie (if needed)
if args.movie:
if args.verbose:
print("Creating movie frames")
pb = ProgressBar(max=alignedData.shape[1] / 2)
else:
pb = None
fig = plt.figure(figsize=(12, 6))
for i in xrange(0, alignedData.shape[1], 2):
fig.clf()
axX = fig.add_subplot(1, 2, 1)
axY = fig.add_subplot(1, 2, 2)
colorsX = 1.0 * (alignedData[0::2, i] +
alignedData[0::2, i + 1]) / 2
colorsY = 1.0 * (alignedData[1::2, i] +
alignedData[1::2, i + 1]) / 2
axX.scatter(standPos[0::2, 0],
standPos[0::2, 1],
c=colorsX,
s=40.0,
vmin=0,
vmax=args.threshold)
axY.scatter(standPos[1::2, 0],
standPos[1::2, 1],
c=colorsY,
s=40.0,
vmin=0,
vmax=args.threshold)
## Add the fence as a dashed line
axX.plot([-59.827, 59.771, 60.148, -59.700, -59.827],
[59.752, 59.864, -59.618, -59.948, 59.752],
linestyle='--',
color='k')
axY.plot([-59.827, 59.771, 60.148, -59.700, -59.827],
[59.752, 59.864, -59.618, -59.948, 59.752],
linestyle='--',
color='k')
## Add the shelter
axX.plot([55.863, 58.144, 58.062, 55.791, 55.863],
[45.946, 45.999, 51.849, 51.838, 45.946],
linestyle='-',
color='k')
axY.plot([55.863, 58.144, 58.062, 55.791, 55.863],
[45.946, 45.999, 51.849, 51.838, 45.946],
linestyle='-',
color='k')
axX.set_xlim([-65, 65])
axX.set_ylim([-65, 65])
axX.set_title('X pol. $\Delta$t = %.1f ns' %
(1.0 * i / fS * 1e9, ))
axX.set_xlabel('$\Delta$ X [m]')
axX.set_ylabel('$\Delta$ Y [m]')
axY.set_xlim([-65, 65])
axY.set_ylim([-65, 65])
axY.set_title('Y pol.')
axY.set_xlabel('$\Delta$ X [m]')
#axY.set_ylabel('$\Delta$ Y [m]')
fig.savefig('burst-%05i.png' % (i / 2, ))
if pb is not None:
pb.inc(amount=1)
if pb.amount != 0 and pb.amount % 10 == 0:
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
del (fig)
if pb is not None:
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
if args.verbose:
print("Creating movie")
os.system(
"mencoder mf://burst-*.png -mf fps=20:type=png -ovc lavc -lavcopts vcodec=mpeg4:aspect=2/1 -o %s-burst.avi -ffourcc DX50 -vf scale=600:1200,expand=600:1200"
% shortname)
def main(args):
# Set the station
station = stations.lwa1
antennas = station.antennas
fh = open(args.filename, "rb")
nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
maxFrames = 30000 * 260
antpols = len(antennas)
nChunks = int(math.ceil(1.0 * nFrames / maxFrames))
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("Chunks: %i" % nChunks)
print("===")
nChunks = 1
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2 * tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Master loop over all of the file chunks
timetags = numpy.zeros((antpols, 30000), dtype=numpy.int64) - 1
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = nFrames
print("Working on chunk %i, %i frames remaining" %
((i + 1), framesRemaining))
# Inner loop that actually reads the frames into the data array
for j in range(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
print("WARNING: Mark 5C sync error on frame #%i" %
(int(fh.tell()) / tbw.FRAME_SIZE - 1))
continue
if not cFrame.header.is_tbw:
continue
stand = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1)
if cFrame.header.frame_count % 10000 == 0:
print("%3i -> %3i %5i %i" %
(stand, aStand, cFrame.header.frame_count,
cFrame.payload.timetag))
# Actually load the data. x pol goes into the even numbers, y pol into the
# odd numbers
count = cFrame.header.frame_count - 1
timetags[aStand, count] = cFrame.payload.timetag
timetags[aStand + 1, count] = cFrame.payload.timetag
# Check for missing frames
missing = numpy.where(timetags < 0)
if len(missing) != 0:
dp1Boards = {}
print(
"Found %i missing frames (%i missing time tags). Missing data from:"
% (len(missing[0]) / 2, len(missing[0])))
for i, f in zip(missing[0], missing[1]):
try:
dp1Boards[antennas[i].board] += 1
except KeyError:
dp1Boards[antennas[i].board] = 1
print(" stand %3i, pol. %1i (dig. %3i) @ frame %5i" %
(antennas[i].stand.id, antennas[i].pol,
antennas[i].digitizer, f + 1))
print("-> DP1 boards with missing frames:")
for k in dp1Boards.keys():
v = dp1Boards[k]
print(" %2i %6i (%7.3f%%)" % (k, v, 100.0 * v / (30000 * 10)))
# Check time tags to make sure every ant/pol as the same time as each frame
for f in xrange(timetags.shape[1]):
## For each frame count value, get the median time tag and use this for comparison.
## If things are really bad, we will get a lot of errors.
frameTime = numpy.median(timetags[:, f])
## Compare all of the antpols at a particular frame count, ignoring the ones that
## are missing.
missing = numpy.where((timetags[:, f] != frameTime)
& (timetags[:, f] >= 0))[0]
## Report any errors
for m in missing:
print("ERROR: t.t. %i @ frame %i != frame median of %i" %
(timetags[m, f], f + 1, frameTime))
print(" -> difference: %i" % (timetags[m, f] - frameTime, ))
# Check time tags to make sure the times increment correctly between frames
for i in xrange(timetags.shape[0]):
for f in xrange(1, timetags.shape[1]):
## Skip missing frames since they always fail
if timetags[i, f] < 0 or timetags[i, f - 1] < 0:
continue
## Compare the current time tag with previous and report an error if there
## is a discrepancy between the two modulo the expected skip.
if timetags[i, f] > (timetags[i, f - 1] + nSamples):
## Too far into the future
print("ERROR: t.t. %i @ frame %i > t.t. %i @ frame %i + skip" %
(timetags[i, f], f + 1, timetags[i, f - 1], f))
print(" -> difference: %i" %
(timetags[i, f] - timetags[i, f - 1], ))
elif timetags[i, f] < (timetags[i, f - 1] + nSamples):
## Not far enough into the future
print("ERROR: t.t. %i @ frame %i < t.t. %i @ frame %i + skip" %
(timetags[i, f], f + 1, timetags[i, f - 1], f))
print(" -> difference: %i" %
(timetags[i, f] - timetags[i, f - 1], ))
else:
## Everything is good if we make it here
pass
