def test_basic_tbn(self):
"""Test building a basic TBN signal"""
testFile = os.path.join(self.testPath, 'tbn.dat')
fh = open(testFile, 'wb')
dp.basic_signal(fh, numpy.array([1,2,3,4]), 2000, mode='TBN', filter=7, start_time=1000)
fh.close()
# Check the file size
fileSize = os.path.getsize(testFile)
nSamples = fileSize // tbn.FRAME_SIZE
self.assertEqual(nSamples, 2000*4*2)
# Check the time of the first frame
fh = open(testFile, 'rb')
frame = tbn.read_frame(fh)
fh.close()
self.assertEqual(frame.payload.timetag, 1000*dp_common.fS)
def __getTBN(self, vanilla=False):
"""Private function to load in the test TBN data and get the frames. If
the keyword 'vanilla' is set to True, gain, sample rate, and frequency meta-
data are not added to the frames."""
fh = open(tbnFile, 'rb')
# Frames 1 through 8
frames = []
for i in range(1, 9):
frames.append(tbn.read_frame(fh))
if not vanilla:
# Set some values for the other meta-data
for frame in frames:
frame.sample_rate = 100000
fh.close()
return frames
def test_point_tbn(self):
"""Test building a point source TBN signal"""
testFile = os.path.join(self.testPath, 'tbn.dat')
station = lwa_common.lwa1
antennas = station.antennas
fh = open(testFile, 'wb')
dp.point_source(fh, antennas[:8:2], self.src, 4, mode='TBN', filter=7, start_time=1000)
fh.close()
# Check the file size
fileSize = os.path.getsize(testFile)
nSamples = fileSize // tbn.FRAME_SIZE
self.assertEqual(nSamples, 4*4*2)
# Check the time of the first frame
fh = open(testFile, 'rb')
frame = tbn.read_frame(fh)
fh.close()
self.assertEqual(frame.payload.timetag, 1000*dp_common.fS)
def test_tbn_math(self):
"""Test mathematical operations on TBN frame data via frames."""
fh = open(tbnFile, 'rb')
# Frames 1 through 29
frames = []
for i in range(1, 30):
frames.append(tbn.read_frame(fh))
fh.close()
# Multiplication
frameT = frames[0] * 2.0
numpy.testing.assert_allclose(frameT.payload.data,
2 * frames[0].payload.data,
atol=1e-6)
frameT *= 2.0
numpy.testing.assert_allclose(frameT.payload.data,
4 * frames[0].payload.data,
atol=1e-6)
frameT = frames[0] * frames[1]
numpy.testing.assert_allclose(frameT.payload.data,
frames[0].payload.data *
frames[1].payload.data,
atol=1e-6)
# Addition
frameA = frames[0] + 2.0
numpy.testing.assert_allclose(frameA.payload.data,
2 + frames[0].payload.data,
atol=1e-6)
frameA += 2.0
numpy.testing.assert_allclose(frameA.payload.data,
4 + frames[0].payload.data,
atol=1e-6)
frameA = frames[0] + frames[1]
numpy.testing.assert_allclose(frameA.payload.data,
frames[0].payload.data +
frames[1].payload.data,
atol=1e-6)
def test_tbn_math(self):
"""Test mathematical operations on TBN frame data via frames."""
fh = open(tbnFile, 'rb')
# Frames 1 through 29
frames = []
for i in range(1, 30):
frames.append(tbn.read_frame(fh))
fh.close()
# Multiplication
frameT = frames[0] * 2.0
for i in range(512):
self.assertAlmostEqual(frameT.payload.data[i],
2 * frames[0].payload.data[i], 2)
frameT *= 2.0
for i in range(512):
self.assertAlmostEqual(frameT.payload.data[i],
4 * frames[0].payload.data[i], 2)
frameT = frames[0] * frames[1]
for i in range(512):
self.assertAlmostEqual(
frameT.payload.data[i],
frames[0].payload.data[i] * frames[1].payload.data[i], 2)
# Addition
frameA = frames[0] + 2.0
for i in range(512):
self.assertAlmostEqual(frameA.payload.data[i],
2 + frames[0].payload.data[i], 2)
frameA += 2.0
for i in range(512):
self.assertAlmostEqual(frameA.payload.data[i],
4 + frames[0].payload.data[i], 2)
frameA = frames[0] + frames[1]
for i in range(512):
self.assertAlmostEqual(
frameA.payload.data[i],
frames[0].payload.data[i] + frames[1].payload.data[i], 2)
def test_frame_data_errors(self):
"""Test the data error scenarios when validating a TBN SimFrame."""
# Read in a TBN frame from the test file
fh = open(tbnFile, 'rb')
origFrame = tbnReader.read_frame(fh)
fh.close()
# Try to validate frame with the wrong data type
fakeFrame = tbnWriter.SimFrame()
fakeFrame.load_frame(copy.deepcopy(origFrame))
fakeFrame.data = fakeFrame.payload.data.real
self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True)
# Try to validate frame with the wrong data size
fakeFrame = tbnWriter.SimFrame()
fakeFrame.load_frame(copy.deepcopy(origFrame))
fakeFrame.data = None
self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True)
fakeFrame = tbnWriter.SimFrame()
fakeFrame.load_frame(copy.deepcopy(origFrame))
fakeFrame.data = fakeFrame.payload.data[0:50]
self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True)
def test_tbn_buffer_flush(self):
"""Test the TBN ring buffer's flush() function."""
fh = open(tbnFile, 'rb')
nFpO = tbn.get_frames_per_obs(fh)
nFpO = nFpO[0] + nFpO[1]
# Create the FrameBuffer instance
frameBuffer = buffer.TBNFrameBuffer(stands=range(1, nFpO // 2 + 1),
pols=[0, 1])
# Go
while True:
try:
cFrame = tbn.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
continue
frameBuffer.append(cFrame)
cFrames = frameBuffer.get()
if cFrames is None:
continue
fh.close()
# Flush the buffer
for cFrames in frameBuffer.flush():
# Make sure the dump has one of the expected time tags
self.assertTrue(cFrames[0].payload.timetag in (119196674956800,
119196675960320))
# Make sure it has the right number of frames
self.assertEqual(len(cFrames), nFpO)
def main(args):
filename = args.filename
sizeB = os.path.getsize(filename)
# Open the file and get some basic info about the data contained
fh = open(filename, 'rb')
sample_rate = tbn.get_sample_rate(fh)
nFramesX, nFramesY = tbn.get_frames_per_obs(fh)
nCaptures = sizeB // tbn.FRAME_SIZE // (nFramesX + nFramesY)
print("Filename: %s" % filename)
print("Size: %.1f MB" % (float(sizeB) / 1024 / 1024))
print("Captures: %i (%.2f seconds)" %
(nCaptures, nCaptures * 512 / sample_rate))
print("Stands: %i (%i x pol., %i y pol.)" %
((nFramesX + nFramesY), nFramesX, nFramesY))
print("Sample Rate: %.2f kHz" % (sample_rate / 1000.0))
print("===")
if args.count > 0:
nCaptures = args.count * sample_rate // 512
else:
nCaptures -= args.offset * sample_rate // 512
args.count = nCaptures * 512 // sample_rate
nSkip = int(args.offset * sample_rate / 512)
print("Seconds to Skip: %.2f (%i captures)" % (args.offset, nSkip))
print("Seconds to Split: %.2f (%i captures)" % (args.count, nCaptures))
# Make sure that the first frame in the file is the first frame if a capture
# (stand 1, pol 0). If not, read in as many frames as necessary to get to
# the beginning of a complete capture.
frame = tbn.read_frame(fh)
stand, pol = frame.id
skip = 0
while (2 * (stand - 1) + pol) != 0:
frame = tbn.read_frame(fh)
stand, pol = frame.id
skip += 1
fh.seek(fh.tell() - tbn.FRAME_SIZE)
if skip != 0:
print("Skipped %i frames at the beginning of the file" % skip)
for c in list(range(nSkip)):
if c < nSkip:
fh.seek(fh.tell() + tbn.FRAME_SIZE * (nFramesX + nFramesY))
continue
nFramesRemaining = (sizeB - fh.tell()) // tbn.FRAME_SIZE
nRecursions = int(nFramesRemaining // (nCaptures * (nFramesX + nFramesY)))
if not args.recursive:
nRecursions = 1
scale = int(math.log10(nRecursions)) + 1
ifString = "Working on #%%%ii of %i (%%s)" % (scale, nRecursions)
for r in range(nRecursions):
if args.date:
filePos = fh.tell()
junkFrame = tbn.read_frame(fh)
fh.seek(filePos)
dt = junkFrame.time.datetime
captFilename = "%s_%s.dat" % (os.path.splitext(
os.path.basename(filename))[0], dt.isoformat())
else:
captFilename = "%s_s%04i_p%%0%ii.dat" % (os.path.splitext(
os.path.basename(filename))[0], args.count, scale)
captFilename = captFilename % r
if not args.recursive:
captFilename = "%s_s%04i.dat" % (os.path.splitext(
os.path.basename(filename))[0], args.count)
print(ifString % (r + 1, captFilename))
t0 = time.time()
fhOut = open(captFilename, 'wb')
split_file(fh, fhOut, nCaptures, nFramesX + nFramesY)
fhOut.close()
t1 = time.time()
print(" Copied %i bytes in %.3f s (%.3f MB/s)" %
(os.path.getsize(captFilename), t1 - t0,
os.path.getsize(captFilename) / 1024.0**2 / (t1 - t0)))
fh.close()
def main(args):
# Set the station
station = stations.lwa1
antennas = station.antennas
fh = open(args.filename, "rb")
nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
maxFrames = 30000 * 260
antpols = len(antennas)
nChunks = int(math.ceil(1.0 * nFrames / maxFrames))
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("Chunks: %i" % nChunks)
print("===")
nChunks = 1
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2 * tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Master loop over all of the file chunks
timetags = numpy.zeros((antpols, 30000), dtype=numpy.int64) - 1
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = nFrames
print("Working on chunk %i, %i frames remaining" %
((i + 1), framesRemaining))
# Inner loop that actually reads the frames into the data array
for j in range(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
print("WARNING: Mark 5C sync error on frame #%i" %
(int(fh.tell()) / tbw.FRAME_SIZE - 1))
continue
if not cFrame.header.is_tbw:
continue
stand = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1)
if cFrame.header.frame_count % 10000 == 0:
print("%3i -> %3i %5i %i" %
(stand, aStand, cFrame.header.frame_count,
cFrame.payload.timetag))
# Actually load the data. x pol goes into the even numbers, y pol into the
# odd numbers
count = cFrame.header.frame_count - 1
timetags[aStand, count] = cFrame.payload.timetag
timetags[aStand + 1, count] = cFrame.payload.timetag
# Check for missing frames
missing = numpy.where(timetags < 0)
if len(missing) != 0:
dp1Boards = {}
print(
"Found %i missing frames (%i missing time tags). Missing data from:"
% (len(missing[0]) / 2, len(missing[0])))
for i, f in zip(missing[0], missing[1]):
try:
dp1Boards[antennas[i].board] += 1
except KeyError:
dp1Boards[antennas[i].board] = 1
print(" stand %3i, pol. %1i (dig. %3i) @ frame %5i" %
(antennas[i].stand.id, antennas[i].pol,
antennas[i].digitizer, f + 1))
print("-> DP1 boards with missing frames:")
for k in dp1Boards.keys():
v = dp1Boards[k]
print(" %2i %6i (%7.3f%%)" % (k, v, 100.0 * v / (30000 * 10)))
# Check time tags to make sure every ant/pol as the same time as each frame
for f in xrange(timetags.shape[1]):
## For each frame count value, get the median time tag and use this for comparison.
## If things are really bad, we will get a lot of errors.
frameTime = numpy.median(timetags[:, f])
## Compare all of the antpols at a particular frame count, ignoring the ones that
## are missing.
missing = numpy.where((timetags[:, f] != frameTime)
& (timetags[:, f] >= 0))[0]
## Report any errors
for m in missing:
print("ERROR: t.t. %i @ frame %i != frame median of %i" %
(timetags[m, f], f + 1, frameTime))
print(" -> difference: %i" % (timetags[m, f] - frameTime, ))
# Check time tags to make sure the times increment correctly between frames
for i in xrange(timetags.shape[0]):
for f in xrange(1, timetags.shape[1]):
## Skip missing frames since they always fail
if timetags[i, f] < 0 or timetags[i, f - 1] < 0:
continue
## Compare the current time tag with previous and report an error if there
## is a discrepancy between the two modulo the expected skip.
if timetags[i, f] > (timetags[i, f - 1] + nSamples):
## Too far into the future
print("ERROR: t.t. %i @ frame %i > t.t. %i @ frame %i + skip" %
(timetags[i, f], f + 1, timetags[i, f - 1], f))
print(" -> difference: %i" %
(timetags[i, f] - timetags[i, f - 1], ))
elif timetags[i, f] < (timetags[i, f - 1] + nSamples):
## Not far enough into the future
print("ERROR: t.t. %i @ frame %i < t.t. %i @ frame %i + skip" %
(timetags[i, f], f + 1, timetags[i, f - 1], f))
print(" -> difference: %i" %
(timetags[i, f] - timetags[i, f - 1], ))
else:
## Everything is good if we make it here
pass
def main(args):
# Set the station
if args.metadata is not None:
station = stations.parse_ssmif(args.metadata)
ssmifContents = open(args.metadata).readlines()
else:
station = stations.lwa1
ssmifContents = open(os.path.join(dataPath,
'lwa1-ssmif.txt')).readlines()
antennas = station.antennas
toKeep = []
for g in (1, 10, 54, 248, 251, 258):
for i, ant in enumerate(antennas):
if ant.stand.id == g and ant.pol == 0:
toKeep.append(i)
for i, j in enumerate(toKeep):
print(i, j, antennas[j].stand.id)
# Length of the FFT
LFFT = args.fft_length
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped
maxFrames = int((30000 * 260) / float(LFFT)) * LFFT
# It seems like that would be a good idea, however... TBW data comes one
# capture at a time so doing something like this actually truncates data
# from the last set of stands for the first integration. So, we really
# should stick with
maxFrames = (30000 * 260)
fh = open(args.filename, "rb")
nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
antpols = len(antennas)
nChunks = int(math.ceil(1.0 * nFrames / maxFrames))
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginTime = junkFrame.time
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("Chunks: %i" % nChunks)
print("===")
nChunks = 1
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2 * tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Master loop over all of the file chunks
masterSpectra = numpy.zeros((nChunks, antpols, LFFT))
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
((i + 1), framesRemaining))
data = numpy.zeros((12, 12000000), dtype=numpy.int16)
# If there are fewer frames than we need to fill an FFT, skip this chunk
if data.shape[1] < 2 * LFFT:
break
# Inner loop that actually reads the frames into the data array
for j in range(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbw.FRAME_SIZE-1))
continue
if not cFrame.header.is_tbw:
continue
stand = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1)
#if cFrame.header.frame_count % 10000 == 0 and config['verbose']:
#print("%3i -> %3i %6.3f %5i %i" % (stand, aStand, cFrame.time, cFrame.header.frame_count, cFrame.payload.timetag))
# Actually load the data. x pol goes into the even numbers, y pol into the
# odd numbers
count = cFrame.header.frame_count - 1
if aStand not in toKeep:
continue
# Convert to reduced index
aStand = 2 * toKeep.index(aStand)
data[aStand, count * nSamples:(count + 1) *
nSamples] = cFrame.payload.data[0, :]
data[aStand + 1, count * nSamples:(count + 1) *
nSamples] = cFrame.payload.data[1, :]
# Time series analysis - mean, std. dev, saturation count
tsMean = data.mean(axis=1)
tsStd = data.std(axis=1)
tsSat = numpy.where((data == 2047) | (data == -2047), 1, 0).sum(axis=1)
# Time series analysis - percentiles
p = [50, 75, 90, 95, 99]
tsPct = numpy.zeros((data.shape[0], len(p)))
for i in xrange(len(p)):
for j in xrange(data.shape[0]):
tsPct[j, i] = percentile(numpy.abs(data[j, :]), p[i])
# Frequency domain analysis - spectra
freq = numpy.fft.fftfreq(2 * args.fft_length, d=1.0 / 196e6)
freq = freq[:args.fft_length]
delays = numpy.zeros((data.shape[0], freq.size))
signalsF, validF = FEngine(data,
freq,
delays,
LFFT=args.fft_length,
Overlap=1,
sample_rate=196e6,
clip_level=0)
# Cleanup to save memory
del validF, data
print(signalsF.shape)
# SK control values
skM = signalsF.shape[2]
skN = 1
# Frequency domain analysis - spectral kurtosis
k = numpy.zeros((signalsF.shape[0], signalsF.shape[1]))
for l in xrange(signalsF.shape[0]):
for m in xrange(freq.size):
k[l, m] = kurtosis.spectral_fft(signalsF[l, m, :])
kl, kh = kurtosis.get_limits(4, skM, skN)
print(kl, kh)
# Integrate the spectra for as long as we can
masterSpectra = (numpy.abs(signalsF)**2).mean(axis=2)
del signalsF
# Mask out bad values (high spectral kurtosis) for the plot
mask = numpy.where((k < kl) | (k > kh), 1, 0)
mask = expandMask(mask, radius=4, merge=True)
masterSpectra = numpy.ma.array(masterSpectra, mask=mask)
# Save the data to an HDF5 file
outname = os.path.splitext(args.filename)[0]
outname = "%s-RFI.hdf5" % outname
f = h5py.File(outname, 'w')
f.attrs['filename'] = args.filename
f.attrs['mode'] = 'TBW'
f.attrs['station'] = 'LWA-1'
f.attrs['dataBits'] = dataBits
f.attrs['startTime'] = beginTime
f.attrs['startTime_units'] = 's'
f.attrs['startTime_sys'] = 'unix'
f.attrs['sample_rate'] = 196e6
f.attrs['sample_rate_units'] = 'Hz'
f.attrs['RBW'] = freq[1] - freq[0]
f.attrs['RBW_Units'] = 'Hz'
f.attrs['SK-M'] = skM
f.attrs['SK-N'] = skN
for l in xrange(len(toKeep)):
antX = antennas[toKeep[l]]
antY = antennas[toKeep[l] + 1]
stand = f.create_group('Stand%03i' % antX.stand.id)
stand['freq'] = freq
stand['freq'].attrs['Units'] = 'Hz'
polX = stand.create_group('X')
polY = stand.create_group('Y')
polX.attrs['tsMean'] = tsMean[2 * l]
polY.attrs['tsMean'] = tsMean[2 * l + 1]
polX.attrs['tsStd'] = tsStd[2 * l]
polY.attrs['tsStd'] = tsStd[2 * l + 1]
polX.attrs['tsSat'] = tsSat[2 * l]
polY.attrs['tsSat'] = tsSat[2 * l + 1]
for i, v in enumerate(p):
polX.attrs['ts%02i' % v] = tsPct[2 * l][i]
polY.attrs['ts%02i' % v] = tsPct[2 * l + 1][i]
polX['spectrum'] = masterSpectra[2 * l, :]
polX['spectrum'].attrs['axis0'] = 'frequency'
polY['spectrum'] = masterSpectra[2 * l + 1, :]
polY['spectrum'].attrs['axis0'] = 'frequency'
polX['kurtosis'] = k[2 * l, :]
polX['kurtosis'].attrs['axis0'] = 'frequency'
polY['kurtosis'] = k[2 * l + 1, :]
polY['kurtosis'].attrs['axis0'] = 'frequency'
# The plot
fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
ax2 = fig.add_subplot(2, 1, 2)
for l in xrange(k.shape[0]):
ant = antennas[toKeep[l / 2]]
ax1.plot(freq / 1e6,
numpy.log10(masterSpectra[l, :]) * 10,
label='Stand %i, Pol %i' %
(ant.stand.id, ant.pol + l % 2))
ax2.plot(freq / 1e6,
k[l, :],
label='Stand %i, Pol %i' %
(ant.stand.id, ant.pol + l % 2))
ax2.hlines(kl,
freq[0] / 1e6,
freq[-1] / 1e6,
linestyle=':',
label='Kurtosis Limit 4$\sigma$')
ax2.hlines(kh,
freq[0] / 1e6,
freq[-1] / 1e6,
linestyle=':',
label='Kurtosis Limit 4$\sigma$')
ax1.set_xlabel('Frequency [MHz]')
ax1.set_ylabel('PSD [arb. dB/RBW]')
ax1.legend(loc=0)
ax2.set_ylim((kl / 2, kh * 2))
ax2.set_xlabel('Frequency [MHz]')
ax2.set_ylabel('Spectral Kurtosis')
ax2.legend(loc=0)
plt.show()
def main(args):
filename = args[0]
stands = [int(i) for i in args[1:]]
antennas = lwa1.antennas
fh = open(filename, "rb")
nFrames = os.path.getsize(filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
antpols = len(antennas)
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginTime = junkFrame.time
beginDate = junkFrame.time.datetime
# Figure out which digitizers to keep
toKeep = []
for a in antennas:
if a.stand.id in stands:
toKeep.append( a.digitizer )
# File summary
print("Filename: %s" % filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("===")
print("Keeping Stands:")
for a in toKeep:
print(" Stand #%3i, pol %i (digitizer %3i)" % (antennas[a-1].stand.id, antennas[a-1].pol, antennas[a-1].digitizer))
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2*tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Create the HDF5 file
outname = os.path.splitext(filename)[0]
outname = "%shdf5" % outname
f = h5py.File(outname, 'w')
f.attrs['filename'] = filename
f.attrs['mode'] = 'TBW'
f.attrs['station'] = 'LWA-1'
f.attrs['dataBits'] = dataBits
f.attrs['startTime'] = beginTime
f.attrs['startTime_units'] = 's'
f.attrs['startTime_sys'] = 'unix'
f.attrs['sample_rate'] = 196e6
f.attrs['sample_rate_units'] = 'Hz'
## Create the digitizer to dataset lookup table and the
standLookup = {}
standData = []
i = 0
for a in toKeep:
if a % 2 == 0:
continue
s = antennas[a-1].stand.id
standLookup[a] = i
temp = f.create_group('Stand%03i' % s)
### Combined status code
temp.attrs['statusCode'] = antennas[a-1].combined_status
### Antenna number
temp.attrs['antennaID'] = antennas[a-1].id
### Cable information
temp.attrs['cableID'] = antennas[a-1].cable.id
temp.attrs['cableLength'] = antennas[a-1].cable.length
temp.attrs['cableLength_units'] = 'm'
### Stand location information
temp.attrs['posX'] = antennas[a-1].stand.x
temp.attrs['posX_units'] = 'm'
temp.attrs['posY'] = antennas[a-1].stand.y
temp.attrs['posY_units'] = 'm'
temp.attrs['posZ'] = antennas[a-1].stand.z
temp.attrs['posZ_units'] = 'm'
### Time series data sets
temp.attrs['axis0'] = 'time'
xpol = temp.create_dataset('X', (12000000,), 'i2', chunks=True)
ypol = temp.create_dataset('Y', (12000000,), 'i2', chunks=True)
standData.append( (xpol, ypol, temp) )
i += 1
# Go!
while True:
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbw.FRAME_SIZE-1))
continue
if not cFrame.header.is_tbw:
continue
# Get the DP "stand" ID and the digitizer number
stand = cFrame.header.id
aStand = 2*(stand-1)
digitizer = aStand + 1
# If we don't need it, skip it
if digitizer not in toKeep:
continue
# Actually load the data.
## Frame count
count = cFrame.header.frame_count - 1
## Which data set
dataset = standLookup[digitizer]
## Load
standData[dataset][0][count*nSamples:(count+1)*nSamples] = cFrame.payload.data[0,:]
standData[dataset][1][count*nSamples:(count+1)*nSamples] = cFrame.payload.data[1,:]
fh.close()
f.close()
def main(args):
# The task at hand
clnfile = args[0]
az = float(args[1])
el = float(args[2])
filename = args[3]
# The station
observer = lwa1.get_observer()
antennas = lwa1.antennas
# The file's parameters
fh = open(filename, 'rb')
nFramesFile = os.path.getsize(filename) / tbn.FRAME_SIZE
srate = tbn.get_sample_rate(fh)
antpols = len(antennas)
# Reference antenna
ref = 258
for a in antennas:
if a.stand.id == ref and a.pol == 0:
refX = a.digitizer
elif a.stand.id == ref and a.pol == 1:
refY = a.digitizer
else:
pass
# Integration time (seconds and frames)
tInt = 5.0
nFrames = int(round(tInt * srate / 512 * antpols))
tInt = nFrames / antpols * 512 / srate
# Total run length
#nChunks = int(round(1.0*nFramesFile / nFrames))
nChunks = 240
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbn.read_frame(fh)
fh.seek(-tbn.FRAME_SIZE, 1)
startFC = junkFrame.header.frame_count
central_freq = junkFrame.central_freq
beginDate = junkFrame.time.datetime
observer.date = beginDate
srcs = []
for line in _srcs:
srcs.append(ephem.readdb(line))
srcs[-1].compute(observer)
if srcs[-1].alt > 0:
print("source %s: alt %.1f degrees, az %.1f degrees" %
(srcs[-1].name, srcs[-1].alt * 180 / numpy.pi,
srcs[-1].az * 180 / numpy.pi))
# File summary
print("Filename: %s" % filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz" % central_freq)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate))
print("---")
print("Integration: %.3f s (%i frames; %i frames per stand/pol)" %
(tInt, nFrames, nFrames / antpols))
print("Chunks: %i" % nChunks)
junkFrame = tbn.read_frame(fh)
while junkFrame.header.frame_count < startFC + 3:
junkFrame = tbn.read_frame(fh)
fh.seek(-tbn.FRAME_SIZE, 1)
# Get the beamformer coefficients - three sets:
# (1) at the requested az, el
# (2) at az, el - 15 degrees
# (3) at the transit location of Cyg A
dataDict = numpy.load(clnfile)
cln = dataDict['cln']
aln1 = []
aln2 = []
aln3 = []
for i in xrange(cln.shape[1]):
gd = getGeoDelay(antennas[i], az, el, central_freq, Degrees=True)
aln1.append(numpy.exp(2j * numpy.pi * central_freq * gd))
gd = getGeoDelay(antennas[i], az, el - 15, central_freq, Degrees=True)
aln2.append(numpy.exp(2j * numpy.pi * central_freq * gd))
gd = getGeoDelay(antennas[i], 0.5, 83.3, central_freq, Degrees=True)
aln3.append(numpy.exp(2j * numpy.pi * central_freq * gd))
aln1 = numpy.array(aln1)
aln2 = numpy.array(aln2)
aln3 = numpy.array(aln3)
bln1 = (cln * aln1).conj() / numpy.abs(cln * aln1)
bln2 = (cln * aln2).conj() / numpy.abs(cln * aln2)
bln3 = (cln * aln3).conj() / numpy.abs(cln * aln3)
for i in xrange(cln.shape[1]):
if antennas[i].combined_status != 33 or antennas[i].stand.id == ref:
bln1[:, i] = 0.0
bln2[:, i] = 0.0
bln3[:, i] = 0.0
# Create the FrameBuffer instance
buffer = TBNFrameBuffer(stands=range(1, antpols / 2 + 1),
pols=[0, 1],
reorder=False)
# Create the beam
times = numpy.zeros(nChunks, dtype=numpy.float64)
beam1 = numpy.zeros((nChunks, 2), dtype=numpy.float64)
beam2 = numpy.zeros((nChunks, 2), dtype=numpy.float64)
beam3 = numpy.zeros((nChunks, 2), dtype=numpy.float64)
# Go!
k = 0
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFramesFile - k
if framesRemaining > nFrames:
framesWork = nFrames
data = numpy.zeros((antpols, framesWork / antpols * 512),
dtype=numpy.complex64)
else:
framesWork = framesRemaining + antpols * buffer.nsegments
data = numpy.zeros((antpols, framesWork / antpols * 512),
dtype=numpy.complex64)
print("Working on chunk %i, %i frames remaining" %
(i + 1, framesRemaining))
count = [0 for a in xrange(antpols)]
j = 0
fillsWork = framesWork / antpols
# Inner loop that actually reads the frames into the data array
while j < fillsWork:
try:
cFrame = tbn.read_frame(fh)
k = k + 1
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbn.FRAME_SIZE-1))
continue
buffer.append(cFrame)
cFrames = buffer.get()
if cFrames is None:
continue
valid = sum(lambda x, y: x + int(y.valid), cFrames, 0)
if valid != antpols:
print(
"WARNING: frame count %i at %i missing %.2f%% of frames" %
(cFrames[0].header.frame_count, cFrames[0].payload.timetag,
float(antpols - valid) / antpols * 100))
continue
for cFrame in cFrames:
stand, pol = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 260. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1) + pol
# Save the time
if j == 0 and aStand == 0:
times[i] = cFrame.time
data[aStand, count[aStand] * 512:(count[aStand] + 1) *
512] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] = count[aStand] + 1
j += 1
# Mask
bad = numpy.where(numpy.abs(data) >= 90)
data[bad] = 0.0
# Beam forming
taskPool = Pool(processes=6)
taskList = []
taskList.append(
(i, 1, 0,
taskPool.apply_async(form_beam,
args=(data[0::2, :], bln1[0, 0::2]))))
taskList.append(
(i, 1, 1,
taskPool.apply_async(form_beam,
args=(data[1::2, :], bln1[0, 1::2]))))
taskList.append(
(i, 2, 0,
taskPool.apply_async(form_beam,
args=(data[0::2, :], bln2[0, 0::2]))))
taskList.append(
(i, 2, 1,
taskPool.apply_async(form_beam,
args=(data[1::2, :], bln2[0, 1::2]))))
taskList.append(
(i, 3, 0,
taskPool.apply_async(form_beam,
args=(data[0::2, :], bln3[0, 0::2]))))
taskList.append(
(i, 3, 1,
taskPool.apply_async(form_beam,
args=(data[1::2, :], bln3[0, 1::2]))))
taskPool.close()
taskPool.join()
for i, b, p, task in taskList:
if b == 1:
beam1[i, p] = task.get()
elif b == 2:
beam2[i, p] = task.get()
else:
beam3[i, p] = task.get()
print('1', beam1[i, 0], '2', beam2[i, 0], '3', beam3[i, 0], '1/2',
beam1[i, 0] / beam2[i, 0], '3/2', beam3[i, 0] / beam2[i, 0])
del data
# Plot the data
print('CygA :', beam1[:, 0])
print('Pointing 2:', beam2[:, 0])
print('Pointing 1:', beam3[:, 0])
fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 2)
ax1.plot(times - times[0], beam1[:, 0])
ax1.plot(times - times[0], beam2[:, 0])
ax1.plot(times - times[0], beam3[:, 0])
ax2.plot(times - times[0], beam1[:, 1])
ax2.plot(times - times[0], beam2[:, 1])
ax2.plot(times - times[0], beam3[:, 1])
plt.show()
def main(args):
# Set the station
if args.lwasv:
station = stations.lwasv
else:
station = stations.lwa1
antennas = station.antennas
fh = open(args.filename, "rb")
nFramesFile = os.path.getsize(args.filename) // tbn.FRAME_SIZE
srate = tbn.get_sample_rate(fh)
#antpols = tbn.get_frames_per_obs(fh)
antpols = len(antennas)
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(args.skip * srate / 512 * antpols)
offset = int(1.0 * offset / antpols) * antpols
args.skip = 1.0 * offset / antpols * 512 / srate
fh.seek(offset * tbn.FRAME_SIZE)
# Number of frames to integrate over
nFrames = int(args.average * srate / 512 * antpols)
args.average = 1.0 * nFrames / antpols * 512 / srate
# Number of remaining chunks
nChunks = int(math.ceil(1.0 * (nFrames) / (200 * 520)))
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbn.read_frame(fh)
fh.seek(0)
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Rate: %i Hz" % srate)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Integration: %.3f s (%i frames; %i frames per stand/pol)" %
(args.average, nFrames, nFrames / antpols))
print("Chunks: %i" % nChunks)
# Sanity check
if offset > nFramesFile:
raise RuntimeError("Requested offset is greater than file length")
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
# Create the FrameBuffer instance
buffer = TBNFrameBuffer(stands=range(1, antpols // 2 + 1), pols=[0, 1])
# Master loop over all of the file chunks
masterCount = [0 for a in xrange(len(antennas))]
# Missing packet control variables
missingPackets = numpy.ones((antpols, 2048), dtype=numpy.int8)
pc = 0
missing = 0
missingList = []
# Figure
fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 2)
k = 0
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time ((200*520)),
# only deal with that chunk
framesRemaining = nFrames - k
if framesRemaining > (200 * 520):
framesWork = (200 * 520)
else:
framesWork = framesRemaining
count = [0 for a in xrange(len(antennas))]
j = 0
fillsWork = framesWork // antpols
# Inner loop that actually reads the frames into the data array
while j < fillsWork:
try:
cFrame = tbn.read_frame(fh)
k = k + 1
except errors.EOFError:
break
except errors.SyncError:
print("WARNING: Mark 5C sync error on frame #%i" %
(int(fh.tell()) / tbn.FRAME_SIZE - 1))
continue
#print(cFrame.header.frame_count, cFrame.payload.timetag, cFrame.id)
buffer.append(cFrame)
cFrames = buffer.get()
if cFrames is None:
continue
valid = sum(lambda x, y: x + int(y.valid), cFrames, 0)
print("Frame #%5i: %.4f seconds with %i valid ant/pols%s" %
(cFrames[0].header.frame_count, cFrames[0].time, valid,
'!' if valid != antpols else ''))
if valid != antpols:
bad = []
for cFrame in cFrames:
if not cFrame.valid:
bad.append(cFrame.id)
missingPackets[2 * (bad[-1][0] - 1) + bad[-1][1],
pc] = 0
bad.sort()
pc += 1
if pc == missingPackets.shape[1]:
plotMissing(ax1, ax2, missingPackets, missingList, antpols)
plt.show()
sys.exit(0)
missing += (antpols - valid)
total = (buffer.full + buffer.partial) * antpols
#print(j, valid, antpols-valid, cFrames[0].header.frame_count, 1.0*missing / total* 100, bad[0], bad[-1], buffer.dropped)
#print(buffer.status())
missingList.append(antpols - valid)
else:
total = (buffer.full + buffer.partial) * antpols
missingList.append(0)
times = numpy.array([f.payload.timetag for f in cFrames],
dtype=numpy.int64)
#print(cFrames[0].header.frame_count, times.min(), times.max(), times.max()-times.min(), "%6.3f%%" % (1.0*missing/total*100,))
for cFrame in cFrames:
stand, pol = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 260. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1) + pol
# Update the counters so that we can average properly later on
count[aStand] = count[aStand] + 1
masterCount[aStand] = masterCount[aStand] + 1
j += 1
# Empty the remaining portion of the buffer and integrate what's left
for cFrames in buffer.flush():
valid = sum(lambda x, y: x + int(y.valid), cFrames, 0)
print("Frame #%5i: %.4f seconds with %i valid ant/pols" %
(cFrames[0].header.frame_count, cFrames[0].time, valid))
if valid != antpols:
bad = []
for cFrame in cFrames:
if not cFrame.valid:
bad.append(cFrame.id)
missingPackets[2 * (bad[-1][0] - 1) + bad[-1][1], pc] = 0
bad.sort()
pc += 1
if pc == missingPackets.shape[1]:
plotMissing(ax1, ax2, missingPackets, missingList, antpols)
plt.show()
sys.exit(0)
missing += (antpols - valid)
total = (buffer.full + buffer.partial) * antpols
#print(j, valid, antpols-valid, cFrames[0].header.frame_count, 1.0*missing / total* 100, bad[0], bad[-1], buffer.dropped)
#print(buffer.status())
missingList.append(antpols - valid)
else:
total = (buffer.full + buffer.partial) * antpols
missingList.append(0)
# Inner loop that actually reads the frames into the data array
for cFrame in cFrames:
stand, pol = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1) + pol
# Update the counters so that we can average properly later on
count[aStand] = count[aStand] + 1
masterCount[aStand] = masterCount[aStand] + 1
j += 1
plotMissing(ax1, ax2, missingPackets, missingList, antpols)
plt.show()
sys.exit(0)
def main(args):
filename = args.filename
# Set the station
if args.lwasv:
station = stations.lwasv
else:
station = stations.lwa1
antennas = station.antennas
fh = open(filename, "rb")
nFramesFile = os.path.getsize(filename) // tbn.FRAME_SIZE
srate = tbn.get_sample_rate(fh)
antpols = len(antennas)
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbn.read_frame(fh)
fh.seek(-tbn.FRAME_SIZE, 1)
central_freq = junkFrame.central_freq
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz" % central_freq)
print(" ")
# Convert chunk length to total frame count
chunkLength = int(args.length * srate / 512 * antpols)
chunkLength = int(1.0 * chunkLength / antpols) * antpols
# Convert chunk skip to total frame count
chunkSkip = int(args.skip * srate / 512 * antpols)
chunkSkip = int(1.0 * chunkSkip / antpols) * antpols
# Create the FrameBuffer instance
buffer = TBNFrameBuffer(stands=range(1,antpols//2+1), pols=[0, 1])
# Output arrays
clipFraction = []
meanPower = []
# Find stands #10
toUse = []
for i in xrange(antpols):
ant = antennas[i]
if ant.stand.id == 10:
toUse.append(i)
# Go!
i = 1
done = False
print(" | Clipping | Power |")
print(" | 10X 10Y | 10X 10Y |")
print("---+-----------------+-------------------+")
while True:
count = [0 for j in xrange(antpols)]
data = numpy.zeros((antpols, chunkLength*512//antpols), dtype=numpy.csingle)
for j in xrange(chunkLength):
try:
cFrame = tbn.read_frame(fh)
except errors.EOFError:
done = True
break
except errors.SyncError:
continue
buffer.append(cFrame)
cFrames = buffer.get()
if cFrames is None:
continue
for cFrame in cFrames:
stand,pol = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 260. So, we
# can use this little trick to populate the data array
aStand = 2*(stand-1)+pol
try:
data[aStand, count[aStand]*512:(count[aStand]+1)*512] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] = count[aStand] + 1
except ValueError:
pass
# Empty the remaining portion of the buffer
for cFrames in buffer.flush():
# Inner loop that actually reads the frames into the data array
for cFrame in cFrames:
stand,pol = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2*(stand-1)+pol
try:
data[aStand, count[aStand]*512:(count[aStand]+1)*512] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] = count[aStand] + 1
except ValueError:
pass
if done:
break
else:
data = numpy.abs(data)**2
data = data.astype(numpy.int32)
clipFraction.append( numpy.zeros(antpols) )
meanPower.append( data.mean(axis=1) )
for j in xrange(antpols):
bad = numpy.nonzero(data[j,:] > args.trim_level)[0]
clipFraction[-1][j] = 1.0*len(bad) / data.shape[1]
clip = clipFraction[-1]
power = meanPower[-1]
print("%2i | %6.2f%% %6.2f%% | %8.2f %8.2f |" % (i, clip[toUse[0]]*100.0, clip[toUse[1]]*100.0, power[toUse[0]], power[toUse[1]]))
i += 1
fh.seek(tbn.FRAME_SIZE*chunkSkip, 1)
clipFraction = numpy.array(clipFraction)
meanPower = numpy.array(meanPower)
clip = clipFraction.mean(axis=0)
power = meanPower.mean(axis=0)
print("---+-----------------+-------------------+")
print("%2s | %6.2f%% %6.2f%% | %8.2f %8.2f |" % ('M', clip[toUse[0]]*100.0, clip[toUse[1]]*100.0, power[toUse[0]], power[toUse[1]]))
def main(args):
# Set the station
if args.metadata is not None:
station = stations.parse_ssmif(args.metadata)
else:
station = stations.lwa1
antennas = station.antennas
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped
maxFrames = int((30000 * 260))
# It seems like that would be a good idea, however... TBW data comes one
# capture at a time so doing something like this actually truncates data
# from the last set of stands for the first integration. So, we really
# should stick with
maxFrames = (30000 * 260)
fh = open(args.filename, "rb")
nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
antpols = len(antennas)
nChunks = int(math.ceil(1.0 * nFrames / maxFrames))
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("Chunks: %i" % nChunks)
print("===")
nChunks = 1
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2 * tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Master loop over all of the file chunks
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
((i + 1), framesRemaining))
#
# NOTE
# Major change here from tbwSpectra.py/stationMaster.py. We are only keeping
# the first 30,000 frames of the TBW file since we don't really need to read
# in all of it to find bursts
#
data = numpy.zeros((antpols, 30000 * nSamples), dtype=numpy.int16)
# Inner loop that actually reads the frames into the data array
for j in range(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
print("WARNING: Mark 5C sync error on frame #%i" %
(int(fh.tell()) // tbw.FRAME_SIZE - 1))
continue
if not cFrame.header.is_tbw:
continue
# Skip frames over 30,000 on all stands
if cFrame.header.frame_count > 30000:
continue
stand = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1)
if cFrame.header.frame_count % 5000 == 0 and args.verbose:
print("%3i -> %3i %6.3f %5i %i" %
(stand, aStand, cFrame.time, cFrame.header.frame_count,
cFrame.payload.timetag))
# Actually load the data. x pol goes into the even numbers, y pol into the
# odd numbers
count = cFrame.header.frame_count - 1
data[aStand, count * nSamples:(count + 1) *
nSamples] = cFrame.payload.data[0, :]
data[aStand + 1, count * nSamples:(count + 1) *
nSamples] = cFrame.payload.data[1, :]
# Compute the power
data = numpy.abs(data)
# We need to various time series data to be aligned so we need to do a
# correction for the cable delays. Using the various Antenna instances,
# we create a array of delays (in seconds) and do everything relative to
# the longest delay.
#
# After this, we can align the data from all of the antennas.
delays = numpy.array([a.cable.delay(30e6) for a in antennas])
delays = numpy.round(delays * fS).astype(numpy.int32)
delays = delays.max() - delays
alignedData = numpy.zeros(
(data.shape[0], data.shape[1] - delays.max()), dtype=data.dtype)
for s in xrange(data.shape[0]):
alignedData[s, :] = data[s, delays[s]:(delays[s] +
alignedData.shape[1])]
del (data)
# Using the good antennas (Antenna.combined_status == 33), we need to find the
# start of the RFI pulses by looking for "large" data values. To make sure
# that we aren't getting stuck on a first partial burst, skip the first one
# and use the second set of "large" data values found.
#
# I was using "large" as saturation/clipping (>= 2047), but the new lower
# value for the ARX gain makes me want to lower to something more like
#
inOne = False
first = 0
status = numpy.array([ant.combined_status for ant in antennas])
good = numpy.where(status == 33)[0]
while first < alignedData.shape[1]:
mv = alignedData[good, first].max()
if mv >= args.threshold:
if not inOne:
first += 5000
inOne = True
else:
break
else:
first += 1
print("Second burst at %i" % first)
# Keep only what would be interesting (200 samples before and 2,800 samples
# afterward) around the burst. This corresponds to a time range from 1
# microsecond before the start of the pulse to 14 microseconds later. Save
# the aligned data snippet to a NPZ file.
alignedData = alignedData[:, first - 200:first + 2800]
standPos = numpy.array([[ant.stand.x, ant.stand.y, ant.stand.z]
for ant in antennas])
junk, basename = os.path.split(args.filename)
shortname, ext = os.path.splitext(basename)
numpy.savez('%s-burst.npz' % shortname,
data=alignedData,
ssmif=args.metadata)
# Make the movie (if needed)
if args.movie:
if args.verbose:
print("Creating movie frames")
pb = ProgressBar(max=alignedData.shape[1] / 2)
else:
pb = None
fig = plt.figure(figsize=(12, 6))
for i in xrange(0, alignedData.shape[1], 2):
fig.clf()
axX = fig.add_subplot(1, 2, 1)
axY = fig.add_subplot(1, 2, 2)
colorsX = 1.0 * (alignedData[0::2, i] +
alignedData[0::2, i + 1]) / 2
colorsY = 1.0 * (alignedData[1::2, i] +
alignedData[1::2, i + 1]) / 2
axX.scatter(standPos[0::2, 0],
standPos[0::2, 1],
c=colorsX,
s=40.0,
vmin=0,
vmax=args.threshold)
axY.scatter(standPos[1::2, 0],
standPos[1::2, 1],
c=colorsY,
s=40.0,
vmin=0,
vmax=args.threshold)
## Add the fence as a dashed line
axX.plot([-59.827, 59.771, 60.148, -59.700, -59.827],
[59.752, 59.864, -59.618, -59.948, 59.752],
linestyle='--',
color='k')
axY.plot([-59.827, 59.771, 60.148, -59.700, -59.827],
[59.752, 59.864, -59.618, -59.948, 59.752],
linestyle='--',
color='k')
## Add the shelter
axX.plot([55.863, 58.144, 58.062, 55.791, 55.863],
[45.946, 45.999, 51.849, 51.838, 45.946],
linestyle='-',
color='k')
axY.plot([55.863, 58.144, 58.062, 55.791, 55.863],
[45.946, 45.999, 51.849, 51.838, 45.946],
linestyle='-',
color='k')
axX.set_xlim([-65, 65])
axX.set_ylim([-65, 65])
axX.set_title('X pol. $\Delta$t = %.1f ns' %
(1.0 * i / fS * 1e9, ))
axX.set_xlabel('$\Delta$ X [m]')
axX.set_ylabel('$\Delta$ Y [m]')
axY.set_xlim([-65, 65])
axY.set_ylim([-65, 65])
axY.set_title('Y pol.')
axY.set_xlabel('$\Delta$ X [m]')
#axY.set_ylabel('$\Delta$ Y [m]')
fig.savefig('burst-%05i.png' % (i / 2, ))
if pb is not None:
pb.inc(amount=1)
if pb.amount != 0 and pb.amount % 10 == 0:
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
del (fig)
if pb is not None:
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
if args.verbose:
print("Creating movie")
os.system(
"mencoder mf://burst-*.png -mf fps=20:type=png -ovc lavc -lavcopts vcodec=mpeg4:aspect=2/1 -o %s-burst.avi -ffourcc DX50 -vf scale=600:1200,expand=600:1200"
% shortname)
def main(args):
# The task at hand
filename = args.filename
# The station
if args.metadata is not None:
site = parse_ssmif(args.metadata)
ssmifContents = open(args.metadata).readlines()
else:
site = lwa1
ssmifContents = open(os.path.join(dataPath,
'lwa1-ssmif.txt')).readlines()
observer = site.get_observer()
antennas = site.antennas
# The file's parameters
fh = open(filename, 'rb')
nFramesFile = os.path.getsize(filename) // tbn.FRAME_SIZE
srate = tbn.get_sample_rate(fh)
antpols = len(antennas)
fh.seek(0)
if srate < 1000:
fh.seek(len(antennas) * 4 * tbn.FRAME_SIZE)
srate = tbn.get_sample_rate(fh)
antpols = len(antennas)
fh.seek(len(antennas) * 4 * tbn.FRAME_SIZE)
# Reference antenna
ref = args.reference
foundRef = False
for i, a in enumerate(antennas):
if a.stand.id == ref and a.pol == 0:
refX = i
foundRef = True
elif a.stand.id == ref and a.pol == 1:
refY = i
else:
pass
if not foundRef:
raise RuntimeError("Cannot file Stand #%i" % ref)
# Integration time (seconds and frames)
tInt = args.average
nFrames = int(round(tInt * srate / 512 * antpols))
tInt = nFrames / antpols * 512 / srate
# Total run length
nChunks = int(1.0 * nFramesFile / antpols * 512 / srate / tInt)
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbn.read_frame(fh)
fh.seek(-tbn.FRAME_SIZE, 1)
startFC = junkFrame.header.frame_count
try:
central_freq = junkFrame.central_freq
except AttributeError:
from lsl.common.dp import fS
central_freq = fS * junkFrame.header.second_count / 2**32
beginDate = junkFrame.time.datetime
observer.date = beginDate
srcs = [
ephem.Sun(),
]
for line in _srcs:
srcs.append(ephem.readdb(line))
for i in xrange(len(srcs)):
srcs[i].compute(observer)
if srcs[i].alt > 0:
print("source %s: alt %.1f degrees, az %.1f degrees" %
(srcs[i].name, srcs[i].alt * 180 / numpy.pi,
srcs[i].az * 180 / numpy.pi))
# File summary
print("Filename: %s" % filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz" % central_freq)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate))
print("---")
print("Integration: %.3f s (%i frames; %i frames per stand/pol)" %
(tInt, nFrames, nFrames // antpols))
print("Chunks: %i" % nChunks)
# Create the FrameBuffer instance
buffer = TBNFrameBuffer(stands=range(1, antpols // 2 + 1), pols=[0, 1])
# Create the phase average and times
LFFT = 512
times = numpy.zeros(nChunks, dtype=numpy.float64)
simpleVis = numpy.zeros((nChunks, antpols), dtype=numpy.complex64)
central_freqs = numpy.zeros(nChunks, dtype=numpy.float64)
# Go!
k = 0
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFramesFile - k
if framesRemaining > nFrames:
framesWork = nFrames
data = numpy.zeros((antpols, framesWork // antpols * 512),
dtype=numpy.complex64)
else:
framesWork = framesRemaining + antpols * buffer.nsegments
data = numpy.zeros((antpols, framesWork // antpols * 512),
dtype=numpy.complex64)
print("Working on chunk %i, %i frames remaining" %
(i + 1, framesRemaining))
count = [0 for a in xrange(len(antennas))]
j = 0
fillsWork = framesWork // antpols
# Inner loop that actually reads the frames into the data array
done = False
while j < fillsWork:
cFrames = deque()
for l in xrange(len(antennas)):
try:
cFrames.append(tbn.read_frame(fh))
k = k + 1
except errors.EOFError:
## Exit at the EOF
done = True
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbn.FRAME_SIZE-1))
## Exit at the first sync error
done = True
break
buffer.append(cFrames)
cFrames = buffer.get()
if cFrames is None:
continue
for cFrame in cFrames:
stand, pol = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 260. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1) + pol
# Save the time
if j == 0 and aStand == 0:
times[i] = cFrame.time
try:
central_freqs[i] = cFrame.central_freq
except AttributeError:
central_freqs[
i] = fS * cFrame.header.second_count / 2**32
if i > 0:
if central_freqs[i] != central_freqs[i - 1]:
print(
"Frequency change from %.3f to %.3f MHz at chunk %i"
% (central_freqs[i - 1] / 1e6,
central_freqs[i] / 1e6, i + 1))
data[aStand, count[aStand] * 512:(count[aStand] + 1) *
512] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] = count[aStand] + 1
j += 1
if done:
break
if done:
break
# Time-domain blanking and cross-correlation with the outlier
simpleVis[i, :] = fringe.Simple(data, refX, refY, args.clip)
fh.close()
# Save the data
outname = os.path.split(filename)[1]
outname = os.path.splitext(outname)[0]
outname = "%s-ref%03i-multi-vis.npz" % (outname, args.reference)
numpy.savez(outname,
ref=ref,
refX=refX,
refY=refY,
tInt=tInt,
central_freqs=central_freqs,
times=times,
simpleVis=simpleVis,
ssmifContents=ssmifContents)
def main(args):
# Set the station
if args.metadata is not None:
station = stations.parse_ssmif(args.metadata)
ssmifContents = open(args.metadata).readlines()
else:
station = stations.lwa1
ssmifContents = open(os.path.join(dataPath,
'lwa1-ssmif.txt')).readlines()
antennas = station.antennas
# Length of the FFT
LFFT = args.fft_length
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped
maxFrames = int((30000 * 260) / float(LFFT)) * LFFT
# It seems like that would be a good idea, however... TBW data comes one
# capture at a time so doing something like this actually truncates data
# from the last set of stands for the first integration. So, we really
# should stick with
maxFrames = (30000 * 260)
fh = open(args.filename, "rb")
nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE
dataBits = tbw.get_data_bits(fh)
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
antpols = len(antennas)
nChunks = int(math.ceil(1.0 * nFrames / maxFrames))
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
junkFrame = tbw.read_frame(fh)
fh.seek(0)
beginDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("Chunks: %i" % nChunks)
print("===")
nChunks = 1
# Skip over any non-TBW frames at the beginning of the file
i = 0
junkFrame = tbw.read_frame(fh)
while not junkFrame.header.is_tbw:
try:
junkFrame = tbw.read_frame(fh)
except errors.SyncError:
fh.seek(0)
while True:
try:
junkFrame = tbn.read_frame(fh)
i += 1
except errors.SyncError:
break
fh.seek(-2 * tbn.FRAME_SIZE, 1)
junkFrame = tbw.read_frame(fh)
i += 1
fh.seek(-tbw.FRAME_SIZE, 1)
print("Skipped %i non-TBW frames at the beginning of the file" % i)
# Setup the window function to use
if args.pfb:
window = fxc.null_window
elif args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
base, ext = os.path.splitext(args.filename)
base = os.path.basename(base)
if (not os.path.exists("%s.npz" % base)) or args.force:
# Master loop over all of the file chunks
masterSpectra = numpy.zeros((nChunks, antpols, LFFT))
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
((i + 1), framesRemaining))
data = numpy.memmap('temp.mmap',
dtype=numpy.int16,
mode='w+',
shape=(antpols,
2 * 30000 * 260 * nSamples // antpols))
# If there are fewer frames than we need to fill an FFT, skip this chunk
if data.shape[1] < 2 * LFFT:
break
# Inner loop that actually reads the frames into the data array
for j in range(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = tbw.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
print("WARNING: Mark 5C sync error on frame #%i" %
(int(fh.tell()) // tbw.FRAME_SIZE - 1))
continue
if not cFrame.header.is_tbw:
continue
stand = cFrame.header.id
# In the current configuration, stands start at 1 and go up to 10. So, we
# can use this little trick to populate the data array
aStand = 2 * (stand - 1)
if cFrame.header.frame_count % 10000 == 0 and args.verbose:
print("%3i -> %3i %6.3f %5i %i" %
(stand, aStand, cFrame.time,
cFrame.header.frame_count, cFrame.payload.timetag))
# Actually load the data. x pol goes into the even numbers, y pol into the
# odd numbers
count = cFrame.header.frame_count - 1
data[aStand, count * nSamples:(count + 1) *
nSamples] = 1 * cFrame.payload.data[0, :]
data[aStand + 1, count * nSamples:(count + 1) *
nSamples] = 1 * cFrame.payload.data[1, :]
del cFrame
# Calculate the spectra for this block of data and then weight the results by
# the total number of frames read. This is needed to keep the averages correct.
# NB: The weighting is the same for the x and y polarizations because of how
# the data are packed in TBW
for j in xrange(0, masterSpectra.shape[1], 4):
tempData = numpy.zeros((4, data.shape[1]), dtype=data.dtype)
tempData = data[j:j + 4, :]
freq, tempSpec = fxc.SpecMaster(tempData,
LFFT=LFFT,
window=window,
verbose=args.verbose,
clip_level=args.clip_level)
masterSpectra[i, j:j + 4, :] = tempSpec
# Compute the 1 ms average power and the data range within each 1 ms window
subSize = 1960
nsegments = data.shape[1] // subSize
print(
"Computing average power and data range in %i-sample intervals"
% subSize)
pb = ProgressBar(max=data.shape[0])
avgPower = numpy.zeros((antpols, nsegments), dtype=numpy.float32)
dataRange = numpy.zeros((antpols, nsegments, 3), dtype=numpy.int16)
for s in xrange(data.shape[0]):
for p in xrange(nsegments):
subData = data[s, (p * subSize):((p + 1) * subSize)]
avgPower[s, p] = numpy.mean(numpy.abs(subData))
dataRange[s, p, 0] = subData.min()
dataRange[s, p, 1] = subData.mean()
dataRange[s, p, 2] = subData.max()
### This little block here looks for likely saturation events and save
### the raw time series around them into individual NPZ files for stand
### number 14.
#if (dataRange[s,p,0] < -1000 or dataRange[s,p,0] > 1000) and antennas[s].stand.id == 14:
#subData = data[s,((p-1)*1960):((p+2)*1960)]
#satFileName = 'stand-14-pol-%i-%i.npz' % (antennas[s].pol, (p-1)*1960)
#print(satFileName)
#numpy.savez(satFileName, start=(p-1)*1960, data=subData)
pb.inc(amount=1)
if pb.amount != 0 and pb.amount % 10 == 0:
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
# We don't really need the data array anymore, so delete it
del (data)
os.unlink('temp.mmap')
# Apply the cable loss corrections, if requested
if True:
for s in xrange(masterSpectra.shape[1]):
currGain = antennas[s].cable.gain(freq)
for c in xrange(masterSpectra.shape[0]):
masterSpectra[c, s, :] /= currGain
# Now that we have read through all of the chunks, perform the final averaging by
# dividing by all of the chunks
spec = masterSpectra.mean(axis=0)
# Estimate the dipole resonance frequencies
print("Computing dipole resonance frequencies")
pb = ProgressBar(max=spec.shape[0])
resFreq = numpy.zeros(spec.shape[0])
toCompare = numpy.where((freq > 31e6) & (freq < 70e6))[0]
for i in xrange(spec.shape[0]):
bestOrder = 0
bestRMS = 1e34
for j in xrange(3, 12):
coeff = numpy.polyfit(freq[toCompare] / 1e6,
numpy.log10(spec[i, toCompare]) * 10, j)
fit = numpy.polyval(coeff, freq[toCompare] / 1e6)
rms = ((fit - numpy.log10(spec[i, toCompare]) * 10)**2).sum()
if rms < bestRMS:
bestOrder = j
bestRMS = rms
coeff = numpy.polyfit(freq[toCompare] / 1e6,
numpy.log10(spec[i, toCompare]) * 10,
bestOrder)
fit = numpy.polyval(coeff, freq[toCompare] / 1e6)
try:
resFreq[i] = freq[toCompare[numpy.where(
fit == fit.max())[0][0]]] / 1e6
except:
pass
pb.inc(amount=1)
if pb.amount != 0 and pb.amount % 10 == 0:
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
numpy.savez("%s.npz" % base,
date=str(beginDate),
freq=freq,
masterSpectra=masterSpectra,
resFreq=resFreq,
avgPower=avgPower,
dataRange=dataRange,
ssmifContents=ssmifContents)
else:
dataDict = numpy.load("%s.npz" % base)
freq = dataDict['freq']
masterSpectra = dataDict['masterSpectra']
resFreq = dataDict['resFreq']
# Now that we have read through all of the chunks, perform the final averaging by
# dividing by all of the chunks
spec = masterSpectra.mean(axis=0)
# Create a good template spectra
specTemplate = numpy.median(spec, axis=0)
specDiff = numpy.zeros(spec.shape[0])
toCompare = numpy.where((freq > 32e6) & (freq < 50e6))[0]
print(len(toCompare))
for i in xrange(spec.shape[0]):
specDiff[i] = (spec[i, toCompare] / specTemplate[toCompare]).mean()
specDiff = numpy.where(specDiff < 2, specDiff, 2)
# Get the station
standPos = numpy.array([[ant.stand.x, ant.stand.y, ant.stand.z]
for ant in antennas if ant.pol == 0])
# Plots
if args.verbose:
fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1)
ax1.scatter(standPos[:, 0],
standPos[:, 1],
c=specDiff[0::2],
s=40.0,
alpha=0.50)
## Add the fence as a dashed line
ax1.plot([-59.827, 59.771, 60.148, -59.700, -59.827],
[59.752, 59.864, -59.618, -59.948, 59.752],
linestyle='--',
color='k')
## Add the shelter
ax1.plot([55.863, 58.144, 58.062, 55.791, 55.863],
[45.946, 45.999, 51.849, 51.838, 45.946],
linestyle='-',
color='k')
## Set the limits to just zoom in on the main stations
ax1.set_xlim([-65, 65])
ax1.set_ylim([-65, 65])
ax2 = fig.add_subplot(1, 2, 2)
ax2.plot(freq / 1e6, numpy.log10(specTemplate) * 10, alpha=0.50)
print("RBW: %.1f Hz" % (freq[1] - freq[0]))
plt.show()
def main(args):
# Set the station
if args.lwasv:
station = stations.lwasv
else:
station = stations.lwa1
antennas = station.antennas
fh = open(args.filename, "rb", buffering=tbn.FRAME_SIZE*10000)
# Get the first frame and find out what the firt time tag is, which the
# first frame number is, and what the sample rate it. From the sample
# rate, estimate how the time tag should advance between frames.
junkFrame = tbn.read_frame(fh)
sample_rate = tbn.get_sample_rate(fh)
antpols = len(antennas)
tagSkip = fS // sample_rate * junkFrame.payload.data.shape[0]
fh.seek(0)
# Store the information about the first frame.
prevTime = junkFrame.payload.timetag
prevDate = junkFrame.time.datetime
prevFrame = junkFrame.header.frame_count
# Report on the file
print("Filename: %s" % os.path.basename(args.filename))
print("Date of first frame: %i -> %s" % (prevTime, str(prevDate)))
print("Sample rate: %i Hz" % sample_rate)
print("Time tag skip per frame: %i" % tagSkip)
# Create the FrameBuffer instance
buffer = TBNFrameBuffer(stands=range(1,antpols//2+1), pols=[0, 1])
j = 0
k = 0
while True:
try:
cFrame = tbn.read_frame(fh)
k += 1
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbn.FRAME_SIZE-1))
continue
buffer.append(cFrame)
cFrames = buffer.get()
if cFrames is None:
continue
valid = reduce(lambda x,y: x+int(y.valid), cFrames, 0)
if valid != antpols:
print("WARNING: frame count %i at %i missing %.2f%% of frames" % (cFrames[0].header.frame_count, cFrames[0].payload.timetag, float(antpols - valid)/antpols*100))
timetags = numpy.zeros(len(cFrames), dtype=numpy.int64) - 1
for cFrame in cFrames:
stand,pol = cFrame.id
timetags[2*(stand-1)+pol] = cFrame.payload.timetag
if j == 0:
prevTime = numpy.median(timetags)
prevDate = cFrames[0].time.datetime
prevFrame = cFrames[0].header.frame_count
j += 1
continue
else:
currTime = numpy.median(timetags)
currDate = cFrames[0].time.datetime
currFrame = cFrames[0].header.frame_count
if currFrame % 1000 == 0:
print("At frame %i t.t. is %i -> %s" % (currFrame, currTime, currDate))
if currTime < prevTime:
print("ERROR: t.t. %i @ frame %i < t.t. %i @ frame %i" % (currTime, currFrame, prevTime, prevFrame))
print(" -> difference: %i (%.5f seconds); %s" % (currTime-prevTime, float(currTime-prevTime)/fS, str(currDate)))
if (currTime-prevTime) > tagSkip:
print("ERROR: t.t. %i @ frame %i > t.t. %i @ frame %i + skip" % (currTime, currFrame, prevTime, prevFrame))
print(" -> difference: %i (%.5f seconds); %s" % (currTime-prevTime, float(currTime-prevTime)/fS, str(currDate)))
for i in xrange(timetags.size):
if timetags[i] != currTime:
print("ERROR: t.t. of dig. %i != frame set median of %i" % (i, currTime))
print(" -> difference: %i" % (currTime-timetags[i],))
prevTime = currTime
prevData = currDate
prevFrame = currFrame
j += 1
