def fine_tune_boundary_start(fh, start, max_frames=4, verbose=True):
fh.seek(start)
# Report
if verbose:
print("Tuning boundary at %i of '%s'..." % (start, os.path.basename(fh.name)))
# Align on the start of a Mark5C packet...
while True:
try:
junkFrame = drx.read_frame(fh)
try:
# ... that has a valid decimation
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE+1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
# ... and save that location
frame_begin = junkFrame.payload.timetag
file_begin = fh.tell()
if verbose:
print(" start @ %i with %i" % (file_begin, frame_begin))
# Get how much the timetags should change and other basic information
fh.seek(file_begin)
ids = []
for i in xrange(24*8):
junkFrame = drx.read_frame(fh)
b,t,p = junkFrame.id
id = (t,p)
if id not in ids:
ids.append(id)
ttStep = 4096*junkFrame.header.decimation
if verbose:
print(" %i frames with a timetag step of %i" % (len(ids), ttStep))
# Load in the times to figure out what to do
fh.seek(file_begin)
timetags = []
for i in xrange(max_frames):
junkFrame = drx.read_frame(fh)
timetags.append( junkFrame.payload.timetag )
skips = [timetags[i]-timetags[i-1] for i in xrange(1, max_frames)]
try:
offset = min([skips.index(0), skips.index(ttStep)])
except ValueError:
try:
offset = skips.index(0)
except ValueError:
offset = 0
if verbose:
print(" -> shifting boundary by %i frame(s)" % offset)
start += drx.FRAME_SIZE*offset
return start
def test_write_frame(self):
"""Test that the DRX data writer works."""
testFile = os.path.join(self.testPath, 'drx-test-W.dat')
nFrames = os.path.getsize(drxFile) // drxReader.FRAME_SIZE
# Read in a TBN frame from the test file
fh = open(drxFile, 'rb')
origFrames = []
for i in range(nFrames):
origFrames.append(drxReader.read_frame(fh))
fh.close()
# Write the data to a TBN test frame
fh = open(testFile, 'wb')
for origFrame in origFrames:
rawFrame = drxWriter.frame_to_frame(origFrame)
rawFrame.tofile(fh)
fh.close()
# Read in the
fh = open(testFile, 'rb')
fakeFrames = []
for i in range(nFrames):
fakeFrames.append(drxReader.read_frame(fh))
fh.close()
for fakeFrame, origFrame in zip(fakeFrames, origFrames):
# Test values returned by info functions
self.assertEqual(fakeFrame.id[0], origFrame.id[0])
self.assertEqual(fakeFrame.id[1], origFrame.id[1])
self.assertEqual(fakeFrame.id[2], origFrame.id[2])
self.assertAlmostEqual(fakeFrame.sample_rate,
origFrame.sample_rate, 4)
# Test raw header values
self.assertEqual(fakeFrame.header.second_count,
origFrame.header.second_count)
self.assertEqual(fakeFrame.header.decimation,
origFrame.header.decimation)
self.assertEqual(fakeFrame.header.time_offset,
origFrame.header.time_offset)
# Test raw data values
self.assertEqual(fakeFrame.payload.timetag,
origFrame.payload.timetag)
self.assertEqual(fakeFrame.payload.flags, origFrame.payload.flags)
for i in range(4096):
self.assertEqual(fakeFrame.payload.data[i].real,
origFrame.payload.data[i].real)
self.assertEqual(fakeFrame.payload.data[i].imag,
origFrame.payload.data[i].imag)
def test_basic_drx(self):
"""Test building a basic DRX signal"""
testFile = os.path.join(self.testPath, 'drx.dat')
fh = open(testFile, 'wb')
dp.basic_signal(fh,
numpy.array([1, 2, 3, 4]),
10,
mode='DRX',
filter=6,
ntuning=2,
start_time=1000)
fh.close()
# Check the file size
fileSize = os.path.getsize(testFile)
nSamples = fileSize // drx.FRAME_SIZE
self.assertEqual(nSamples, 10 * 4 * 2 * 2)
# Check the file size
fh = open(testFile, 'rb')
frame = drx.read_frame(fh)
fh.close()
self.assertEqual(frame.payload.timetag, 1000 * dp_common.fS)
self.assertEqual(frame.header.frame_count, 0)
self.assertEqual(frame.header.second_count, 0)
def test_drx_read(self):
"""Test reading in a frame from a DRX file."""
fh = open(drxFile, 'rb')
# First frame is really DRX and stores the IDs
frame1 = drx.read_frame(fh)
beam, tune, pol = frame1.id
self.assertEqual(beam, 4)
self.assertEqual(tune, 1)
self.assertEqual(pol, 1)
# Second frame
frame2 = drx.read_frame(fh)
beam, tune, pol = frame2.id
self.assertEqual(beam, 4)
self.assertEqual(tune, 2)
self.assertEqual(pol, 0)
fh.close()
def test_drx_reorder(self):
"""Test the reorder function of the TBN ring buffer."""
fh = open(drxFile, 'rb')
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
fh.seek(-drx.FRAME_SIZE, 1)
# Create the FrameBuffer instance
frameBuffer = buffer.DRXFrameBuffer(beams=[
b,
],
tunes=[1, 2],
pols=[0, 1],
nsegments=2,
reorder=True)
# Go
while True:
try:
cFrame = drx.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
continue
frameBuffer.append(cFrame)
cFrames = frameBuffer.get()
if cFrames is None:
continue
# Make sure it has the right number of frames
self.assertEqual(len(cFrames), 4)
# Check the order
for i in range(1, len(cFrames)):
pB, pT, pP = cFrames[i - 1].id
cB, cT, cP = cFrames[i].id
pID = 4 * pB + 2 * (pT - 1) + pP
cID = 4 * cB + 2 * (cT - 1) + cP
self.assertTrue(cID > pID)
fh.close()
def test_drx_buffer_flush(self):
"""Test the DRX ring buffer's flush() function."""
fh = open(drxFile, 'rb')
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
fh.seek(-drx.FRAME_SIZE, 1)
# Create the FrameBuffer instance
frameBuffer = buffer.DRXFrameBuffer(beams=[
b,
],
tunes=[1, 2],
pols=[0, 1],
nsegments=2)
# Go
while True:
try:
cFrame = drx.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
continue
frameBuffer.append(cFrame)
cFrames = frameBuffer.get()
if cFrames is None:
continue
fh.close()
# Flush the buffer
for cFrames in frameBuffer.flush():
# Make sure the dump has one of the expected time tags
self.assertTrue(
cFrames[0].payload.timetag in (257355782095346056, ))
# Make sure it has the right number of frames
self.assertEqual(len(cFrames), 4)
def test_sim_frame(self):
"""Test the drx.SimFrame class."""
# Read in a DRX frame from the test file
fh = open(drxFile, 'rb')
origFrame = drxReader.read_frame(fh)
fh.close()
fakeFrame = drxWriter.SimFrame()
fakeFrame.load_frame(origFrame)
# Test the validity of the SimFrame
self.assertTrue(fakeFrame.is_valid())
def test_drx_rate(self):
"""Test finding out the DRX sample rate."""
fh = open(drxFile, 'rb')
cFrame = drx.read_frame(fh)
fh.seek(0)
# Sample rate
self.assertEqual(cFrame.sample_rate, drx.get_sample_rate(fh))
# Filter code
self.assertEqual(cFrame.filter_code,
drx.get_sample_rate(fh, filter_code=True))
fh.close()
def test_drx_sort(self):
"""Test sorting DRX frames by time tags."""
fh = open(drxFile, 'rb')
# Frames 1 through 10
frames = []
for i in range(1, 11):
frames.append(drx.read_frame(fh))
fh.close()
frames.sort()
frames = frames[::-1]
for i in range(1, len(frames)):
self.assertTrue(frames[i - 1] >= frames[i])
def test_drx_comps(self):
"""Test the DRX frame comparison operators (>, <, etc.) for time tags."""
fh = open(drxFile, 'rb')
# Frames 1 through 10
frames = []
for i in range(1, 11):
frames.append(drx.read_frame(fh))
fh.close()
self.assertTrue(0 < frames[0])
self.assertFalse(0 > frames[0])
self.assertTrue(frames[-1] >= frames[0])
self.assertFalse(frames[-1] <= frames[0])
self.assertTrue(frames[0] == frames[0])
self.assertFalse(frames[0] == frames[-1])
self.assertFalse(frames[0] != frames[0])
def test_drx_errors(self):
"""Test reading in all frames from a truncated DRX file."""
fh = open(drxFile, 'rb')
# Frames 1 through 32
for i in range(1, 33):
frame = drx.read_frame(fh)
# Last frame should be an error (errors.EOFError)
self.assertRaises(errors.EOFError, drx.read_frame, fh)
fh.close()
# If we offset in the file by 1 byte, we should be a
# sync error (errors.SyncError).
fh = open(drxFile, 'rb')
fh.seek(1)
self.assertRaises(errors.SyncError, drx.read_frame, fh)
fh.close()
def test_frame_header_errors(self):
"""Test the header error scenarios when validating a DRX SimFrame."""
# Read in a DRX frame from the test file
fh = open(drxFile, 'rb')
origFrame = drxReader.read_frame(fh)
fh.close()
# Try to validate frame with the wrong beam number
fakeFrame = drxWriter.SimFrame()
fakeFrame.load_frame(copy.deepcopy(origFrame))
fakeFrame.beam = 5
self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True)
# Try to validate frame with the wrong tuning number
fakeFrame = drxWriter.SimFrame()
fakeFrame.load_frame(copy.deepcopy(origFrame))
fakeFrame.tune = 3
self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True)
def test_drx_math(self):
"""Test mathematical operations on DRX frame data via frames."""
fh = open(drxFile, 'rb')
# Frames 1 through 10
frames = []
for i in range(1, 11):
frames.append(drx.read_frame(fh))
fh.close()
# Multiplication
frameT = frames[0] * 2.0
numpy.testing.assert_allclose(frameT.payload.data,
2 * frames[0].payload.data,
atol=1e-6)
frameT *= 2.0
numpy.testing.assert_allclose(frameT.payload.data,
4 * frames[0].payload.data,
atol=1e-6)
frameT = frames[0] * frames[1]
numpy.testing.assert_allclose(frameT.payload.data,
frames[0].payload.data *
frames[1].payload.data,
atol=1e-6)
# Addition
frameA = frames[0] + 2.0
numpy.testing.assert_allclose(frameA.payload.data,
2 + frames[0].payload.data,
atol=1e-6)
frameA += 2.0
numpy.testing.assert_allclose(frameA.payload.data,
4 + frames[0].payload.data,
atol=1e-6)
frameA = frames[0] + frames[1]
numpy.testing.assert_allclose(frameA.payload.data,
frames[0].payload.data +
frames[1].payload.data,
atol=1e-6)
def test_drx_math(self):
"""Test mathematical operations on DRX frame data via frames."""
fh = open(drxFile, 'rb')
# Frames 1 through 10
frames = []
for i in range(1, 11):
frames.append(drx.read_frame(fh))
fh.close()
# Multiplication
frameT = frames[0] * 2.0
for i in range(4096):
self.assertAlmostEqual(frameT.payload.data[i],
2 * frames[0].payload.data[i], 2)
frameT *= 2.0
for i in range(4096):
self.assertAlmostEqual(frameT.payload.data[i],
4 * frames[0].payload.data[i], 2)
frameT = frames[0] * frames[1]
for i in range(4096):
self.assertAlmostEqual(
frameT.payload.data[i],
frames[0].payload.data[i] * frames[1].payload.data[i], 2)
# Addition
frameA = frames[0] + 2.0
for i in range(4096):
self.assertAlmostEqual(frameA.payload.data[i],
2 + frames[0].payload.data[i], 2)
frameA += 2.0
for i in range(4096):
self.assertAlmostEqual(frameA.payload.data[i],
4 + frames[0].payload.data[i], 2)
frameA = frames[0] + frames[1]
for i in range(4096):
self.assertAlmostEqual(
frameA.payload.data[i],
frames[0].payload.data[i] + frames[1].payload.data[i], 2)
def test_frame_data_errors(self):
"""Test the data error scenarios when validating a DRX SimFrame."""
# Read in a DRX frame from the test file
fh = open(drxFile, 'rb')
origFrame = drxReader.read_frame(fh)
fh.close()
# Try to validate frame with the wrong data type
fakeFrame = drxWriter.SimFrame()
fakeFrame.load_frame(copy.deepcopy(origFrame))
fakeFrame.data = fakeFrame.payload.data.real
self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True)
# Try to validate frame with the wrong data size
fakeFrame = drxWriter.SimFrame()
fakeFrame.load_frame(copy.deepcopy(origFrame))
fakeFrame.data = None
self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True)
fakeFrame = drxWriter.SimFrame()
fakeFrame.load_frame(copy.deepcopy(origFrame))
fakeFrame.data = fakeFrame.payload.data[0:50]
self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True)
def main(args):
LFFT = args.fft_length
stand1 = int(args.dipole_id_x)
stand2 = int(args.dipole_id_y)
filenames = args.filename
# Build up the station
if args.lwasv:
site = stations.lwasv
else:
site = stations.lwa1
# Figure out which antennas we need
antennas = []
for ant in site.antennas:
if ant.stand.id == stand1 and ant.pol == 0:
antennas.append(ant)
for ant in site.antennas:
if ant.stand.id == stand2 and ant.pol == 0:
antennas.append(ant)
# Loop through the input files...
for filename in filenames:
fh = open(filename, "rb")
nFramesFile = os.path.getsize(filename) // drx.FRAME_SIZE
#junkFrame = drx.read_frame(fh)
#fh.seek(0)
while True:
try:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
t0 = junkFrame.time
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE + 1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
beam, tune, pol = junkFrame.id
srate = junkFrame.sample_rate
tunepols = drx.get_frames_per_obs(fh)
tunepols = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepols
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(args.skip * srate / 4096 * beampols)
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * drx.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = drx.read_frame(fh)
srate = junkFrame.sample_rate
t1 = junkFrame.time
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
fh.seek(-drx.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1 - (t0 + args.skip)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 8.0
cOffset = int(tCorr * srate / 4096 * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * drx.FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1 - t0
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
tnom = junkFrame.header.time_offset
tStart = junkFrame.time
# Get the DRX frequencies
cFreq1 = 0.0
cFreq2 = 0.0
for i in xrange(4):
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
if p == 0 and t == 1:
cFreq1 = junkFrame.central_freq
elif p == 0 and t == 2:
cFreq2 = junkFrame.central_freq
else:
pass
fh.seek(-4 * drx.FRAME_SIZE, 1)
# Align the files as close as possible by the time tags and then make sure that
# the first frame processed is from tuning 1, pol 0.
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
pair = 2 * (tune - 1) + pol
j = 0
while pair != 0:
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
pair = 2 * (tune - 1) + pol
j += 1
fh.seek(-drx.FRAME_SIZE, 1)
print("Shifted beam %i data by %i frames (%.4f s)" %
(beam, j, j * 4096 / srate / 4))
# Set integration time
tInt = args.avg_time
nFrames = int(round(tInt * srate / 4096))
tInt = nFrames * 4096 / srate
# Read in some data
tFile = nFramesFile / 4 * 4096 / srate
# Report
print("Filename: %s" % filename)
print(" Sample Rate: %i Hz" % srate)
print(" Tuning 1: %.1f Hz" % cFreq1)
print(" Tuning 2: %.1f Hz" % cFreq2)
print(" ===")
print(" Integration Time: %.3f s" % tInt)
print(" Integrations in File: %i" % int(tFile / tInt))
nChunks = int(tFile / tInt)
pb = ProgressBar(max=nChunks)
for i in xrange(nChunks):
junkFrame = drx.read_frame(fh)
tStart = junkFrame.time
fh.seek(-drx.FRAME_SIZE, 1)
count1 = [0, 0]
data1 = numpy.zeros((2, 4096 * nFrames), dtype=numpy.complex64)
count2 = [0, 0]
data2 = numpy.zeros((2, 4096 * nFrames), dtype=numpy.complex64)
for j in xrange(nFrames):
for k in xrange(4):
cFrame = drx.read_frame(fh)
beam, tune, pol = cFrame.id
pair = 2 * (tune - 1) + pol
if tune == 1:
data1[pol, count1[pol] * 4096:(count1[pol] + 1) *
4096] = cFrame.payload.data
count1[pol] += 1
else:
data2[pol, count2[pol] * 4096:(count2[pol] + 1) *
4096] = cFrame.payload.data
count2[pol] += 1
# Correlate
blList1, freq1, vis1 = fxc.FXMaster(data1,
antennas,
LFFT=LFFT,
overlap=1,
include_auto=True,
verbose=False,
sample_rate=srate,
central_freq=cFreq1,
pol='XX',
return_baselines=True,
gain_correct=False,
clip_level=0)
blList2, freq2, vis2 = fxc.FXMaster(data2,
antennas,
LFFT=LFFT,
overlap=1,
include_auto=True,
verbose=False,
sample_rate=srate,
central_freq=cFreq2,
pol='XX',
return_baselines=True,
gain_correct=False,
clip_level=0)
if nChunks != 1:
outfile = os.path.split(filename)[1]
outfile = os.path.splitext(outfile)[0]
outfile = "%s-vis-%04i.npz" % (outfile, i + 1)
else:
outfile = os.path.split(filename)[1]
outfile = os.path.splitext(outfile)[0]
outfile = "%s-vis.npz" % outfile
numpy.savez(outfile,
srate=srate,
freq1=freq1,
vis1=vis1,
freq2=freq2,
vis2=vis2,
tStart=tStart,
tInt=tInt,
stands=numpy.array([stand1, stand2]))
del data1
del data2
pb.inc(amount=1)
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
# Plot
fig = plt.figure()
i = 0
for bl, vi in zip(blList1, vis1):
ax = fig.add_subplot(4, 3, i + 1)
ax.plot(freq1 / 1e6, numpy.unwrap(numpy.angle(vi)))
ax.set_title('Stand %i - Stand %i' %
(bl[0].stand.id, bl[1].stand.id))
ax = fig.add_subplot(4, 3, i + 4)
ax.plot(freq1 / 1e6, numpy.abs(vi))
i += 1
coeff = numpy.polyfit(freq1, numpy.unwrap(numpy.angle(vi)), 1)
#print(coeff[0]/2/numpy.pi*1e9, coeff[1]*180/numpy.pi)
i = 6
for bl, vi in zip(blList2, vis2):
ax = fig.add_subplot(4, 3, i + 1)
ax.plot(freq2 / 1e6, numpy.unwrap(numpy.angle(vi)))
ax.set_title('Stand %i - Stand %i' %
(bl[0].stand.id, bl[1].stand.id))
ax = fig.add_subplot(4, 3, i + 4)
ax.plot(freq2 / 1e6, numpy.abs(vi))
i += 1
coeff = numpy.polyfit(freq2, numpy.unwrap(numpy.angle(vi)), 1)
def test_drx_basic(self):
"""Test the DRX ring buffer."""
fh = open(drxFile, 'rb')
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
fh.seek(-drx.FRAME_SIZE, 1)
# Create the FrameBuffer instance
frameBuffer = buffer.DRXFrameBuffer(beams=[
b,
],
tunes=[1, 2],
pols=[0, 1],
nsegments=2)
# Go
dumped = []
while True:
try:
cFrame = drx.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
continue
frameBuffer.append(cFrame)
cFrames = frameBuffer.get()
if cFrames is None:
continue
dumped.append(cFrames[0].payload.timetag)
fh.close()
# Make sure we have the right number of frames in the buffer
nFrames = 0
for key in frameBuffer.buffer.keys():
nFrames = nFrames + len(frameBuffer.buffer[key])
self.assertEqual(nFrames, 1)
self.assertEqual(nFrames + len(dumped) * 4, 32 + 1)
# Make sure nothing has happened that shouldn't have
self.assertEqual(frameBuffer.full, 7)
self.assertEqual(frameBuffer.partial, 1)
self.assertEqual(frameBuffer.missing, 1)
self.assertEqual(frameBuffer.dropped, 0)
# Make sure we have the right keys
for key in dumped:
self.assertTrue(key in (257355782095018376, 257355782095059336,
257355782095100296, 257355782095141256,
257355782095182216, 257355782095223176,
257355782095264136, 257355782095305096))
for key in frameBuffer.buffer.keys():
self.assertTrue(key in (257355782095346056, ))
# Make sure the buffer keys have the right sizes
self.assertEqual(len(frameBuffer.buffer[257355782095346056]), 1)
def main(args):
fh = open(args.filename, "rb")
nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE
while True:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
fh.seek(-drx.FRAME_SIZE, 1)
print(junkFrame.header.time_offset)
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
args.skip = 1.0 * offset / beampols * 4096 / srate
fh.seek(offset * drx.FRAME_SIZE)
# Make sure that the file chunk size contains is an intger multiple
# of the beampols.
maxFrames = int(19144 / beampols) * beampols
# Number of frames to integrate over
toClip = False
oldAverage = args.plot_range
if args.plot_range < 4096 / srate:
toClip = True
args.plot_range = 4096 / srate
nFrames = int(args.plot_range * srate / 4096 * beampols)
nFrames = int(1.0 * nFrames / beampols) * beampols
args.plot_range = 1.0 * nFrames / beampols * 4096 / srate
# Number of remaining chunks
nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames))
# File summary
print("Filename: %s" % args.filename)
print("Beams: %i" % beams)
print("Tune/Pols: %i %i %i %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Plot time: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.plot_range, nFrames, nFrames // beampols))
print("Chunks: %i" % nChunks)
# Sanity check
if offset > nFramesFile:
raise RuntimeError("Requested offset is greater than file length")
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
while 2 * (t - 1) + p != 0:
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
print(b, t, p)
print(fh.tell())
fh.seek(-drx.FRAME_SIZE, 1)
# Master loop over all of the file chuncks
standMapper = []
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
(i, framesRemaining))
count = {}
data = numpy.zeros((beampols, framesWork * 4096 // beampols),
dtype=numpy.csingle)
# Inner loop that actually reads the frames into the data array
print("Working on %.1f ms of data" %
((framesWork * 4096 / beampols / srate) * 1000.0))
t0 = time.time()
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1))
continue
beam, tune, pol = cFrame.id
aStand = 4 * (beam - 1) + 2 * (tune - 1) + pol
#print(aStand, beam, tune, pol)
if aStand not in standMapper:
standMapper.append(aStand)
oStand = 1 * aStand
aStand = standMapper.index(aStand)
print(
"Mapping beam %i, tune. %1i, pol. %1i (%2i) to array index %3i"
% (beam, tune, pol, oStand, aStand))
else:
aStand = standMapper.index(aStand)
if aStand not in count.keys():
count[aStand] = 0
#if cFrame.header.frame_count % 10000 == 0 and args.verbose:
# print("%2i,%1i,%1i -> %2i %5i %i" % (beam, tune, pol, aStand, cFrame.header.frame_count, cFrame.payload.timetag))
#print(data.shape, count[aStand]*4096, (count[aStand]+1)*4096, cFrame.payload.data.shape)
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] += 1
# The plots: This is setup for the current configuration of 20 beampols
fig = plt.figure()
figsX = int(round(math.sqrt(beampols)))
figsY = beampols // figsX
t1X = 1
t1Y = 1
offset = 0
samples = 65536
for sec in xrange(data.shape[1] // samples):
if toClip:
print("Plotting only the first %i samples (%.3f ms) of data" %
(samples, oldAverage * 1000.0))
sortedMapper = sorted(standMapper)
for k, aStand in enumerate(sortedMapper):
i = standMapper.index(aStand)
if standMapper[i] % 2 == 0:
ref = data[0, :]
t1R = t1X
else:
ref = data[1, :]
t1R = t1Y
(lag, cc), junkI, junkQ = crossCorrelate(
data[i, sec * samples:(sec + 1) * samples],
ref[offset + sec * samples:offset + (sec + 1) * samples])
best = numpy.where(cc == cc.max())[0][0]
if args.verbose:
print('tune %i pol. %s' %
(standMapper[i] % 4 // 2 + 1, standMapper[i] % 2))
print(' -> best peak of %.0f at a lag of %i samples' %
(cc.max(), lag[best]))
print(' -> NCM with tuning 1 of %.3f' % (cc.max() / t1R))
# Plot
ax = fig.add_subplot(figsX, figsY, k + 1)
ax.plot(lag, cc, label='Same', color='blue')
# Center on the peak
best = numpy.where(cc == cc.max())[0][0]
ax.set_xlim([lag[best - 50], lag[best + 50]])
ax.set_title('Beam %i, Tune. %i, Pol. %i' %
(standMapper[i] // 4 + 1,
standMapper[i] % 4 // 2 + 1, standMapper[i] % 2))
ax.set_xlabel('Lag [samples]')
ax.set_ylabel('Analysis Sets')
# Save the tuning 1 values for the peak of the CC function
if standMapper[i] % 4 / 2 + 1 == 1:
if standMapper[i] % 2 == 0:
t1X = cc.max()
else:
t1Y = cc.max()
plt.show()
# Save image if requested
if args.output is not None:
fig.savefig(args.output)
def main(args):
# Get the file names
meta = args.metadata
data = args.filename
# Get all observations and their start times
try:
## LWA-1
sdf = metabundle.get_sdf(meta)
ses = metabundle.get_session_spec(meta)
obs = metabundle.get_observation_spec(meta)
except:
## LWA-SV
### Try again
sdf = metabundleADP.get_sdf(meta)
ses = metabundleADP.get_session_spec(meta)
obs = metabundleADP.get_observation_spec(meta)
obs.sort(_obs_comp)
tStart = []
oDetails = []
for i,o in enumerate(obs):
tStart.append( mjdmpm_to_datetime(o['mjd'], o['mpm']) )
oDetails.append( {'m': o['mode'], 'd': o['dur'] / 1000.0, 'f': o['bw'],
'p': o['project_id'], 's': o['session_id'], 'o': o['obs_id'],
't': sdf.sessions[0].observations[o['obs_id']-1].target} )
print("Observation #%i" % (o['obs_id']))
print(" Start: %i, %i -> %s" % (o['mjd'], o['mpm'], tStart[-1]))
print(" Mode: %s" % mode_to_string(o['mode']))
print(" BW: %i" % o['bw'])
print(" Target: %s" % sdf.sessions[0].observations[o['obs_id']-1].target)
print(" ")
# Figure out where in the file the various bits are.
fh = open(data, 'rb')
lf = drx.read_frame(fh)
beam, j, k = lf.id
if beam != obs[0]['drx_beam']:
print('ERROR: Beam mis-match, metadata is for #%i, file is for #%i' % (obs[0]['drx_beam'], beam))
sys.exit()
firstFrame = lf.time.datetime
if abs(firstFrame - min(tStart)) > timedelta(seconds=30):
print('ERROR: Time mis-match, metadata is for %s, file is for %s' % (min(tStart), firstFrame))
sys.exit()
fh.seek(0)
for i in range(len(tStart)):
eof = False
## Get observation properties
oStart = tStart[i]
oMode = mode_to_string(oDetails[i]['m'])
oDur = oDetails[i]['d']
oBW = oDetails[i]['f']
print("Seeking %s observation of %.3f seconds at %s" % (oMode, oDur, oStart))
## Get the correct reader to use
if oMode == 'TBW':
reader = tbw
bwKey = None
bwMult = 520.0 / 400
fCount = 400
elif oMode == 'TBN':
reader = tbn
bwKey = tbn.FILTER_CODES
bwMult = 520.0 / 512
fCount = 512
else:
reader = drx
bwKey = drx.FILTER_CODES
bwMult = 4.0 / 4096
fCount = 4096
## Jump ahead to where the next frame should be, if needed
if i != 0:
pDur = oDetails[i-1]['d']
pBW = oDetails[i-1]['f']
nFramesSkip = int(pDur*bwKey[pBW]*bwMult)
fh.seek(nFramesSkip*reader.FRAME_SIZE, 1)
if fh.tell() >= os.path.getsize(data):
fh.seek(-10*reader.FRAME_SIZE, 2)
## Figure out where we are and make sure we line up on a frame
## NOTE: This should never be needed
fail = True
while fail:
try:
frame = reader.read_frame(fh)
fail = False
except errors.SyncError:
fh.seek(1, 1)
except errors.EOFError:
break
fh.seek(-reader.FRAME_SIZE, 1)
## Go in search of the start of the observation
if frame.time.datetime < oStart:
### We aren't at the beginning yet, seek fowards
print("-> At byte %i, time is %s < %s" % (fh.tell(), frame.time.datetime, oStart))
while frame.time.datetime < oStart:
try:
frame = reader.read_frame(fh)
except errors.SyncError:
fh.seek(1, 1)
except errors.EOFError:
break
#print(frame.time.datetime, oStart)
elif frame.time.datetime > oStart:
### We've gone too far, seek backwards
print("-> At byte %i, time is %s > %s" % (fh.tell(), frame.time.datetime, oStart))
while frame.time.datetime > oStart:
if fh.tell() == 0:
break
fh.seek(-2*reader.FRAME_SIZE, 1)
try:
frame = reader.read_frame(fh)
except errors.SyncError:
fh.seek(-1, 1)
except errors.EOFError:
break
#print(frame.time.datetime, oStart)
else:
### We're there already
print("-> At byte %i, time is %s = %s" % (fh.tell(), frame.time.datetime, oStart))
## Jump back exactly one frame so that the filehandle is in a position
## to read the first frame that is part of the observation
try:
frame = reader.read_frame(fh)
print("-> At byte %i, time is %s = %s" % (fh.tell(), frame.time.datetime, oStart))
fh.seek(-reader.FRAME_SIZE, 1)
except errors.EOFError:
pass
## Update the bytes ranges
if fh.tell() < os.path.getsize(data):
oDetails[i]['b'] = fh.tell()
oDetails[i]['e'] = -1
else:
oDetails[i]['b'] = -1
oDetails[i]['e'] = -1
if i != 0:
oDetails[i-1]['e'] = fh.tell()
## Progress report
if oDetails[i]['b'] >= 0:
print('-> Obs.', oDetails[i]['o'], 'starts at byte', oDetails[i]['b'])
else:
print('-> Obs.', oDetails[i]['o'], 'starts after the end of the file')
print(" ")
# Report
for i in range(len(tStart)):
if oDetails[i]['b'] < 0:
print("%s, Session %i, Observation %i: not found" % (oDetails[i]['p'], oDetails[i]['s'], oDetails[i]['o']))
else:
print("%s, Session %i, Observation %i: %i to %i (%i bytes)" % (oDetails[i]['p'], oDetails[i]['s'], oDetails[i]['o'], oDetails[i]['b'], oDetails[i]['e'], (oDetails[i]['e'] - oDetails[i]['b'])))
print(" ")
# Split
if not args.list:
for i in range(len(tStart)):
if oDetails[i]['b'] < 0:
continue
## Report
print("Working on Observation %i" % (i+1,))
## Create the output name
if args.source:
outname = '%s_%i_%s.dat' % (oDetails[i]['p'], oDetails[i]['s'], oDetails[i]['t'].replace(' ', '').replace('/','').replace('&','and'))
else:
outname = '%s_%i_%i.dat' % (oDetails[i]['p'], oDetails[i]['s'], oDetails[i]['o'])
oMode = mode_to_string(oDetails[i]['m'])
## Get the correct reader to use
if oMode == 'TBW':
reader = tbw
elif oMode == 'TBN':
reader = tbn
else:
reader = drx
## Get the number of frames
if oDetails[i]['e'] > 0:
nFramesRead = (oDetails[i]['e'] - oDetails[i]['b']) // reader.FRAME_SIZE
else:
nFramesRead = (os.path.getsize(data) - oDetails[i]['b']) // reader.FRAME_SIZE
## Split
if os.path.exists(outname):
if not args.force:
yn = input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
print("WARNING: output file '%s' already exists, skipping" % outname)
continue
fh.seek(oDetails[i]['b'])
t0 = time.time()
oh = open(outname, 'wb')
for sl in [2**i for i in range(17)[::-1]]:
while nFramesRead >= sl:
temp = fh.read(sl*reader.FRAME_SIZE)
oh.write(temp)
nFramesRead -= sl
oh.close()
t1 = time.time()
print(" Copied %i bytes in %.3f s (%.3f MB/s)" % (os.path.getsize(outname), t1-t0, os.path.getsize(outname)/1024.0**2/(t1-t0)))
print(" ")
def identify_section(fh, start=0, stop=-1, strict=True, min_frames=4096, verbose=True):
if stop <= start:
stop = os.path.getsize(fh.name)
fh.seek(start)
# Report
if verbose:
print("Working on %i to %i of '%s'..." % (start, stop, os.path.basename(fh.name)))
# Make sure this is enough to work with
if stop-start < drx.FRAME_SIZE*min_frames:
if verbose:
print(" too small for analysis, skipping")
return None
# Align on the start of a Mark5C packet...
while True:
try:
junkFrame = drx.read_frame(fh)
try:
# ... that has a valid decimation
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE+1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
# ... and save that location
frame_begin = junkFrame.payload.timetag
file_begin = fh.tell()
if verbose:
print(" start @ %i with %i" % (file_begin, frame_begin))
# Find the last valid Mark5C packet...
fh.seek(stop-drx.FRAME_SIZE)
while True:
try:
junkFrame = drx.read_frame(fh)
try:
# ... that has a valid decimation
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE-1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
# ... and save that location
frame_end = junkFrame.payload.timetag
file_end = fh.tell() + drx.FRAME_SIZE
if verbose:
print(" stop @ %i with %i" % (file_end, frame_end))
# Get how much the timetags should change and other basic information
fh.seek(file_begin)
ids = []
for i in xrange(24*8):
junkFrame = drx.read_frame(fh)
b,t,p = junkFrame.id
id = (t,p)
if id not in ids:
ids.append(id)
ttStep = 4096*junkFrame.header.decimation
if verbose:
print(" %i frames with a timetag step of %i" % (len(ids), ttStep))
# Difference
nBytes = file_end - file_begin
nFrames = nBytes // drx.FRAME_SIZE
ttDiffFound = frame_end - frame_begin
ttDiffExpected = nFrames // len(ids) * ttStep
if verbose:
print(" -> found timetag difference of %i" % ttDiffFound)
print(" -> expected timetag difference is %i" % ttDiffExpected)
# Decide what to do
if abs(ttDiffFound - ttDiffExpected) > ttStep*(1-strict):
if verbose:
print(" ====> mis-match, subsampling")
file_middle = file_begin + (nFrames // 2) * drx.FRAME_SIZE
parts0 = identify_section(fh, file_begin, file_middle, strict=strict, min_frames=min_frames, verbose=verbose)
parts1 = identify_section(fh, file_middle, file_end, strict=strict, min_frames=min_frames, verbose=verbose)
else:
if verbose:
print(" ====> good, done")
parts0 = [[file_begin, file_end],]
parts1 = None
# Sort and merge
partsList = []
for parts in (parts0, parts1):
if parts is None:
continue
for part in parts:
partsList.append( part )
partsList.sort()
# Merge
parts = []
if len(partsList) > 0:
parts.append( partsList[0] )
for part in partsList[1:]:
if part[0] == parts[-1][1]:
parts[-1][1] = part[1]
else:
parts.append(part)
return parts
def main(args):
filename = args.filename
fh = open(filename, "rb")
nFramesFile = os.path.getsize(filename) // drx.FRAME_SIZE
while True:
try:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE + 1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
beam, tune, pol = junkFrame.id
tunepols = max(drx.get_frames_per_obs(fh))
# Date & Central Frequnecy
beginDate = junkFrame.time.datetime
central_freq1 = 0.0
central_freq2 = 0.0
for i in xrange(32):
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
if p == 0 and t == 1:
central_freq1 = junkFrame.central_freq
elif p == 0 and t == 2:
central_freq2 = junkFrame.central_freq
else:
pass
fh.seek(-32 * drx.FRAME_SIZE, 1)
# Report on the file
print("Filename: %s" % filename)
print("Date of First Frame: %s" % str(beginDate))
print("Beam: %i" % beam)
print("Tune/Pols: %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" %
(central_freq1, central_freq2))
print(" ")
# Convert chunk length to total frame count
chunkLength = int(args.length * srate / 4096 * tunepols)
chunkLength = int(1.0 * chunkLength / tunepols) * tunepols
# Convert chunk skip to total frame count
chunkSkip = int(args.skip * srate / 4096 * tunepols)
chunkSkip = int(1.0 * chunkSkip / tunepols) * tunepols
# Output arrays
clipFraction = []
meanPower = []
# Go!
i = 1
done = False
print(" | Clipping | Power |")
print(" | 1X 1Y 2X 2Y | 1X 1Y 2X 2Y |")
print("---+---------------------------------+-------------------------+")
while True:
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.empty((4, chunkLength * 4096 // tunepols),
dtype=numpy.csingle)
for j in xrange(chunkLength):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
done = True
break
except errors.SyncError:
continue
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
try:
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] += 1
except ValueError:
pass
if done:
break
else:
data = numpy.abs(data)**2
data = data.astype(numpy.int32)
clipFraction.append(numpy.zeros(4))
meanPower.append(data.mean(axis=1))
for j in xrange(4):
bad = numpy.nonzero(data[j, :] > args.trim_level)[0]
clipFraction[-1][j] = 1.0 * len(bad) / data.shape[1]
clip = clipFraction[-1]
power = meanPower[-1]
print(
"%2i | %6.2f%% %6.2f%% %6.2f%% %6.2f%% | %5.2f %5.2f %5.2f %5.2f |"
% (i, clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0,
clip[3] * 100.0, power[0], power[1], power[2], power[3]))
i += 1
fh.seek(drx.FRAME_SIZE * chunkSkip, 1)
clipFraction = numpy.array(clipFraction)
meanPower = numpy.array(meanPower)
clip = clipFraction.mean(axis=0)
power = meanPower.mean(axis=0)
print("---+---------------------------------+-------------------------+")
print("%2s | %6.2f%% %6.2f%% %6.2f%% %6.2f%% | %5.2f %5.2f %5.2f %5.2f |" %
('M', clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0,
clip[3] * 100.0, power[0], power[1], power[2], power[3]))
def main(args):
# Parse the command line
filenames = args.filename
# Check if the first argument on the command line is a directory. If so,
# use what is in that directory
if os.path.isdir(filenames[0]):
filenames = [
os.path.join(filenames[0], filename)
for filename in os.listdir(filenames[0])
]
filenames.sort()
# Convert the filenames to absolute paths
filenames = [os.path.abspath(filename) for filename in filenames]
# Open the database connection to NRAO to find the antenna locations
try:
db = database('params')
except Exception as e:
sys.stderr.write("WARNING: %s" % str(e))
sys.stderr.flush()
db = None
# Pass 1 - Get the LWA metadata so we know where we are pointed
context = {
'observer': 'Unknown',
'project': 'Unknown',
'session': None,
'vlaref': None
}
setup = None
sources = []
metadata = {}
lwasite = {}
for filename in filenames:
# Figure out what to do with the file
ext = os.path.splitext(filename)[1]
if ext == '.tgz':
## LWA Metadata
try:
## Extract the SDF
if len(sources) == 0:
try:
sdf = metabundle.get_sdf(filename)
except Exception as e:
sdf = metabundleADP.get_sdf(filename)
context['observer'] = sdf.observer.name
context['project'] = sdf.id
context['session'] = sdf.sessions[0].id
comments = sdf.project_office.sessions[0]
mtch = CORR_CHANNELS.search(comments)
if mtch is not None:
corr_channels = int(mtch.group('channels'), 10)
else:
corr_channels = None
mtch = CORR_INTTIME.search(comments)
if mtch is not None:
corr_inttime = float(mtch.group('inttime'))
else:
corr_inttime = None
mtch = CORR_BASIS.search(comments)
if mtch is not None:
corr_basis = mtch.group('basis')
else:
sys.stderr.write(
"WARNING: No output correlation polarization basis defined, assuming 'linear'.\n"
)
corr_basis = 'linear'
if corr_channels is not None and corr_inttime is not None:
setup = {
'channels': corr_channels,
'inttime': corr_inttime,
'basis': corr_basis
}
else:
sys.stderr.write(
"WARNING: No or incomplete correlation configuration defined, setting to be defined at correlation time.\n"
)
for o, obs in enumerate(sdf.sessions[0].observations):
if type(obs).__name__ == 'Solar':
name = 'Sun'
intent = 'target'
ra = None
dec = None
elif type(obs).__name__ == 'Jovian':
name = 'Jupiter'
intent = 'target'
ra = None
dec = None
else:
name = obs.target
intent = obs.name
ra = ephem.hours(str(obs.ra))
dec = ephem.degrees(str(obs.dec))
tStart = mjdmpm_to_datetime(obs.mjd, obs.mpm)
tStop = mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
sources.append({
'name': name,
'intent': intent,
'ra2000': ra,
'dec2000': dec,
'start': tStart,
'stop': tStop
})
### Alternate phase centers
comments = sdf.project_office.observations[0][o]
alts = {}
for mtch in ALT_TARGET.finditer(comments):
alt_id = int(mtch.group('id'), 10)
alt_name = mtch.group('target')
try:
alts[alt_id]['name'] = alt_name
except KeyError:
alts[alt_id] = {
'name': alt_name,
'intent': 'dummy',
'ra': None,
'dec': None
}
for mtch in ALT_INTENT.finditer(comments):
alt_id = int(mtch.group('id'), 10)
alt_intent = mtch.group('intent')
try:
alts[alt_id]['intent'] = alt_intent
except KeyError:
alts[alt_id] = {
'name': None,
'intent': alt_intent,
'ra': None,
'dec': None
}
for mtch in ALT_RA.finditer(comments):
alt_id = int(mtch.group('id'), 10)
alt_ra = ephem.hours(mtch.group('ra'))
try:
alts[alt_id]['ra'] = alt_ra
except KeyError:
alts[alt_id] = {
'name': None,
'intent': 'dummy',
'ra': alt_ra,
'dec': None
}
for mtch in ALT_DEC.finditer(comments):
alt_id = int(mtch.group('id'), 10)
alt_dec = ephem.degrees(mtch.group('dec'))
try:
alts[alt_id]['dec'] = alt_dec
except KeyError:
alts[alt_id] = {
'name': None,
'intent': 'dummy',
'ra': None,
'dec': alt_dec
}
for alt_id in sorted(alts.keys()):
alt_name, alt_ra, alt_dec = alts[alt_id]
if alt_name is None or alt_ra is None or alt_dec is None:
sys.stderr.write(
"WARNING: Incomplete alternate phase center %i, skipping.\n"
% alt_id)
else:
sources.append({
'name': alt_name,
'ra2000': alt_ra,
'dec2000': alt_dec,
'start': tStart,
'stop': tStop
})
## Extract the file information so that we can pair things together
fileInfo = metabundle.get_session_metadata(filename)
for obsID in fileInfo.keys():
metadata[fileInfo[obsID]['tag']] = filename
## Figure out LWA1 vs LWA-SV
try:
cs = metabundle.get_command_script(filename)
for c in cs:
if c['subsystem_id'] == 'DP':
site = 'LWA1'
break
elif c['subsystem_id'] == 'ADP':
site = 'LWA-SV'
break
except (RuntimeError, ValueError):
site = 'LWA-SV'
for obsID in fileInfo.keys():
lwasite[fileInfo[obsID]['tag']] = site
except Exception as e:
sys.stderr.write("ERROR reading metadata file: %s\n" % str(e))
sys.stderr.flush()
# Setup what we need to write out a configuration file
corrConfig = {
'context': context,
'setup': setup,
'source': {
'name': '',
'ra2000': '',
'dec2000': ''
},
'inputs': []
}
metadata = {}
for filename in filenames:
#print("%s:" % os.path.basename(filename))
# Skip over empty files
if os.path.getsize(filename) == 0:
continue
# Open the file
fh = open(filename, 'rb')
# Figure out what to do with the file
ext = os.path.splitext(filename)[1]
if ext == '':
## DRX
try:
## Get the site
try:
sitename = lwasite[os.path.basename(filename)]
except KeyError:
sitename = 'LWA1'
## Get the location so that we can set site-specific parameters
if sitename == 'LWA1':
xyz = LWA1_ECEF
off = args.lwa1_offset
elif sitename == 'LWA-SV':
xyz = LWASV_ECEF
off = args.lwasv_offset
else:
raise RuntimeError("Unknown LWA site '%s'" % site)
## Move into the LWA1 coordinate system
### ECEF to LWA1
rho = xyz - LWA1_ECEF
sez = numpy.dot(LWA1_ROT, rho)
enz = sez[[1, 0, 2]] # pylint: disable=invalid-sequence-index
enz[1] *= -1
## Read in the first few frames to get the start time
frames = [drx.read_frame(fh) for i in xrange(1024)]
streams = []
freq1, freq2 = 0.0, 0.0
for frame in frames:
beam, tune, pol = frame.id
if tune == 1:
freq1 = frame.central_freq
else:
freq2 = frame.central_freq
if (beam, tune, pol) not in streams:
streams.append((beam, tune, pol))
tStart = frames[0].time.datetime
tStartAlt = (frames[-1].time - 1023 // len(streams) * 4096 /
frames[-1].sample_rate).datetime
tStartDiff = tStart - tStartAlt
if abs(tStartDiff) > timedelta(microseconds=10000):
sys.stderr.write(
"WARNING: Stale data found at the start of '%s', ignoring\n"
% os.path.basename(filename))
sys.stderr.flush()
tStart = tStartAlt
### ^ Adjustment to the start time to deal with occasional problems
### with stale data in the DR buffers at LWA-SV
## Read in the last few frames to find the end time
fh.seek(os.path.getsize(filename) - 1024 * drx.FRAME_SIZE)
backed = 0
while backed < 2 * drx.FRAME_SIZE:
try:
drx.read_frame(fh)
fh.seek(-drx.FRAME_SIZE, 1)
break
except errors.SyncError:
backed += 1
fh.seek(-drx.FRAME_SIZE - 1, 1)
for i in xrange(32):
try:
frame = drx.read_frame(fh)
beam, tune, _ = frame.id
if tune == 1:
freq1 = frame.central_freq
else:
freq2 = frame.central_freq
except errors.SyncError:
continue
tStop = frame.time.datetime
## Save
corrConfig['inputs'].append({
'file':
filename,
'type':
'DRX',
'antenna':
sitename,
'pols':
'X, Y',
'location': (enz[0], enz[1], enz[2]),
'clockoffset': (off, off),
'fileoffset':
0,
'beam':
beam,
'tstart':
tStart,
'tstop':
tStop,
'freq': (freq1, freq2)
})
except Exception as e:
sys.stderr.write("ERROR reading DRX file: %s\n" % str(e))
sys.stderr.flush()
elif ext == '.vdif':
## VDIF
try:
## Read in the GUPPI header
header = vdif.read_guppi_header(fh)
## Read in the first frame
vdif.FRAME_SIZE = vdif.get_frame_size(fh)
frame = vdif.read_frame(fh)
antID = frame.id[0] - 12300
tStart = frame.time.datetime
nThread = vdif.get_thread_count(fh)
## Read in the last frame
nJump = int(os.path.getsize(filename) / vdif.FRAME_SIZE)
nJump -= 30
fh.seek(nJump * vdif.FRAME_SIZE, 1)
mark = fh.tell()
while True:
try:
frame = vdif.read_frame(fh)
tStop = frame.time.datetime
except Exception as e:
break
## Find the antenna location
pad, edate = db.get_pad('EA%02i' % antID, tStart)
x, y, z = db.get_xyz(pad, tStart)
#print(" Pad: %s" % pad)
#print(" VLA relative XYZ: %.3f, %.3f, %.3f" % (x,y,z))
## Move into the LWA1 coordinate system
### relative to ECEF
xyz = numpy.array([x, y, z])
xyz += VLA_ECEF
### ECEF to LWA1
rho = xyz - LWA1_ECEF
sez = numpy.dot(LWA1_ROT, rho)
enz = sez[[1, 0, 2]] # pylint: disable=invalid-sequence-index
enz[1] *= -1
## Set an apparent position if WiDAR is already applying a delay model
apparent_enz = (None, None, None)
if args.no_vla_delay_model:
apparent_xyz = VLA_ECEF
apparent_rho = apparent_xyz - LWA1_ECEF
apparent_sez = numpy.dot(LWA1_ROT, apparent_rho)
apparent_enz = apparent_sez[[1, 0, 2]] # pylint: disable=invalid-sequence-index
apparent_enz[1] *= -1
## VLA time offset
off = args.vla_offset
## Save
corrConfig['context']['observer'] = header['OBSERVER']
try:
corrConfig['context']['project'] = header[
'BASENAME'].split('_')[0]
corrConfig['context']['session'] = header[
'BASENAME'].split('_')[1].replace('sb', '')
except IndexError:
corrConfig['context']['project'] = header[
'BASENAME'].split('.')[0]
corrConfig['context']['session'] = header[
'BASENAME'].split('.')[1].replace('sb', '')
corrConfig['context']['vlaref'] = re.sub(
'\.[0-9]+\.[0-9]+\.[AB][CD]-.*', '', header['BASENAME'])
corrConfig['source']['name'] = header['SRC_NAME']
corrConfig['source']['intent'] = 'target'
corrConfig['source']['ra2000'] = header['RA_STR']
corrConfig['source']['dec2000'] = header['DEC_STR']
corrConfig['inputs'].append({
'file':
filename,
'type':
'VDIF',
'antenna':
'EA%02i' % antID,
'pols':
'Y, X',
'location': (enz[0], enz[1], enz[2]),
'apparent_location':
(apparent_enz[0], apparent_enz[1], apparent_enz[2]),
'clockoffset': (off, off),
'fileoffset':
0,
'pad':
pad,
'tstart':
tStart,
'tstop':
tStop,
'freq':
header['OBSFREQ']
})
except Exception as e:
sys.stderr.write("ERROR reading VDIF file: %s\n" % str(e))
sys.stderr.flush()
elif ext == '.tgz':
## LWA Metadata
try:
## Extract the file information so that we can pair things together
fileInfo = metabundle.get_session_metadata(filename)
for obsID in fileInfo.keys():
metadata[fileInfo[obsID]['tag']] = filename
except Exception as e:
sys.stderr.write("ERROR reading metadata file: %s\n" % str(e))
sys.stderr.flush()
# Done
fh.close()
# Close out the connection to NRAO
try:
db.close()
except AttributeError:
pass
# Choose a VDIF reference file, if there is one, and mark whether or
# not DRX files were found
vdifRefFile = None
isDRX = False
for cinp in corrConfig['inputs']:
if cinp['type'] == 'VDIF':
if vdifRefFile is None:
vdifRefFile = cinp
elif cinp['type'] == 'DRX':
isDRX = True
# Set a state variable so that we can generate a warning about missing
# DRX files
drxFound = False
# Purge DRX files that don't make sense
toPurge = []
drxFound = False
lwasvFound = False
for cinp in corrConfig['inputs']:
### Sort out multiple DRX files - this only works if we have only one LWA station
if cinp['type'] == 'DRX':
if vdifRefFile is not None:
l0, l1 = cinp['tstart'], cinp['tstop']
v0, v1 = vdifRefFile['tstart'], vdifRefFile['tstop']
ve = (v1 - v0).total_seconds()
overlapWithVDIF = (v0 >= l0 and v0 < l1) or (l0 >= v0
and l0 < v1)
lvo = (min([v1, l1]) - max([v0, l0])).total_seconds()
if not overlapWithVDIF or lvo < 0.25 * ve:
toPurge.append(cinp)
drxFound = True
if cinp['antenna'] == 'LWA-SV':
lwasvFound = True
for cinp in toPurge:
del corrConfig['inputs'][corrConfig['inputs'].index(cinp)]
# Sort the inputs based on the antenna name - this puts LWA1 first,
# LWA-SV second, and the VLA at the end in 'EA' antenna order, i.e.,
# EA01, EA02, etc.
corrConfig['inputs'].sort(key=lambda x: 0 if x['antenna'] == 'LWA1' else (
1 if x['antenna'] == 'LWA-SV' else int(x['antenna'][2:], 10)))
# VDIF/DRX warning check/report
if vdifRefFile is not None and isDRX and not drxFound:
sys.stderr.write(
"WARNING: DRX files provided but none overlapped with VDIF data")
# Duplicate antenna check
antCounts = {}
for cinp in corrConfig['inputs']:
try:
antCounts[cinp['antenna']] += 1
except KeyError:
antCounts[cinp['antenna']] = 1
for ant in antCounts.keys():
if antCounts[ant] != 1:
sys.stderr.write("WARNING: Antenna '%s' is defined %i times" %
(ant, antCounts[ant]))
# Update the file offsets to get things lined up better
tMax = max([cinp['tstart'] for cinp in corrConfig['inputs']])
for cinp in corrConfig['inputs']:
diff = tMax - cinp['tstart']
offset = diff.days * 86400 + diff.seconds + diff.microseconds / 1e6
cinp['fileoffset'] = max([0, offset])
# Reconcile the source lists for when we have eLWA data. This is needed so
# that we use the source information contained in the VDIF files rather than
# the stub information contained in the SDFs
if len(sources) <= 1:
if corrConfig['source']['name'] != '':
## Update the source information with what comes from the VLA
try:
sources[0] = corrConfig['source']
except IndexError:
sources.append(corrConfig['source'])
# Update the dwell time using the minimum on-source time for all inputs if
# there is only one source, i.e., for full eLWA runs
if len(sources) == 1:
sources[0]['start'] = max(
[cinp['tstart'] for cinp in corrConfig['inputs']])
sources[0]['stop'] = min(
[cinp['tstop'] for cinp in corrConfig['inputs']])
# Render the configuration
startRef = sources[0]['start']
s = 0
for source in sources:
startOffset = source['start'] - startRef
startOffset = startOffset.total_seconds()
dur = source['stop'] - source['start']
dur = dur.total_seconds()
## Skip over dummy scans and scans that start after the files end
if source['intent'] in (None, 'dummy'):
continue
if source['start'] > max(
[cinp['tstop'] for cinp in corrConfig['inputs']]):
print(
"Skipping scan of %s which starts at %s, %.3f s after the data end"
% (source['name'], source['start'],
(source['start'] -
max([cinp['tstop']
for cinp in corrConfig['inputs']])).total_seconds()))
continue
## Small correction for the first scan to compensate for stale data at LWA-SV
if lwasvFound and s == 0:
startOffset += 10.0
dur -= 10.0
## Skip over scans that are too short
if dur < args.minimum_scan_length:
continue
## Setup
if args.output is None:
fh = sys.stdout
else:
outname = args.output
if len(sources) > 1:
outname += str(s + 1)
fh = open(outname, 'w')
try:
repo = git.Repo(os.path.dirname(os.path.abspath(__file__)))
try:
branch = repo.active_branch.name
hexsha = repo.active_branch.commit.hexsha
except TypeError:
branch = '<detached>'
hexsha = repo.head.commit.hexsha
shortsha = hexsha[-7:]
dirty = ' (dirty)' if repo.is_dirty() else ''
except git.exc.GitError:
branch = 'unknown'
hexsha = 'unknown'
shortsha = 'unknown'
dirty = ''
## Preamble
fh.write("# Created\n")
fh.write("# on %s\n" % datetime.now())
fh.write("# using %s, revision %s.%s%s\n" %
(os.path.basename(__file__), branch, shortsha, dirty))
fh.write("\n")
## Observation context
fh.write("Context\n")
fh.write(" Observer %s\n" % corrConfig['context']['observer'])
fh.write(" Project %s\n" % corrConfig['context']['project'])
if corrConfig['context']['session'] is not None:
fh.write(" Session %s\n" % corrConfig['context']['session'])
if corrConfig['context']['vlaref'] is not None:
fh.write(" VLARef %s\n" % corrConfig['context']['vlaref'])
fh.write("EndContext\n")
fh.write("\n")
## Configuration, if present
if corrConfig['setup'] is not None:
fh.write("Configuration\n")
fh.write(" Channels %i\n" % corrConfig['setup']['channels'])
fh.write(" IntTime %.3f\n" % corrConfig['setup']['inttime'])
fh.write(" PolBasis %s\n" % corrConfig['setup']['basis'])
fh.write("EndConfiguration\n")
fh.write("\n")
## Source
fh.write("Source\n")
fh.write("# Observation start is %s\n" % source['start'])
fh.write("# Duration is %s\n" % (source['stop'] - source['start'], ))
fh.write(" Name %s\n" % source['name'])
fh.write(" Intent %s\n" % source['intent'].lower())
if source['name'] not in ('Sun', 'Jupiter'):
fh.write(" RA2000 %s\n" % source['ra2000'])
fh.write(" Dec2000 %s\n" % source['dec2000'])
fh.write(" Duration %.3f\n" % dur)
fh.write("SourceDone\n")
fh.write("\n")
## Input files
for cinp in corrConfig['inputs']:
fh.write("Input\n")
fh.write("# Start time is %s\n" % cinp['tstart'])
fh.write("# Stop time is %s\n" % cinp['tstop'])
try:
fh.write("# Beam is %i\n" % cinp['beam'])
except KeyError:
pass
try:
fh.write("# VLA pad is %s\n" % cinp['pad'])
except KeyError:
pass
try:
fh.write("# Frequency tuning 1 is %.3f Hz\n" % cinp['freq'][0])
fh.write("# Frequency tuning 2 is %.3f Hz\n" % cinp['freq'][1])
except TypeError:
fh.write("# Frequency tuning is %.3f Hz\n" % cinp['freq'])
fh.write(" File %s\n" % cinp['file'])
try:
metaname = metadata[os.path.basename(cinp['file'])]
fh.write(" MetaData %s\n" % metaname)
except KeyError:
if cinp['type'] == 'DRX':
sys.stderr.write(
"WARNING: No metadata found for '%s', source %i\n" %
(os.path.basename(cinp['file']), s + 1))
sys.stderr.flush()
pass
fh.write(" Type %s\n" % cinp['type'])
fh.write(" Antenna %s\n" % cinp['antenna'])
fh.write(" Pols %s\n" % cinp['pols'])
fh.write(" Location %.6f, %.6f, %.6f\n" %
cinp['location'])
try:
if cinp['apparent_location'][0] is not None:
fh.write(" ApparentLocation %.6f, %.6f, %.6f\n" %
cinp['apparent_location'])
except KeyError:
pass
fh.write(" ClockOffset %s, %s\n" % cinp['clockoffset'])
fh.write(" FileOffset %.3f\n" %
(startOffset + cinp['fileoffset'], ))
fh.write("InputDone\n")
fh.write("\n")
if fh != sys.stdout:
fh.close()
# Increment the source/file counter
s += 1
def main(args):
filename = args[0]
fh = open(filename, "rb")
nFramesFile = os.path.getsize(filename) / drx.FRAME_SIZE
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
while 2 * (tune - 1) + pol != 0:
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
fh.seek(fh.tell() - drx.FRAME_SIZE)
srate = junkFrame.sample_rate
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# File summary
out = "Filename: %s" % filename
out += "\nBeams: %i" % beams
out += "\nTune/Pols: %i %i %i %i" % tunepols
out += "\nSample Rate: %i Hz" % srate
out += "\nFrames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile /
beampols * 4096 / srate)
out += "\n==="
print(out)
tuningOffset = numpy.zeros(nFramesFile / 8, dtype=numpy.int64)
try:
screen = curses.initscr()
curses.noecho()
curses.cbreak()
screen.nodelay(1)
strdict = {'preamble': out}
for i in xrange(tuningOffset.size):
screen.clear()
beamIDs = [0, 0, 0, 0]
timetags = numpy.zeros(4, dtype=numpy.int64) - 1
time_offsets = numpy.zeros(4, dtype=numpy.int64) - 1
timeValues = numpy.zeros(4, dtype=numpy.float64)
for j in xrange(4):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1))
continue
## Save the time time, time offset, and computed time values
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
if timetags[aStand] == -1:
beamIDs[aStand] = (beam, tune, pol)
timetags[aStand] = cFrame.payload.timetag
time_offsets[aStand] = cFrame.header.time_offset
timeValues[aStand] = cFrame.time
k = 0
for id, tt, to, tv in zip(beamIDs, timetags, time_offsets,
timeValues):
strdict['b%i' % k] = id[0]
strdict['t%i' % k] = id[1]
strdict['p%i' % k] = id[2]
strdict['tt%i' % k] = tt
strdict['os%i' % k] = to
strdict['tv%i' % k] = tv
k += 1
t1t = timetags[0] - time_offsets[0]
t2t = timetags[3] - time_offsets[3]
tuningOffset[i] = t2t - t1t
strdict['ttd'] = t2t - t1t
strdict['tvd'] = (t2t - t1t) / fS
screen.addstr(0, 0, display.safe_substitute(strdict))
screen.refresh()
# Check for keypress and exit if Q
c = screen.getch()
if (c > 0):
if chr(c) == 'q':
break
if chr(c) == 'Q':
break
curses.nocbreak()
curses.echo()
curses.endwin()
except KeyboardInterrupt:
curses.nocbreak()
curses.echo()
curses.endwin()
tuningOffset = tuningOffset[0:i]
print(display.safe_substitute(strdict))
print(
"T2-T1 time tag offset range: %i to %i (based on %i sets of frames)" %
(tuningOffset.min(), tuningOffset.max(), len(tuningOffset)))
def main(args):
skip = args.skip
fh = open(args.filename, "rb")
# Get the first frame and find out what the firt time tag is, which the
# first frame number is, and what the sample rate it. From the sample
# rate, estimate how the time tag should advance between frames.
while True:
junkFrame = drx.read_frame(fh)
try:
sample_rate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
tagSkip = int(fS / sample_rate * junkFrame.payload.data.shape[0])
fh.seek(-drx.FRAME_SIZE, 1)
# Store the information about the first frame.
prevTime = junkFrame.payload.timetag
prevDate = junkFrame.time.datetime
prevFrame = junkFrame.header.frame_count
# Skip ahead
fh.seek(int(skip*sample_rate/4096)*4*drx.FRAME_SIZE)
# Report on the file
print("Filename: %s" % os.path.basename(args.filename))
print("Date of first frame: %i -> %s" % (prevTime, str(prevDate)))
print("Sample rate: %i Hz" % sample_rate)
print("Time tag skip per frame: %i" % tagSkip)
if skip != 0:
print("Skipping ahead %i frames (%.6f seconds)" % (int(skip*sample_rate/4096)*4, int(skip*sample_rate/4096)*4096/sample_rate))
k = 0
#k = 1
prevTime = [0, 0, 0, 0]
prevDate = ['', '', '', '']
prevNumb = [0, 0, 0, 0]
for i in xrange(4):
currFrame = drx.read_frame(fh)
beam, tune, pol = currFrame.id
rID = 2*(tune-1) + pol
prevTime[rID] = currFrame.payload.timetag
prevDate[rID] = currFrame.time.datetime
prevNumb[rID] = 1 + k // 4
#prevNumb[rID] = k
k += 1
while True:
try:
currFrame = drx.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
currNumb = 1 + k // 4
print("ERROR: invalid frame (sync. word error) @ frame %8i" % currNumb)
continue
beam, tune, pol = currFrame.id
rID = 2*(tune-1) + pol
currTime = currFrame.payload.timetag
currDate = currFrame.time.datetime
currNumb = 1 + k // 4
#currNumb = k
if tune == 1 and pol == 0 and currNumb % 50000 == 0:
print("Beam %i, tune %i, pol %i: frame %8i -> %i (%s)" % (beam, tune, pol, currNumb, currTime, currDate))
if currTime < prevTime[rID]:
print("ERROR: t.t. %i @ frame %i < t.t. %i @ frame %i" % (currTime, currNumb, prevTime[rID], prevNumb[rID]))
print(" -> difference: %i (%.3f frames; %.5f seconds); %s" % (currTime-prevTime[rID], float(currTime-prevTime[rID])/tagSkip, float(currTime-prevTime[rID])/fS, str(currDate)))
print(" -> beam %i, tuning %i, pol %i" % (beam, tune, pol))
elif currTime > (prevTime[rID] + tagSkip):
print("ERROR: t.t. %i @ frame %i > t.t. %i @ frame %i + skip" % (currTime, currNumb, prevTime[rID], prevNumb[rID]))
print(" -> difference: %i (%.3f frames; %.5f seconds); %s" % (currTime-prevTime[rID], float(currTime-prevTime[rID])/tagSkip, float(currTime-prevTime[rID])/fS, str(currDate)))
print(" -> beam %i, tuning %i, pol %i" % (beam, tune, pol))
elif currTime < (prevTime[rID] + tagSkip):
print("ERROR: t.t %i @ frame %i < t.t. %i @ frame %i + skip" % (currTime, currNumb, prevTime[rID], prevNumb[rID]))
print(" -> difference: %i (%.3f frames; %.5f seconds; %s" % (currTime-prevTime[rID], float(currTime-prevTime[rID])/tagSkip, float(currTime-prevTime[rID])/fS, str(currDate)))
print(" -> beam %i, tuning %i, pol %i" % (beam, tune, pol))
else:
pass
prevTime[rID] = currTime
prevDate[rID] = currDate
prevNumb[rID] = currNumb
k += 1
del currFrame
fh.close()
def main(args):
# Set the site
site = None
if args.lwa1:
site = 'lwa1'
elif args.lwasv:
site = 'lwasv'
# Open the file and file good data (not raw DRX data)
fh = open(args.filename, 'rb')
try:
for i in xrange(5):
junkFrame = drx.read_frame(fh)
raise RuntimeError("ERROR: '%s' appears to be a raw DRX file, not a DR spectrometer file" % args.filename)
except errors.SyncError:
fh.seek(0)
# Interrogate the file to figure out what frames sizes to expect, now many
# frames there are, and what the transform length is
FRAME_SIZE = drspec.get_frame_size(fh)
nFrames = os.path.getsize(args.filename) // FRAME_SIZE
nChunks = nFrames
LFFT = drspec.get_transform_size(fh)
# Read in the first frame to figure out the DP information
junkFrame = drspec.read_frame(fh)
fh.seek(-FRAME_SIZE, 1)
srate = junkFrame.sample_rate
t0 = junkFrame.time
tInt = junkFrame.header.nints*LFFT/srate
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.skip / tInt))
fh.seek(offset*FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = drspec.read_frame(fh)
srate = junkFrame.sample_rate
t1 = junkFrame.time
tInt = junkFrame.header.nints*LFFT/srate
fh.seek(-FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1 - (t0 + args.skip)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 2.0
cOffset = int(round(tCorr / tInt))
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset*FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1 - t0
nChunks = (os.path.getsize(args.filename) - fh.tell()) // FRAME_SIZE
# Update the file contents
beam = junkFrame.id
central_freq1, central_freq2 = junkFrame.central_freq
srate = junkFrame.sample_rate
data_products = junkFrame.data_products
t0 = junkFrame.time
tInt = junkFrame.header.nints*LFFT/srate
beginDate = junkFrame.time.datetime
# Report
print("Filename: %s" % args.filename)
if args.metadata is not None:
print("Metadata: %s" % args.metadata)
elif args.sdf is not None:
print("SDF: %s" % args.sdf)
print("Date of First Frame: %s" % beginDate)
print("Beam: %i" % beam)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2))
print("Data Products: %s" % ','.join(data_products))
print("Frames: %i (%.3f s)" % (nFrames, nFrames*tInt))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Transform Length: %i" % LFFT)
print("Integration: %.3f s" % tInt)
# Setup the output file
outname = os.path.split(args.filename)[1]
outname = os.path.splitext(outname)[0]
outname = '%s-waterfall.hdf5' % outname
if os.path.exists(outname):
if not args.force:
yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
raise RuntimeError("Output file '%s' already exists" % outname)
f = hdfData.create_new_file(outname)
obsList = {}
if args.metadata is not None:
try:
project = metabundle.get_sdf(args.metadata)
except Exception as e:
if adpReady:
project = metabundleADP.get_sdf(args.metadata)
else:
raise e
sdfBeam = project.sessions[0].drx_beam
spcSetup = project.sessions[0].spcSetup
if sdfBeam != beam:
raise RuntimeError("Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam))
for i,obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsChunks = int(numpy.ceil(obs.dur/1000.0 * drx.FILTER_CODES[obs.filter] / (spcSetup[0]*spcSetup[1])))
obsList[i+1] = (sdfStart, sdfStop, obsChunks)
hdfData.fill_from_metabundle(f, args.metadata)
elif args.sdf is not None:
try:
project = sdf.parse_sdf(args.sdf)
except Exception as e:
if adpReady:
project = sdfADP.parse_sdf(args.sdf)
else:
raise e
sdfBeam = project.sessions[0].drx_beam
spcSetup = project.sessions[0].spcSetup
if sdfBeam != beam:
raise RuntimeError("Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam))
for i,obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsChunks = int(numpy.ceil(obs.dur/1000.0 * drx.FILTER_CODES[obs.filter] / (spcSetup[0]*spcSetup[1])))
obsList[i+1] = (sdfStart, sdfStop, obsChunks)
hdfData.fill_from_sdf(f, args.sdf, station=site)
else:
obsList[1] = (beginDate, datetime(2222,12,31,23,59,59), nChunks)
hdfData.fill_minimum(f, 1, beam, srate, station=site)
data_products = junkFrame.data_products
for o in sorted(obsList.keys()):
for t in (1,2):
hdfData.create_observation_set(f, o, t, numpy.arange(LFFT, dtype=numpy.float64), obsList[o][2], data_products)
f.attrs['FileGenerator'] = 'drspec2hdf.py'
f.attrs['InputData'] = os.path.basename(args.filename)
# Create the various HDF group holders
ds = {}
for o in sorted(obsList.keys()):
obs = hdfData.get_observation_set(f, o)
ds['obs%i' % o] = obs
ds['obs%i-time' % o] = hdfData.get_time(f, o)
for t in (1,2):
ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq')
for p in data_products:
ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p)
ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'Saturation')
# Loop over DR spectrometer frames to fill in the HDF5 file
pbar = progress.ProgressBar(max=nChunks)
o = 1
j = 0
firstPass = True
for i in xrange(nChunks):
frame = drspec.read_frame(fh)
cTime = frame.time.datetime
if cTime > obsList[o][1]:
# Increment to the next observation
o += 1
# If we have reached the end, exit...
try:
obsList[o]
firstPass = True
except KeyError:
sys.stdout.write('%s\r' % (' '*pbar.span))
sys.stdout.flush()
print("End of observing block according to SDF, exiting")
break
if cTime < obsList[o][0]:
# Skip over data that occurs before the start of the observation
continue
try:
if frame.time > oTime + 1.001*tInt:
print('Warning: Time tag error at frame %i; %.3f > %.3f + %.3f' % (i, frame.time, oTime, tInt))
except NameError:
pass
oTime = frame.time
if firstPass:
# Otherwise, continue on...
central_freq1, central_freq2 = frame.central_freq
srate = frame.sample_rate
tInt = frame.header.nints*LFFT/srate
freq = numpy.fft.fftshift( numpy.fft.fftfreq(LFFT, d=1.0/srate) )
freq = freq.astype(numpy.float64)
sys.stdout.write('%s\r' % (' '*pbar.span))
sys.stdout.flush()
print("Switching to Obs. #%i" % o)
print("-> Tunings: %.1f Hz, %.1f Hz" % (central_freq1, central_freq2))
print("-> Sample Rate: %.1f Hz" % srate)
print("-> Integration Time: %.3f s" % tInt)
sys.stdout.write(pbar.show()+'\r')
sys.stdout.flush()
j = 0
ds['obs%i-freq1' % o][:] = freq + central_freq1
ds['obs%i-freq2' % o][:] = freq + central_freq2
obs = ds['obs%i' % o]
obs.attrs['tInt'] = tInt
obs.attrs['tInt_Units'] = 's'
obs.attrs['LFFT'] = LFFT
obs.attrs['nChan'] = LFFT
obs.attrs['RBW'] = freq[1]-freq[0]
obs.attrs['RBW_Units'] = 'Hz'
firstPass = False
# Load the data from the spectrometer frame into the HDF5 group
ds['obs%i-time' % o][j] = (frame.time[0], frame.time[1])
ds['obs%i-Saturation1' % o][j,:] = frame.payload.saturations[0:2]
ds['obs%i-Saturation2' % o][j,:] = frame.payload.saturations[2:4]
for t in (1,2):
for p in data_products:
ds['obs%i-%s%i' % (o, p, t)][j,:] = getattr(frame.payload, "%s%i" % (p, t-1), None)
j += 1
# Update the progress bar
pbar.inc()
if i % 10 == 0:
sys.stdout.write(pbar.show()+'\r')
sys.stdout.flush()
sys.stdout.write(pbar.show()+'\n')
sys.stdout.flush()
# Done
fh.close()
# Save the output to a HDF5 file
f.close()
def test_drx_buffer_reorder_flush(self):
"""Test the DRX ring buffer's flush() function with reordering."""
fh = open(drxFile, 'rb')
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
fh.seek(-drx.FRAME_SIZE, 1)
# Create the FrameBuffer instance
frameBuffer = buffer.DRXFrameBuffer(beams=[
b,
],
tunes=[1, 2],
pols=[0, 1],
nsegments=2,
reorder=True)
# Go
while True:
try:
cFrame = drx.read_frame(fh)
except errors.EOFError:
break
except errors.SyncError:
continue
frameBuffer.append(cFrame)
cFrames = frameBuffer.get()
if cFrames is None:
continue
# Make sure it has the right number of frames
self.assertEqual(len(cFrames), 4)
# Check the order
for i in range(1, len(cFrames)):
pB, pT, pP = cFrames[i - 1].id
cB, cT, cP = cFrames[i].id
pID = 4 * pB + 2 * (pT - 1) + pP
cID = 4 * cB + 2 * (cT - 1) + cP
self.assertTrue(cID > pID)
fh.close()
# Flush the buffer
for cFrames in frameBuffer.flush():
# Make sure the dump has one of the expected time tags
self.assertTrue(
cFrames[0].payload.timetag in (257355782095346056, ))
# Make sure it has the right number of frames
self.assertEqual(len(cFrames), 4)
# Check the order
for i in range(1, len(cFrames)):
pB, pT, pP = cFrames[i - 1].id
cB, cT, cP = cFrames[i].id
pID = 4 * pB + 2 * (pT - 1) + pP
cID = 4 * cB + 2 * (cT - 1) + cP
self.assertTrue(cID > pID)
def main(args):
fh = open(args.filename, "rb")
try:
for i in xrange(5):
junkFrame = drx.read_frame(fh)
raise RuntimeError(
"ERROR: '%s' appears to be a raw DRX file, not a DR spectrometer file"
% args.filename)
except errors.SyncError:
fh.seek(0)
# Interrogate the file to figure out what frames sizes to expect, now many
# frames there are, and what the transform length is
FRAME_SIZE = drspec.get_frame_size(fh)
nFrames = os.path.getsize(args.filename) / FRAME_SIZE
nChunks = nFrames
LFFT = drspec.get_transform_size(fh)
# Read in the first frame to figure out the DP information
junkFrame = drspec.read_frame(fh)
fh.seek(-FRAME_SIZE, 1)
srate = junkFrame.sample_rate
t0 = junkFrame.time
tInt = junkFrame.header.nints * LFFT / srate
# Update the file contents
beam = junkFrame.id
central_freq1, central_freq2 = junkFrame.central_freq
srate = junkFrame.sample_rate
data_products = junkFrame.data_products
t0 = junkFrame.time
tInt = junkFrame.header.nints * LFFT / srate
beginDate = junkFrame.time.datetime
# Report
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % beginDate)
print("Beam: %i" % beam)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" %
(central_freq1, central_freq2))
print("Data Products: %s" % ','.join(data_products))
print("Frames: %i (%.3f s)" % (nFrames, nFrames * tInt))
print("---")
print("Transform Length: %i" % LFFT)
print("Integration: %.3f s" % tInt)
# Convert chunk length to total frame count
chunkLength = int(args.length / tInt)
# Convert chunk skip to total frame count
chunkSkip = int(args.skip / tInt)
# Output arrays
clipFraction = []
meanPower = []
# Go!
i = 1
done = False
print(" |%sClipping%s |%sPower %s |" %
(" " * (8 * len(data_products) - 4), " " *
(8 * len(data_products) - 4), " " *
(6 * len(data_products) - 3), " " * (6 * len(data_products) - 3)))
out = " | 1X 1Y 2X 2Y |"
for t in (1, 2):
for dp in data_products:
out += "%6s" % ("%i%s" % (t, dp))
out += " |"
print(out)
print("-" * len(out))
while True:
count = {0: 0, 1: 0, 2: 0, 3: 0}
sats = numpy.empty((4, chunkLength), dtype=numpy.float32)
data = numpy.empty((2 * len(data_products), chunkLength * LFFT),
dtype=numpy.float32)
for j in xrange(chunkLength):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = frame = drspec.read_frame(fh)
except errors.EOFError:
done = True
break
except errors.SyncError:
continue
for t in (1, 2):
for p, dp in enumerate(data_products):
l = len(data_products) * (t - 1) + p
data[l, j * LFFT:(j + 1) * LFFT] = getattr(
cFrame.payload, '%s%i' % (dp, t - 1))
sats[:,
j] = numpy.array(cFrame.payload.saturations) / (tInt * srate)
if done:
break
else:
clipFraction.append(sats.mean(axis=1))
meanPower.append(data.mean(axis=1))
clip = clipFraction[-1]
power = meanPower[-1]
out = "%2i | %6.2f%% %6.2f%% %6.2f%% %6.2f%% |" % (
i, clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0,
clip[3] * 100.0)
for t in (1, 2):
for p in xrange(len(data_products)):
out += " %5.2f" % (power[len(data_products) *
(t - 1) + p], )
out += " |"
print(out)
i += 1
fh.seek(FRAME_SIZE * chunkSkip, 1)
clipFraction = numpy.array(clipFraction)
meanPower = numpy.array(meanPower)
clip = clipFraction.mean(axis=0)
power = meanPower.mean(axis=0)
print("-" * len(out))
out = "%2s | %6.2f%% %6.2f%% %6.2f%% %6.2f%% |" % (
'M', clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0,
clip[3] * 100.0)
for t in (1, 2):
for p in xrange(len(data_products)):
out += " %5.2f" % (power[len(data_products) * (t - 1) + p], )
out += " |"
print(out)
def main(args):
for filename in args:
with open(filename, 'rb') as fh:
nframe = os.path.getsize(filename) // drx.FRAME_SIZE
frame0 = drx.read_frame(fh)
fh.seek((nframe-1)*drx.FRAME_SIZE, 0)
frameN = drx.read_frame(fh)
fh.seek(0)
print('Time Range:')
print(' Start:', frame0.time.dp_timetag, '->', frame0.time.datetime)
print(' Stop: ', frameN.time.dp_timetag, '->', frameN.time.datetime)
print(' Total frames:', nframe)
sets = []
times = {}
tuning_word1, tuning_word2 = None, None
central_freq1, central_freq2 = None, None
sample_rate1, sample_rate2 = None, None
for i in range(1024):
try:
frame = drx.read_frame(fh)
except errors.EOFError:
break
id = frame.id
if id not in sets:
sets.append(id)
tt = frame.time.dp_timetag
try:
times[id].append(tt)
except KeyError:
times[id] = [tt,]
beam, tune, pol = id
if tune == 1:
tuning_word1 = frame.payload.tuning_word
central_freq1 = frame.central_freq
sample_rate1 = frame.sample_rate
else:
tuning_word2 = frame.payload.tuning_word
central_freq2 = frame.central_freq
sample_rate2 = frame.sample_rate
sets.sort()
expected_step = int(round(4096 * (196e6 / sample_rate1)))
print('Frequency Range:')
if central_freq1 is not None:
print(' Tuning 1:', tuning_word1, '->', '%.3f MHz' % (central_freq1/1e6,), '@', '%.3f MHz' % (sample_rate1/1e6,))
else:
print(' Tuning 1:', 'not found')
if central_freq2 is not None:
print(' Tuning 2:', tuning_word2, '->', '%.3f MHz' % (central_freq2/1e6,), '@', '%.3f MHz' % (sample_rate2/1e6,))
else:
print(' Tuning 2:', 'not found')
print('Frame Sets:')
for id in sets:
print(' Beam %i, tuning %i, pol. %i' % id)
print('Time Flow:')
for id in sets:
tts = times[id]
offsets = {}
for i in range(1, len(tts)):
step = tts[i] - tts[i-1]
try:
offsets[step] += 1
except KeyError:
offsets[step] = 1
print(' Beam %i, tuning %i, pol. %i' % id)
for step in sorted(list(offsets.keys())):
print(' Timetag step of %i:' % step, '%i occurances' % offsets[step], '*' if step == expected_step else '')
def main(args):
fh = open(args.filename, "rb")
nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE
while True:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
fh.seek(-drx.FRAME_SIZE, 1)
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.offset * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
args.offset = 1.0 * offset / beampols * 4096 / srate
fh.seek(offset * drx.FRAME_SIZE)
# Make sure that the file chunk size contains is an intger multiple
# of the beampols.
maxFrames = int((19144 * 4) / beampols) * beampols
# Setup the statistics data set
if args.stats:
if args.plot_range < 0.1:
args.plot_range = 0.5
# Number of frames to integrate over
nFrames = int(args.plot_range * srate / 4096 * beampols)
nFrames = int(1.0 * nFrames / beampols) * beampols
args.plot_range = 1.0 * nFrames / beampols * 4096 / srate
# Number of remaining chunks
nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames))
# File summary
print("Filename: %s" % args.filename)
print("Beams: %i" % beams)
print("Tune/Pols: %i %i %i %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.offset, offset))
print("Plot time: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.plot_range, nFrames, nFrames // beampols))
print("Chunks: %i" % nChunks)
# Sanity check
if offset > nFramesFile:
raise RuntimeError("Requested offset is greater than file length")
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
# Align the file handle so that the first frame read in the
# main analysis loop is from tuning 1, polarization 0
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
while 2 * (t - 1) + p != 0:
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
fh.seek(-drx.FRAME_SIZE, 1)
# Master loop over all of the file chuncks
standMapper = []
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
(i, framesRemaining))
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros((beampols, framesWork * 4096 // beampols),
dtype=numpy.csingle)
# Inner loop that actually reads the frames into the data array
print("Working on %.1f ms of data" %
((framesWork * 4096 / beampols / srate) * 1000.0))
t0 = time.time()
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1))
continue
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = numpy.abs(cFrame.payload.data)**2
# Update the counters so that we can average properly later on
count[aStand] += 1
# Statistics
print("Running robust statistics")
means = [robust.mean(data[i, :]) for i in xrange(data.shape[0])]
stds = [robust.std(data[i, :]) for i in xrange(data.shape[0])]
if args.stats:
## Report statistics
print("Mean: %s" % ' '.join(["%.3f" % m for m in means]))
print("StdD: %s" % ' '.join(["%.3f" % s for s in stds]))
print("Levels:")
## Count'em up
j = 0
counts = [
1,
] * data.shape[0]
while (means[i] + j * stds[i] <= 98) and max(counts) != 0:
counts = [
len(
numpy.where(
numpy.abs(data[i, :] - means[i]) >= j *
stds[i])[0]) for i in xrange(data.shape[0])
]
print(" %2isigma (%5.1f%%): %s" %
(j, 100.0 * (1 - erf(j / numpy.sqrt(2))), ' '.join([
"%7i (%5.1f%%)" % (c, 100.0 * c / data.shape[1])
for c in counts
])))
j += 1
## Why j-2? Well, j is 1 more than the last iteration. So, that last iteration
## is j-1, which is always filled with 0s by construction. So, the last crazy
## bin is j-2.
jP = j - 2
if jP > 20:
counts = [
len(
numpy.where(
numpy.abs(data[i, :] - means[i]) >= jP *
stds[i])[0]) for i in xrange(data.shape[0])
]
for i in xrange(data.shape[0]):
if counts[i] > 0:
break
if counts[i] == 1:
print(
" -> Clip-o-rama likely occuring with %i %i-sigma detection on tuning %i, pol %i"
% (counts[i], jP, i // 2 + 1, i % 2))
else:
print(
" -> Clip-o-rama likely occuring with %i %i-sigma detections on tuning %i, pol %i"
% (counts[i], jP, i // 2 + 1, i % 2))
else:
outfile = os.path.splitext(args.filename)[0]
outfile = '%s.txt' % outfile
fh = open(outfile, 'w')
# Plot possible clip-o-rama and flag it
print("Computing power derivatives w.r.t. time")
deriv = numpy.zeros_like(data)
for i in xrange(data.shape[0]):
deriv[i, :] = numpy.roll(data[i, :], -1) - data[i, :]
# The plots: This is setup for the current configuration of 20 beampols
print("Plotting")
fig = plt.figure()
figsX = int(round(math.sqrt(beampols)))
figsY = beampols // figsX
for i in xrange(data.shape[0]):
ax = fig.add_subplot(figsX, figsY, i + 1)
ax.plot(args.offset + numpy.arange(0, data.shape[1]) / srate,
data[i, :])
## Mark areas of crazy derivatives
bad = numpy.where(
deriv[i, :] > 20 * stds[i] * numpy.sqrt(2))[0]
for j in bad:
fh.write(
"Clip-o-rama on tuning %i, pol. %i at %.6f seconds\n" %
(i // 2 + 1, i % 2, args.offset + j / srate))
print("Clip-o-rama on tuning %i, pol. %i at %.6f seconds" %
(i // 2 + 1, i % 2, args.offset + j / srate))
ax.vlines(args.offset + j / srate,
-10,
100,
linestyle='--',
color='red',
linewidth=2.0)
## Mark areas of crazy power levels
bad = numpy.where(data[i, :] == 98)[0]
for j in bad:
fh.write(
"Saturation on tuning %i, pol. %i at %.6f seconds\n" %
(i // 2 + 1, i % 2, args.offset + j / srate))
print("Saturation on tuning %i, pol. %i at %.6f seconds" %
(i // 2 + 1, i % 2, args.offset + j / srate))
ax.vlines(args.offset + j / srate,
-10,
100,
linestyle='-.',
color='red')
ax.set_ylim([-10, 100])
ax.set_title('Beam %i, Tune. %i, Pol. %i' %
(beam, i // 2 + 1, i % 2))
ax.set_xlabel('Time [seconds]')
ax.set_ylabel('I$^2$ + Q$^2$')
fh.close()
if args.do_plot:
plt.show()
def main(args):
LFFT = args.fft_length
stand1 = 0
stand2 = int(args.dipole_id_y)
filenames = args.filename
# Build up the station
if args.lwasv:
site = stations.lwasv
else:
site = stations.lwa1
# Get the antennas we need (and a fake one for the beam)
rawAntennas = site.antennas
antennas = []
dipole = None
xyz = numpy.zeros((len(rawAntennas), 3))
i = 0
for ant in rawAntennas:
if ant.stand.id == stand2 and ant.pol == 0:
dipole = ant
xyz[i, 0] = ant.stand.x
xyz[i, 1] = ant.stand.y
xyz[i, 2] = ant.stand.z
i += 1
arrayX = xyz[:, 0].mean()
arrayY = xyz[:, 1].mean()
arrayZ = xyz[:, 2].mean()
## Fake one down here...
beamStand = stations.Stand(0, arrayX, arrayY, arrayZ)
beamFEE = stations.FEE('Beam', 0, gain1=0, gain2=0, status=3)
beamCable = stations.Cable('Beam', 0, vf=1.0)
beamAntenna = stations.Antenna(0,
stand=beamStand,
pol=0,
theta=0,
phi=0,
status=3)
beamAntenna.fee = beamFEE
beamAntenna.feePort = 1
beamAntenna.cable = beamCable
antennas.append(beamAntenna)
## Dipole down here...
### NOTE
### Here we zero out the cable length for the dipole since the delay
### setup that is used for these observations already takes the
### cable/geometric delays into account. We shouldn't need anything
### else to get good fringes.
dipole.cable.length = 0
antennas.append(dipole)
# Loop over the input files...
for filename in filenames:
fh = open(filename, "rb")
nFramesFile = os.path.getsize(filename) // drx.FRAME_SIZE
#junkFrame = drx.read_frame(fh)
#fh.seek(0)
while True:
try:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
t0 = junkFrame.time
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE + 1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
beam, tune, pol = junkFrame.id
srate = junkFrame.sample_rate
tunepols = drx.get_frames_per_obs(fh)
tunepols = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepols
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(args.skip * srate / 4096 * beampols)
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * drx.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = drx.read_frame(fh)
srate = junkFrame.sample_rate
t1 = junkFrame.time
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
fh.seek(-drx.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1 - (t0 + args.skip)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 8.0
cOffset = int(tCorr * srate / 4096 * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * drx.FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1 - t0
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
tnom = junkFrame.header.time_offset
tStart = junkFrame.time
# Get the DRX frequencies
cFreq1 = 0.0
cFreq2 = 0.0
for i in xrange(32):
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
if p == 0 and t == 1:
cFreq1 = junkFrame.central_freq
elif p == 0 and t == 2:
cFreq2 = junkFrame.central_freq
else:
pass
fh.seek(-32 * drx.FRAME_SIZE, 1)
# Align the files as close as possible by the time tags and then make sure that
# the first frame processed is from tuning 1, pol 0.
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
pair = 2 * (tune - 1) + pol
j = 0
while pair != 0:
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
pair = 2 * (tune - 1) + pol
j += 1
fh.seek(-drx.FRAME_SIZE, 1)
print("Shifted beam %i data by %i frames (%.4f s)" %
(beam, j, j * 4096 / srate / 4))
# Set integration time
tInt = args.avg_time
nFrames = int(round(tInt * srate / 4096))
tInt = nFrames * 4096 / srate
# Read in some data
tFile = nFramesFile / 4 * 4096 / srate
# Report
print("Filename: %s" % filename)
print(" Sample Rate: %i Hz" % srate)
print(" Tuning 1: %.1f Hz" % cFreq1)
print(" Tuning 2: %.1f Hz" % cFreq2)
print(" ===")
print(" Integration Time: %.3f s" % tInt)
print(" Integrations in File: %i" % int(tFile / tInt))
print(" Duration of File: %f" % tFile)
print(" Offset: %f s" % offset)
if args.duration != 0:
nChunks = int(round(args.duration / tInt))
else:
nChunks = int(tFile / tInt)
print("Processing: %i integrations" % nChunks)
# Here we start the HDF5 file
outname = os.path.split(filename)[1]
outname = os.path.splitext(outname)[0]
outname = "%s.hdf5" % outname
outfile = h5py.File(outname)
group1 = outfile.create_group("Time")
group2 = outfile.create_group("Frequencies")
group3 = outfile.create_group("Visibilities")
out = raw_input("Target Name: ")
outfile.attrs["OBJECT"] = out
out = raw_input("Polarization (X/Y): ")
outfile.attrs["POLARIZATION"] = out
dset1 = group1.create_dataset("Timesteps", (nChunks, 3),
numpy.float64,
maxshape=(nChunks, 3))
dset2 = group2.create_dataset("Tuning1", (LFFT, ),
numpy.float64,
maxshape=(LFFT, ))
dset3 = group2.create_dataset("Tuning2", (LFFT, ),
numpy.float64,
maxshape=(LFFT, ))
dset4 = group3.create_dataset("Tuning1", (nChunks, 3, LFFT),
numpy.complex64,
maxshape=(nChunks, 3, LFFT))
dset5 = group3.create_dataset("Tuning2", (nChunks, 3, LFFT),
numpy.complex64,
maxshape=(nChunks, 3, LFFT))
drxBuffer = buffer.DRXFrameBuffer(beams=[
beam,
],
tunes=[1, 2],
pols=[0, 1])
data = numpy.zeros((2, 2, 4096 * nFrames), dtype=numpy.complex64)
pb = ProgressBarPlus(max=nChunks)
tsec = numpy.zeros(1, dtype=numpy.float64)
for i in xrange(nChunks):
j = 0
while j < nFrames:
for k in xrange(4):
try:
cFrame = drx.read_frame(fh)
drxBuffer.append(cFrame)
except errors.SyncError:
pass
cFrames = drxBuffer.get()
if cFrames is None:
continue
for cFrame in cFrames:
if j == 0:
tStart = cFrame.time
beam, tune, pol = cFrame.id
pair = 2 * (tune - 1) + pol
if tune == 1:
data[0, pol,
j * 4096:(j + 1) * 4096] = cFrame.payload.data
else:
data[1, pol,
j * 4096:(j + 1) * 4096] = cFrame.payload.data
j += 1
# Correlate
blList1, freq1, vis1 = fxc.FXMaster(data[0, :, :],
antennas,
LFFT=LFFT,
overlap=1,
include_auto=True,
verbose=False,
sample_rate=srate,
central_freq=cFreq1,
pol='XX',
return_baselines=True,
gain_correct=False,
clip_level=0)
blList2, freq2, vis2 = fxc.FXMaster(data[1, :, :],
antennas,
LFFT=LFFT,
overlap=1,
include_auto=True,
verbose=False,
sample_rate=srate,
central_freq=cFreq2,
pol='XX',
return_baselines=True,
gain_correct=False,
clip_level=0)
if i == 0:
tsec = tInt / 2
outfile.attrs["STANDS"] = numpy.array([stand1, stand2])
outfile.attrs["SRATE"] = srate
date = datetime.fromtimestamp(tStart).date()
outfile.attrs["DATE"] = str(date)
dset2.write_direct(freq1)
dset3.write_direct(freq2)
else:
tsec += tInt
temp = numpy.zeros(3, dtype=numpy.float64)
temp[0] = tStart
temp[1] = tInt
temp[2] = tsec
dset1.write_direct(temp, dest_sel=numpy.s_[i])
dset4.write_direct(vis1, dest_sel=numpy.s_[i])
dset5.write_direct(vis2, dest_sel=numpy.s_[i])
pb.inc(amount=1)
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
outfile.close()
# Plot
fig = plt.figure()
i = 0
for bl, vi in zip(blList1, vis1):
ax = fig.add_subplot(4, 3, i + 1)
ax.plot(freq1 / 1e6, numpy.unwrap(numpy.angle(vi)))
ax.set_title('Stand %i - Stand %i' %
(bl[0].stand.id, bl[1].stand.id))
ax = fig.add_subplot(4, 3, i + 4)
ax.plot(freq1 / 1e6, numpy.abs(vi))
i += 1
coeff = numpy.polyfit(freq1, numpy.unwrap(numpy.angle(vi)), 1)
#print(coeff[0]/2/numpy.pi*1e9, coeff[1]*180/numpy.pi)
i = 6
for bl, vi in zip(blList2, vis2):
ax = fig.add_subplot(4, 3, i + 1)
ax.plot(freq2 / 1e6, numpy.unwrap(numpy.angle(vi)))
ax.set_title('Stand %i - Stand %i' %
(bl[0].stand.id, bl[1].stand.id))
ax = fig.add_subplot(4, 3, i + 4)
ax.plot(freq2 / 1e6, numpy.abs(vi))
i += 1
coeff = numpy.polyfit(freq2, numpy.unwrap(numpy.angle(vi)), 1)
