def test_drx_block(self):
"""Test finding out how many tunings/pols. per beam are in a DRX file."""
fh = open(drxFile, 'rb')
b1, b2, b3, b4 = drx.get_frames_per_obs(fh)
self.assertEqual(b1, 0)
self.assertEqual(b2, 0)
self.assertEqual(b3, 0)
self.assertEqual(b4, 4)
fh.close()
def main(args):
fh = open(args.filename, "rb")
nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE
while True:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
fh.seek(-drx.FRAME_SIZE, 1)
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.offset * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
args.offset = 1.0 * offset / beampols * 4096 / srate
fh.seek(offset * drx.FRAME_SIZE)
# Make sure that the file chunk size contains is an intger multiple
# of the beampols.
maxFrames = int((19144 * 4) / beampols) * beampols
# Setup the statistics data set
if args.stats:
if args.plot_range < 0.1:
args.plot_range = 0.5
# Number of frames to integrate over
nFrames = int(args.plot_range * srate / 4096 * beampols)
nFrames = int(1.0 * nFrames / beampols) * beampols
args.plot_range = 1.0 * nFrames / beampols * 4096 / srate
# Number of remaining chunks
nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames))
# File summary
print("Filename: %s" % args.filename)
print("Beams: %i" % beams)
print("Tune/Pols: %i %i %i %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.offset, offset))
print("Plot time: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.plot_range, nFrames, nFrames // beampols))
print("Chunks: %i" % nChunks)
# Sanity check
if offset > nFramesFile:
raise RuntimeError("Requested offset is greater than file length")
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
# Align the file handle so that the first frame read in the
# main analysis loop is from tuning 1, polarization 0
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
while 2 * (t - 1) + p != 0:
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
fh.seek(-drx.FRAME_SIZE, 1)
# Master loop over all of the file chuncks
standMapper = []
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
(i, framesRemaining))
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros((beampols, framesWork * 4096 // beampols),
dtype=numpy.csingle)
# Inner loop that actually reads the frames into the data array
print("Working on %.1f ms of data" %
((framesWork * 4096 / beampols / srate) * 1000.0))
t0 = time.time()
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1))
continue
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = numpy.abs(cFrame.payload.data)**2
# Update the counters so that we can average properly later on
count[aStand] += 1
# Statistics
print("Running robust statistics")
means = [robust.mean(data[i, :]) for i in xrange(data.shape[0])]
stds = [robust.std(data[i, :]) for i in xrange(data.shape[0])]
if args.stats:
## Report statistics
print("Mean: %s" % ' '.join(["%.3f" % m for m in means]))
print("StdD: %s" % ' '.join(["%.3f" % s for s in stds]))
print("Levels:")
## Count'em up
j = 0
counts = [
1,
] * data.shape[0]
while (means[i] + j * stds[i] <= 98) and max(counts) != 0:
counts = [
len(
numpy.where(
numpy.abs(data[i, :] - means[i]) >= j *
stds[i])[0]) for i in xrange(data.shape[0])
]
print(" %2isigma (%5.1f%%): %s" %
(j, 100.0 * (1 - erf(j / numpy.sqrt(2))), ' '.join([
"%7i (%5.1f%%)" % (c, 100.0 * c / data.shape[1])
for c in counts
])))
j += 1
## Why j-2? Well, j is 1 more than the last iteration. So, that last iteration
## is j-1, which is always filled with 0s by construction. So, the last crazy
## bin is j-2.
jP = j - 2
if jP > 20:
counts = [
len(
numpy.where(
numpy.abs(data[i, :] - means[i]) >= jP *
stds[i])[0]) for i in xrange(data.shape[0])
]
for i in xrange(data.shape[0]):
if counts[i] > 0:
break
if counts[i] == 1:
print(
" -> Clip-o-rama likely occuring with %i %i-sigma detection on tuning %i, pol %i"
% (counts[i], jP, i // 2 + 1, i % 2))
else:
print(
" -> Clip-o-rama likely occuring with %i %i-sigma detections on tuning %i, pol %i"
% (counts[i], jP, i // 2 + 1, i % 2))
else:
outfile = os.path.splitext(args.filename)[0]
outfile = '%s.txt' % outfile
fh = open(outfile, 'w')
# Plot possible clip-o-rama and flag it
print("Computing power derivatives w.r.t. time")
deriv = numpy.zeros_like(data)
for i in xrange(data.shape[0]):
deriv[i, :] = numpy.roll(data[i, :], -1) - data[i, :]
# The plots: This is setup for the current configuration of 20 beampols
print("Plotting")
fig = plt.figure()
figsX = int(round(math.sqrt(beampols)))
figsY = beampols // figsX
for i in xrange(data.shape[0]):
ax = fig.add_subplot(figsX, figsY, i + 1)
ax.plot(args.offset + numpy.arange(0, data.shape[1]) / srate,
data[i, :])
## Mark areas of crazy derivatives
bad = numpy.where(
deriv[i, :] > 20 * stds[i] * numpy.sqrt(2))[0]
for j in bad:
fh.write(
"Clip-o-rama on tuning %i, pol. %i at %.6f seconds\n" %
(i // 2 + 1, i % 2, args.offset + j / srate))
print("Clip-o-rama on tuning %i, pol. %i at %.6f seconds" %
(i // 2 + 1, i % 2, args.offset + j / srate))
ax.vlines(args.offset + j / srate,
-10,
100,
linestyle='--',
color='red',
linewidth=2.0)
## Mark areas of crazy power levels
bad = numpy.where(data[i, :] == 98)[0]
for j in bad:
fh.write(
"Saturation on tuning %i, pol. %i at %.6f seconds\n" %
(i // 2 + 1, i % 2, args.offset + j / srate))
print("Saturation on tuning %i, pol. %i at %.6f seconds" %
(i // 2 + 1, i % 2, args.offset + j / srate))
ax.vlines(args.offset + j / srate,
-10,
100,
linestyle='-.',
color='red')
ax.set_ylim([-10, 100])
ax.set_title('Beam %i, Tune. %i, Pol. %i' %
(beam, i // 2 + 1, i % 2))
ax.set_xlabel('Time [seconds]')
ax.set_ylabel('I$^2$ + Q$^2$')
fh.close()
if args.do_plot:
plt.show()
def main(args):
fh = open(args.filename, "rb")
nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE
while True:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
fh.seek(-drx.FRAME_SIZE, 1)
print(junkFrame.header.time_offset)
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
args.skip = 1.0 * offset / beampols * 4096 / srate
fh.seek(offset * drx.FRAME_SIZE)
# Make sure that the file chunk size contains is an intger multiple
# of the beampols.
maxFrames = int(19144 / beampols) * beampols
# Number of frames to integrate over
toClip = False
oldAverage = args.plot_range
if args.plot_range < 4096 / srate:
toClip = True
args.plot_range = 4096 / srate
nFrames = int(args.plot_range * srate / 4096 * beampols)
nFrames = int(1.0 * nFrames / beampols) * beampols
args.plot_range = 1.0 * nFrames / beampols * 4096 / srate
# Number of remaining chunks
nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames))
# File summary
print("Filename: %s" % args.filename)
print("Beams: %i" % beams)
print("Tune/Pols: %i %i %i %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Plot time: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.plot_range, nFrames, nFrames // beampols))
print("Chunks: %i" % nChunks)
# Sanity check
if offset > nFramesFile:
raise RuntimeError("Requested offset is greater than file length")
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
while 2 * (t - 1) + p != 0:
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
print(b, t, p)
print(fh.tell())
fh.seek(-drx.FRAME_SIZE, 1)
# Master loop over all of the file chuncks
standMapper = []
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
(i, framesRemaining))
count = {}
data = numpy.zeros((beampols, framesWork * 4096 // beampols),
dtype=numpy.csingle)
# Inner loop that actually reads the frames into the data array
print("Working on %.1f ms of data" %
((framesWork * 4096 / beampols / srate) * 1000.0))
t0 = time.time()
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1))
continue
beam, tune, pol = cFrame.id
aStand = 4 * (beam - 1) + 2 * (tune - 1) + pol
#print(aStand, beam, tune, pol)
if aStand not in standMapper:
standMapper.append(aStand)
oStand = 1 * aStand
aStand = standMapper.index(aStand)
print(
"Mapping beam %i, tune. %1i, pol. %1i (%2i) to array index %3i"
% (beam, tune, pol, oStand, aStand))
else:
aStand = standMapper.index(aStand)
if aStand not in count.keys():
count[aStand] = 0
#if cFrame.header.frame_count % 10000 == 0 and args.verbose:
# print("%2i,%1i,%1i -> %2i %5i %i" % (beam, tune, pol, aStand, cFrame.header.frame_count, cFrame.payload.timetag))
#print(data.shape, count[aStand]*4096, (count[aStand]+1)*4096, cFrame.payload.data.shape)
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] += 1
# The plots: This is setup for the current configuration of 20 beampols
fig = plt.figure()
figsX = int(round(math.sqrt(beampols)))
figsY = beampols // figsX
t1X = 1
t1Y = 1
offset = 0
samples = 65536
for sec in xrange(data.shape[1] // samples):
if toClip:
print("Plotting only the first %i samples (%.3f ms) of data" %
(samples, oldAverage * 1000.0))
sortedMapper = sorted(standMapper)
for k, aStand in enumerate(sortedMapper):
i = standMapper.index(aStand)
if standMapper[i] % 2 == 0:
ref = data[0, :]
t1R = t1X
else:
ref = data[1, :]
t1R = t1Y
(lag, cc), junkI, junkQ = crossCorrelate(
data[i, sec * samples:(sec + 1) * samples],
ref[offset + sec * samples:offset + (sec + 1) * samples])
best = numpy.where(cc == cc.max())[0][0]
if args.verbose:
print('tune %i pol. %s' %
(standMapper[i] % 4 // 2 + 1, standMapper[i] % 2))
print(' -> best peak of %.0f at a lag of %i samples' %
(cc.max(), lag[best]))
print(' -> NCM with tuning 1 of %.3f' % (cc.max() / t1R))
# Plot
ax = fig.add_subplot(figsX, figsY, k + 1)
ax.plot(lag, cc, label='Same', color='blue')
# Center on the peak
best = numpy.where(cc == cc.max())[0][0]
ax.set_xlim([lag[best - 50], lag[best + 50]])
ax.set_title('Beam %i, Tune. %i, Pol. %i' %
(standMapper[i] // 4 + 1,
standMapper[i] % 4 // 2 + 1, standMapper[i] % 2))
ax.set_xlabel('Lag [samples]')
ax.set_ylabel('Analysis Sets')
# Save the tuning 1 values for the peak of the CC function
if standMapper[i] % 4 / 2 + 1 == 1:
if standMapper[i] % 2 == 0:
t1X = cc.max()
else:
t1Y = cc.max()
plt.show()
# Save image if requested
if args.output is not None:
fig.savefig(args.output)
def main(args):
LFFT = args.fft_length
stand1 = 0
stand2 = int(args.dipole_id_y)
filenames = args.filename
# Build up the station
if args.lwasv:
site = stations.lwasv
else:
site = stations.lwa1
# Get the antennas we need (and a fake one for the beam)
rawAntennas = site.antennas
antennas = []
dipole = None
xyz = numpy.zeros((len(rawAntennas), 3))
i = 0
for ant in rawAntennas:
if ant.stand.id == stand2 and ant.pol == 0:
dipole = ant
xyz[i, 0] = ant.stand.x
xyz[i, 1] = ant.stand.y
xyz[i, 2] = ant.stand.z
i += 1
arrayX = xyz[:, 0].mean()
arrayY = xyz[:, 1].mean()
arrayZ = xyz[:, 2].mean()
## Fake one down here...
beamStand = stations.Stand(0, arrayX, arrayY, arrayZ)
beamFEE = stations.FEE('Beam', 0, gain1=0, gain2=0, status=3)
beamCable = stations.Cable('Beam', 0, vf=1.0)
beamAntenna = stations.Antenna(0,
stand=beamStand,
pol=0,
theta=0,
phi=0,
status=3)
beamAntenna.fee = beamFEE
beamAntenna.feePort = 1
beamAntenna.cable = beamCable
antennas.append(beamAntenna)
## Dipole down here...
### NOTE
### Here we zero out the cable length for the dipole since the delay
### setup that is used for these observations already takes the
### cable/geometric delays into account. We shouldn't need anything
### else to get good fringes.
dipole.cable.length = 0
antennas.append(dipole)
# Loop over the input files...
for filename in filenames:
fh = open(filename, "rb")
nFramesFile = os.path.getsize(filename) // drx.FRAME_SIZE
#junkFrame = drx.read_frame(fh)
#fh.seek(0)
while True:
try:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
t0 = junkFrame.time
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE + 1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
beam, tune, pol = junkFrame.id
srate = junkFrame.sample_rate
tunepols = drx.get_frames_per_obs(fh)
tunepols = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepols
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(args.skip * srate / 4096 * beampols)
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * drx.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = drx.read_frame(fh)
srate = junkFrame.sample_rate
t1 = junkFrame.time
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
fh.seek(-drx.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1 - (t0 + args.skip)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 8.0
cOffset = int(tCorr * srate / 4096 * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * drx.FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1 - t0
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
tnom = junkFrame.header.time_offset
tStart = junkFrame.time
# Get the DRX frequencies
cFreq1 = 0.0
cFreq2 = 0.0
for i in xrange(32):
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
if p == 0 and t == 1:
cFreq1 = junkFrame.central_freq
elif p == 0 and t == 2:
cFreq2 = junkFrame.central_freq
else:
pass
fh.seek(-32 * drx.FRAME_SIZE, 1)
# Align the files as close as possible by the time tags and then make sure that
# the first frame processed is from tuning 1, pol 0.
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
pair = 2 * (tune - 1) + pol
j = 0
while pair != 0:
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
pair = 2 * (tune - 1) + pol
j += 1
fh.seek(-drx.FRAME_SIZE, 1)
print("Shifted beam %i data by %i frames (%.4f s)" %
(beam, j, j * 4096 / srate / 4))
# Set integration time
tInt = args.avg_time
nFrames = int(round(tInt * srate / 4096))
tInt = nFrames * 4096 / srate
# Read in some data
tFile = nFramesFile / 4 * 4096 / srate
# Report
print("Filename: %s" % filename)
print(" Sample Rate: %i Hz" % srate)
print(" Tuning 1: %.1f Hz" % cFreq1)
print(" Tuning 2: %.1f Hz" % cFreq2)
print(" ===")
print(" Integration Time: %.3f s" % tInt)
print(" Integrations in File: %i" % int(tFile / tInt))
print(" Duration of File: %f" % tFile)
print(" Offset: %f s" % offset)
if args.duration != 0:
nChunks = int(round(args.duration / tInt))
else:
nChunks = int(tFile / tInt)
print("Processing: %i integrations" % nChunks)
# Here we start the HDF5 file
outname = os.path.split(filename)[1]
outname = os.path.splitext(outname)[0]
outname = "%s.hdf5" % outname
outfile = h5py.File(outname)
group1 = outfile.create_group("Time")
group2 = outfile.create_group("Frequencies")
group3 = outfile.create_group("Visibilities")
out = raw_input("Target Name: ")
outfile.attrs["OBJECT"] = out
out = raw_input("Polarization (X/Y): ")
outfile.attrs["POLARIZATION"] = out
dset1 = group1.create_dataset("Timesteps", (nChunks, 3),
numpy.float64,
maxshape=(nChunks, 3))
dset2 = group2.create_dataset("Tuning1", (LFFT, ),
numpy.float64,
maxshape=(LFFT, ))
dset3 = group2.create_dataset("Tuning2", (LFFT, ),
numpy.float64,
maxshape=(LFFT, ))
dset4 = group3.create_dataset("Tuning1", (nChunks, 3, LFFT),
numpy.complex64,
maxshape=(nChunks, 3, LFFT))
dset5 = group3.create_dataset("Tuning2", (nChunks, 3, LFFT),
numpy.complex64,
maxshape=(nChunks, 3, LFFT))
drxBuffer = buffer.DRXFrameBuffer(beams=[
beam,
],
tunes=[1, 2],
pols=[0, 1])
data = numpy.zeros((2, 2, 4096 * nFrames), dtype=numpy.complex64)
pb = ProgressBarPlus(max=nChunks)
tsec = numpy.zeros(1, dtype=numpy.float64)
for i in xrange(nChunks):
j = 0
while j < nFrames:
for k in xrange(4):
try:
cFrame = drx.read_frame(fh)
drxBuffer.append(cFrame)
except errors.SyncError:
pass
cFrames = drxBuffer.get()
if cFrames is None:
continue
for cFrame in cFrames:
if j == 0:
tStart = cFrame.time
beam, tune, pol = cFrame.id
pair = 2 * (tune - 1) + pol
if tune == 1:
data[0, pol,
j * 4096:(j + 1) * 4096] = cFrame.payload.data
else:
data[1, pol,
j * 4096:(j + 1) * 4096] = cFrame.payload.data
j += 1
# Correlate
blList1, freq1, vis1 = fxc.FXMaster(data[0, :, :],
antennas,
LFFT=LFFT,
overlap=1,
include_auto=True,
verbose=False,
sample_rate=srate,
central_freq=cFreq1,
pol='XX',
return_baselines=True,
gain_correct=False,
clip_level=0)
blList2, freq2, vis2 = fxc.FXMaster(data[1, :, :],
antennas,
LFFT=LFFT,
overlap=1,
include_auto=True,
verbose=False,
sample_rate=srate,
central_freq=cFreq2,
pol='XX',
return_baselines=True,
gain_correct=False,
clip_level=0)
if i == 0:
tsec = tInt / 2
outfile.attrs["STANDS"] = numpy.array([stand1, stand2])
outfile.attrs["SRATE"] = srate
date = datetime.fromtimestamp(tStart).date()
outfile.attrs["DATE"] = str(date)
dset2.write_direct(freq1)
dset3.write_direct(freq2)
else:
tsec += tInt
temp = numpy.zeros(3, dtype=numpy.float64)
temp[0] = tStart
temp[1] = tInt
temp[2] = tsec
dset1.write_direct(temp, dest_sel=numpy.s_[i])
dset4.write_direct(vis1, dest_sel=numpy.s_[i])
dset5.write_direct(vis2, dest_sel=numpy.s_[i])
pb.inc(amount=1)
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
outfile.close()
# Plot
fig = plt.figure()
i = 0
for bl, vi in zip(blList1, vis1):
ax = fig.add_subplot(4, 3, i + 1)
ax.plot(freq1 / 1e6, numpy.unwrap(numpy.angle(vi)))
ax.set_title('Stand %i - Stand %i' %
(bl[0].stand.id, bl[1].stand.id))
ax = fig.add_subplot(4, 3, i + 4)
ax.plot(freq1 / 1e6, numpy.abs(vi))
i += 1
coeff = numpy.polyfit(freq1, numpy.unwrap(numpy.angle(vi)), 1)
#print(coeff[0]/2/numpy.pi*1e9, coeff[1]*180/numpy.pi)
i = 6
for bl, vi in zip(blList2, vis2):
ax = fig.add_subplot(4, 3, i + 1)
ax.plot(freq2 / 1e6, numpy.unwrap(numpy.angle(vi)))
ax.set_title('Stand %i - Stand %i' %
(bl[0].stand.id, bl[1].stand.id))
ax = fig.add_subplot(4, 3, i + 4)
ax.plot(freq2 / 1e6, numpy.abs(vi))
i += 1
coeff = numpy.polyfit(freq2, numpy.unwrap(numpy.angle(vi)), 1)
def main(args):
filename = args[0]
fh = open(filename, "rb")
nFramesFile = os.path.getsize(filename) / drx.FRAME_SIZE
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
while 2 * (tune - 1) + pol != 0:
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
fh.seek(fh.tell() - drx.FRAME_SIZE)
srate = junkFrame.sample_rate
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# File summary
out = "Filename: %s" % filename
out += "\nBeams: %i" % beams
out += "\nTune/Pols: %i %i %i %i" % tunepols
out += "\nSample Rate: %i Hz" % srate
out += "\nFrames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile /
beampols * 4096 / srate)
out += "\n==="
print(out)
tuningOffset = numpy.zeros(nFramesFile / 8, dtype=numpy.int64)
try:
screen = curses.initscr()
curses.noecho()
curses.cbreak()
screen.nodelay(1)
strdict = {'preamble': out}
for i in xrange(tuningOffset.size):
screen.clear()
beamIDs = [0, 0, 0, 0]
timetags = numpy.zeros(4, dtype=numpy.int64) - 1
time_offsets = numpy.zeros(4, dtype=numpy.int64) - 1
timeValues = numpy.zeros(4, dtype=numpy.float64)
for j in xrange(4):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1))
continue
## Save the time time, time offset, and computed time values
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
if timetags[aStand] == -1:
beamIDs[aStand] = (beam, tune, pol)
timetags[aStand] = cFrame.payload.timetag
time_offsets[aStand] = cFrame.header.time_offset
timeValues[aStand] = cFrame.time
k = 0
for id, tt, to, tv in zip(beamIDs, timetags, time_offsets,
timeValues):
strdict['b%i' % k] = id[0]
strdict['t%i' % k] = id[1]
strdict['p%i' % k] = id[2]
strdict['tt%i' % k] = tt
strdict['os%i' % k] = to
strdict['tv%i' % k] = tv
k += 1
t1t = timetags[0] - time_offsets[0]
t2t = timetags[3] - time_offsets[3]
tuningOffset[i] = t2t - t1t
strdict['ttd'] = t2t - t1t
strdict['tvd'] = (t2t - t1t) / fS
screen.addstr(0, 0, display.safe_substitute(strdict))
screen.refresh()
# Check for keypress and exit if Q
c = screen.getch()
if (c > 0):
if chr(c) == 'q':
break
if chr(c) == 'Q':
break
curses.nocbreak()
curses.echo()
curses.endwin()
except KeyboardInterrupt:
curses.nocbreak()
curses.echo()
curses.endwin()
tuningOffset = tuningOffset[0:i]
print(display.safe_substitute(strdict))
print(
"T2-T1 time tag offset range: %i to %i (based on %i sets of frames)" %
(tuningOffset.min(), tuningOffset.max(), len(tuningOffset)))
def main(args):
LFFT = args.fft_length
stand1 = int(args.dipole_id_x)
stand2 = int(args.dipole_id_y)
filenames = args.filename
# Build up the station
if args.lwasv:
site = stations.lwasv
else:
site = stations.lwa1
# Figure out which antennas we need
antennas = []
for ant in site.antennas:
if ant.stand.id == stand1 and ant.pol == 0:
antennas.append(ant)
for ant in site.antennas:
if ant.stand.id == stand2 and ant.pol == 0:
antennas.append(ant)
# Loop through the input files...
for filename in filenames:
fh = open(filename, "rb")
nFramesFile = os.path.getsize(filename) // drx.FRAME_SIZE
#junkFrame = drx.read_frame(fh)
#fh.seek(0)
while True:
try:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
t0 = junkFrame.time
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE + 1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
beam, tune, pol = junkFrame.id
srate = junkFrame.sample_rate
tunepols = drx.get_frames_per_obs(fh)
tunepols = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepols
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(args.skip * srate / 4096 * beampols)
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * drx.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = drx.read_frame(fh)
srate = junkFrame.sample_rate
t1 = junkFrame.time
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
fh.seek(-drx.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1 - (t0 + args.skip)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 8.0
cOffset = int(tCorr * srate / 4096 * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * drx.FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1 - t0
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
tnom = junkFrame.header.time_offset
tStart = junkFrame.time
# Get the DRX frequencies
cFreq1 = 0.0
cFreq2 = 0.0
for i in xrange(4):
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
if p == 0 and t == 1:
cFreq1 = junkFrame.central_freq
elif p == 0 and t == 2:
cFreq2 = junkFrame.central_freq
else:
pass
fh.seek(-4 * drx.FRAME_SIZE, 1)
# Align the files as close as possible by the time tags and then make sure that
# the first frame processed is from tuning 1, pol 0.
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
pair = 2 * (tune - 1) + pol
j = 0
while pair != 0:
junkFrame = drx.read_frame(fh)
beam, tune, pol = junkFrame.id
pair = 2 * (tune - 1) + pol
j += 1
fh.seek(-drx.FRAME_SIZE, 1)
print("Shifted beam %i data by %i frames (%.4f s)" %
(beam, j, j * 4096 / srate / 4))
# Set integration time
tInt = args.avg_time
nFrames = int(round(tInt * srate / 4096))
tInt = nFrames * 4096 / srate
# Read in some data
tFile = nFramesFile / 4 * 4096 / srate
# Report
print("Filename: %s" % filename)
print(" Sample Rate: %i Hz" % srate)
print(" Tuning 1: %.1f Hz" % cFreq1)
print(" Tuning 2: %.1f Hz" % cFreq2)
print(" ===")
print(" Integration Time: %.3f s" % tInt)
print(" Integrations in File: %i" % int(tFile / tInt))
nChunks = int(tFile / tInt)
pb = ProgressBar(max=nChunks)
for i in xrange(nChunks):
junkFrame = drx.read_frame(fh)
tStart = junkFrame.time
fh.seek(-drx.FRAME_SIZE, 1)
count1 = [0, 0]
data1 = numpy.zeros((2, 4096 * nFrames), dtype=numpy.complex64)
count2 = [0, 0]
data2 = numpy.zeros((2, 4096 * nFrames), dtype=numpy.complex64)
for j in xrange(nFrames):
for k in xrange(4):
cFrame = drx.read_frame(fh)
beam, tune, pol = cFrame.id
pair = 2 * (tune - 1) + pol
if tune == 1:
data1[pol, count1[pol] * 4096:(count1[pol] + 1) *
4096] = cFrame.payload.data
count1[pol] += 1
else:
data2[pol, count2[pol] * 4096:(count2[pol] + 1) *
4096] = cFrame.payload.data
count2[pol] += 1
# Correlate
blList1, freq1, vis1 = fxc.FXMaster(data1,
antennas,
LFFT=LFFT,
overlap=1,
include_auto=True,
verbose=False,
sample_rate=srate,
central_freq=cFreq1,
pol='XX',
return_baselines=True,
gain_correct=False,
clip_level=0)
blList2, freq2, vis2 = fxc.FXMaster(data2,
antennas,
LFFT=LFFT,
overlap=1,
include_auto=True,
verbose=False,
sample_rate=srate,
central_freq=cFreq2,
pol='XX',
return_baselines=True,
gain_correct=False,
clip_level=0)
if nChunks != 1:
outfile = os.path.split(filename)[1]
outfile = os.path.splitext(outfile)[0]
outfile = "%s-vis-%04i.npz" % (outfile, i + 1)
else:
outfile = os.path.split(filename)[1]
outfile = os.path.splitext(outfile)[0]
outfile = "%s-vis.npz" % outfile
numpy.savez(outfile,
srate=srate,
freq1=freq1,
vis1=vis1,
freq2=freq2,
vis2=vis2,
tStart=tStart,
tInt=tInt,
stands=numpy.array([stand1, stand2]))
del data1
del data2
pb.inc(amount=1)
sys.stdout.write(pb.show() + '\r')
sys.stdout.flush()
sys.stdout.write(pb.show() + '\r')
sys.stdout.write('\n')
sys.stdout.flush()
# Plot
fig = plt.figure()
i = 0
for bl, vi in zip(blList1, vis1):
ax = fig.add_subplot(4, 3, i + 1)
ax.plot(freq1 / 1e6, numpy.unwrap(numpy.angle(vi)))
ax.set_title('Stand %i - Stand %i' %
(bl[0].stand.id, bl[1].stand.id))
ax = fig.add_subplot(4, 3, i + 4)
ax.plot(freq1 / 1e6, numpy.abs(vi))
i += 1
coeff = numpy.polyfit(freq1, numpy.unwrap(numpy.angle(vi)), 1)
#print(coeff[0]/2/numpy.pi*1e9, coeff[1]*180/numpy.pi)
i = 6
for bl, vi in zip(blList2, vis2):
ax = fig.add_subplot(4, 3, i + 1)
ax.plot(freq2 / 1e6, numpy.unwrap(numpy.angle(vi)))
ax.set_title('Stand %i - Stand %i' %
(bl[0].stand.id, bl[1].stand.id))
ax = fig.add_subplot(4, 3, i + 4)
ax.plot(freq2 / 1e6, numpy.abs(vi))
i += 1
coeff = numpy.polyfit(freq2, numpy.unwrap(numpy.angle(vi)), 1)
def main(args):
filename = args.filename
fh = open(filename, "rb")
nFramesFile = os.path.getsize(filename) // drx.FRAME_SIZE
while True:
try:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE + 1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
beam, tune, pol = junkFrame.id
tunepols = max(drx.get_frames_per_obs(fh))
# Date & Central Frequnecy
beginDate = junkFrame.time.datetime
central_freq1 = 0.0
central_freq2 = 0.0
for i in xrange(32):
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
if p == 0 and t == 1:
central_freq1 = junkFrame.central_freq
elif p == 0 and t == 2:
central_freq2 = junkFrame.central_freq
else:
pass
fh.seek(-32 * drx.FRAME_SIZE, 1)
# Report on the file
print("Filename: %s" % filename)
print("Date of First Frame: %s" % str(beginDate))
print("Beam: %i" % beam)
print("Tune/Pols: %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" %
(central_freq1, central_freq2))
print(" ")
# Convert chunk length to total frame count
chunkLength = int(args.length * srate / 4096 * tunepols)
chunkLength = int(1.0 * chunkLength / tunepols) * tunepols
# Convert chunk skip to total frame count
chunkSkip = int(args.skip * srate / 4096 * tunepols)
chunkSkip = int(1.0 * chunkSkip / tunepols) * tunepols
# Output arrays
clipFraction = []
meanPower = []
# Go!
i = 1
done = False
print(" | Clipping | Power |")
print(" | 1X 1Y 2X 2Y | 1X 1Y 2X 2Y |")
print("---+---------------------------------+-------------------------+")
while True:
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.empty((4, chunkLength * 4096 // tunepols),
dtype=numpy.csingle)
for j in xrange(chunkLength):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
done = True
break
except errors.SyncError:
continue
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
try:
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] += 1
except ValueError:
pass
if done:
break
else:
data = numpy.abs(data)**2
data = data.astype(numpy.int32)
clipFraction.append(numpy.zeros(4))
meanPower.append(data.mean(axis=1))
for j in xrange(4):
bad = numpy.nonzero(data[j, :] > args.trim_level)[0]
clipFraction[-1][j] = 1.0 * len(bad) / data.shape[1]
clip = clipFraction[-1]
power = meanPower[-1]
print(
"%2i | %6.2f%% %6.2f%% %6.2f%% %6.2f%% | %5.2f %5.2f %5.2f %5.2f |"
% (i, clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0,
clip[3] * 100.0, power[0], power[1], power[2], power[3]))
i += 1
fh.seek(drx.FRAME_SIZE * chunkSkip, 1)
clipFraction = numpy.array(clipFraction)
meanPower = numpy.array(meanPower)
clip = clipFraction.mean(axis=0)
power = meanPower.mean(axis=0)
print("---+---------------------------------+-------------------------+")
print("%2s | %6.2f%% %6.2f%% %6.2f%% %6.2f%% | %5.2f %5.2f %5.2f %5.2f |" %
('M', clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0,
clip[3] * 100.0, power[0], power[1], power[2], power[3]))
def main(args):
filename = args.filename
sizeB = os.path.getsize(filename)
# Open the file and get some basic info about the data contained
fh = open(filename, 'rb')
nFramesFile = sizeB / drx.FRAME_SIZE
while True:
try:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
t0 = junkFrame.time
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE + 1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.offset * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * drx.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = drx.read_frame(fh)
srate = junkFrame.sample_rate
t1 = junkFrame.time
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
fh.seek(-drx.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1 - (t0 + args.offset)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 2.0
cOffset = int(tCorr * srate / 4096 * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * drx.FRAME_SIZE, 1)
# Update the offset actually used
args.offset = t1 - t0
nCaptures = nFramesFile / beampols
print("Filename: %s" % filename)
print("Size: %.1f MB" % (float(sizeB) / 1024 / 1024))
print("Captures: %i (%.2f seconds)" %
(nCaptures, nCaptures * 4096 / srate))
print("Tuning/Pols.: %i " % tunepol)
print("Sample Rate: %.2f MHz" % (srate / 1e6))
print("===")
if args.count > 0:
nCaptures = args.count * srate // 4096
else:
nCaptures -= args.offset * srate // 4096
args.count = nCaptures * 4096 // srate
print("Seconds to Skip: %.2f (%i captures)" %
(args.offset, offset / beampols))
print("Seconds to Split: %.2f (%i captures)" % (args.count, nCaptures))
# Make sure that the first frame in the file is the first frame of a capture
# (tuning 1, pol 0). If not, read in as many frames as necessary to get to
# the beginning of a complete capture.
beam, tune, pol = junkFrame.id
skip = 0
while (2 * (tune - 1) + pol) != 0:
frame = drx.read_frame(fh)
beam, tune, pol = frame.id
skip += 1
nFramesRemaining = (sizeB - fh.tell()) // drx.FRAME_SIZE
nRecursions = int(nFramesRemaining // (nCaptures * beampols))
if not args.recursive:
nRecursions = 1
scale = int(math.log10(nRecursions)) + 1
ifString = "Working on #%%%ii of %i (%%s)" % (scale, nRecursions)
for r in range(nRecursions):
if args.date:
filePos = fh.tell()
junkFrame = drx.read_frame(fh)
fh.seek(filePos)
dt = junkFrame.time.datetime
captFilename = "%s_%s.dat" % (os.path.splitext(
os.path.basename(filename))[0], dt.isoformat())
else:
captFilename = "%s_s%04i_p%%0%ii.dat" % (os.path.splitext(
os.path.basename(filename))[0], args.count, scale)
captFilename = captFilename % r
if not args.recursive:
captFilename = "%s_s%04i.dat" % (os.path.splitext(
os.path.basename(filename))[0], args.count)
print(ifString % (r + 1, captFilename))
t0 = time.time()
fhOut = open(captFilename, 'wb')
split_file(fh, fhOut, nCaptures, beampols)
fhOut.close()
t1 = time.time()
print(" Copied %i bytes in %.3f s (%.3f MB/s)" %
(os.path.getsize(captFilename), t1 - t0,
os.path.getsize(captFilename) / 1024.0**2 / (t1 - t0)))
fh.close()
def main(args):
fh = open(args.filename, "rb")
nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE
while True:
try:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
t0i, t0f = junkFrame.time
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-drx.FRAME_SIZE + 1, 1)
fh.seek(-drx.FRAME_SIZE, 1)
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * drx.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = drx.read_frame(fh)
srate = junkFrame.sample_rate
t1i, t1f = junkFrame.time
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
fh.seek(-drx.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1i - (t0i + args.skip) + (t1f - t0f)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 2.0
cOffset = int(tCorr * srate / 4096 * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * drx.FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1i - t0i + t1f - t0f
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
# Make sure that the file chunk size contains is an intger multiple
# of the beampols.
maxFrames = int(19144 / beampols) * beampols
# Number of frames to integrate over
toClip = False
oldAverage = args.plot_range
if args.plot_range < 4096 / srate:
toClip = True
args.plot_range = 4096 / srate
nFrames = int(args.plot_range * srate / 4096 * beampols)
nFrames = int(1.0 * nFrames / beampols) * beampols
args.plot_range = 1.0 * nFrames / beampols * 4096 / srate
# Number of remaining chunks
nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames))
# File summary
print("Filename: %s" % args.filename)
print("Beams: %i" % beams)
print("Tune/Pols: %i %i %i %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Plot time: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.plot_range, nFrames, nFrames // beampols))
print("Chunks: %i" % nChunks)
# Sanity check
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
# Align the file handle so that the first frame read in the
# main analysis loop is from tuning 1, polarization 0
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
while 2 * (t - 1) + p != 0:
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
fh.seek(-drx.FRAME_SIZE, 1)
# Master loop over all of the file chuncks
standMapper = []
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
(i, framesRemaining))
count = {0: 0, 1: 0, 2: 0, 3: 0}
tt = numpy.zeros(
(beampols, framesWork // beampols), dtype=numpy.int64) - 1
data = numpy.zeros((beampols, framesWork * 4096 // beampols),
dtype=numpy.csingle)
# Inner loop that actually reads the frames into the data array
print("Working on %.1f ms of data" %
((framesWork * 4096 / beampols / srate) * 1000.0))
t0 = time.time()
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())//drx.FRAME_SIZE-1))
continue
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
tt[aStand, count[aStand]] = cFrame.payload.timetag
if args.instantaneous_power:
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = numpy.abs(cFrame.payload.data)**2
else:
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] += 1
# The plots: This is setup for the current configuration of 20 beampols
fig = plt.figure()
figsX = int(round(math.sqrt(beampols)))
figsY = beampols // figsX
samples = int(oldAverage * srate)
if toClip:
print("Plotting only the first %i samples (%.3f ms) of data" %
(samples, oldAverage * 1000.0))
sortedMapper = sorted(standMapper)
for i in xrange(data.shape[0]):
ax = fig.add_subplot(figsX, figsY, i + 1)
if args.instantaneous_power:
limits = (-10, 100)
if toClip:
ax.plot(args.skip + numpy.arange(0, samples) / srate,
data[i, 0:samples])
else:
ax.plot(args.skip + numpy.arange(0, data.shape[1]) / srate,
data[i, :])
else:
limits = (-8, 8)
if toClip:
ax.plot(args.skip + numpy.arange(0, samples) / srate,
data[i, 0:samples].real,
label='I')
ax.plot(args.skip + numpy.arange(0, samples) / srate,
data[i, 0:samples].imag,
label='Q')
else:
ax.plot(args.skip + numpy.arange(0, data.shape[1]) / srate,
data[i, :].real,
label='I')
ax.plot(args.skip + numpy.arange(0, data.shape[1]) / srate,
data[i, :].imag,
label='Q')
ax.legend(loc=0)
if args.mark_frames:
for j in xrange(0, samples - 4096, 4096):
ax.vlines(float(j) / srate,
limits[0],
limits[1],
color='k',
label='%i' % tt[i, j // 4096])
ax.set_ylim(limits)
ax.set_title('Beam %i, Tune. %i, Pol. %i' %
(beam, i // 2 + 1, i % 2))
ax.set_xlabel('Time [seconds]')
if args.instantaneous_power:
ax.set_ylabel('I$^2$ + Q$^2$')
else:
ax.set_ylabel('Output Level')
plt.show()
# Save image if requested
if args.output is not None:
fig.savefig(args.output)
def main(args):
fh = open(args.filename, "rb")
nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE
while True:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
t0i, t0f = junkFrame.time
break
except ZeroDivisionError:
pass
fh.seek(-drx.FRAME_SIZE, 1)
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * drx.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = drx.read_frame(fh)
srate = junkFrame.sample_rate
t1i, t1f = junkFrame.time
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
fh.seek(-drx.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1i - (t0i + args.skip) + (t1f - t0f)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 2.0
cOffset = int(tCorr * srate / 4096 * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * drx.FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1i - t0i + t1f - t0f
offset = int(round(args.skip * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
# Make sure that the file chunk size contains is an intger multiple
# of the beampols.
maxFrames = int(round(args.average * srate / 4096)) * beampols
if maxFrames < beampols:
maxFrames = beampols
args.average = 1.0 * maxFrames / beampols * 4096 / srate
# Number of remaining chunks
nChunks = int(round(args.duration / args.average))
nFrames = maxFrames * nChunks
# Store the information about the first frame.
prevTime = junkFrame.payload.timetag
prevDate = junkFrame.time.datetime
# File summary
print("Filename: %s" % args.filename)
print("Beams: %i" % beams)
print("Tune/Pols: %i %i %i %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Date of first frame: %i -> %s" % (prevTime, str(prevDate)))
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Integration: %.4f s (%i frames; %i frames per beam/tune/pol)" %
(args.average, maxFrames, maxFrames // beampols))
print("Duration: %.4f s (%i frames; %i frames per beam/tune/pol)" %
(args.average * nChunks, nFrames, nFrames // beampols))
print(" ")
# Sanity check
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
# Align the file handle so that the first frame read in the
# main analysis loop is from tuning 1, polarization 0
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
while 2 * (t - 1) + p != 0:
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
fh.seek(-drx.FRAME_SIZE, 1)
# Master loop over all of the file chuncks
masterTimes = numpy.zeros((nChunks, 4), dtype=numpy.float64)
masterData = numpy.zeros((nChunks, 4), dtype=numpy.float32)
masterData2 = numpy.zeros((nChunks, 4), dtype=numpy.float32)
masterProcessed = [0, 0, 0, 0]
masterRemoved = [0, 0, 0, 0]
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
#print("Working on chunk %i, %i frames remaining" % (i, framesRemaining))
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros((4, framesWork * 4096 // beampols),
dtype=numpy.float32)
data2 = numpy.zeros((4, framesWork * 4096 // beampols),
dtype=numpy.float32)
## Inner loop that actually reads the frames into the data array
#print("Working on %.2f ms of data" % ((framesWork*4096/beampols/srate)*1000.0))
t0 = time.time()
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1))
continue
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
if j < 4:
masterTimes[i, aStand] = cFrame.time
try:
framePower = numpy.abs(cFrame.payload.data)**2
# Calculate the clipping
mask = numpy.where(framePower <= args.trim_level, 1, 0)
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = framePower
data2[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = framePower * mask
masterProcessed[aStand] += mask.size
masterRemoved[aStand] += (mask.size - mask.sum())
# Update the counters so that we can average properly later on
count[aStand] += 1
except ValueError:
pass
# Save the data
masterData[i, :] = data.sum(axis=1)
masterData2[i, :] = data2.sum(axis=1)
# Really save the data to a NPZ file
if args.write_npz:
outfile = os.path.split(args.filename)[1]
outfile = os.path.splitext(outfile)[0]
outfile = '%s-power.npz' % outfile
numpy.savez(outfile,
beam=beam,
avgPowerFull=masterData,
avgPowerTrim=masterData2,
times=masterTimes,
trimLevel=args.trim_level)
# Report on the clipping
print("Summary:")
for i in xrange(4):
print(" Tuning %i, Pol. %s:" % (i / 2 + 1, 'X' if i % 2 else 'Y'))
print(" Processed: %i samples" % masterProcessed[i])
print(" Clipped: %i samples" % masterRemoved[i])
print(" -> %.1f%% blanked" %
(100.0 * masterRemoved[i] / masterProcessed[i], ))
# The plots: This is setup for the current configuration of 20 beampols
fig = plt.figure()
figsX = int(round(math.sqrt(4)))
figsY = 4 // figsX
for i in xrange(masterData.shape[1]):
ax = fig.add_subplot(figsX, figsY, i + 1)
ax.plot(numpy.arange(0, masterData.shape[0]) * args.average,
masterData[:, i],
label='Full')
ax.plot(numpy.arange(0, masterData2.shape[0]) * args.average,
masterData2[:, i],
label='Trimmed')
ax.set_ylim([0, masterData.max()])
ax.set_title('Beam %i, Tune. %i, Pol. %i' % (beam, i // 2 + 1, i % 2))
ax.set_xlabel('Time [seconds]')
ax.set_ylabel('Output Power Level')
ax.legend(loc=0)
# Part 2, polarization stuff
fig2 = plt.figure()
ax = fig2.add_subplot(3, 2, 1)
ax.plot(
numpy.arange(0, masterData.shape[0]) * args.average,
numpy.sqrt(masterData[:, 0]**2 + masterData[:, 1]**2))
ax.set_title('$\\sqrt{X1^2 + Y1^2}$')
ax.set_xlabel('Time [seconds]')
ax = fig2.add_subplot(3, 2, 2)
ax.plot(
numpy.arange(0, masterData.shape[0]) * args.average,
masterData[:, 1] / masterData[:, 0])
ax.set_title('$Y1 / X1$')
ax.set_xlabel('Time [seconds]')
ax = fig2.add_subplot(3, 2, 3)
ax.plot(
numpy.arange(0, masterData.shape[0]) * args.average,
numpy.sqrt(masterData[:, 2]**2 + masterData[:, 3]**2))
ax.set_title('$\\sqrt{X2^2 + Y2^2}$')
ax.set_xlabel('Time [seconds]')
ax = fig2.add_subplot(3, 2, 4)
ax.plot(
numpy.arange(0, masterData.shape[0]) * args.average,
masterData[:, 3] / masterData[:, 2])
ax.set_title('$Y2 / X2$')
ax.set_xlabel('Time [seconds]')
ax = fig2.add_subplot(3, 2, 5)
ax.plot(
numpy.arange(0, masterData.shape[0]) * args.average,
numpy.sqrt(masterData[:, 2]**2 + masterData[:, 3]**2) /
numpy.sqrt(masterData[:, 0]**2 + masterData[:, 1]**2))
ax.set_title('$\\sqrt{X2^2 + Y2^2} / \\sqrt{X1^2 + Y1^2}$')
ax.set_xlabel('Time [seconds]')
ax = fig2.add_subplot(3, 2, 6)
ax.plot(
numpy.arange(0, masterData.shape[0]) * args.average,
(masterData[:, 3] / masterData[:, 2]) /
(masterData[:, 1] / masterData[:, 0]))
ax.set_title('$(Y2 / X2) / (Y1 / X1)$')
ax.set_xlabel('Time [seconds]')
plt.show()
# Save image if requested
if args.output is not None:
fig.savefig(args.output)
def main(args):
files = args
fh = []
nFramesFile = []
srate = []
beams = []
tunepols = []
beampols = []
tnom = []
tStart = []
# Open the various files and get basic data information: beam, number of
# frames per obs, T_NOM, time tag of first frame
for filename in files:
fh.append( open(filename, "rb") )
nFramesFile.append( os.path.getsize(filename) / drx.FRAME_SIZE )
while True:
junkFrame = drx.read_frame(fh[-1])
try:
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
fh[-1].seek(-drx.FRAME_SIZE, 1)
beam, tune, pol = junkFrame.id
beams.append( beam )
tunepols.append( drx.get_frames_per_obs(fh[-1]) )
tunepols.append( tunepols[-1][0] + tunepols[-1][1] + tunepols[-1][2] + tunepols[-1][3] )
beampols.append( tunepols[-1] )
tnom.append( junkFrame.header.time_offset )
tStart.append( junkFrame.payload.timetag )
# Align the files as close as possible by the time tags and then make sure that
# the first frame processed is from tuning 1, pol 0.
for i in xrange(len(files)):
junkFrame = drx.read_frame(fh[i])
beam, tune, pol = junkFrame.id
pair = 2*(tune-1) + pol
j = 0
while junkFrame.payload.timetag < max(tStart):
junkFrame = drx.read_frame(fh[i])
beam, tune, pol = junkFrame.id
pair = 2*(tune-1) + pol
j += 1
while pair != 0:
junkFrame = drx.read_frame(fh[i])
beam, tune, pol = junkFrame.id
pair = 2*(tune-1) + pol
j += 1
fh[i].seek(-drx.FRAME_SIZE, 1)
print("Shifted beam %i data by %i frames (%.4f s)" % (i, j, j*4096/srate[i]/4))
# Date
beginDate = junkFrame.time.datetime
# File summary
print(" ")
print("Filenames: %s" % ' '.join([os.path.split(f)[1] for f in files]))
print("Date of First Frame: ~%s" % str(beginDate))
print("Beams: %s" % ' '.join([str(b) for b in beams]))
print("Sample Rate: %s Hz" % ' '.join([str(s) for s in srate]))
print(" ")
# Main reader loop - save the data to the `data` list and the raw time tag values
# to the `times` list.
nFrames = 2000
data = numpy.zeros((len(files), 4, 4096*nFrames), dtype=numpy.csingle)
times = numpy.zeros((len(files), 4, nFrames), dtype=numpy.int64)
for i in xrange(len(files)):
for j in xrange(nFrames):
for k in xrange(4):
frame = drx.read_frame(fh[i])
beam, tune, pol = frame.id
pair = 2*(tune-1) + pol
data[i,pair,j*4096:(j+1)*4096] = frame.payload.data
times[i,pair,j] = frame.payload.timetag
#print(i, j, k, beam, tune, pol, frame.payload.timetag)
# Cross-correlate
refs = [0,0,0,0]
for i in xrange(len(files)):
for j in xrange(i, len(files)):
if i != j:
fig = plt.figure()
for k in xrange(4):
lag, cc = crossCorrelate(data[j,k,:], data[i,k,:])
best = numpy.where( cc == cc.max() )[0][0]
if i == j:
refs[k] = cc.max()
else:
ccOffset = lag[best]*fS/srate[i]
rtOffset = times[i,k,0] - times[j,k,0]
ctOffset = (times[i,k,0] - tnom[i]) - (times[j,k,0] - tnom[j])
print("Beams %i & %i, Tuning %i, Pol. %i" % (beams[i], beams[j], k/2+1, k%2))
print(" T_NOM%i: %i ticks" % (beams[i], tnom[i]))
print(" T_NOM%i: %i ticks" % (beams[j], tnom[j]))
print(" -> T_NOM%i - T_NOM%i: %i ticks" % (beams[j], beams[i], tnom[j] - tnom[i]))
print(" NCM: %.3f" % (cc.max() / refs[k]))
print(" CC Offset: %i ticks" % ccOffset)
print(" raw time tag Offset: %i ticks" % rtOffset)
print(" cor time tag Offset: %i ticks" % ctOffset)
print(" -> CC - raw: %i ticks" % (ccOffset - rtOffset))
print(" -> CC - cor: %i ticks" % (ccOffset - ctOffset))
ax = fig.add_subplot(2, 2, k+1)
ax.plot(lag[best-50:best+50], cc[best-50:best+50])
ax.set_title('Beams %i & %i, Tuning %i, Pol. %i' % (beams[i], beams[j], k/2+1, k%2))
ax.set_xlabel('Lag [samples]')
ax.set_ylabel('Analysis Sets')
if i != j:
plt.draw()
plt.show()
for f in fh:
f.close()
