def main(args):
idf = LWA1DataFile(args[0])
nFramesFile = idf.getInfo('nFrames')
srate = idf.getInfo('sampleRate')
beam = idf.getInfo('beam')
beampols = idf.getInfo('beampols')
# Date
beginDate = ephem.Date(unix_to_utcjd(idf.getInfo('tStart')) - DJD_OFFSET)
# File summary
print "Filename: %s" % args[0]
print "Date of First Frame: %s" % str(beginDate)
print "Beam: %i" % beam
print "Tune/Pols: %ii" % beampols
print "Sample Rate: %i Hz" % srate
print "Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate)
print "---"
def meta_to_txt(filename, outdir='./', station='lwasv'):
"""Pulls metadata from TBN file and puts it into a txt file of the same name
Parameters
----------
filename : string
name of file to be read (may end in dat, tbn, or nothing)
outdir : string
name of output directory to save textfile in
station : string
one of 'lwasv' or 'lwa1' for now
"""
simple_name = filename.split('/')[-1]
simple_name = simple_name.split('.')[0]
if not outdir.endswith('/'):
outdir = outdir + '/'
if not pathlib.Path(outdir).exists():
os.mkdir(outdir)
print("{} TBN Size: {} kB".format(filename,
os.path.getsize(filename) / 1024))
if station == 'lwasv':
idfN = LWASVDataFile(filename)
simple_name = simple_name + '-LWASV'
elif station == 'lwa1':
idfN = LWA1DataFile(filename)
simple_name = simple_name + '-LWA1'
else:
raise NotImplementedError(
"I haven't implemented that type of station yet")
print("{} is of type: {}".format(filename, type(idfN)))
# Poll the TBN file for its specifics
with open(outdir + simple_name + ".txt", 'w') as meta:
meta.write('TBN Metadata:\n')
for key, value in idfN.get_info().items():
meta.write(" %s: %s\n" % (str(key), str(value)))
idfN.close()
def main(args):
# Setup the LWA station information
if args.metadata is not None:
try:
station = stations.parse_ssmif(args.metadata)
except ValueError:
station = metabundle.get_station(args.metadata, apply_sdm=True)
else:
station = stations.lwana
antennas = []
for a in station.antennas:
if a.digitizer != 0:
antennas.append(a)
# Length of the FFT
LFFT = args.fft_length
idf = LWA1DataFile(args.filename)
nFramesFile = idf.get_info('nframe')
srate = idf.get_info('sample_rate')
antpols = len(antennas)
# Offset in frames for beampols beam/tuning/pol. sets
args.skip = idf.offset(args.skip)
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped. This needs to
# take into account the number of antpols in the data, the FFT length,
# and the number of samples per frame.
maxFrames = int(
(2 * 10 * 750) / antpols * 512 / float(LFFT)) * LFFT / 512 * antpols
# Number of frames to integrate over
nFrames = int(args.average * srate / 512 * antpols)
nFrames = int(
1.0 * nFrames / antpols * 512 / float(LFFT)) * LFFT / 512 * antpols
args.average = 1.0 * nFrames / antpols * 512 / srate
# Number of remaining chunks
nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames))
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
beginDate = ephem.Date(
unix_to_utcjd(idf.get_info('start_time')) - DJD_OFFSET)
central_freq = idf.get_info('freq1')
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz" % central_freq)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate))
print("---")
print("Offset: %.3f s (%i frames)" %
(args.skip, args.skip * srate * antpols / 512))
print("Integration: %.3f s (%i frames; %i frames per stand/pol)" %
(args.average, nFrames, nFrames / antpols))
print("Chunks: %i" % nChunks)
# Sanity check
if args.skip * srate * antpols / 512 > nFramesFile:
raise RuntimeError("Requested offset is greater than file length")
if nFrames > (nFramesFile - args.skip * srate * antpols / 512):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
# Setup the window function to use
if args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
# Master loop over all of the file chunks
masterWeight = numpy.zeros((nChunks, antpols, LFFT))
masterSpectra = numpy.zeros((nChunks, antpols, LFFT))
for i in range(nChunks):
print("Working on chunk #%i of %i" % (i + 1, nChunks))
try:
readT, t, data = idf.read(args.average / nChunks)
except Exception as e:
print("Error: %s" % str(e))
continue
# Calculate the spectra for this block of data and then weight the results by
# the total number of frames read. This is needed to keep the averages correct.
freq, tempSpec = fxc.SpecMaster(data,
LFFT=LFFT,
window=window,
pfb=args.pfb,
verbose=args.verbose,
sample_rate=srate)
for stand in range(tempSpec.shape[0]):
masterSpectra[i, stand, :] = tempSpec[stand, :]
masterWeight[i, stand, :] = int(readT * srate / LFFT)
# Apply the cable loss corrections, if requested
if False:
for s in range(masterSpectra.shape[1]):
currGain = antennas[s].cable.gain(freq)
for c in range(masterSpectra.shape[0]):
masterSpectra[c, s, :] /= currGain
# Now that we have read through all of the chunks, perform the final averaging by
# dividing by all of the chunks
spec = numpy.squeeze(
(masterWeight * masterSpectra).sum(axis=0) / masterWeight.sum(axis=0))
# Put the frequencies in the best units possible
freq += central_freq
freq, units = _best_freq_units(freq)
# Deal with the `keep` options
if args.keep == 'all':
antpolsDisp = int(numpy.ceil(antpols / 20))
js = [i for i in range(antpols)]
else:
antpolsDisp = int(numpy.ceil(len(args.keep) * 2 / 20))
if antpolsDisp < 1:
antpolsDisp = 1
js = []
for k in args.keep:
for i, ant in enumerate(antennas):
if ant.stand.id == k:
js.append(i)
nPlot = len(js)
if nPlot < 20:
if nPlot % 4 == 0 and nPlot != 4:
figsY = 4
else:
figsY = 2
figsX = int(numpy.ceil(1.0 * nPlot / figsY))
else:
figsY = 4
figsX = 5
figsN = figsX * figsY
for i in range(antpolsDisp):
# Normal plotting
fig = plt.figure()
for k in range(i * figsN, i * figsN + figsN):
try:
j = js[k]
currSpectra = numpy.squeeze(numpy.log10(spec[j, :]) * 10.0)
except IndexError:
break
ax = fig.add_subplot(figsX, figsY, (k % figsN) + 1)
ax.plot(
freq,
currSpectra,
label='Stand: %i, Pol: %i (Dig: %i)' %
(antennas[j].stand.id, antennas[j].pol, antennas[j].digitizer))
# If there is more than one chunk, plot the difference between the global
# average and each chunk
if nChunks > 1 and not args.disable_chunks:
for k in range(nChunks):
# Some files are padded by zeros at the end and, thus, carry no
# weight in the average spectra. Skip over those.
if masterWeight[k, j, :].sum() == 0:
continue
# Calculate the difference between the spectra and plot
subspectra = numpy.squeeze(
numpy.log10(masterSpectra[k, j, :]) * 10.0)
diff = subspectra - currSpectra
ax.plot(freq, diff)
ax.set_title('Stand: %i (%i); Dig: %i [%i]' %
(antennas[j].stand.id, antennas[j].pol,
antennas[j].digitizer, antennas[j].combined_status))
ax.set_xlabel('Frequency [%s]' % units)
ax.set_ylabel('P.S.D. [dB/RBW]')
ax.set_ylim([-10, 30])
# Save spectra image if requested
if args.output is not None:
base, ext = os.path.splitext(args.output)
outFigure = "%s-%02i%s" % (base, i + 1, ext)
fig.savefig(outFigure)
plt.draw()
print("RBW: %.4f %s" % ((freq[1] - freq[0]), units))
plt.show()
def main(args):
# Setup the LWA station information
if args.metadata is not None:
try:
station = stations.parse_ssmif(args.metadata)
except ValueError:
station = metabundle.get_station(args.metadata, apply_sdm=True)
else:
station = stations.lwa1
antennas = station.antennas
# Length of the FFT
LFFT = args.fft_length
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped
maxFrames = int((30000 * 260) / float(LFFT)) * LFFT
# It seems like that would be a good idea, however... TBW data comes one
# capture at a time so doing something like this actually truncates data
# from the last set of stands for the first integration. So, we really
# should stick with
maxFrames = (30000 * 260)
idf = LWA1DataFile(args.filename)
if not isinstance(idf, TBWFile):
raise RuntimeError("File '%s' does not appear to be a valid TBW file" %
os.path.basename(filename))
nFrames = idf.get_info('nframe')
srate = idf.get_info('sample_rate')
dataBits = idf.get_info('data_bits')
# The number of ant/pols in the file is hard coded because I cannot figure out
# a way to get this number in a systematic fashion
antpols = len(antennas)
nChunks = int(math.ceil(1.0 * nFrames / maxFrames))
if dataBits == 12:
nSamples = 400
else:
nSamples = 1200
# Read in the first frame and get the date/time of the first sample
# of the frame. This is needed to get the list of stands.
beginDate = idf.get_info('start_time').datetime
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Ant/Pols: %i" % antpols)
print("Sample Length: %i-bit" % dataBits)
print("Frames: %i" % nFrames)
print("Chunks: %i" % nChunks)
print("===")
# Setup the window function to use
if args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
# Master loop over all of the file chunks
nChunks = 1
masterSpectra = numpy.zeros((nChunks, antpols, LFFT))
masterWeight = numpy.zeros((nChunks, antpols, LFFT))
readT, t, data = idf.read(0.061)
# Calculate the spectra for this block of data and then weight the results by
# the total number of frames read. This is needed to keep the averages correct.
# NB: The weighting is the same for the x and y polarizations because of how
# the data are packed in TBW
freq, tempSpec = fxc.SpecMaster(data,
LFFT=LFFT,
window=window,
pfb=args.pfb,
verbose=args.verbose)
for stand in range(masterSpectra.shape[1]):
masterSpectra[0, stand, :] = tempSpec[stand, :]
masterWeight[0, stand, :] = int(readT * srate / LFFT)
# We don't really need the data array anymore, so delete it
del (data)
# Apply the cable loss corrections, if requested
if args.gain_correct:
for s in range(masterSpectra.shape[1]):
currGain = antennas[s].cable.gain(freq)
for c in range(masterSpectra.shape[0]):
masterSpectra[c, s, :] /= currGain
# Now that we have read through all of the chunks, perform the final averaging by
# dividing by all of the chunks
spec = masterSpectra.mean(axis=0)
# The plots: This is setup for the current configuration of 20 antpols
if args.gain_correct & args.stack:
# Stacked spectra - only if cable loss corrections are to be applied
colors = [
'blue', 'green', 'red', 'cyan', 'magenta', 'black', 'purple',
'salmon', 'olive', 'maroon', 'saddlebrown', 'yellowgreen', 'teal',
'steelblue', 'seagreen', 'slategray', 'mediumorchid', 'lime',
'dodgerblue', 'darkorange'
]
for f in range(int(numpy.ceil(antpols / 20.))):
fig = plt.figure()
ax1 = fig.add_subplot(1, 1, 1)
for i in range(f * 20, f * 20 + 20):
currSpectra = numpy.squeeze(numpy.log10(spec[i, :]) * 10.0)
ax1.plot(freq / 1e6,
currSpectra,
label='%i,%i' %
(antennas[i].stand.id, antennas[i].pol),
color=colors[i % 20])
ax1.set_xlabel('Frequency [MHz]')
ax1.set_ylabel('P.S.D. [dB/RBW]')
ax1.set_xlim([20, 88])
#ax1.set_ylim([10,90])
leg = ax1.legend(loc=0, ncol=3)
for l in leg.get_lines():
l.set_linewidth(1.7) # the legend line width
else:
for f in range(int(numpy.ceil(antpols / 20))):
# Normal plotting
fig = plt.figure()
figsY = 4
figsX = 5
fig.subplots_adjust(left=0.06,
bottom=0.06,
right=0.94,
top=0.94,
wspace=0.20,
hspace=0.50)
for i in range(f * 20, f * 20 + 20):
ax = fig.add_subplot(figsX, figsY, (i % 20) + 1)
try:
currSpectra = numpy.squeeze(numpy.log10(spec[i, :]) * 10.0)
except IndexError:
break
ax.plot(freq / 1e6,
currSpectra,
label='Stand: %i, Pol: %i (Dig: %i)' %
(antennas[i].stand.id, antennas[i].pol,
antennas[i].digitizer))
# If there is more than one chunk, plot the difference between the global
# average and each chunk
if nChunks > 1:
for j in range(nChunks):
# Some files are padded by zeros at the end and, thus, carry no
# weight in the average spectra. Skip over those.
if masterWeight[j, i, :].sum() == 0:
continue
# Calculate the difference between the spectra and plot
subspectra = numpy.squeeze(
numpy.log10(masterSpectra[j, i, :]) * 10.0)
diff = subspectra - currSpectra
ax.plot(freq / 1e6, diff)
ax.set_title(
'Stand: %i (%i); Dig: %i [%i]' %
(antennas[i].stand.id, antennas[i].pol,
antennas[i].digitizer, antennas[i].combined_status))
ax.set_xlabel('Frequency [MHz]')
ax.set_ylabel('P.S.D. [dB/RBW]')
ax.set_xlim([10, 90])
ax.set_ylim([10, 80])
# Save spectra image if requested
if args.output is not None:
base, ext = os.path.splitext(args.output)
outFigure = "%s-%02i%s" % (base, f + 1, ext)
fig.savefig(outFigure)
plt.draw()
print("RBW: %.1f Hz" % (freq[1] - freq[0]))
plt.show()
def main(args):
# Parse command line options
filename = args.filename
# Length of the FFT
LFFT = args.fft_length
# Setup the LWA station information
if args.metadata is not None:
try:
station = stations.parse_ssmif(args.metadata)
except ValueError:
station = metabundle.get_station(args.metadata, apply_sdm=True)
else:
station = stations.lwana
antennas = station.antennas
idf = LWA1DataFile(filename)
jd = astro.unix_to_utcjd(idf.get_info('start_time'))
date = str(ephem.Date(jd - astro.DJD_OFFSET))
nFpO = len(antennas)
sample_rate = idf.get_info('sample_rate')
nInts = idf.get_info('nframe') // nFpO
# Get valid stands for both polarizations
goodX = []
goodY = []
for i in range(len(antennas)):
ant = antennas[i]
if ant.combined_status != 33 and not args.all:
pass
else:
if ant.pol == 0:
goodX.append(ant)
else:
goodY.append(ant)
# Now combine both lists to come up with stands that
# are in both so we can form the cross-polarization
# products if we need to
good = []
for antX in goodX:
for antY in goodY:
if antX.stand.id == antY.stand.id:
good.append(antX.digitizer - 1)
good.append(antY.digitizer - 1)
# Report on the valid stands found. This is a little verbose,
# but nice to see.
print("Found %i good stands to use" % (len(good) // 2, ))
for i in good:
print("%3i, %i" % (antennas[i].stand.id, antennas[i].pol))
# Number of frames to read in at once and average
nFrames = int(args.avg_time * sample_rate / 512)
args.offset = idf.offset(args.offset)
nSets = idf.get_info('nframe') // nFpO // nFrames
nSets = nSets - int(args.offset * sample_rate / 512) // nFrames
central_freq = idf.get_info('freq1')
print("Data type: %s" % type(idf))
print("Samples per observations: %i per pol." % (nFpO / 2))
print("Sampling rate: %i Hz" % sample_rate)
print("Tuning frequency: %.3f Hz" % central_freq)
print("Captures in file: %i (%.1f s)" % (nInts, nInts * 512 / sample_rate))
print("==")
print("Station: %s" % station.name)
print("Date observed: %s" % date)
print("Julian day: %.5f" % jd)
print("Offset: %.3f s (%i frames)" %
(args.offset, args.offset * sample_rate / 512))
print("Integration Time: %.3f s" % (512 * nFrames / sample_rate))
print("Number of integrations in file: %i" % nSets)
# Make sure we don't try to do too many sets
if args.samples > nSets:
args.samples = nSets
# Loop over junks of 300 integrations to make sure that we don't overflow
# the FITS IDI memory buffer
s = 0
leftToDo = args.samples
basename = os.path.split(filename)[1]
basename, ext = os.path.splitext(basename)
while leftToDo > 0:
fitsFilename = "%s.FITS_%i" % (
basename,
(s + 1),
)
if leftToDo > 100:
chunk = 100
else:
chunk = leftToDo
process_chunk(idf,
station,
good,
fitsFilename,
int_time=args.avg_time,
LFFT=args.fft_length,
overlap=1,
pfb=args.pfb,
pols=args.products,
chunk_size=chunk)
s += 1
leftToDo = leftToDo - chunk
idf.close()
def main(args):
# Length of the FFT and the window to use
LFFT = args.fft_length
if args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
args.window = window
# Open the file and find good data
idf = LWA1DataFile(args.filename,
ignore_timetag_errors=args.ignore_time_errors)
# Metadata
nFramesFile = idf.get_info('nframe')
srate = idf.get_info('sample_rate')
antpols = idf.get_info('nantenna')
# Number of frames to integrate over
nFramesAvg = int(args.average * srate / 512) * antpols
nFramesAvg = int(1.0 * (nFramesAvg // antpols) * 512 /
float(LFFT)) * LFFT / 512 * antpols
args.average = 1.0 * (nFramesAvg // antpols) * 512 / srate
maxFrames = nFramesAvg
# Offset into the file, if needed
offset = idf.offset(args.skip)
# Number of remaining chunks (and the correction to the number of
# frames to read in).
if args.metadata is not None:
args.duration = 0
if args.duration == 0:
args.duration = 1.0 * nFramesFile / antpols * 512 / srate
args.duration -= args.skip
else:
args.duration = int(
round(args.duration * srate * antpols / 512) // antpols * 512 //
srate)
nChunks = int(round(args.duration / args.average))
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg * nChunks
# Date & Central Frequency
t1 = idf.get_info('start_time')
beginDate = t1.datetime
central_freq1 = idf.get_info('freq1')
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Antenna/Pols: %i" % antpols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz" % (central_freq1, ))
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Integration: %.3f s (%i frames; %i frames per antenna/pol)" %
(args.average, nFramesAvg, nFramesAvg // antpols))
print("Duration: %.3f s (%i frames; %i frames per antenna/pol)" %
(args.average * nChunks, nFrames, nFrames // antpols))
print("Chunks: %i" % nChunks)
print(" ")
# Estimate clip level (if needed)
if args.estimate_clip_level:
estimate = idf.estimate_levels(sigma=5.0)
clip1 = 1.0 * sum(estimate) / len(estimate)
else:
clip1 = args.clip_level
# Get the antennas for Stokes calculation
if args.metadata is not None:
try:
project = metabundle.get_sdf(args.metadata)
station = stations.lwa1
except Exception as e:
project = metabundleADP.get_sdf(args.metadata)
station = stations.lwasv
elif args.lwasv:
station = stations.lwasv
else:
station = stations.lwa1
antennas = station.antennas
# Setup the output file
outname = os.path.split(args.filename)[1]
outname = os.path.splitext(outname)[0]
outname = '%s-tbn-waterfall.hdf5' % outname
if os.path.exists(outname):
if not args.force:
yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
raise RuntimeError("Output file '%s' already exists" % outname)
f = hdfData.create_new_file(outname)
# Look at the metadata and come up with a list of observations. If
# there are no metadata, create a single "observation" that covers the
# whole file.
obsList = {}
if args.metadata is not None:
try:
project = metabundle.get_sdf(args.metadata)
except Exception as e:
project = metabundleADP.get_sdf(args.metadata)
sdfBeam = project.sessions[0].drx_beam
if sdfBeam != 5:
raise RuntimeError(
"Metadata is for beam #%i, but data is from beam #%i" %
(sdfBeam, 5))
for i, obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsDur = obs.dur / 1000.0
obsSR = tbn.FILTER_CODES[obs.filter]
obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR)
print("Observations:")
for i in sorted(obsList.keys()):
obs = obsList[i]
print(" #%i: %s to %s (%.3f s) at %.3f MHz" %
(i, obs[0], obs[1], obs[2], obs[3] / 1e6))
print(" ")
hdfData.fill_from_metabundle(f, args.metadata)
elif args.sdf is not None:
try:
project = sdf.parse_sdf(args.sdf)
except Exception as e:
project = sdfADP.parse_sdf(args.sdf)
sdfBeam = project.sessions[0].drx_beam
if sdfBeam != 5:
raise RuntimeError(
"Metadata is for beam #%i, but data is from beam #%i" %
(sdfBeam, 5))
for i, obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsDur = obs.dur / 1000.0
obsSR = tbn.FILTER_CODES[obs.filter]
obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR)
site = 'lwa1'
if args.lwasv:
site = 'lwasv'
hdfData.fill_from_sdf(f, args.sdf, station=site)
else:
obsList[1] = (datetime.utcfromtimestamp(t1),
datetime(2222, 12, 31, 23, 59, 59), args.duration, srate)
site = 'lwa1'
if args.lwasv:
site = 'lwasv'
hdfData.fill_minimum(f, 1, 5, srate, station=site)
if (not args.stokes):
data_products = ['XX', 'YY']
else:
data_products = ['I', 'Q', 'U', 'V']
for o in sorted(obsList.keys()):
for t in range(len(antennas) // 2):
hdfData.create_observation_set(
f, o, t + 1, numpy.arange(LFFT, dtype=numpy.float64),
int(round(obsList[o][2] / args.average)), data_products)
f.attrs['FileGenerator'] = 'tbnWaterfall.py'
f.attrs['InputData'] = os.path.basename(args.filename)
# Create the various HDF group holders
ds = {}
for o in sorted(obsList.keys()):
obs = hdfData.get_observation_set(f, o)
ds['obs%i' % o] = obs
ds['obs%i-time' % o] = hdfData.get_time(f, o)
for t in range(len(antennas) // 2):
ds['obs%i-freq%i' % (o, t + 1)] = hdfData.get_data_set(
f, o, t + 1, 'freq')
for p in data_products:
ds["obs%i-%s%i" % (o, p, t + 1)] = hdfData.get_data_set(
f, o, t + 1, p)
ds['obs%i-Saturation%i' % (o, t + 1)] = hdfData.get_data_set(
f, o, t + 1, 'Saturation')
# Load in the correct analysis function
if (not args.stokes):
process_data = process_data_to_linear
else:
process_data = process_data_to_stokes
# Go!
for o in sorted(obsList.keys()):
try:
process_data(idf,
antennas,
obsList[o][0],
obsList[o][2],
obsList[o][3],
args,
ds,
obsID=o,
clip1=clip1)
except RuntimeError as e:
print("Observation #%i: %s, abandoning this observation" %
(o, str(e)))
# Save the output to a HDF5 file
f.close()
# Close out the data file
idf.close()
def main(args):
# Length of the FFT and the window to use
LFFT = args.fft_length
if args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
args.window = window
# Open the file and find good data (not spectrometer data)
fh = open(args.filename, "rb")
try:
for i in xrange(5):
junkFrame = drspec.read_frame(fh)
raise RuntimeError(
"ERROR: '%s' appears to be a DR spectrometer file, not a raw DRX file"
% args.filename)
except errors.SyncError:
fh.seek(0)
# Good, we seem to have a real DRX file, switch over to the LDP interface
fh.close()
idf = LWA1DataFile(args.filename,
ignore_timetag_errors=args.ignore_time_errors)
# Metadata
nFramesFile = idf.get_info('nframe')
beam = idf.get_info('beam')
srate = idf.get_info('sample_rate')
beampols = idf.get_info('nbeampol')
beams = max([1, beampols // 4])
# Number of frames to integrate over
nFramesAvg = int(args.average * srate / 4096) * beampols
nFramesAvg = int(1.0 * (nFramesAvg // beampols) * 4096 /
float(LFFT)) * LFFT / 4096 * beampols
args.average = 1.0 * (nFramesAvg // beampols) * 4096 / srate
maxFrames = nFramesAvg
# Offset into the file, if needed
offset = idf.offset(args.skip)
# Number of remaining chunks (and the correction to the number of
# frames to read in).
if args.metadata is not None:
args.duration = 0
if args.duration == 0:
args.duration = 1.0 * nFramesFile / beampols * 4096 / srate
args.duration -= args.skip
else:
args.duration = int(
round(args.duration * srate * beampols / 4096) / beampols * 4096 /
srate)
nChunks = int(round(args.duration / args.average))
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg * nChunks
# Date & Central Frequency
t1 = idf.get_info('start_time')
beginDate = t1.datetime
central_freq1 = idf.get_info('freq1')
central_freq2 = idf.get_info('freq2')
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Beams: %i" % beams)
print("Tune/Pols: %i" % beampols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" %
(central_freq1, central_freq2))
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Integration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average, nFramesAvg, nFramesAvg / beampols))
print("Duration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average * nChunks, nFrames, nFrames / beampols))
print("Chunks: %i" % nChunks)
print(" ")
# Estimate clip level (if needed)
if args.estimate_clip_level:
estimate = idf.estimate_levels(fh, sigma=5.0)
clip1 = (estimate[0] + estimate[1]) / 2.0
clip2 = (estimate[2] + estimate[3]) / 2.0
else:
clip1 = args.clip_level
clip2 = args.clip_level
# Make the pseudo-antennas for Stokes calculation
antennas = []
for i in xrange(4):
if i // 2 == 0:
newAnt = stations.Antenna(1)
else:
newAnt = stations.Antenna(2)
if i % 2 == 0:
newAnt.pol = 0
else:
newAnt.pol = 1
antennas.append(newAnt)
# Setup the output file
outname = os.path.split(args.filename)[1]
outname = os.path.splitext(outname)[0]
outname = '%s-waterfall.hdf5' % outname
if os.path.exists(outname):
if not args.force:
yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
raise RuntimeError("Output file '%s' already exists" % outname)
f = hdfData.create_new_file(outname)
# Look at the metadata and come up with a list of observations. If
# there are no metadata, create a single "observation" that covers the
# whole file.
obsList = {}
if args.metadata is not None:
try:
project = metabundle.get_sdf(args.metadata)
except Exception as e:
project = metabundleADP.get_sdf(args.metadata)
sdfBeam = project.sessions[0].drx_beam
spcSetup = project.sessions[0].spcSetup
if sdfBeam != beam:
raise RuntimeError(
"Metadata is for beam #%i, but data is from beam #%i" %
(sdfBeam, beam))
for i, obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsDur = obs.dur / 1000.0
obsSR = drx.FILTER_CODES[obs.filter]
obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR)
print("Observations:")
for i in sorted(obsList.keys()):
obs = obsList[i]
print(" #%i: %s to %s (%.3f s) at %.3f MHz" %
(i, obs[0], obs[1], obs[2], obs[3] / 1e6))
print(" ")
hdfData.fill_from_metabundle(f, args.metadata)
elif args.sdf is not None:
try:
project = sdf.parse_sdf(args.sdf)
except Exception as e:
project = sdfADP.parse_sdf(args.sdf)
sdfBeam = project.sessions[0].drx_beam
spcSetup = project.sessions[0].spcSetup
if sdfBeam != beam:
raise RuntimeError(
"Metadata is for beam #%i, but data is from beam #%i" %
(sdfBeam, beam))
for i, obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsDur = obs.dur / 1000.0
obsSR = drx.FILTER_CODES[obs.filter]
obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR)
site = 'lwa1'
if args.lwasv:
site = 'lwasv'
hdfData.fill_from_sdf(f, args.sdf, station=site)
else:
obsList[1] = (datetime.utcfromtimestamp(t1),
datetime(2222, 12, 31, 23, 59, 59), args.duration, srate)
site = 'lwa1'
if args.lwasv:
site = 'lwasv'
hdfData.fill_minimum(f, 1, beam, srate, station=site)
if (not args.stokes):
data_products = ['XX', 'YY']
else:
data_products = ['I', 'Q', 'U', 'V']
for o in sorted(obsList.keys()):
for t in (1, 2):
hdfData.create_observation_set(
f, o, t, numpy.arange(LFFT, dtype=numpy.float64),
int(round(obsList[o][2] / args.average)), data_products)
f.attrs['FileGenerator'] = 'hdfWaterfall.py'
f.attrs['InputData'] = os.path.basename(args.filename)
# Create the various HDF group holders
ds = {}
for o in sorted(obsList.keys()):
obs = hdfData.get_observation_set(f, o)
ds['obs%i' % o] = obs
ds['obs%i-time' % o] = hdfData.get_time(f, o)
for t in (1, 2):
ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq')
for p in data_products:
ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p)
ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set(
f, o, t, 'Saturation')
# Load in the correct analysis function
if (not args.stokes):
processDataBatch = processDataBatchLinear
else:
processDataBatch = processDataBatchStokes
# Go!
for o in sorted(obsList.keys()):
try:
processDataBatch(idf,
antennas,
obsList[o][0],
obsList[o][2],
obsList[o][3],
args,
ds,
obsID=o,
clip1=clip1,
clip2=clip2)
except RuntimeError as e:
print("Observation #%i: %s, abandoning this observation" %
(o, str(e)))
# Save the output to a HDF5 file
f.close()
# Close out the data file
idf.close()
def main(args):
# Parse command line options
filename = args.filename
# Setup the LWA station information
if args.metadata is not None:
try:
station = stations.parse_ssmif(args.metadata)
except ValueError:
station = metabundle.get_station(args.metadata, apply_sdm=True)
else:
station = stations.lwa1
antennas = station.antennas
idf = LWA1DataFile(filename)
if not isinstance(idf, TBWFile):
raise RuntimeError("File '%s' does not appear to be a valid TBW file" %
os.path.basename(filename))
jd = idf.get_info('start_time').jd
date = idf.get_info('start_time').datetime
sample_rate = idf.get_info('sample_rate')
nInts = idf.get_info('nframe') // (30000 * len(antennas) // 2)
# Get valid stands for both polarizations
goodX = []
goodY = []
for i in range(len(antennas)):
ant = antennas[i]
if ant.combined_status != 33 and not args.all:
pass
else:
if ant.pol == 0:
goodX.append(ant)
else:
goodY.append(ant)
# Select which polarization to use
good = []
for antX in goodX:
for antY in goodY:
if antX.stand.id == antY.stand.id:
good.append(antX.digitizer - 1)
good.append(antY.digitizer - 1)
break
# Report on the valid stands found. This is a little verbose,
# but nice to see.
print("Found %i good stands to use" % (len(good) // 2, ))
for i in good:
print("%3i, %i" % (antennas[i].stand.id, antennas[i].pol))
# Number of frames to read in at once and average
nFrames = 30000
nSets = idf.get_info('nframe') // (30000 * len(antennas) // 2)
print("Data type: %s" % type(idf))
print("Captures in file: %i (%.3f s)" %
(nInts, nInts * 30000 * 400 / sample_rate))
print("==")
print("Station: %s" % station.name)
print("Date observed: %s" % date)
print("Julian day: %.5f" % jd)
print("Integration Time: %.3f s" % (400 * nFrames / sample_rate))
print("Number of integrations in file: %i" % nSets)
print("==")
basename = os.path.split(filename)[1]
basename, ext = os.path.splitext(basename)
if args.casa:
fitsFilename = "%s.ms_1" % (basename, )
else:
fitsFilename = "%s.FITS_1" % (basename, )
process_chunk(idf,
station,
good,
fitsFilename,
LFFT=args.fft_length,
overlap=1,
pfb=args.pfb,
pols=args.products)
idf.close()