def test_vdif_frames_per_second(self):
"""Test reading a VDIF file with a GUPPI header."""
filename = glob.glob('*.vdif')[0]
# Open the file
fh = open(filename, 'rb')
# Read the GUPPI header and a frame
header = vdif.read_guppi_header(fh)
frame0 = vdif.read_frame(fh)
# Find the number of frames per second
fps = vdif.get_frames_per_second(fh)
# Does it make sense?
bw = header['OBSBW']
bw *= 1 if frame0.header.is_complex else 2
calculated_fps = int(bw / frame0.payload.data.size)
self.assertEqual(fps, calculated_fps)
# Find the sample rate
sample_rate = vdif.get_sample_rate(fh)
self.assertAlmostEqual(sample_rate, bw, 6)
fh.close()
def test_vdif_guppi_header(self):
"""Test reading a VDIF file with a GUPPI header."""
filename = glob.glob('*.vdif')[0]
# Open the file
fh = open(filename, 'rb')
# Make sure it is there
self.assertTrue(vdif.has_guppi_header(fh))
# Read the GUPPI header
header = vdif.read_guppi_header(fh)
self.assertEqual(header['NBITS'], 4)
self.assertEqual(header['PKTSIZE'], 5000)
self.assertAlmostEqual(header['OBSBW'], 8.0e6, 6)
# And now read some frames to make sure that everything is ok
frame0 = vdif.read_frame(fh)
frame1 = vdif.read_frame(fh)
self.assertEqual(frame0.payload.data.size, header['PKTSIZE']*8//header['NBITS'])
fh.close()
def main(args):
if args[0] == '-s':
skip = float(args[1])
filename = args[2]
else:
skip = 0
filename = args[0]
fh = open(filename, 'rb')
header = vdif.read_guppi_header(fh)
vdif.FRAME_SIZE = vdif.get_frame_size(fh)
nThreads = get_thread_count(fh)
nFramesFile = os.path.getsize(filename) // vdif.FRAME_SIZE
nFramesSecond = get_frames_per_second(fh)
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
sample_rate = junkFrame.sample_rate
vdif.DATA_LENGTH = junkFrame.payload.data.size
nSampsFrame = vdif.DATA_LENGTH
station, thread = junkFrame.id
tunepols = nThreads
beampols = tunepols
fh.seek(-vdif.FRAME_SIZE, 1)
# Store the information about the first frame
prevDate = junkFrame.time.datetime
prevTime = junkFrame.header.seconds_from_epoch
prevFrame = junkFrame.header.frame_in_second
# Skip ahead
fh.seek(int(skip * sample_rate / vdif.DATA_LENGTH) * vdif.FRAME_SIZE, 1)
# Report on the file
print("Filename: %s" % os.path.basename(args[0]))
print(" Station: %i" % station)
print(" Thread count: %i" % nThreads)
print(" Date of first frame: %i -> %s" % (prevTime, str(prevDate)))
print(" Samples per frame: %i" % vdif.DATA_LENGTH)
print(" Frames per second: %i" % nFramesSecond)
print(" Sample rate: %i Hz" % sample_rate)
print(" Bit Depth: %i" % junkFrame.header.bits_per_sample)
print(" ")
if skip != 0:
print("Skipping ahead %i frames (%.6f seconds)" %
(int(skip * sample_rate / vdif.DATA_LENGTH) * 4,
int(skip * sample_rate / vdif.DATA_LENGTH) * vdif.DATA_LENGTH /
sample_rate))
print(" ")
prevDate = ['' for i in xrange(nThreads)]
prevTime = [0 for i in xrange(nThreads)]
prevFrame = [0 for i in xrange(nThreads)]
for i in xrange(nThreads):
currFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
station, thread = currFrame.id
prevDate[thread] = currFrame.time.datetime
prevTime[thread] = currFrame.header.seconds_from_epoch
prevFrame[thread] = currFrame.header.frame_in_second
inError = False
while True:
try:
currFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
if inError:
print("ERROR: sync. error cleared @ byte %i" %
(fh.tell() - vdif.FRAME_SIZE, ))
inError = False
except errors.SyncError:
if not inError:
print("ERROR: invalid frame (sync. word error) @ byte %i" %
fh.tell())
inError = True
fh.seek(vdif.FRAME_SIZE, 1)
continue
except errors.EOFError:
break
station, thread = currFrame.id
currDate = currFrame.time.datetime
currTime = currFrame.header.seconds_from_epoch
currFrame = currFrame.header.frame_in_second
if thread == 0 and currTime % 10 == 0 and currFrame == 0:
print("station %i, thread %i: t.t. %i @ frame %i -> %s" %
(station, thread, currTime, currFrame, currDate))
deltaT = (currTime - prevTime[thread]) * nFramesSecond + (
currFrame - prevFrame[thread])
#print(station, thread, deltaT, fh.tell())
if deltaT < 1:
print("ERROR: t.t. %i @ frame %i < t.t. %i @ frame %i + 1" %
(currTime, currFrame, prevTime[thread], prevFrame[thread]))
print(" -> difference: %i (%.5f seconds); %s" %
(deltaT, deltaT * nSampsFrame / sample_rate, str(currDate)))
print(" -> station %i, thread %i" % (station, thread))
elif deltaT > 1:
print("ERROR: t.t. %i @ frame %i > t.t. %i @ frame %i + 1" %
(currTime, currFrame, prevTime[thread], prevFrame[thread]))
print(" -> difference: %i (%.5f seconds); %s" %
(deltaT, deltaT * nSampsFrame / sample_rate, str(currDate)))
print(" -> station %i, thread %i" % (station, thread))
else:
pass
prevDate[thread] = currDate
prevTime[thread] = currTime
prevFrame[thread] = currFrame
del currFrame
fh.close()
def main(args):
# Parse the command line
filename = args.filename
fh = open(filename, 'rb')
header = vdif.read_guppi_header(fh)
vdif.FRAME_SIZE = vdif.get_frame_size(fh)
nFramesFile = os.path.getsize(filename) // vdif.FRAME_SIZE
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
srate = junkFrame.sample_rate
vdif.DATA_LENGTH = junkFrame.payload.data.size
beam, pol = junkFrame.id
tunepols = vdif.get_thread_count(fh)
beampols = tunepols
if args.skip != 0:
print("Skipping forward %.3f s" % args.skip)
print("-> %.6f (%s)" % (junkFrame.time, junkFrame.time.datetime))
offset = int(args.skip * srate / vdif.DATA_LENGTH)
fh.seek(beampols * vdif.FRAME_SIZE * offset, 1)
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
fh.seek(-vdif.FRAME_SIZE, 1)
print("-> %.6f (%s)" % (junkFrame.time, junkFrame.time.datetime))
tStart = junkFrame.time
# Get the frequencies
cFreq = 0.0
for j in xrange(4):
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
s, p = junkFrame.id
if p == 0:
cFreq = junkFrame.central_freq
# Set integration time
tInt = args.avg_time
nFrames = int(round(tInt * srate / vdif.DATA_LENGTH))
tInt = nFrames * vdif.DATA_LENGTH / srate
nFrames = int(round(tInt * srate / vdif.DATA_LENGTH))
# Read in some data
tFile = nFramesFile / beampols * vdif.DATA_LENGTH / srate
# Date
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
fh.seek(-vdif.FRAME_SIZE, 1)
beginDate = junkFrame.time.datetime
# Report
print("Filename: %s" % os.path.basename(filename))
print(" Date of First Frame: %s" % beginDate)
print(" Station: %i" % beam)
print(" Sample Rate: %i Hz" % srate)
print(" Tuning 1: %.1f Hz" % cFreq)
print(" Bit Depth: %i" % junkFrame.header.bits_per_sample)
print(" Integration Time: %.3f s" % tInt)
print(" Integrations in File: %i" % int(tFile / tInt))
print(" ")
# Go!
data = numpy.zeros((beampols, vdif.DATA_LENGTH * nFrames),
dtype=numpy.complex64)
count = [0 for i in xrange(data.shape[0])]
for i in xrange(beampols * nFrames):
try:
cFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
except errors.SyncError:
print("Error @ %i" % i)
fh.seek(vdif.FRAME_SIZE, 1)
continue
std, pol = cFrame.id
sid = pol
data[sid, count[sid] * vdif.DATA_LENGTH:(count[sid] + 1) *
vdif.DATA_LENGTH] = cFrame.payload.data
count[sid] += 1
# Plot
nBins = 2**junkFrame.header.bits_per_sample
weights = numpy.ones(data.shape[1],
dtype=numpy.float32) * 100.0 / data.shape[1]
fig = plt.figure()
## X
ax = fig.add_subplot(2, 1, 1)
c, b, o = ax.hist(data[0, :].real, bins=nBins, weights=weights)
ax.set_xlim((b[0], b[-1]))
ax.set_xlabel('Value - X')
ax.set_ylabel('Fraction [%]')
## Y
ax = fig.add_subplot(2, 1, 2)
c, b, o = ax.hist(data[1, :].real, bins=nBins, weights=weights)
ax.set_xlim((b[0], b[-1]))
ax.set_xlabel('Value - Y')
ax.set_ylabel('Fraction [%]')
plt.show()
# Done
fh.close()
def main(args):
# Parse the command line
filenames = args.filename
# Check if the first argument on the command line is a directory. If so,
# use what is in that directory
if os.path.isdir(filenames[0]):
filenames = [
os.path.join(filenames[0], filename)
for filename in os.listdir(filenames[0])
]
filenames.sort()
# Convert the filenames to absolute paths
filenames = [os.path.abspath(filename) for filename in filenames]
# Open the database connection to NRAO to find the antenna locations
try:
db = database('params')
except Exception as e:
sys.stderr.write("WARNING: %s" % str(e))
sys.stderr.flush()
db = None
# Pass 1 - Get the LWA metadata so we know where we are pointed
context = {
'observer': 'Unknown',
'project': 'Unknown',
'session': None,
'vlaref': None
}
setup = None
sources = []
metadata = {}
lwasite = {}
for filename in filenames:
# Figure out what to do with the file
ext = os.path.splitext(filename)[1]
if ext == '.tgz':
## LWA Metadata
try:
## Extract the SDF
if len(sources) == 0:
try:
sdf = metabundle.get_sdf(filename)
except Exception as e:
sdf = metabundleADP.get_sdf(filename)
context['observer'] = sdf.observer.name
context['project'] = sdf.id
context['session'] = sdf.sessions[0].id
comments = sdf.project_office.sessions[0]
mtch = CORR_CHANNELS.search(comments)
if mtch is not None:
corr_channels = int(mtch.group('channels'), 10)
else:
corr_channels = None
mtch = CORR_INTTIME.search(comments)
if mtch is not None:
corr_inttime = float(mtch.group('inttime'))
else:
corr_inttime = None
mtch = CORR_BASIS.search(comments)
if mtch is not None:
corr_basis = mtch.group('basis')
else:
sys.stderr.write(
"WARNING: No output correlation polarization basis defined, assuming 'linear'.\n"
)
corr_basis = 'linear'
if corr_channels is not None and corr_inttime is not None:
setup = {
'channels': corr_channels,
'inttime': corr_inttime,
'basis': corr_basis
}
else:
sys.stderr.write(
"WARNING: No or incomplete correlation configuration defined, setting to be defined at correlation time.\n"
)
for o, obs in enumerate(sdf.sessions[0].observations):
if type(obs).__name__ == 'Solar':
name = 'Sun'
intent = 'target'
ra = None
dec = None
elif type(obs).__name__ == 'Jovian':
name = 'Jupiter'
intent = 'target'
ra = None
dec = None
else:
name = obs.target
intent = obs.name
ra = ephem.hours(str(obs.ra))
dec = ephem.degrees(str(obs.dec))
tStart = mjdmpm_to_datetime(obs.mjd, obs.mpm)
tStop = mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
sources.append({
'name': name,
'intent': intent,
'ra2000': ra,
'dec2000': dec,
'start': tStart,
'stop': tStop
})
### Alternate phase centers
comments = sdf.project_office.observations[0][o]
alts = {}
for mtch in ALT_TARGET.finditer(comments):
alt_id = int(mtch.group('id'), 10)
alt_name = mtch.group('target')
try:
alts[alt_id]['name'] = alt_name
except KeyError:
alts[alt_id] = {
'name': alt_name,
'intent': 'dummy',
'ra': None,
'dec': None
}
for mtch in ALT_INTENT.finditer(comments):
alt_id = int(mtch.group('id'), 10)
alt_intent = mtch.group('intent')
try:
alts[alt_id]['intent'] = alt_intent
except KeyError:
alts[alt_id] = {
'name': None,
'intent': alt_intent,
'ra': None,
'dec': None
}
for mtch in ALT_RA.finditer(comments):
alt_id = int(mtch.group('id'), 10)
alt_ra = ephem.hours(mtch.group('ra'))
try:
alts[alt_id]['ra'] = alt_ra
except KeyError:
alts[alt_id] = {
'name': None,
'intent': 'dummy',
'ra': alt_ra,
'dec': None
}
for mtch in ALT_DEC.finditer(comments):
alt_id = int(mtch.group('id'), 10)
alt_dec = ephem.degrees(mtch.group('dec'))
try:
alts[alt_id]['dec'] = alt_dec
except KeyError:
alts[alt_id] = {
'name': None,
'intent': 'dummy',
'ra': None,
'dec': alt_dec
}
for alt_id in sorted(alts.keys()):
alt_name, alt_ra, alt_dec = alts[alt_id]
if alt_name is None or alt_ra is None or alt_dec is None:
sys.stderr.write(
"WARNING: Incomplete alternate phase center %i, skipping.\n"
% alt_id)
else:
sources.append({
'name': alt_name,
'ra2000': alt_ra,
'dec2000': alt_dec,
'start': tStart,
'stop': tStop
})
## Extract the file information so that we can pair things together
fileInfo = metabundle.get_session_metadata(filename)
for obsID in fileInfo.keys():
metadata[fileInfo[obsID]['tag']] = filename
## Figure out LWA1 vs LWA-SV
try:
cs = metabundle.get_command_script(filename)
for c in cs:
if c['subsystem_id'] == 'DP':
site = 'LWA1'
break
elif c['subsystem_id'] == 'ADP':
site = 'LWA-SV'
break
except (RuntimeError, ValueError):
site = 'LWA-SV'
for obsID in fileInfo.keys():
lwasite[fileInfo[obsID]['tag']] = site
except Exception as e:
sys.stderr.write("ERROR reading metadata file: %s\n" % str(e))
sys.stderr.flush()
# Setup what we need to write out a configuration file
corrConfig = {
'context': context,
'setup': setup,
'source': {
'name': '',
'ra2000': '',
'dec2000': ''
},
'inputs': []
}
metadata = {}
for filename in filenames:
#print("%s:" % os.path.basename(filename))
# Skip over empty files
if os.path.getsize(filename) == 0:
continue
# Open the file
fh = open(filename, 'rb')
# Figure out what to do with the file
ext = os.path.splitext(filename)[1]
if ext == '':
## DRX
try:
## Get the site
try:
sitename = lwasite[os.path.basename(filename)]
except KeyError:
sitename = 'LWA1'
## Get the location so that we can set site-specific parameters
if sitename == 'LWA1':
xyz = LWA1_ECEF
off = args.lwa1_offset
elif sitename == 'LWA-SV':
xyz = LWASV_ECEF
off = args.lwasv_offset
else:
raise RuntimeError("Unknown LWA site '%s'" % site)
## Move into the LWA1 coordinate system
### ECEF to LWA1
rho = xyz - LWA1_ECEF
sez = numpy.dot(LWA1_ROT, rho)
enz = sez[[1, 0, 2]] # pylint: disable=invalid-sequence-index
enz[1] *= -1
## Read in the first few frames to get the start time
frames = [drx.read_frame(fh) for i in xrange(1024)]
streams = []
freq1, freq2 = 0.0, 0.0
for frame in frames:
beam, tune, pol = frame.id
if tune == 1:
freq1 = frame.central_freq
else:
freq2 = frame.central_freq
if (beam, tune, pol) not in streams:
streams.append((beam, tune, pol))
tStart = frames[0].time.datetime
tStartAlt = (frames[-1].time - 1023 // len(streams) * 4096 /
frames[-1].sample_rate).datetime
tStartDiff = tStart - tStartAlt
if abs(tStartDiff) > timedelta(microseconds=10000):
sys.stderr.write(
"WARNING: Stale data found at the start of '%s', ignoring\n"
% os.path.basename(filename))
sys.stderr.flush()
tStart = tStartAlt
### ^ Adjustment to the start time to deal with occasional problems
### with stale data in the DR buffers at LWA-SV
## Read in the last few frames to find the end time
fh.seek(os.path.getsize(filename) - 1024 * drx.FRAME_SIZE)
backed = 0
while backed < 2 * drx.FRAME_SIZE:
try:
drx.read_frame(fh)
fh.seek(-drx.FRAME_SIZE, 1)
break
except errors.SyncError:
backed += 1
fh.seek(-drx.FRAME_SIZE - 1, 1)
for i in xrange(32):
try:
frame = drx.read_frame(fh)
beam, tune, _ = frame.id
if tune == 1:
freq1 = frame.central_freq
else:
freq2 = frame.central_freq
except errors.SyncError:
continue
tStop = frame.time.datetime
## Save
corrConfig['inputs'].append({
'file':
filename,
'type':
'DRX',
'antenna':
sitename,
'pols':
'X, Y',
'location': (enz[0], enz[1], enz[2]),
'clockoffset': (off, off),
'fileoffset':
0,
'beam':
beam,
'tstart':
tStart,
'tstop':
tStop,
'freq': (freq1, freq2)
})
except Exception as e:
sys.stderr.write("ERROR reading DRX file: %s\n" % str(e))
sys.stderr.flush()
elif ext == '.vdif':
## VDIF
try:
## Read in the GUPPI header
header = vdif.read_guppi_header(fh)
## Read in the first frame
vdif.FRAME_SIZE = vdif.get_frame_size(fh)
frame = vdif.read_frame(fh)
antID = frame.id[0] - 12300
tStart = frame.time.datetime
nThread = vdif.get_thread_count(fh)
## Read in the last frame
nJump = int(os.path.getsize(filename) / vdif.FRAME_SIZE)
nJump -= 30
fh.seek(nJump * vdif.FRAME_SIZE, 1)
mark = fh.tell()
while True:
try:
frame = vdif.read_frame(fh)
tStop = frame.time.datetime
except Exception as e:
break
## Find the antenna location
pad, edate = db.get_pad('EA%02i' % antID, tStart)
x, y, z = db.get_xyz(pad, tStart)
#print(" Pad: %s" % pad)
#print(" VLA relative XYZ: %.3f, %.3f, %.3f" % (x,y,z))
## Move into the LWA1 coordinate system
### relative to ECEF
xyz = numpy.array([x, y, z])
xyz += VLA_ECEF
### ECEF to LWA1
rho = xyz - LWA1_ECEF
sez = numpy.dot(LWA1_ROT, rho)
enz = sez[[1, 0, 2]] # pylint: disable=invalid-sequence-index
enz[1] *= -1
## Set an apparent position if WiDAR is already applying a delay model
apparent_enz = (None, None, None)
if args.no_vla_delay_model:
apparent_xyz = VLA_ECEF
apparent_rho = apparent_xyz - LWA1_ECEF
apparent_sez = numpy.dot(LWA1_ROT, apparent_rho)
apparent_enz = apparent_sez[[1, 0, 2]] # pylint: disable=invalid-sequence-index
apparent_enz[1] *= -1
## VLA time offset
off = args.vla_offset
## Save
corrConfig['context']['observer'] = header['OBSERVER']
try:
corrConfig['context']['project'] = header[
'BASENAME'].split('_')[0]
corrConfig['context']['session'] = header[
'BASENAME'].split('_')[1].replace('sb', '')
except IndexError:
corrConfig['context']['project'] = header[
'BASENAME'].split('.')[0]
corrConfig['context']['session'] = header[
'BASENAME'].split('.')[1].replace('sb', '')
corrConfig['context']['vlaref'] = re.sub(
'\.[0-9]+\.[0-9]+\.[AB][CD]-.*', '', header['BASENAME'])
corrConfig['source']['name'] = header['SRC_NAME']
corrConfig['source']['intent'] = 'target'
corrConfig['source']['ra2000'] = header['RA_STR']
corrConfig['source']['dec2000'] = header['DEC_STR']
corrConfig['inputs'].append({
'file':
filename,
'type':
'VDIF',
'antenna':
'EA%02i' % antID,
'pols':
'Y, X',
'location': (enz[0], enz[1], enz[2]),
'apparent_location':
(apparent_enz[0], apparent_enz[1], apparent_enz[2]),
'clockoffset': (off, off),
'fileoffset':
0,
'pad':
pad,
'tstart':
tStart,
'tstop':
tStop,
'freq':
header['OBSFREQ']
})
except Exception as e:
sys.stderr.write("ERROR reading VDIF file: %s\n" % str(e))
sys.stderr.flush()
elif ext == '.tgz':
## LWA Metadata
try:
## Extract the file information so that we can pair things together
fileInfo = metabundle.get_session_metadata(filename)
for obsID in fileInfo.keys():
metadata[fileInfo[obsID]['tag']] = filename
except Exception as e:
sys.stderr.write("ERROR reading metadata file: %s\n" % str(e))
sys.stderr.flush()
# Done
fh.close()
# Close out the connection to NRAO
try:
db.close()
except AttributeError:
pass
# Choose a VDIF reference file, if there is one, and mark whether or
# not DRX files were found
vdifRefFile = None
isDRX = False
for cinp in corrConfig['inputs']:
if cinp['type'] == 'VDIF':
if vdifRefFile is None:
vdifRefFile = cinp
elif cinp['type'] == 'DRX':
isDRX = True
# Set a state variable so that we can generate a warning about missing
# DRX files
drxFound = False
# Purge DRX files that don't make sense
toPurge = []
drxFound = False
lwasvFound = False
for cinp in corrConfig['inputs']:
### Sort out multiple DRX files - this only works if we have only one LWA station
if cinp['type'] == 'DRX':
if vdifRefFile is not None:
l0, l1 = cinp['tstart'], cinp['tstop']
v0, v1 = vdifRefFile['tstart'], vdifRefFile['tstop']
ve = (v1 - v0).total_seconds()
overlapWithVDIF = (v0 >= l0 and v0 < l1) or (l0 >= v0
and l0 < v1)
lvo = (min([v1, l1]) - max([v0, l0])).total_seconds()
if not overlapWithVDIF or lvo < 0.25 * ve:
toPurge.append(cinp)
drxFound = True
if cinp['antenna'] == 'LWA-SV':
lwasvFound = True
for cinp in toPurge:
del corrConfig['inputs'][corrConfig['inputs'].index(cinp)]
# Sort the inputs based on the antenna name - this puts LWA1 first,
# LWA-SV second, and the VLA at the end in 'EA' antenna order, i.e.,
# EA01, EA02, etc.
corrConfig['inputs'].sort(key=lambda x: 0 if x['antenna'] == 'LWA1' else (
1 if x['antenna'] == 'LWA-SV' else int(x['antenna'][2:], 10)))
# VDIF/DRX warning check/report
if vdifRefFile is not None and isDRX and not drxFound:
sys.stderr.write(
"WARNING: DRX files provided but none overlapped with VDIF data")
# Duplicate antenna check
antCounts = {}
for cinp in corrConfig['inputs']:
try:
antCounts[cinp['antenna']] += 1
except KeyError:
antCounts[cinp['antenna']] = 1
for ant in antCounts.keys():
if antCounts[ant] != 1:
sys.stderr.write("WARNING: Antenna '%s' is defined %i times" %
(ant, antCounts[ant]))
# Update the file offsets to get things lined up better
tMax = max([cinp['tstart'] for cinp in corrConfig['inputs']])
for cinp in corrConfig['inputs']:
diff = tMax - cinp['tstart']
offset = diff.days * 86400 + diff.seconds + diff.microseconds / 1e6
cinp['fileoffset'] = max([0, offset])
# Reconcile the source lists for when we have eLWA data. This is needed so
# that we use the source information contained in the VDIF files rather than
# the stub information contained in the SDFs
if len(sources) <= 1:
if corrConfig['source']['name'] != '':
## Update the source information with what comes from the VLA
try:
sources[0] = corrConfig['source']
except IndexError:
sources.append(corrConfig['source'])
# Update the dwell time using the minimum on-source time for all inputs if
# there is only one source, i.e., for full eLWA runs
if len(sources) == 1:
sources[0]['start'] = max(
[cinp['tstart'] for cinp in corrConfig['inputs']])
sources[0]['stop'] = min(
[cinp['tstop'] for cinp in corrConfig['inputs']])
# Render the configuration
startRef = sources[0]['start']
s = 0
for source in sources:
startOffset = source['start'] - startRef
startOffset = startOffset.total_seconds()
dur = source['stop'] - source['start']
dur = dur.total_seconds()
## Skip over dummy scans and scans that start after the files end
if source['intent'] in (None, 'dummy'):
continue
if source['start'] > max(
[cinp['tstop'] for cinp in corrConfig['inputs']]):
print(
"Skipping scan of %s which starts at %s, %.3f s after the data end"
% (source['name'], source['start'],
(source['start'] -
max([cinp['tstop']
for cinp in corrConfig['inputs']])).total_seconds()))
continue
## Small correction for the first scan to compensate for stale data at LWA-SV
if lwasvFound and s == 0:
startOffset += 10.0
dur -= 10.0
## Skip over scans that are too short
if dur < args.minimum_scan_length:
continue
## Setup
if args.output is None:
fh = sys.stdout
else:
outname = args.output
if len(sources) > 1:
outname += str(s + 1)
fh = open(outname, 'w')
try:
repo = git.Repo(os.path.dirname(os.path.abspath(__file__)))
try:
branch = repo.active_branch.name
hexsha = repo.active_branch.commit.hexsha
except TypeError:
branch = '<detached>'
hexsha = repo.head.commit.hexsha
shortsha = hexsha[-7:]
dirty = ' (dirty)' if repo.is_dirty() else ''
except git.exc.GitError:
branch = 'unknown'
hexsha = 'unknown'
shortsha = 'unknown'
dirty = ''
## Preamble
fh.write("# Created\n")
fh.write("# on %s\n" % datetime.now())
fh.write("# using %s, revision %s.%s%s\n" %
(os.path.basename(__file__), branch, shortsha, dirty))
fh.write("\n")
## Observation context
fh.write("Context\n")
fh.write(" Observer %s\n" % corrConfig['context']['observer'])
fh.write(" Project %s\n" % corrConfig['context']['project'])
if corrConfig['context']['session'] is not None:
fh.write(" Session %s\n" % corrConfig['context']['session'])
if corrConfig['context']['vlaref'] is not None:
fh.write(" VLARef %s\n" % corrConfig['context']['vlaref'])
fh.write("EndContext\n")
fh.write("\n")
## Configuration, if present
if corrConfig['setup'] is not None:
fh.write("Configuration\n")
fh.write(" Channels %i\n" % corrConfig['setup']['channels'])
fh.write(" IntTime %.3f\n" % corrConfig['setup']['inttime'])
fh.write(" PolBasis %s\n" % corrConfig['setup']['basis'])
fh.write("EndConfiguration\n")
fh.write("\n")
## Source
fh.write("Source\n")
fh.write("# Observation start is %s\n" % source['start'])
fh.write("# Duration is %s\n" % (source['stop'] - source['start'], ))
fh.write(" Name %s\n" % source['name'])
fh.write(" Intent %s\n" % source['intent'].lower())
if source['name'] not in ('Sun', 'Jupiter'):
fh.write(" RA2000 %s\n" % source['ra2000'])
fh.write(" Dec2000 %s\n" % source['dec2000'])
fh.write(" Duration %.3f\n" % dur)
fh.write("SourceDone\n")
fh.write("\n")
## Input files
for cinp in corrConfig['inputs']:
fh.write("Input\n")
fh.write("# Start time is %s\n" % cinp['tstart'])
fh.write("# Stop time is %s\n" % cinp['tstop'])
try:
fh.write("# Beam is %i\n" % cinp['beam'])
except KeyError:
pass
try:
fh.write("# VLA pad is %s\n" % cinp['pad'])
except KeyError:
pass
try:
fh.write("# Frequency tuning 1 is %.3f Hz\n" % cinp['freq'][0])
fh.write("# Frequency tuning 2 is %.3f Hz\n" % cinp['freq'][1])
except TypeError:
fh.write("# Frequency tuning is %.3f Hz\n" % cinp['freq'])
fh.write(" File %s\n" % cinp['file'])
try:
metaname = metadata[os.path.basename(cinp['file'])]
fh.write(" MetaData %s\n" % metaname)
except KeyError:
if cinp['type'] == 'DRX':
sys.stderr.write(
"WARNING: No metadata found for '%s', source %i\n" %
(os.path.basename(cinp['file']), s + 1))
sys.stderr.flush()
pass
fh.write(" Type %s\n" % cinp['type'])
fh.write(" Antenna %s\n" % cinp['antenna'])
fh.write(" Pols %s\n" % cinp['pols'])
fh.write(" Location %.6f, %.6f, %.6f\n" %
cinp['location'])
try:
if cinp['apparent_location'][0] is not None:
fh.write(" ApparentLocation %.6f, %.6f, %.6f\n" %
cinp['apparent_location'])
except KeyError:
pass
fh.write(" ClockOffset %s, %s\n" % cinp['clockoffset'])
fh.write(" FileOffset %.3f\n" %
(startOffset + cinp['fileoffset'], ))
fh.write("InputDone\n")
fh.write("\n")
if fh != sys.stdout:
fh.close()
# Increment the source/file counter
s += 1
def main(args):
# Select the multirate module to use
if args.jit:
from jit import multirate
else:
import multirate
# Build up the station
site = stations.lwa1
## Updated 2018/3/8 with solutions from the 2018 Feb 28 eLWA
## run. See createConfigFile.py for details.
site.lat = 34.068956328 * numpy.pi / 180
site.long = -107.628103026 * numpy.pi / 180
site.elev = 2132.96837346
observer = site.get_observer()
# Parse the correlator configuration
config, refSrc, filenames, metanames, foffsets, readers, antennas = read_correlator_configuration(
args.filename)
try:
args.fft_length = config['channels']
args.dump_time = config['inttime']
print(
"NOTE: Set FFT length to %i and dump time to %.3f s per user defined configuration"
% (args.fft_length, args.dump_time))
except (TypeError, KeyError):
pass
if args.duration == 0.0:
args.duration = refSrc.duration
args.duration = min([args.duration, refSrc.duration])
# Length of the FFT
LFFT = args.fft_length
# Get the raw configuration
fh = open(args.filename, 'r')
rawConfig = fh.readlines()
fh.close()
# Antenna report
print("Antennas:")
for ant in antennas:
print(" Antenna %i: Stand %i, Pol. %i (%.3f us offset)" %
(ant.id, ant.stand.id, ant.pol, ant.cable.clock_offset * 1e6))
# Open and align the files
fh = []
nFramesFile = []
srate = []
beams = []
tunepols = []
beampols = []
tStart = []
cFreqs = []
bitDepths = []
buffers = []
grossOffsets = []
for i, (filename, metaname,
foffset) in enumerate(zip(filenames, metanames, foffsets)):
fh.append(open(filename, "rb"))
go = numpy.int32(antennas[2 * i].cable.clock_offset)
antennas[2 * i + 0].cable.clock_offset -= go
antennas[2 * i + 1].cable.clock_offset -= go
grossOffsets.append(go)
if go != 0:
print("Correcting time tags for gross offset of %i s" %
grossOffsets[i])
print(" Antenna clock offsets are now at %.3f us, %.3f us" %
(antennas[2 * i + 0].cable.clock_offset * 1e6,
antennas[2 * i + 1].cable.clock_offset * 1e6))
if readers[i] is vdif:
header = vdif.read_guppi_header(fh[i])
readers[i].FRAME_SIZE = readers[i].get_frame_size(fh[i])
nFramesFile.append(os.path.getsize(filename) // readers[i].FRAME_SIZE)
if readers[i] is vdif:
junkFrame = readers[i].read_frame(fh[i],
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] *
2.0)
readers[i].DATA_LENGTH = junkFrame.payload.data.size
beam, pol = junkFrame.id
elif readers[i] is drx:
junkFrame = readers[i].read_frame(fh[i])
while junkFrame.header.decimation == 0:
junkFrame = readers[i].read_frame(fh[i])
readers[i].DATA_LENGTH = junkFrame.payload.data.size
beam, tune, pol = junkFrame.id
fh[i].seek(-readers[i].FRAME_SIZE, 1)
beams.append(beam)
srate.append(junkFrame.sample_rate)
if readers[i] is vdif:
tunepols.append(readers[i].get_thread_count(fh[i]))
beampols.append(tunepols[i])
elif readers[i] is drx:
beampols.append(max(readers[i].get_frames_per_obs(fh[i])))
skip = args.skip + foffset
if skip != 0:
print("Skipping forward %.3f s" % skip)
print("-> %.6f (%s)" % (junkFrame.time, junkFrame.time.datetime))
offset = int(skip * srate[i] / readers[i].DATA_LENGTH)
fh[i].seek(beampols[i] * readers[i].FRAME_SIZE * offset, 1)
if readers[i] is vdif:
junkFrame = readers[i].read_frame(
fh[i],
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
else:
junkFrame = readers[i].read_frame(fh[i])
fh[i].seek(-readers[i].FRAME_SIZE, 1)
print("-> %.6f (%s)" % (junkFrame.time, junkFrame.time.datetime))
tStart.append(junkFrame.time + grossOffsets[i])
# Get the frequencies
cFreq1 = 0.0
cFreq2 = 0.0
for j in xrange(64):
if readers[i] is vdif:
junkFrame = readers[i].read_frame(
fh[i],
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
s, p = junkFrame.id
if p == 0:
cFreq1 = junkFrame.central_freq
else:
pass
elif readers[i] is drx:
junkFrame = readers[i].read_frame(fh[i])
b, t, p = junkFrame.id
if p == 0:
if t == 1:
cFreq1 = junkFrame.central_freq
else:
cFreq2 = junkFrame.central_freq
else:
pass
fh[i].seek(-64 * readers[i].FRAME_SIZE, 1)
cFreqs.append([cFreq1, cFreq2])
try:
bitDepths.append(junkFrame.header.bits_per_sample)
except AttributeError:
bitDepths.append(8)
# Setup the frame buffers
if readers[i] is vdif:
buffers.append(VDIFFrameBuffer(threads=[0, 1]))
elif readers[i] is drx:
buffers.append(
DRXFrameBuffer(beams=[
beam,
],
tunes=[1, 2],
pols=[0, 1],
nsegments=16))
for i in xrange(len(filenames)):
# Align the files as close as possible by the time tags
if readers[i] is vdif:
timetags = []
for k in xrange(16):
junkFrame = readers[i].read_frame(fh[i])
timetags.append(junkFrame.header.frame_in_second)
fh[i].seek(-16 * readers[i].FRAME_SIZE, 1)
j = 0
while (timetags[j + 0] != timetags[j + 1]):
j += 1
fh[i].seek(readers[i].FRAME_SIZE, 1)
nFramesFile[i] -= j
elif readers[i] is drx:
pass
# Align the files as close as possible by the time tags
if readers[i] is vdif:
junkFrame = readers[i].read_frame(fh[i],
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] *
2.0)
else:
junkFrame = readers[i].read_frame(fh[i])
fh[i].seek(-readers[i].FRAME_SIZE, 1)
j = 0
while junkFrame.time + grossOffsets[i] < max(tStart):
if readers[i] is vdif:
for k in xrange(beampols[i]):
try:
junkFrame = readers[i].read_frame(
fh[i],
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
except errors.SyncError:
print("Error - VDIF @ %i" % (i, ))
fh[i].seek(readers[i].FRAME_SIZE, 1)
continue
else:
for k in xrange(beampols[i]):
junkFrame = readers[i].read_frame(fh[i])
j += beampols[i]
jTime = j * readers[i].DATA_LENGTH / srate[i] / beampols[i]
print("Shifted beam %i data by %i frames (%.4f s)" %
(beams[i], j, jTime))
# Set integration time
tRead = 1.0
nFrames = int(round(tRead * srate[-1] / readers[-1].DATA_LENGTH))
tRead = nFrames * readers[-1].DATA_LENGTH / srate[-1]
nFramesV = tRead * srate[0] / readers[0].DATA_LENGTH
nFramesD = nFrames
while nFramesV != int(nFramesV):
nFrames += 1
tRead = nFrames * readers[-1].DATA_LENGTH / srate[-1]
nFramesV = tRead * srate[0] / readers[0].DATA_LENGTH
nFramesD = nFrames
nFramesV = int(nFramesV)
# Read in some data
tFileV = nFramesFile[0] / beampols[0] * readers[0].DATA_LENGTH / srate[0]
tFileD = nFramesFile[-1] / beampols[-1] * readers[-1].DATA_LENGTH / srate[
-1]
tFile = min([tFileV, tFileD])
if args.duration > 0.0:
duration = args.duration
duration = tRead * int(round(duration / tRead))
tFile = duration
# Date
beginMJDs = []
beginDates = []
for i in xrange(len(filenames)):
if readers[i] is vdif:
junkFrame = readers[i].read_frame(fh[i],
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] *
2.0)
else:
junkFrame = readers[i].read_frame(fh[i])
fh[i].seek(-readers[i].FRAME_SIZE, 1)
beginMJDs.append(junkFrame.time.mjd)
beginDates.append(junkFrame.time.datetime)
# Set the output base filename
if args.tag is None:
outbase = os.path.basename(filenames[0])
outbase = os.path.splitext(outbase)[0][:8]
else:
outbase = args.tag
# Report
for i in xrange(len(filenames)):
print("Filename: %s" % os.path.basename(filenames[i]))
print(" Type/Reader: %s" % readers[i].__name__)
print(" Date of First Frame: %s" % beginDates[i])
print(" Sample Rate: %i Hz" % srate[i])
print(" Tuning 1: %.3f Hz" % cFreqs[i][0])
print(" Tuning 2: %.3f Hz" % cFreqs[i][1])
print(" Bit Depth: %i" % bitDepths[i])
print(" ===")
print(" Phase Center:")
print(" Name: %s" % refSrc.name)
print(" RA: %s" % refSrc._ra)
print(" Dec: %s" % refSrc._dec)
print(" ===")
print(" Data Read Time: %.3f s" % tRead)
print(" Data Reads in File: %i" % int(tFile / tRead))
print(" ")
nVDIFInputs = sum([1 for reader in readers if reader is vdif])
nDRXInputs = sum([1 for reader in readers if reader is drx])
print("Processing %i VDIF and %i DRX input streams" %
(nVDIFInputs, nDRXInputs))
print(" ")
nFramesV = int(round(tRead * srate[0] / readers[0].DATA_LENGTH))
framesPerSecondV = int(srate[0] / readers[0].DATA_LENGTH)
nFramesB = nFrames
framesPerSecondB = srate[-1] / readers[-1].DATA_LENGTH
if nVDIFInputs:
print("VDIF Frames/s: %.6f" % framesPerSecondV)
print("VDIF Frames/Integration: %i" % nFramesV)
if nDRXInputs:
print("DRX Frames/s: %.6f" % framesPerSecondB)
print("DRX Frames/Integration: %i" % nFramesB)
if nVDIFInputs * nDRXInputs:
print("Sample Count Ratio: %.6f" %
(1.0 * (nFramesV * readers[0].DATA_LENGTH) /
(nFramesB * 4096), ))
print("Sample Rate Ratio: %.6f" % (srate[0] / srate[-1], ))
print(" ")
vdifLFFT = LFFT * (2 if nVDIFInputs else 1
) # Fix to deal with LWA-only correlations
drxLFFT = vdifLFFT * srate[-1] / srate[0]
while drxLFFT != int(drxLFFT):
vdifLFFT += 1
drxLFFT = vdifLFFT * srate[-1] / srate[0]
vdifLFFT = vdifLFFT // (2 if nVDIFInputs else 1
) # Fix to deal with LWA-only correlations
drxLFFT = int(drxLFFT)
if nVDIFInputs:
print("VDIF Transform Size: %i" % vdifLFFT)
if nDRXInputs:
print("DRX Transform Size: %i" % drxLFFT)
print(" ")
vdifPivot = 1
if abs(cFreqs[0][0] - cFreqs[-1][1]) < abs(cFreqs[0][0] - cFreqs[-1][0]):
vdifPivot = 2
if nVDIFInputs == 0 and args.which != 0:
vdifPivot = args.which
if nVDIFInputs * nDRXInputs:
print("VDIF appears to correspond to tuning #%i in DRX" % vdifPivot)
elif nDRXInputs:
print("Correlating DRX tuning #%i" % vdifPivot)
print(" ")
nChunks = int(tFile / tRead)
tSub = args.subint_time
tSub = tRead / int(round(tRead / tSub))
tDump = args.dump_time
tDump = tSub * int(round(tDump / tSub))
nDump = int(tDump / tSub)
tDump = nDump * tSub
nInt = int((nChunks * tRead) / tDump)
print("Sub-integration time is: %.3f s" % tSub)
print("Integration (dump) time is: %.3f s" % tDump)
print(" ")
if args.gpu is not None:
try:
from jit import xcupy
xcupy.select_gpu(args.gpu)
xcupy.set_memory_usage_limit(1.5 * 1024**3)
multirate.xengine = xcupy.xengine
multirate.xengine_full = xcupy.xengine_full
print(
"Loaded GPU X-engine support on GPU #%i with %.2f GB of device memory"
% (args.gpu, xcupy.get_memory_usage_limit() / 1024.0**3))
except ImportError as e:
pass
subIntTimes = []
subIntCount = 0
fileCount = 0
wallStart = time.time()
done = False
oldStartRel = [0 for i in xrange(nVDIFInputs + nDRXInputs)]
username = getpass.getuser()
for i in xrange(nChunks):
wallTime = time.time()
tStart = []
tStartB = []
vdifRef = [0 for j in xrange(nVDIFInputs * 2)]
drxRef = [0 for j in xrange(nDRXInputs * 2)]
# Read in the data
with InterProcessLock('/dev/shm/sc-reader-%s' % username) as lock:
try:
dataV *= 0.0
dataD *= 0.0
except NameError:
dataV = numpy.zeros(
(len(vdifRef), readers[0].DATA_LENGTH * nFramesV),
dtype=numpy.float32)
dataD = numpy.zeros(
(len(drxRef), readers[-1].DATA_LENGTH * nFramesD),
dtype=numpy.complex64)
for j, f in enumerate(fh):
if readers[j] is vdif:
## VDIF
k = 0
while k < beampols[j] * nFramesV:
try:
cFrame = readers[j].read_frame(
f,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
buffers[j].append(cFrame)
except errors.SyncError:
print("Error - VDIF @ %i, %i" % (i, j))
f.seek(readers[j].FRAME_SIZE, 1)
continue
except errors.EOFError:
done = True
break
frames = buffers[j].get()
if frames is None:
continue
for cFrame in frames:
std, pol = cFrame.id
sid = 2 * j + pol
if k == 0:
tStart.append(cFrame.time)
tStart[-1] = tStart[-1] + grossOffsets[j]
tStartB.append(get_better_time(cFrame))
tStartB[-1][
0] = tStart[-1][0] + grossOffsets[j]
for p in (0, 1):
psid = 2 * j + p
vdifRef[
psid] = cFrame.header.seconds_from_epoch * framesPerSecondV + cFrame.header.frame_in_second
count = cFrame.header.seconds_from_epoch * framesPerSecondV + cFrame.header.frame_in_second
count -= vdifRef[sid]
dataV[sid,
count * readers[j].DATA_LENGTH:(count + 1) *
readers[j].DATA_LENGTH] = cFrame.payload.data
k += 1
elif readers[j] is drx:
## DRX
k = 0
while k < beampols[j] * nFramesD:
try:
cFrame = readers[j].read_frame(f)
buffers[j].append(cFrame)
except errors.SyncError:
print("Error - DRX @ %i, %i" % (i, j))
continue
except errors.EOFError:
done = True
break
frames = buffers[j].get()
if frames is None:
continue
for cFrame in frames:
beam, tune, pol = cFrame.id
if tune != vdifPivot:
continue
bid = 2 * (j - nVDIFInputs) + pol
if k == 0:
tStart.append(cFrame.time)
tStart[-1] = tStart[-1] + grossOffsets[j]
tStartB.append(get_better_time(cFrame))
tStartB[-1][
0] = tStart[-1][0] + grossOffsets[j]
for p in (0, 1):
pbid = 2 * (j - nVDIFInputs) + p
drxRef[pbid] = cFrame.payload.timetag
count = cFrame.payload.timetag
count -= drxRef[bid]
count //= (4096 * int(196e6 / srate[-1]))
### Fix from some LWA-SV files that seem to cause the current LSL
### ring buffer problems
if count < 0:
continue
try:
dataD[bid, count *
readers[j].DATA_LENGTH:(count + 1) *
readers[j].
DATA_LENGTH] = cFrame.payload.data
k += beampols[j] // 2
except ValueError:
k = beampols[j] * nFramesD
break
print('RR - Read finished in %.3f s for %.3fs of data' %
(time.time() - wallTime, tRead))
# Figure out which DRX tuning corresponds to the VDIF data
if nDRXInputs > 0:
dataD /= 7.0
# Time tag alignment (sample based)
## Initial time tags for each stream and the relative start time for each stream
if args.verbose:
### TT = time tag
print('TT - Start', tStartB)
tStartMin = min([sec for sec, frac in tStartB])
tStartRel = [(sec - tStartMin) + frac for sec, frac in tStartB]
## Sample offsets between the streams
offsets = []
for j in xrange(nVDIFInputs + nDRXInputs):
offsets.append(
int(round(nsround(max(tStartRel) - tStartRel[j]) * srate[j])))
if args.verbose:
print('TT - Offsets', offsets)
## Roll the data to apply the sample offsets and then trim the ends to get rid
## of the rolled part
for j, offset in enumerate(offsets):
if j < nVDIFInputs:
if offset != 0:
idx0 = 2 * j + 0
idx1 = 2 * j + 1
tStart[j] += offset / (srate[j])
tStartB[j][1] += offset / (srate[j])
dataV[idx0, :] = numpy.roll(dataV[idx0, :], -offset)
dataV[idx1, :] = numpy.roll(dataV[idx1, :], -offset)
else:
if offset != 0:
idx0 = 2 * (j - nVDIFInputs) + 0
idx1 = 2 * (j - nVDIFInputs) + 1
tStart[j] += offset / (srate[j])
tStartB[j][1] += offset / (srate[j])
dataD[idx0, :] = numpy.roll(dataD[idx0, :], -offset)
dataD[idx1, :] = numpy.roll(dataD[idx1, :], -offset)
vdifOffsets = offsets[:nVDIFInputs]
drxOffsets = offsets[nVDIFInputs:]
## Apply the corrections to the original time tags and report on the sub-sample
## residuals
if args.verbose:
print('TT - Adjusted', tStartB)
tStartMinSec = min([sec for sec, frac in tStartB])
tStartMinFrac = min([frac for sec, frac in tStartB])
tStartRel = [(sec - tStartMinSec) + (frac - tStartMinFrac)
for sec, frac in tStartB]
if args.verbose:
print('TT - Residual',
["%.1f ns" % (r * 1e9, ) for r in tStartRel])
for k in xrange(len(tStartRel)):
antennas[2 * k +
0].cable.clock_offset -= tStartRel[k] - oldStartRel[k]
antennas[2 * k +
1].cable.clock_offset -= tStartRel[k] - oldStartRel[k]
oldStartRel = tStartRel
# Setup everything we need to loop through the sub-integrations
nSub = int(tRead / tSub)
nSampV = int(srate[0] * tSub)
nSampD = int(srate[-1] * tSub)
#tV = i*tRead + numpy.arange(dataV.shape[1]-max(vdifOffsets), dtype=numpy.float64)/srate[ 0]
if nDRXInputs > 0:
tD = i * tRead + numpy.arange(dataD.shape[1] - max(drxOffsets),
dtype=numpy.float64) / srate[-1]
# Loop over sub-integrations
for j in xrange(nSub):
## Select the data to work with
tSubInt = tStart[0] + (
j + 1) * nSampV / srate[0] - nSampV // 2 / srate[0]
#tVSub = tV[j*nSampV:(j+1)*nSampV]
if nDRXInputs > 0:
tDSub = tD[j * nSampD:(j + 1) * nSampD]
dataVSub = dataV[:, j * nSampV:(j + 1) * nSampV]
#if dataVSub.shape[1] != tVSub.size:
# dataVSub = dataVSub[:,:tVSub.size]
#if tVSub.size == 0:
# continue
dataDSub = dataD[:, j * nSampD:(j + 1) * nSampD]
if nDRXInputs > 0:
if dataDSub.shape[1] != tDSub.size:
dataDSub = dataDSub[:, :tDSub.size]
if tDSub.size == 0:
continue
## Update the observation
observer.date = astro.unix_to_utcjd(tSubInt) - astro.DJD_OFFSET
refSrc.compute(observer)
## Correct for the LWA dipole power pattern
if nDRXInputs > 0:
dipoleX, dipoleY = jones.get_lwa_antenna_gain(
observer, refSrc, freq=cFreqs[-1][vdifPivot - 1])
dataDSub[0::2, :] /= numpy.sqrt(dipoleX)
dataDSub[1::2, :] /= numpy.sqrt(dipoleY)
## Get the Jones matrices and apply
## NOTE: This moves the LWA into the frame of the VLA
if nVDIFInputs * nDRXInputs > 0:
lwaToSky = jones.get_matrix_lwa(observer, refSrc)
skyToVLA = jones.get_matrix_vla(observer, refSrc, inverse=True)
dataDSub = jones.apply_matrix(
dataDSub,
numpy.matrix(skyToVLA) * numpy.matrix(lwaToSky))
## Correlate
delayPadding = multirate.get_optimal_delay_padding(
antennas[:2 * nVDIFInputs],
antennas[2 * nVDIFInputs:],
LFFT=drxLFFT,
sample_rate=srate[-1],
central_freq=cFreqs[-1][vdifPivot - 1],
pol='*',
phase_center=refSrc)
if nVDIFInputs > 0:
freqV, feoV, veoV, deoV = multirate.fengine(
dataVSub,
antennas[:2 * nVDIFInputs],
LFFT=vdifLFFT,
sample_rate=srate[0],
central_freq=cFreqs[0][0] - srate[0] / 4,
pol='*',
phase_center=refSrc,
delayPadding=delayPadding)
if nDRXInputs > 0:
freqD, feoD, veoD, deoD = multirate.fengine(
dataDSub,
antennas[2 * nVDIFInputs:],
LFFT=drxLFFT,
sample_rate=srate[-1],
central_freq=cFreqs[-1][vdifPivot - 1],
pol='*',
phase_center=refSrc,
delayPadding=delayPadding)
## Rotate the phase in time to deal with frequency offset between the VLA and LWA
if nDRXInputs * nVDIFInputs > 0:
subChanFreqOffset = (cFreqs[0][0] - cFreqs[-1][vdifPivot - 1]
) % (freqD[1] - freqD[0])
if i == 0 and j == 0:
## FC = frequency correction
tv, tu = bestFreqUnits(subChanFreqOffset)
print(
"FC - Applying fringe rotation rate of %.3f %s to the DRX data"
% (tv, tu))
freqD += subChanFreqOffset
for w in xrange(feoD.shape[2]):
feoD[:, :,
w] *= numpy.exp(-2j * numpy.pi * subChanFreqOffset *
tDSub[w * drxLFFT])
## Sort out what goes where (channels and antennas) if we don't already know
try:
if nVDIFInputs > 0:
freqV = freqV[goodV]
feoV = numpy.roll(feoV, -goodV[0],
axis=1)[:, :len(goodV), :]
if nDRXInputs > 0:
freqD = freqD[goodD]
feoD = numpy.roll(feoD, -goodD[0],
axis=1)[:, :len(goodD), :]
except NameError:
### Frequency overlap
fMin, fMax = -1e12, 1e12
if nVDIFInputs > 0:
fMin, fMax = max([fMin,
freqV.min()]), min([fMax,
freqV.max()])
if nDRXInputs > 0:
fMin, fMax = max([fMin,
freqD.min()]), min([fMax,
freqD.max()])
### Channels and antennas (X vs. Y)
if nVDIFInputs > 0:
goodV = numpy.where((freqV >= fMin) & (freqV <= fMax))[0]
aXV = [
k for (k, a) in enumerate(antennas[:2 * nVDIFInputs])
if a.pol == 0
]
aYV = [
k for (k, a) in enumerate(antennas[:2 * nVDIFInputs])
if a.pol == 1
]
if nDRXInputs > 0:
goodD = numpy.where((freqD >= fMin) & (freqD <= fMax))[0]
aXD = [
k for (k, a) in enumerate(antennas[2 * nVDIFInputs:])
if a.pol == 0
]
aYD = [
k for (k, a) in enumerate(antennas[2 * nVDIFInputs:])
if a.pol == 1
]
### Validate the channel alignent and fix it if needed
if nVDIFInputs * nDRXInputs != 0:
pd = freqV[goodV[0]] - freqD[goodD[0]]
# Need to shift?
if abs(pd) >= 1.01 * abs(subChanFreqOffset):
## Need to shift
if pd < 0.0:
goodV = goodV[1:]
else:
goodD = goodD[1:]
# Need to trim?
if len(goodV) > len(goodD):
## Yes, goodV is too long
goodV = goodV[:len(goodD)]
elif len(goodD) > len(goodV):
## Yes, goodD is too long
goodD = goodD[:len(goodV)]
else:
## No, nothing needs to be done
pass
# Validate
fd = freqV[goodV] - freqD[goodD]
try:
assert (fd.min() >= -1.01 * subChanFreqOffset)
assert (fd.max() <= 1.01 * subChanFreqOffset)
## FS = frequency selection
tv, tu = bestFreqUnits(freqV[1] - freqV[0])
print("FS - Found %i, %.3f %s overalapping channels" %
(len(goodV), tv, tu))
tv, tu = bestFreqUnits(freqV[goodV[-1]] -
freqV[goodV[0]])
print("FS - Bandwidth is %.3f %s" % (tv, tu))
print("FS - Channels span %.3f MHz to %.3f MHz" %
(freqV[goodV[0]] / 1e6, freqV[goodV[-1]] / 1e6))
except AssertionError:
raise RuntimeError(
"Cannot find a common frequency set between the input data: offsets range between %.3f Hz and %.3f Hz, expected %.3f Hz"
% (fd.min(), fd.max(), subChanFreqOffset))
### Apply
if nVDIFInputs > 0:
freqV = freqV[goodV]
feoV = numpy.roll(feoV, -goodV[0],
axis=1)[:, :len(goodV), :]
if nDRXInputs > 0:
freqD = freqD[goodD]
feoD = numpy.roll(feoD, -goodD[0],
axis=1)[:, :len(goodD), :]
try:
nchan = freqV.size
fdt = feoV.dtype
vdt = veoV.dtype
except NameError:
nchan = freqD.size
fdt = feoD.dtype
vdt = veoD.dtype
## Setup the intermediate F-engine products and trim the data
### Figure out the minimum number of windows
nWin = 1e12
if nVDIFInputs > 0:
nWin = min([nWin, feoV.shape[2]])
nWin = min(
[nWin,
numpy.argmax(numpy.cumsum(veoV.sum(axis=0))) + 1])
if nDRXInputs > 0:
nWin = min([nWin, feoD.shape[2]])
nWin = min(
[nWin,
numpy.argmax(numpy.cumsum(veoD.sum(axis=0))) + 1])
### Initialize the intermediate arrays
try:
assert (feoX.shape[2] == nWin)
except (NameError, AssertionError):
feoX = numpy.zeros((nVDIFInputs + nDRXInputs, nchan, nWin),
dtype=fdt)
feoY = numpy.zeros((nVDIFInputs + nDRXInputs, nchan, nWin),
dtype=fdt)
veoX = numpy.zeros((nVDIFInputs + nDRXInputs, nWin), dtype=vdt)
veoY = numpy.zeros((nVDIFInputs + nDRXInputs, nWin), dtype=vdt)
### Trim
if nVDIFInputs > 0:
feoV = feoV[:, :, :nWin]
veoV = veoV[:, :nWin]
if nDRXInputs > 0:
feoD = feoD[:, :, :nWin]
veoD = veoD[:, :nWin]
## Sort it all out by polarization
for k in xrange(nVDIFInputs):
feoX[k, :, :] = feoV[aXV[k], :, :]
feoY[k, :, :] = feoV[aYV[k], :, :]
veoX[k, :] = veoV[aXV[k], :]
veoY[k, :] = veoV[aYV[k], :]
for k in xrange(nDRXInputs):
feoX[k + nVDIFInputs, :, :] = feoD[aXD[k], :, :]
feoY[k + nVDIFInputs, :, :] = feoD[aYD[k], :, :]
veoX[k + nVDIFInputs, :] = veoD[aXD[k], :]
veoY[k + nVDIFInputs, :] = veoD[aYD[k], :]
## Cross multiply
try:
sfreqXX = freqV
sfreqYY = freqV
except NameError:
sfreqXX = freqD
sfreqYY = freqD
svisXX, svisXY, svisYX, svisYY = multirate.xengine_full(
feoX, veoX, feoY, veoY)
## Accumulate
if subIntCount == 0:
subIntTimes = [
tSubInt,
]
freqXX = sfreqXX
freqYY = sfreqYY
visXX = svisXX / nDump
visXY = svisXY / nDump
visYX = svisYX / nDump
visYY = svisYY / nDump
else:
subIntTimes.append(tSubInt)
visXX += svisXX / nDump
visXY += svisXY / nDump
visYX += svisYX / nDump
visYY += svisYY / nDump
subIntCount += 1
## Save
if subIntCount == nDump:
subIntCount = 0
fileCount += 1
### CD = correlator dump
outfile = "%s-vis2-%05i.npz" % (outbase, fileCount)
numpy.savez(outfile,
config=rawConfig,
srate=srate[0] / 2.0,
freq1=freqXX,
vis1XX=visXX,
vis1XY=visXY,
vis1YX=visYX,
vis1YY=visYY,
tStart=numpy.mean(
numpy.array(subIntTimes, dtype=numpy.float64)),
tInt=tDump)
print(
"CD - writing integration %i to disk, timestamp is %.3f s"
% (fileCount,
numpy.mean(numpy.array(subIntTimes,
dtype=numpy.float64))))
if fileCount == 1:
print("CD - each integration is %.1f MB on disk" %
(os.path.getsize(outfile) / 1024.0**2, ))
if (fileCount - 1) % 25 == 0:
print(
"CD - average processing time per integration is %.3f s"
% ((time.time() - wallStart) / fileCount, ))
etc = (nInt - fileCount) * (time.time() -
wallStart) / fileCount
eth = int(etc / 60.0) // 60
etm = int(etc / 60.0) % 60
ets = etc % 60
print(
"CD - estimated time to completion is %i:%02i:%04.1f" %
(eth, etm, ets))
if done:
break
# Cleanup
etc = time.time() - wallStart
eth = int(etc / 60.0) // 60
etm = int(etc / 60.0) % 60
ets = etc % 60
print("Processing finished after %i:%02i:%04.1f" % (eth, etm, ets))
print("Average time per integration was %.3f s" % (etc / fileCount, ))
for f in fh:
f.close()
def main(args):
# Parse the command line
filename = args.filename
fh = open(filename, 'rb')
header = vdif.read_guppi_header(fh)
vdif.FRAME_SIZE = vdif.get_frame_size(fh)
nFramesFile = os.path.getsize(filename) // vdif.FRAME_SIZE
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
srate = junkFrame.sample_rate
vdif.DATA_LENGTH = junkFrame.payload.data.size
beam, pol = junkFrame.id
tunepols = vdif.get_thread_count(fh)
beampols = tunepols
# Get the frequencies
cFreq = 0.0
for j in xrange(4):
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
s, p = junkFrame.id
if p == 0:
cFreq = junkFrame.central_freq
# Date
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
fh.seek(-vdif.FRAME_SIZE, 1)
beginDate = junkFrame.time.datetime
# Report
print("Filename: %s" % os.path.basename(filename))
print(" Date of First Frame: %s" % beginDate)
print(" Station: %i" % beam)
print(" Sample Rate: %i Hz" % srate)
print(" Bit Depth: %i" % junkFrame.header.bits_per_sample)
print(" Tuning 1: %.1f Hz" % cFreq)
print(" ")
# Determine the clip level
if args.trim_level is None:
if junkFrame.header.bits_per_sample == 1:
args.trim_level = abs(1.0)**2
elif junkFrame.header.bits_per_sample == 2:
args.trim_level = abs(3.3359)**2
elif junkFrame.header.bits_per_sample == 4:
args.trim_level = abs(7 / 2.95)**2
elif junkFrame.header.bits_per_sample == 8:
args.trim_level = abs(255 / 256.)**2
else:
args.trim_level = 1.0
print("Setting clip level to %.3f" % args.trim_level)
print(" ")
# Convert chunk length to total frame count
chunkLength = int(args.length * srate / vdif.DATA_LENGTH * tunepols)
chunkLength = int(1.0 * chunkLength / tunepols) * tunepols
# Convert chunk skip to total frame count
chunkSkip = int(args.skip * srate / vdif.DATA_LENGTH * tunepols)
chunkSkip = int(1.0 * chunkSkip / tunepols) * tunepols
# Output arrays
clipFraction = []
meanPower = []
meanRMS = []
# Go!
i = 1
done = False
print(" | Clipping | Power | RMS |")
print(" | 1X 1Y | 1X 1Y | 1X 1Y |")
print("----+-----------------+---------------+---------------+")
while True:
count = {0: 0, 1: 0}
data = numpy.empty((2, chunkLength * vdif.DATA_LENGTH // tunepols),
dtype=numpy.float32)
for j in xrange(chunkLength):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0,
verbose=False)
except errors.EOFError:
done = True
break
except errors.SyncError:
continue
beam, pol = cFrame.id
aStand = pol
try:
data[aStand,
count[aStand] * vdif.DATA_LENGTH:(count[aStand] + 1) *
vdif.DATA_LENGTH] = cFrame.payload.data
# Update the counters so that we can average properly later on
count[aStand] += 1
except ValueError:
pass
if done:
break
else:
rms = numpy.sqrt((data**2).mean(axis=1))
data = numpy.abs(data)**2
clipFraction.append(numpy.zeros(2))
meanPower.append(data.mean(axis=1))
meanRMS.append(rms)
for j in xrange(2):
bad = numpy.nonzero(data[j, :] > args.trim_level)[0]
clipFraction[-1][j] = 1.0 * len(bad) / data.shape[1]
clip = clipFraction[-1]
power = meanPower[-1]
print("%3i | %6.2f%% %6.2f%% | %6.3f %6.3f | %6.3f %6.3f |" %
(i, clip[0] * 100.0, clip[1] * 100.0, power[0], power[1],
rms[0], rms[1]))
i += 1
fh.seek(vdif.FRAME_SIZE * chunkSkip, 1)
clipFraction = numpy.array(clipFraction)
meanPower = numpy.array(meanPower)
meanRMS = numpy.array(meanRMS)
clip = clipFraction.mean(axis=0)
power = meanPower.mean(axis=0)
rms = meanRMS.mean(axis=0)
print("----+-----------------+---------------+---------------+")
print("%3s | %6.2f%% %6.2f%% | %6.3f %6.3f | %6.3f %6.3f |" %
('M', clip[0] * 100.0, clip[1] * 100.0, power[0], power[1], rms[0],
rms[1]))
def main(args):
# Parse the command line
filename = args.filename
# Length of the FFT
LFFT = args.fft_length
fh = open(filename, 'rb')
header = vdif.read_guppi_header(fh)
vdif.FRAME_SIZE = vdif.get_frame_size(fh)
nFramesFile = os.path.getsize(filename) // vdif.FRAME_SIZE
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
srate = junkFrame.sample_rate
vdif.DATA_LENGTH = junkFrame.payload.data.size
beam, pol = junkFrame.id
tunepols = vdif.get_thread_count(fh)
beampols = tunepols
if args.skip != 0:
print("Skipping forward %.3f s" % args.skip)
print("-> %.6f (%s)" % (junkFrame.time, junkFrame.time.datetime))
offset = int(args.skip * srate / vdif.DATA_LENGTH)
fh.seek(beampols * vdif.FRAME_SIZE * offset, 1)
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
fh.seek(-vdif.FRAME_SIZE, 1)
print("-> %.6f (%s)" % (junkFrame.time, junkFrame.time.datetime))
tStart = junkFrame.time
# Get the frequencies
cFreq = 0.0
for j in xrange(4):
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
s, p = junkFrame.id
if p == 0:
cFreq = junkFrame.central_freq
# Set integration time
tInt = args.avg_time
nFrames = int(round(tInt * srate / vdif.DATA_LENGTH))
tInt = nFrames * vdif.DATA_LENGTH / srate
nFrames = int(round(tInt * srate / vdif.DATA_LENGTH))
# Read in some data
tFile = nFramesFile / beampols * vdif.DATA_LENGTH / srate
# Date
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
fh.seek(-vdif.FRAME_SIZE, 1)
beginDate = junkFrame.time.datetime
# Report
print("Filename: %s" % os.path.basename(filename))
print(" Date of First Frame: %s" % beginDate)
print(" Station: %i" % beam)
print(" Sample Rate: %i Hz" % srate)
print(" Tuning 1: %.1f Hz" % cFreq)
print(" Bit Depth: %i" % junkFrame.header.bits_per_sample)
print(" Integration Time: %.3f s" % tInt)
print(" Integrations in File: %i" % int(tFile / tInt))
print(" ")
# Go!
data = numpy.zeros((beampols, vdif.DATA_LENGTH * nFrames),
dtype=numpy.complex64)
count = [0 for i in xrange(data.shape[0])]
for i in xrange(beampols * nFrames):
try:
cFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
except errors.SyncError:
print("Error @ %i" % i)
fh.seek(vdif.FRAME_SIZE, 1)
continue
std, pol = cFrame.id
sid = pol
data[sid, count[sid] * vdif.DATA_LENGTH:(count[sid] + 1) *
vdif.DATA_LENGTH] = cFrame.payload.data
count[sid] += 1
# Transform and trim off the negative frequencies
freq, psd = fxc.SpecMaster(data,
LFFT=2 * LFFT,
sample_rate=srate,
central_freq=header['OBSFREQ'] - srate / 4)
freq, psd = freq[LFFT:], psd[:, LFFT:]
# Plot
fig = plt.figure()
ax = fig.gca()
for i in xrange(psd.shape[0]):
ax.plot(freq / 1e6, numpy.log10(psd[i, :]) * 10, label='%i' % i)
ax.set_title('%i' % beam)
ax.set_xlabel('Frequency [MHz]')
ax.set_ylabel('PSD [arb. dB]')
ax.legend(loc=0)
plt.show()
# Done
fh.close()
def main(args):
# Length of the FFT
LFFT = args.fft_length
if args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
args.window = window
# Open the file and find good data (not spectrometer data)
filename = args.filename
fh = open(filename, "rb")
header = vdif.read_guppi_header(fh)
vdif.FRAME_SIZE = vdif.get_frame_size(fh)
nFramesFile = os.path.getsize(filename) // vdif.FRAME_SIZE
while True:
try:
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
try:
srate = junkFrame.sample_rate
t0 = junkFrame.time
vdif.DATA_LENGTH = junkFrame.payload.data.size
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-vdif.FRAME_SIZE + 1, 1)
fh.seek(-vdif.FRAME_SIZE, 1)
beam, pol = junkFrame.id
beams = 1
tunepols = vdif.get_thread_count(fh)
tunepol = tunepols
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(args.skip * srate / vdif.DATA_LENGTH * beampols)
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * vdif.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
srate = junkFrame.sample_rate
t1 = junkFrame.time
tunepols = (vdif.get_thread_count(fh), )
tunepol = tunepols[0]
beampols = tunepol
fh.seek(-vdif.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1 - (t0 + args.skip)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 2.0
cOffset = int(tCorr * srate / vdif.DATA_LENGTH * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * vdif.FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1 - t0
offset = int(round(args.skip * srate / vdif.DATA_LENGTH * beampols))
offset = int(1.0 * offset / beampols) * beampols
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped. This needs to
# take into account the number of beampols in the data, the FFT length,
# and the number of samples per frame.
maxFrames = int(1.0 * 28000 / beampols * vdif.DATA_LENGTH /
float(2 * LFFT)) * 2 * LFFT / vdif.DATA_LENGTH * beampols
# Number of frames to integrate over
nFramesAvg = int(args.average * srate / vdif.DATA_LENGTH * beampols)
nFramesAvg = int(1.0 * nFramesAvg / beampols * vdif.DATA_LENGTH /
float(2 * LFFT)) * 2 * LFFT / vdif.DATA_LENGTH * beampols
args.average = 1.0 * nFramesAvg / beampols * vdif.DATA_LENGTH / srate
maxFrames = nFramesAvg
# Number of remaining chunks (and the correction to the number of
# frames to read in).
if args.duration == 0:
args.duration = 1.0 * nFramesFile / beampols * vdif.DATA_LENGTH / srate
args.duration -= args.skip
else:
args.duration = int(
round(args.duration * srate * beampols / vdif.DATA_LENGTH) /
beampols * vdif.DATA_LENGTH / srate)
nChunks = int(round(args.duration / args.average))
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg * nChunks
# Date & Central Frequency
t1 = junkFrame.time
beginDate = junkFrame.time.datetime
central_freq1 = 0.0
central_freq2 = 0.0
for i in xrange(4):
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
b, p = junkFrame.id
if p == 0:
central_freq1 = junkFrame.central_freq
elif p == 0:
central_freq2 = junkFrame.central_freq
else:
pass
fh.seek(-4 * vdif.FRAME_SIZE, 1)
# File summary
print("Filename: %s" % filename)
print("Date of First Frame: %s" % str(beginDate))
print("Beams: %i" % beams)
print("Tune/Pols: %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Bit Depth: %i" % junkFrame.header.bits_per_sample)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" %
(central_freq1, central_freq2))
print(
"Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * vdif.DATA_LENGTH / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Integration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average, nFramesAvg, nFramesAvg // beampols))
print("Duration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average * nChunks, nFrames, nFrames // beampols))
print("Chunks: %i" % nChunks)
print(" ")
# Get the clip levels
clip1 = args.clip_level
clip2 = args.clip_level
# Make the pseudo-antennas for Stokes calculation
antennas = []
for i in xrange(4):
if i // 2 == 0:
newAnt = stations.Antenna(1)
else:
newAnt = stations.Antenna(2)
if i % 2 == 0:
newAnt.pol = 0
else:
newAnt.pol = 1
antennas.append(newAnt)
# Setup the output file
outname = os.path.split(filename)[1]
outname = os.path.splitext(outname)[0]
outname = '%s-waterfall.hdf5' % outname
if os.path.exists(outname):
if not args.force:
yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
raise RuntimeError("Output file '%s' already exists" % outname)
f = hdfData.createNewFile(outname)
# Look at the metadata and come up with a list of observations. If
# there are no metadata, create a single "observation" that covers the
# whole file.
obsList = {}
obsList[1] = (datetime.utcfromtimestamp(t1),
datetime(2222, 12, 31, 23, 59, 59), args.duration, srate)
hdfData.fillMinimum(f, 1, beam, srate)
if (not args.stokes):
data_products = ['XX', 'YY']
else:
data_products = ['I', 'Q', 'U', 'V']
for o in sorted(obsList.keys()):
for t in (1, 2):
hdfData.createDataSets(
f, o, t,
numpy.arange(LFFT -
1 if float(fxc.__version__) < 0.8 else LFFT,
dtype=numpy.float32),
int(round(obsList[o][2] / args.average)), data_products)
f.attrs['FileGenerator'] = 'hdfWaterfall.py'
f.attrs['InputData'] = os.path.basename(filename)
# Create the various HDF group holders
ds = {}
for o in sorted(obsList.keys()):
obs = hdfData.getObservationSet(f, o)
ds['obs%i' % o] = obs
ds['obs%i-time' % o] = obs.create_dataset(
'time', (int(round(obsList[o][2] / args.average)), ), 'f8')
for t in (1, 2):
ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq')
for p in data_products:
ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p)
ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set(
f, o, t, 'Saturation')
# Load in the correct analysis function
if (not args.stokes):
processDataBatch = processDataBatchLinear
else:
processDataBatch = processDataBatchStokes
# Go!
for o in sorted(obsList.keys()):
try:
processDataBatch(fh,
header,
antennas,
obsList[o][0],
obsList[o][2],
obsList[o][3],
args,
ds,
obsID=o,
clip1=clip1,
clip2=clip2)
except RuntimeError as e:
print("Observation #%i: %s, abandoning this observation" %
(o, str(e)))
# Save the output to a HDF5 file
f.close()