def test_os(self):
"""Test the observation specification utilities."""
obs1 = metabundle.get_observation_spec(mdbFile)
obs2 = metabundle.get_observation_spec(mdbFile, obs_id=1)
# Check if the right observation is returned
self.assertEqual(obs1[0], obs2)
# Check the mode
self.assertEqual(obs2['Mode'], 1)
# Check the time
self.assertEqual(obs2['MJD'], 56742)
self.assertEqual(obs2['MPM'], 4919000)
def test_os(self):
"""Test the observation specification utilities."""
obs1 = metabundle.get_observation_spec(mdbFileOld1)
obs2 = metabundle.get_observation_spec(mdbFileOld1, obs_id=1)
# Check if the right observation is returned
self.assertEqual(obs1[0], obs2)
# Check the mode
self.assertEqual(obs2['mode'], 1)
# Check the time
self.assertEqual(obs2['mjd'], 56492)
self.assertEqual(obs2['mpm'], 69000000)
def test_ss(self):
"""Test the session specification utilties."""
ses = metabundle.get_session_spec(mdbFile)
obs = metabundle.get_observation_spec(mdbFile)
# Check session start time
self.assertEqual(ses['MJD'], 56742)
self.assertEqual(ses['MPM'], 4914000)
# Check the duration
self.assertEqual(ses['Dur'], obs[0]['Dur'] + 10000)
# Check the number of observations
self.assertEqual(ses['nObs'], len(obs))
def test_ss(self):
"""Test the session specification utilties."""
ses = metabundle.get_session_spec(mdbFileOld1)
obs = metabundle.get_observation_spec(mdbFileOld1)
# Check session start time
self.assertEqual(ses['mjd'], 56492)
self.assertEqual(ses['mpm'], 68995000)
# Check the duration
self.assertEqual(ses['dur'], obs[0]['dur'] + 10000)
# Check the number of observations
self.assertEqual(ses['nobs'], len(obs))
def main(args):
# Get the site and observer
site = stations.lwa1
observer = site.get_observer()
# Filenames in an easier format
inputTGZ = args.filename
# Parse the input file and get the dates of the observations. Be default
# this is for LWA1 but we switch over to LWA-SV if an error occurs.
try:
# LWA1
project = metabundle.get_sdf(inputTGZ)
obsImpl = metabundle.get_observation_spec(inputTGZ)
fileInfo = metabundle.get_session_metadata(inputTGZ)
aspConfigB = metabundle.get_asp_configuration_summary(
inputTGZ, which='Beginning')
aspConfigE = metabundle.get_asp_configuration_summary(inputTGZ,
which='End')
except:
# LWA-SV
## Site changes
site = stations.lwasv
observer = site.get_observer()
## Try again
project = metabundleADP.get_sdf(inputTGZ)
obsImpl = metabundleADP.get_observation_spec(inputTGZ)
fileInfo = metabundleADP.get_session_metadata(inputTGZ)
aspConfigB = metabundleADP.get_asp_configuration_summary(
inputTGZ, which='Beginning')
aspConfigE = metabundleADP.get_asp_configuration_summary(inputTGZ,
which='End')
nObs = len(project.sessions[0].observations)
tStart = [
None,
] * nObs
for i in range(nObs):
tStart[i] = utcjd_to_unix(project.sessions[0].observations[i].mjd +
MJD_OFFSET)
tStart[i] += project.sessions[0].observations[i].mpm / 1000.0
tStart[i] = datetime.utcfromtimestamp(tStart[i])
tStart[i] = _UTC.localize(tStart[i])
# Get the LST at the start
observer.date = (min(tStart)).strftime('%Y/%m/%d %H:%M:%S')
lst = observer.sidereal_time()
# Report on the file
print("Filename: %s" % inputTGZ)
print(" Project ID: %s" % project.id)
print(" Session ID: %i" % project.sessions[0].id)
print(" Observations appear to start at %s" %
(min(tStart)).strftime(_FORMAT_STRING))
print(" -> LST at %s for this date/time is %s" % (site.name, lst))
lastDur = project.sessions[0].observations[nObs - 1].dur
lastDur = timedelta(seconds=int(lastDur / 1000),
microseconds=(lastDur * 1000) % 1000000)
sessionDur = max(tStart) - min(tStart) + lastDur
print(" ")
print(" Total Session Duration: %s" % sessionDur)
print(" -> First observation starts at %s" %
min(tStart).strftime(_FORMAT_STRING))
print(" -> Last observation ends at %s" %
(max(tStart) + lastDur).strftime(_FORMAT_STRING))
if project.sessions[0].observations[0].mode not in ('TBW', 'TBN'):
drspec = 'No'
if project.sessions[0].spcSetup[0] != 0 and project.sessions[
0].spcSetup[1] != 0:
drspec = 'Yes'
drxBeam = project.sessions[0].drx_beam
if drxBeam < 1:
drxBeam = "MCS decides"
else:
drxBeam = "%i" % drxBeam
print(" DRX Beam: %s" % drxBeam)
print(" DR Spectrometer used? %s" % drspec)
if drspec == 'Yes':
print(" -> %i channels, %i windows/integration" %
tuple(project.sessions[0].spcSetup))
else:
tbnCount = 0
tbwCount = 0
for obs in project.sessions[0].observations:
if obs.mode == 'TBW':
tbwCount += 1
else:
tbnCount += 1
if tbwCount > 0 and tbnCount == 0:
print(" Transient Buffer Mode: TBW")
elif tbwCount == 0 and tbnCount > 0:
print(" Transient Buffer Mode: TBN")
else:
print(" Transient Buffer Mode: both TBW and TBN")
print(" ")
print("File Information:")
for obsID in fileInfo.keys():
print(" Obs. #%i: %s" % (obsID, fileInfo[obsID]['tag']))
print(" ")
print("ASP Configuration:")
print(' Beginning')
for k, v in aspConfigB.items():
print(' %s: %i' % (k, v))
print(' End')
for k, v in aspConfigE.items():
print(' %s: %i' % (k, v))
print(" ")
print(" Number of observations: %i" % nObs)
print(" Observation Detail:")
for i in range(nObs):
currDur = project.sessions[0].observations[i].dur
currDur = timedelta(seconds=int(currDur / 1000),
microseconds=(currDur * 1000) % 1000000)
print(" Observation #%i" % (i + 1, ))
currObs = None
for j in range(len(obsImpl)):
if obsImpl[j]['obsID'] == i + 1:
currObs = obsImpl[j]
break
## Basic setup
print(" Target: %s" % project.sessions[0].observations[i].target)
print(" Mode: %s" % project.sessions[0].observations[i].mode)
print(" Start:")
print(" MJD: %i" % project.sessions[0].observations[i].mjd)
print(" MPM: %i" % project.sessions[0].observations[i].mpm)
print(" -> %s" % get_observation_start_stop(
project.sessions[0].observations[i])[0].strftime(_FORMAT_STRING))
print(" Duration: %s" % currDur)
## DP setup
if project.sessions[0].observations[i].mode not in ('TBW', ):
print(" Tuning 1: %.3f MHz" %
(project.sessions[0].observations[i].frequency1 / 1e6, ))
if project.sessions[0].observations[i].mode not in ('TBW', 'TBN'):
print(" Tuning 2: %.3f MHz" %
(project.sessions[0].observations[i].frequency2 / 1e6, ))
if project.sessions[0].observations[i].mode not in ('TBW', ):
print(" Filter code: %i" %
project.sessions[0].observations[i].filter)
if currObs is not None:
if project.sessions[0].observations[i].mode not in ('TBW', ):
if project.sessions[0].observations[i].mode == 'TBN':
print(" Gain setting: %i" % currObs['tbnGain'])
else:
print(" Gain setting: %i" % currObs['drxGain'])
else:
print(
" WARNING: observation specification not found for this observation"
)
## Comments/notes
print(" Observer Comments: %s" %
project.sessions[0].observations[i].comments)
def main(args):
# Get the file names
meta = args.metadata
data = args.filename
# Get all observations and their start times
try:
## LWA-1
sdf = metabundle.get_sdf(meta)
ses = metabundle.get_session_spec(meta)
obs = metabundle.get_observation_spec(meta)
except:
## LWA-SV
### Try again
sdf = metabundleADP.get_sdf(meta)
ses = metabundleADP.get_session_spec(meta)
obs = metabundleADP.get_observation_spec(meta)
obs.sort(_obs_comp)
tStart = []
oDetails = []
for i,o in enumerate(obs):
tStart.append( mjdmpm_to_datetime(o['mjd'], o['mpm']) )
oDetails.append( {'m': o['mode'], 'd': o['dur'] / 1000.0, 'f': o['bw'],
'p': o['project_id'], 's': o['session_id'], 'o': o['obs_id'],
't': sdf.sessions[0].observations[o['obs_id']-1].target} )
print("Observation #%i" % (o['obs_id']))
print(" Start: %i, %i -> %s" % (o['mjd'], o['mpm'], tStart[-1]))
print(" Mode: %s" % mode_to_string(o['mode']))
print(" BW: %i" % o['bw'])
print(" Target: %s" % sdf.sessions[0].observations[o['obs_id']-1].target)
print(" ")
# Figure out where in the file the various bits are.
fh = open(data, 'rb')
lf = drx.read_frame(fh)
beam, j, k = lf.id
if beam != obs[0]['drx_beam']:
print('ERROR: Beam mis-match, metadata is for #%i, file is for #%i' % (obs[0]['drx_beam'], beam))
sys.exit()
firstFrame = lf.time.datetime
if abs(firstFrame - min(tStart)) > timedelta(seconds=30):
print('ERROR: Time mis-match, metadata is for %s, file is for %s' % (min(tStart), firstFrame))
sys.exit()
fh.seek(0)
for i in range(len(tStart)):
eof = False
## Get observation properties
oStart = tStart[i]
oMode = mode_to_string(oDetails[i]['m'])
oDur = oDetails[i]['d']
oBW = oDetails[i]['f']
print("Seeking %s observation of %.3f seconds at %s" % (oMode, oDur, oStart))
## Get the correct reader to use
if oMode == 'TBW':
reader = tbw
bwKey = None
bwMult = 520.0 / 400
fCount = 400
elif oMode == 'TBN':
reader = tbn
bwKey = tbn.FILTER_CODES
bwMult = 520.0 / 512
fCount = 512
else:
reader = drx
bwKey = drx.FILTER_CODES
bwMult = 4.0 / 4096
fCount = 4096
## Jump ahead to where the next frame should be, if needed
if i != 0:
pDur = oDetails[i-1]['d']
pBW = oDetails[i-1]['f']
nFramesSkip = int(pDur*bwKey[pBW]*bwMult)
fh.seek(nFramesSkip*reader.FRAME_SIZE, 1)
if fh.tell() >= os.path.getsize(data):
fh.seek(-10*reader.FRAME_SIZE, 2)
## Figure out where we are and make sure we line up on a frame
## NOTE: This should never be needed
fail = True
while fail:
try:
frame = reader.read_frame(fh)
fail = False
except errors.SyncError:
fh.seek(1, 1)
except errors.EOFError:
break
fh.seek(-reader.FRAME_SIZE, 1)
## Go in search of the start of the observation
if frame.time.datetime < oStart:
### We aren't at the beginning yet, seek fowards
print("-> At byte %i, time is %s < %s" % (fh.tell(), frame.time.datetime, oStart))
while frame.time.datetime < oStart:
try:
frame = reader.read_frame(fh)
except errors.SyncError:
fh.seek(1, 1)
except errors.EOFError:
break
#print(frame.time.datetime, oStart)
elif frame.time.datetime > oStart:
### We've gone too far, seek backwards
print("-> At byte %i, time is %s > %s" % (fh.tell(), frame.time.datetime, oStart))
while frame.time.datetime > oStart:
if fh.tell() == 0:
break
fh.seek(-2*reader.FRAME_SIZE, 1)
try:
frame = reader.read_frame(fh)
except errors.SyncError:
fh.seek(-1, 1)
except errors.EOFError:
break
#print(frame.time.datetime, oStart)
else:
### We're there already
print("-> At byte %i, time is %s = %s" % (fh.tell(), frame.time.datetime, oStart))
## Jump back exactly one frame so that the filehandle is in a position
## to read the first frame that is part of the observation
try:
frame = reader.read_frame(fh)
print("-> At byte %i, time is %s = %s" % (fh.tell(), frame.time.datetime, oStart))
fh.seek(-reader.FRAME_SIZE, 1)
except errors.EOFError:
pass
## Update the bytes ranges
if fh.tell() < os.path.getsize(data):
oDetails[i]['b'] = fh.tell()
oDetails[i]['e'] = -1
else:
oDetails[i]['b'] = -1
oDetails[i]['e'] = -1
if i != 0:
oDetails[i-1]['e'] = fh.tell()
## Progress report
if oDetails[i]['b'] >= 0:
print('-> Obs.', oDetails[i]['o'], 'starts at byte', oDetails[i]['b'])
else:
print('-> Obs.', oDetails[i]['o'], 'starts after the end of the file')
print(" ")
# Report
for i in range(len(tStart)):
if oDetails[i]['b'] < 0:
print("%s, Session %i, Observation %i: not found" % (oDetails[i]['p'], oDetails[i]['s'], oDetails[i]['o']))
else:
print("%s, Session %i, Observation %i: %i to %i (%i bytes)" % (oDetails[i]['p'], oDetails[i]['s'], oDetails[i]['o'], oDetails[i]['b'], oDetails[i]['e'], (oDetails[i]['e'] - oDetails[i]['b'])))
print(" ")
# Split
if not args.list:
for i in range(len(tStart)):
if oDetails[i]['b'] < 0:
continue
## Report
print("Working on Observation %i" % (i+1,))
## Create the output name
if args.source:
outname = '%s_%i_%s.dat' % (oDetails[i]['p'], oDetails[i]['s'], oDetails[i]['t'].replace(' ', '').replace('/','').replace('&','and'))
else:
outname = '%s_%i_%i.dat' % (oDetails[i]['p'], oDetails[i]['s'], oDetails[i]['o'])
oMode = mode_to_string(oDetails[i]['m'])
## Get the correct reader to use
if oMode == 'TBW':
reader = tbw
elif oMode == 'TBN':
reader = tbn
else:
reader = drx
## Get the number of frames
if oDetails[i]['e'] > 0:
nFramesRead = (oDetails[i]['e'] - oDetails[i]['b']) // reader.FRAME_SIZE
else:
nFramesRead = (os.path.getsize(data) - oDetails[i]['b']) // reader.FRAME_SIZE
## Split
if os.path.exists(outname):
if not args.force:
yn = input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
print("WARNING: output file '%s' already exists, skipping" % outname)
continue
fh.seek(oDetails[i]['b'])
t0 = time.time()
oh = open(outname, 'wb')
for sl in [2**i for i in range(17)[::-1]]:
while nFramesRead >= sl:
temp = fh.read(sl*reader.FRAME_SIZE)
oh.write(temp)
nFramesRead -= sl
oh.close()
t1 = time.time()
print(" Copied %i bytes in %.3f s (%.3f MB/s)" % (os.path.getsize(outname), t1-t0, os.path.getsize(outname)/1024.0**2/(t1-t0)))
print(" ")