def test_drspec_metadata(self): """Test finding out the DR spectrometer metadata.""" fh = open(drspecFile, 'rb') cFrame = drspec.read_frame(fh) fh.seek(0) # Beam self.assertEqual(cFrame.id, 1) # Sample rate self.assertAlmostEqual(cFrame.sample_rate, 19.6e6, 1) self.assertAlmostEqual(cFrame.sample_rate, drspec.get_sample_rate(fh), 1) # Filter code self.assertEqual(cFrame.filter_code, 7) self.assertEqual(cFrame.filter_code, drspec.get_sample_rate(fh, filter_code=True)) # FFT windows per integration self.assertEqual(cFrame.ffts_per_integration, 6144) self.assertEqual(cFrame.ffts_per_integration, drspec.get_ffts_per_integration(fh)) # Transform size self.assertEqual(cFrame.transform_size, 1024) self.assertEqual(cFrame.transform_size, drspec.get_transform_size(fh)) # Integration time self.assertAlmostEqual(cFrame.integration_time, 0.32099265, 8) self.assertAlmostEqual(cFrame.integration_time, drspec.get_integration_time(fh), 8) fh.close()
def test_drspec_sort(self): """Test sorting DR spectrometer frames by time tags.""" fh = open(drspecFile, 'rb') # Frames 1 through 7 frames = [] for i in range(1, 8): frames.append(drspec.read_frame(fh)) frames.sort() frames = frames[::-1] for i in range(1, len(frames)): self.assertTrue(frames[i - 1] >= frames[i]) fh.close()
def test_drspec_read(self): """Test reading in a frame from a DR spectrometer file.""" fh = open(drspecFile, 'rb') # First frame is really DR spectrometer and stores the IDs frame1 = drspec.read_frame(fh) beam = frame1.id self.assertEqual(beam, 1) # Second frame frame2 = drspec.read_frame(fh) beam = frame2.id self.assertEqual(beam, 1) fh.close() # The special "data" attribute data = frame2.payload.data self.assertEqual(len(data.shape), 1) self.assertEqual(data.shape[0], 1024) for attr in ('XX0', 'XX1', 'YY0', 'YY1'): d0 = getattr(frame2.payload, attr, None) d1 = data[attr] for i in range(1024): self.assertAlmostEqual(d0[i], d1[i], 6)
def test_drspec_comps(self): """Test the DR spectrometer frame comparison operators (>, <, etc.) for time tags.""" fh = open(drspecFile, 'rb') # Frames 1 through 7 frames = [] for i in range(1, 8): frames.append(drspec.read_frame(fh)) fh.close() self.assertTrue(0 < frames[0]) self.assertFalse(0 > frames[0]) self.assertTrue(frames[-1] >= frames[0]) self.assertFalse(frames[-1] <= frames[0]) self.assertTrue(frames[0] == frames[0]) self.assertFalse(frames[0] == frames[-1]) self.assertFalse(frames[0] != frames[0])
def test_drspec_errors(self): """Test reading in all frames from a truncated DR spectrometer file.""" fh = open(drspecFile, 'rb') # Frames 1 through 8 for i in range(1, 8): frame = drspec.read_frame(fh) # Last frame should be an error (errors.EOFError) self.assertRaises(errors.EOFError, drspec.read_frame, fh) fh.close() # If we offset in the file by 1 byte, we should be a # sync error (errors.SyncError). fh = open(drspecFile, 'rb') fh.seek(1) self.assertRaises(errors.SyncError, drspec.read_frame, fh) fh.close()
def test_drspec_math(self): """Test mathematical operations on DR spectrometer frame data via frames.""" fh = open(drspecFile, 'rb') # Frames 1 through 7 frames = [] for i in range(1, 8): frames.append(drspec.read_frame(fh)) fh.close() npts = frames[0].payload.XX0.size # Multiplication frameT = frames[0] * 2.0 for i in range(npts): self.assertAlmostEqual(frameT.payload.XX0[i], 2 * frames[0].payload.XX0[i], 2) frameT *= 2.0 for i in range(npts): self.assertAlmostEqual(frameT.payload.XX1[i], 4 * frames[0].payload.XX1[i], 2) frameT = frames[0] * frames[1] for i in range(npts): self.assertAlmostEqual( frameT.payload.YY0[i], frames[0].payload.YY0[i] * frames[1].payload.YY0[i], 2) # Addition frameA = frames[0] + 2.0 for i in range(npts): self.assertAlmostEqual(frameA.payload.XX0[i], 2 + frames[0].payload.XX0[i], 2) frameA += 2.0 for i in range(npts): self.assertAlmostEqual(frameA.payload.XX1[i], 4 + frames[0].payload.XX1[i], 2) frameA = frames[0] + frames[1] for i in range(npts): self.assertAlmostEqual( frameA.payload.YY0[i], frames[0].payload.YY0[i] + frames[1].payload.YY0[i], 2)
def test_drspec_math(self): """Test mathematical operations on DR spectrometer frame data via frames.""" fh = open(drspecFile, 'rb') # Frames 1 through 7 frames = [] for i in range(1, 8): frames.append(drspec.read_frame(fh)) fh.close() npts = frames[0].payload.XX0.size # Multiplication frameT = frames[0] * 2.0 numpy.testing.assert_allclose(frameT.payload.XX0, 2 * frames[0].payload.XX0, atol=1e-6) frameT *= 2.0 numpy.testing.assert_allclose(frameT.payload.XX1, 4 * frames[0].payload.XX1, atol=1e-6) frameT = frames[0] * frames[1] numpy.testing.assert_allclose(frameT.payload.YY0, frames[0].payload.YY0 * frames[1].payload.YY0, atol=1e-6) # Addition frameA = frames[0] + 2.0 numpy.testing.assert_allclose(frameA.payload.XX0, 2 + frames[0].payload.XX0, atol=1e-6) frameA += 2.0 numpy.testing.assert_allclose(frameA.payload.XX0, 4 + frames[0].payload.XX0, atol=1e-6) frameA = frames[0] + frames[1] numpy.testing.assert_allclose(frameA.payload.YY0, frames[0].payload.YY0 + frames[1].payload.YY0, atol=1e-6)
def main(args): # Set the site site = None if args.lwa1: site = 'lwa1' elif args.lwasv: site = 'lwasv' # Open the file and file good data (not raw DRX data) fh = open(args.filename, 'rb') try: for i in xrange(5): junkFrame = drx.read_frame(fh) raise RuntimeError("ERROR: '%s' appears to be a raw DRX file, not a DR spectrometer file" % args.filename) except errors.SyncError: fh.seek(0) # Interrogate the file to figure out what frames sizes to expect, now many # frames there are, and what the transform length is FRAME_SIZE = drspec.get_frame_size(fh) nFrames = os.path.getsize(args.filename) // FRAME_SIZE nChunks = nFrames LFFT = drspec.get_transform_size(fh) # Read in the first frame to figure out the DP information junkFrame = drspec.read_frame(fh) fh.seek(-FRAME_SIZE, 1) srate = junkFrame.sample_rate t0 = junkFrame.time tInt = junkFrame.header.nints*LFFT/srate # Offset in frames for beampols beam/tuning/pol. sets offset = int(round(args.skip / tInt)) fh.seek(offset*FRAME_SIZE, 1) # Iterate on the offsets until we reach the right point in the file. This # is needed to deal with files that start with only one tuning and/or a # different sample rate. while True: ## Figure out where in the file we are and what the current tuning/sample ## rate is junkFrame = drspec.read_frame(fh) srate = junkFrame.sample_rate t1 = junkFrame.time tInt = junkFrame.header.nints*LFFT/srate fh.seek(-FRAME_SIZE, 1) ## See how far off the current frame is from the target tDiff = t1 - (t0 + args.skip) ## Half that to come up with a new seek parameter tCorr = -tDiff / 2.0 cOffset = int(round(tCorr / tInt)) offset += cOffset ## If the offset is zero, we are done. Otherwise, apply the offset ## and check the location in the file again/ if cOffset is 0: break fh.seek(cOffset*FRAME_SIZE, 1) # Update the offset actually used args.skip = t1 - t0 nChunks = (os.path.getsize(args.filename) - fh.tell()) // FRAME_SIZE # Update the file contents beam = junkFrame.id central_freq1, central_freq2 = junkFrame.central_freq srate = junkFrame.sample_rate data_products = junkFrame.data_products t0 = junkFrame.time tInt = junkFrame.header.nints*LFFT/srate beginDate = junkFrame.time.datetime # Report print("Filename: %s" % args.filename) if args.metadata is not None: print("Metadata: %s" % args.metadata) elif args.sdf is not None: print("SDF: %s" % args.sdf) print("Date of First Frame: %s" % beginDate) print("Beam: %i" % beam) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2)) print("Data Products: %s" % ','.join(data_products)) print("Frames: %i (%.3f s)" % (nFrames, nFrames*tInt)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Transform Length: %i" % LFFT) print("Integration: %.3f s" % tInt) # Setup the output file outname = os.path.split(args.filename)[1] outname = os.path.splitext(outname)[0] outname = '%s-waterfall.hdf5' % outname if os.path.exists(outname): if not args.force: yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname) else: yn = 'y' if yn not in ('n', 'N'): os.unlink(outname) else: raise RuntimeError("Output file '%s' already exists" % outname) f = hdfData.create_new_file(outname) obsList = {} if args.metadata is not None: try: project = metabundle.get_sdf(args.metadata) except Exception as e: if adpReady: project = metabundleADP.get_sdf(args.metadata) else: raise e sdfBeam = project.sessions[0].drx_beam spcSetup = project.sessions[0].spcSetup if sdfBeam != beam: raise RuntimeError("Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam)) for i,obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsChunks = int(numpy.ceil(obs.dur/1000.0 * drx.FILTER_CODES[obs.filter] / (spcSetup[0]*spcSetup[1]))) obsList[i+1] = (sdfStart, sdfStop, obsChunks) hdfData.fill_from_metabundle(f, args.metadata) elif args.sdf is not None: try: project = sdf.parse_sdf(args.sdf) except Exception as e: if adpReady: project = sdfADP.parse_sdf(args.sdf) else: raise e sdfBeam = project.sessions[0].drx_beam spcSetup = project.sessions[0].spcSetup if sdfBeam != beam: raise RuntimeError("Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam)) for i,obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsChunks = int(numpy.ceil(obs.dur/1000.0 * drx.FILTER_CODES[obs.filter] / (spcSetup[0]*spcSetup[1]))) obsList[i+1] = (sdfStart, sdfStop, obsChunks) hdfData.fill_from_sdf(f, args.sdf, station=site) else: obsList[1] = (beginDate, datetime(2222,12,31,23,59,59), nChunks) hdfData.fill_minimum(f, 1, beam, srate, station=site) data_products = junkFrame.data_products for o in sorted(obsList.keys()): for t in (1,2): hdfData.create_observation_set(f, o, t, numpy.arange(LFFT, dtype=numpy.float64), obsList[o][2], data_products) f.attrs['FileGenerator'] = 'drspec2hdf.py' f.attrs['InputData'] = os.path.basename(args.filename) # Create the various HDF group holders ds = {} for o in sorted(obsList.keys()): obs = hdfData.get_observation_set(f, o) ds['obs%i' % o] = obs ds['obs%i-time' % o] = hdfData.get_time(f, o) for t in (1,2): ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq') for p in data_products: ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p) ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'Saturation') # Loop over DR spectrometer frames to fill in the HDF5 file pbar = progress.ProgressBar(max=nChunks) o = 1 j = 0 firstPass = True for i in xrange(nChunks): frame = drspec.read_frame(fh) cTime = frame.time.datetime if cTime > obsList[o][1]: # Increment to the next observation o += 1 # If we have reached the end, exit... try: obsList[o] firstPass = True except KeyError: sys.stdout.write('%s\r' % (' '*pbar.span)) sys.stdout.flush() print("End of observing block according to SDF, exiting") break if cTime < obsList[o][0]: # Skip over data that occurs before the start of the observation continue try: if frame.time > oTime + 1.001*tInt: print('Warning: Time tag error at frame %i; %.3f > %.3f + %.3f' % (i, frame.time, oTime, tInt)) except NameError: pass oTime = frame.time if firstPass: # Otherwise, continue on... central_freq1, central_freq2 = frame.central_freq srate = frame.sample_rate tInt = frame.header.nints*LFFT/srate freq = numpy.fft.fftshift( numpy.fft.fftfreq(LFFT, d=1.0/srate) ) freq = freq.astype(numpy.float64) sys.stdout.write('%s\r' % (' '*pbar.span)) sys.stdout.flush() print("Switching to Obs. #%i" % o) print("-> Tunings: %.1f Hz, %.1f Hz" % (central_freq1, central_freq2)) print("-> Sample Rate: %.1f Hz" % srate) print("-> Integration Time: %.3f s" % tInt) sys.stdout.write(pbar.show()+'\r') sys.stdout.flush() j = 0 ds['obs%i-freq1' % o][:] = freq + central_freq1 ds['obs%i-freq2' % o][:] = freq + central_freq2 obs = ds['obs%i' % o] obs.attrs['tInt'] = tInt obs.attrs['tInt_Units'] = 's' obs.attrs['LFFT'] = LFFT obs.attrs['nChan'] = LFFT obs.attrs['RBW'] = freq[1]-freq[0] obs.attrs['RBW_Units'] = 'Hz' firstPass = False # Load the data from the spectrometer frame into the HDF5 group ds['obs%i-time' % o][j] = (frame.time[0], frame.time[1]) ds['obs%i-Saturation1' % o][j,:] = frame.payload.saturations[0:2] ds['obs%i-Saturation2' % o][j,:] = frame.payload.saturations[2:4] for t in (1,2): for p in data_products: ds['obs%i-%s%i' % (o, p, t)][j,:] = getattr(frame.payload, "%s%i" % (p, t-1), None) j += 1 # Update the progress bar pbar.inc() if i % 10 == 0: sys.stdout.write(pbar.show()+'\r') sys.stdout.flush() sys.stdout.write(pbar.show()+'\n') sys.stdout.flush() # Done fh.close() # Save the output to a HDF5 file f.close()
def main(args): fh = open(args.filename, "rb") try: for i in xrange(5): junkFrame = drx.read_frame(fh) raise RuntimeError( "ERROR: '%s' appears to be a raw DRX file, not a DR spectrometer file" % args.filename) except errors.SyncError: fh.seek(0) # Interrogate the file to figure out what frames sizes to expect, now many # frames there are, and what the transform length is FRAME_SIZE = drspec.get_frame_size(fh) nFrames = os.path.getsize(args.filename) / FRAME_SIZE nChunks = nFrames LFFT = drspec.get_transform_size(fh) # Read in the first frame to figure out the DP information junkFrame = drspec.read_frame(fh) fh.seek(-FRAME_SIZE, 1) srate = junkFrame.sample_rate t0 = junkFrame.time tInt = junkFrame.header.nints * LFFT / srate # Update the file contents beam = junkFrame.id central_freq1, central_freq2 = junkFrame.central_freq srate = junkFrame.sample_rate data_products = junkFrame.data_products t0 = junkFrame.time tInt = junkFrame.header.nints * LFFT / srate beginDate = junkFrame.time.datetime # Report print("Filename: %s" % args.filename) print("Date of First Frame: %s" % beginDate) print("Beam: %i" % beam) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2)) print("Data Products: %s" % ','.join(data_products)) print("Frames: %i (%.3f s)" % (nFrames, nFrames * tInt)) print("---") print("Transform Length: %i" % LFFT) print("Integration: %.3f s" % tInt) # Convert chunk length to total frame count chunkLength = int(args.length / tInt) # Convert chunk skip to total frame count chunkSkip = int(args.skip / tInt) # Output arrays clipFraction = [] meanPower = [] # Go! i = 1 done = False print(" |%sClipping%s |%sPower %s |" % (" " * (8 * len(data_products) - 4), " " * (8 * len(data_products) - 4), " " * (6 * len(data_products) - 3), " " * (6 * len(data_products) - 3))) out = " | 1X 1Y 2X 2Y |" for t in (1, 2): for dp in data_products: out += "%6s" % ("%i%s" % (t, dp)) out += " |" print(out) print("-" * len(out)) while True: count = {0: 0, 1: 0, 2: 0, 3: 0} sats = numpy.empty((4, chunkLength), dtype=numpy.float32) data = numpy.empty((2 * len(data_products), chunkLength * LFFT), dtype=numpy.float32) for j in xrange(chunkLength): # Read in the next frame and anticipate any problems that could occur try: cFrame = frame = drspec.read_frame(fh) except errors.EOFError: done = True break except errors.SyncError: continue for t in (1, 2): for p, dp in enumerate(data_products): l = len(data_products) * (t - 1) + p data[l, j * LFFT:(j + 1) * LFFT] = getattr( cFrame.payload, '%s%i' % (dp, t - 1)) sats[:, j] = numpy.array(cFrame.payload.saturations) / (tInt * srate) if done: break else: clipFraction.append(sats.mean(axis=1)) meanPower.append(data.mean(axis=1)) clip = clipFraction[-1] power = meanPower[-1] out = "%2i | %6.2f%% %6.2f%% %6.2f%% %6.2f%% |" % ( i, clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0, clip[3] * 100.0) for t in (1, 2): for p in xrange(len(data_products)): out += " %5.2f" % (power[len(data_products) * (t - 1) + p], ) out += " |" print(out) i += 1 fh.seek(FRAME_SIZE * chunkSkip, 1) clipFraction = numpy.array(clipFraction) meanPower = numpy.array(meanPower) clip = clipFraction.mean(axis=0) power = meanPower.mean(axis=0) print("-" * len(out)) out = "%2s | %6.2f%% %6.2f%% %6.2f%% %6.2f%% |" % ( 'M', clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0, clip[3] * 100.0) for t in (1, 2): for p in xrange(len(data_products)): out += " %5.2f" % (power[len(data_products) * (t - 1) + p], ) out += " |" print(out)
def main(args): # Length of the FFT and the window to use LFFT = args.fft_length if args.bartlett: window = numpy.bartlett elif args.blackman: window = numpy.blackman elif args.hanning: window = numpy.hanning else: window = fxc.null_window args.window = window # Open the file and find good data (not spectrometer data) fh = open(args.filename, "rb") try: for i in xrange(5): junkFrame = drspec.read_frame(fh) raise RuntimeError( "ERROR: '%s' appears to be a DR spectrometer file, not a raw DRX file" % args.filename) except errors.SyncError: fh.seek(0) # Good, we seem to have a real DRX file, switch over to the LDP interface fh.close() idf = LWA1DataFile(args.filename, ignore_timetag_errors=args.ignore_time_errors) # Metadata nFramesFile = idf.get_info('nframe') beam = idf.get_info('beam') srate = idf.get_info('sample_rate') beampols = idf.get_info('nbeampol') beams = max([1, beampols // 4]) # Number of frames to integrate over nFramesAvg = int(args.average * srate / 4096) * beampols nFramesAvg = int(1.0 * (nFramesAvg // beampols) * 4096 / float(LFFT)) * LFFT / 4096 * beampols args.average = 1.0 * (nFramesAvg // beampols) * 4096 / srate maxFrames = nFramesAvg # Offset into the file, if needed offset = idf.offset(args.skip) # Number of remaining chunks (and the correction to the number of # frames to read in). if args.metadata is not None: args.duration = 0 if args.duration == 0: args.duration = 1.0 * nFramesFile / beampols * 4096 / srate args.duration -= args.skip else: args.duration = int( round(args.duration * srate * beampols / 4096) / beampols * 4096 / srate) nChunks = int(round(args.duration / args.average)) if nChunks == 0: nChunks = 1 nFrames = nFramesAvg * nChunks # Date & Central Frequency t1 = idf.get_info('start_time') beginDate = t1.datetime central_freq1 = idf.get_info('freq1') central_freq2 = idf.get_info('freq2') # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Beams: %i" % beams) print("Tune/Pols: %i" % beampols) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2)) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Integration: %.3f s (%i frames; %i frames per beam/tune/pol)" % (args.average, nFramesAvg, nFramesAvg / beampols)) print("Duration: %.3f s (%i frames; %i frames per beam/tune/pol)" % (args.average * nChunks, nFrames, nFrames / beampols)) print("Chunks: %i" % nChunks) print(" ") # Estimate clip level (if needed) if args.estimate_clip_level: estimate = idf.estimate_levels(fh, sigma=5.0) clip1 = (estimate[0] + estimate[1]) / 2.0 clip2 = (estimate[2] + estimate[3]) / 2.0 else: clip1 = args.clip_level clip2 = args.clip_level # Make the pseudo-antennas for Stokes calculation antennas = [] for i in xrange(4): if i // 2 == 0: newAnt = stations.Antenna(1) else: newAnt = stations.Antenna(2) if i % 2 == 0: newAnt.pol = 0 else: newAnt.pol = 1 antennas.append(newAnt) # Setup the output file outname = os.path.split(args.filename)[1] outname = os.path.splitext(outname)[0] outname = '%s-waterfall.hdf5' % outname if os.path.exists(outname): if not args.force: yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname) else: yn = 'y' if yn not in ('n', 'N'): os.unlink(outname) else: raise RuntimeError("Output file '%s' already exists" % outname) f = hdfData.create_new_file(outname) # Look at the metadata and come up with a list of observations. If # there are no metadata, create a single "observation" that covers the # whole file. obsList = {} if args.metadata is not None: try: project = metabundle.get_sdf(args.metadata) except Exception as e: project = metabundleADP.get_sdf(args.metadata) sdfBeam = project.sessions[0].drx_beam spcSetup = project.sessions[0].spcSetup if sdfBeam != beam: raise RuntimeError( "Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam)) for i, obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsDur = obs.dur / 1000.0 obsSR = drx.FILTER_CODES[obs.filter] obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR) print("Observations:") for i in sorted(obsList.keys()): obs = obsList[i] print(" #%i: %s to %s (%.3f s) at %.3f MHz" % (i, obs[0], obs[1], obs[2], obs[3] / 1e6)) print(" ") hdfData.fill_from_metabundle(f, args.metadata) elif args.sdf is not None: try: project = sdf.parse_sdf(args.sdf) except Exception as e: project = sdfADP.parse_sdf(args.sdf) sdfBeam = project.sessions[0].drx_beam spcSetup = project.sessions[0].spcSetup if sdfBeam != beam: raise RuntimeError( "Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam)) for i, obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsDur = obs.dur / 1000.0 obsSR = drx.FILTER_CODES[obs.filter] obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR) site = 'lwa1' if args.lwasv: site = 'lwasv' hdfData.fill_from_sdf(f, args.sdf, station=site) else: obsList[1] = (datetime.utcfromtimestamp(t1), datetime(2222, 12, 31, 23, 59, 59), args.duration, srate) site = 'lwa1' if args.lwasv: site = 'lwasv' hdfData.fill_minimum(f, 1, beam, srate, station=site) if (not args.stokes): data_products = ['XX', 'YY'] else: data_products = ['I', 'Q', 'U', 'V'] for o in sorted(obsList.keys()): for t in (1, 2): hdfData.create_observation_set( f, o, t, numpy.arange(LFFT, dtype=numpy.float64), int(round(obsList[o][2] / args.average)), data_products) f.attrs['FileGenerator'] = 'hdfWaterfall.py' f.attrs['InputData'] = os.path.basename(args.filename) # Create the various HDF group holders ds = {} for o in sorted(obsList.keys()): obs = hdfData.get_observation_set(f, o) ds['obs%i' % o] = obs ds['obs%i-time' % o] = hdfData.get_time(f, o) for t in (1, 2): ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq') for p in data_products: ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p) ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set( f, o, t, 'Saturation') # Load in the correct analysis function if (not args.stokes): processDataBatch = processDataBatchLinear else: processDataBatch = processDataBatchStokes # Go! for o in sorted(obsList.keys()): try: processDataBatch(idf, antennas, obsList[o][0], obsList[o][2], obsList[o][3], args, ds, obsID=o, clip1=clip1, clip2=clip2) except RuntimeError as e: print("Observation #%i: %s, abandoning this observation" % (o, str(e))) # Save the output to a HDF5 file f.close() # Close out the data file idf.close()
def main(args): skip = args.skip fh = open(args.filename, "rb") try: for i in xrange(5): junkFrame = drx.read_frame(fh) raise RuntimeError( "ERROR: '%s' appears to be a raw DRX file, not a DR spectrometer file" % args.filename) except errors.SyncError: fh.seek(0) # Interrogate the file to figure out what frames sizes to expect, now many # frames there are, and what the transform length is FRAME_SIZE = drspec.get_frame_size(fh) nFrames = os.path.getsize(args.filename) / FRAME_SIZE nChunks = nFrames LFFT = drspec.get_transform_size(fh) # Read in the first frame to figure out the DP information junkFrame = drspec.read_frame(fh) fh.seek(-FRAME_SIZE, 1) srate = junkFrame.sample_rate t0 = junkFrame.time tInt = junkFrame.header.nints * LFFT / srate # Offset in frames for beampols beam/tuning/pol. sets offset = int(round(skip / tInt)) fh.seek(offset * FRAME_SIZE, 1) # Iterate on the offsets until we reach the right point in the file. This # is needed to deal with files that start with only one tuning and/or a # different sample rate. while True: ## Figure out where in the file we are and what the current tuning/sample ## rate is junkFrame = drspec.read_frame(fh) srate = junkFrame.sample_rate t1 = junkFrame.time tInt = junkFrame.header.nints * LFFT / srate fh.seek(-FRAME_SIZE, 1) ## See how far off the current frame is from the target tDiff = t1 - (t0 + skip) ## Half that to come up with a new seek parameter tCorr = -tDiff / 2.0 cOffset = int(round(tCorr / tInt)) offset += cOffset ## If the offset is zero, we are done. Otherwise, apply the offset ## and check the location in the file again/ if cOffset is 0: break fh.seek(cOffset * FRAME_SIZE, 1) # Update the offset actually used skip = t1 - t0 nChunks = (os.path.getsize(args.filename) - fh.tell()) // FRAME_SIZE # Update the file contents beam = junkFrame.id central_freq1, central_freq2 = junkFrame.central_freq srate = junkFrame.sample_rate data_products = junkFrame.data_products t0 = junkFrame.time tInt = junkFrame.header.nints * LFFT / srate beginDate = junkFrame.time.datetime # Report print("Filename: %s" % args.filename) print("Date of First Frame: %s" % beginDate) print("Beam: %i" % beam) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2)) print("Data Products: %s" % ','.join(data_products)) print("Frames: %i (%.3f s)" % (nFrames, nFrames * tInt)) print("---") print("Transform Length: %i" % LFFT) print("Integration: %.3f s" % tInt) for i in xrange(nChunks): frame = drspec.read_frame(fh) cTime = frame.time if i % 1000 == 0: print("Frame %i: %s" % (i, datetime.utcfromtimestamp(cTime))) try: if cTime > oTime + 1.001 * tInt: print( 'Warning: Time tag error at frame %i; %.3f > %.3f + %.3f' % (i, cTime, oTime, tInt)) except NameError: pass oTime = frame.time cFreq1, cFreq2 = frame.central_freq try: if cFreq1 != oFreq1: print( 'Warning: Tuning 1 frequncy changed at frame %i; %.3f Hz != %.3f Hz' % (i, cFreq1, oFreq1)) if cFreq2 != oFreq2: print( 'Warning: Tuning 2 frequncy changed at frame %i; %.3f Hz != %.3f Hz' % (i, cFreq2, oFreq2)) except NameError: pass oFreq1, oFreq2 = frame.central_freq del frame if i % 100 == 0: gc.collect() # Done fh.close()