def test_tbf_read(self): """Test reading in a frame from a TBF file.""" fh = open(tbfFile, 'rb') # First frame is really TBF and stores the first channel frame1 = tbf.read_frame(fh) self.assertTrue(frame1.header.is_tbf) self.assertEqual(frame1.header.first_chan, 2348) # Second frame frame2 = tbf.read_frame(fh) self.assertTrue(frame2.header.is_tbf) self.assertEqual(frame2.header.first_chan, 2360) fh.close()
def test_tbf_buffer_flush(self): """Test the TBF ring buffer's flush() function.""" fh = open(tbfFile, 'rb') nFpO = tbf.get_frames_per_obs(fh) # Create the FrameBuffer instance frameBuffer = buffer.TBFFrameBuffer( chans=[2348, 2360, 2372, 2384, 2396]) # Go while True: try: cFrame = tbf.read_frame(fh) except errors.EOFError: break except errors.SyncError: continue frameBuffer.append(cFrame) cFrames = frameBuffer.get() if cFrames is None: continue fh.close() # Flush the buffer for cFrames in frameBuffer.flush(): # Make sure the dump has one of the expected time tags self.assertTrue( cFrames[0].payload.timetag in (283685766952000000, )) # Make sure it has the right number of frames self.assertEqual(len(cFrames), nFpO)
def test_tbf_math(self): """Test mathematical operations on TBF frame data via frames.""" fh = open(tbfFile, 'rb') # Frames 1 through 3 frames = [] for i in range(1, 4): frames.append(tbf.read_frame(fh)) fh.close() # Multiplication frameT = frames[0] * 2.0 for i in range(12 * 256 * 2): c = i // 2 // 256 s = i // 2 % 256 p = i % 2 self.assertAlmostEqual(frameT.payload.data[c, s, p], 2 * frames[0].payload.data[c, s, p], 2) frameT *= 2.0 for i in range(12 * 256 * 2): c = i // 2 // 256 s = i // 2 % 256 p = i % 2 self.assertAlmostEqual(frameT.payload.data[c, s, p], 4 * frames[0].payload.data[c, s, p], 2) frameT = frames[0] * frames[1] for i in range(12 * 256 * 2): c = i // 2 // 256 s = i // 2 % 256 p = i % 2 self.assertAlmostEqual( frameT.payload.data[c, s, p], frames[0].payload.data[c, s, p] * frames[1].payload.data[c, s, p], 2) # Addition frameA = frames[0] + 2.0 for i in range(800): c = i // 2 // 256 s = i // 2 % 256 p = i % 2 self.assertAlmostEqual(frameA.payload.data[c, s, p], 2 + frames[0].payload.data[c, s, p], 2) frameA += 2.0 for i in range(800): c = i // 2 // 256 s = i // 2 % 256 p = i % 2 self.assertAlmostEqual(frameA.payload.data[c, s, p], 4 + frames[0].payload.data[c, s, p], 2) frameA = frames[0] + frames[1] for i in range(800): c = i // 2 // 256 s = i // 2 % 256 p = i % 2 self.assertAlmostEqual( frameA.payload.data[c, s, p], frames[0].payload.data[c, s, p] + frames[1].payload.data[c, s, p], 2)
def test_tbf_sort(self): """Test sorting TBF frames by time tags.""" fh = open(tbfFile, 'rb') # Frames 1 through 3 frames = [] for i in range(1, 4): frames.append(tbf.read_frame(fh)) fh.close() frames.sort() frames = frames[::-1] for i in range(1, len(frames)): self.assertTrue(frames[i - 1] >= frames[i])
def test_tbf_comps(self): """Test the TBF frame comparison operators (>, <, etc.) for time tags.""" fh = open(tbfFile, 'rb') # Frames 1 through 4 frames = [] for i in range(1, 5): frames.append(tbf.read_frame(fh)) fh.close() self.assertTrue(0 < frames[0]) self.assertFalse(0 > frames[0]) self.assertTrue(frames[-1] >= frames[0]) self.assertTrue(frames[-1] <= frames[0]) self.assertTrue(frames[0] == frames[0]) self.assertTrue(frames[0] == frames[-1]) self.assertFalse(frames[0] != frames[0])
def test_tbf_default(self): """Test the TBF ring buffer with the default values.""" fh = open(tbfFile, 'rb') nFpO = tbf.get_frames_per_obs(fh) # Create the FrameBuffer instance frameBuffer = buffer.TBFFrameBuffer( chans=[2348, 2360, 2372, 2384, 2396]) # Go while True: try: cFrame = tbf.read_frame(fh) except errors.EOFError: break except errors.SyncError: continue frameBuffer.append(cFrame) cFrames = frameBuffer.get() if cFrames is None: continue fh.close() # Make sure we have the right number of frames in the buffer nFrames = 0 for key in frameBuffer.buffer.keys(): nFrames = nFrames + len(frameBuffer.buffer[key]) self.assertEqual(nFrames, 5) # Make sure nothing has happened that shouldn't have self.assertEqual(frameBuffer.full, 0) self.assertEqual(frameBuffer.partial, 0) self.assertEqual(frameBuffer.missing, 0) self.assertEqual(frameBuffer.dropped, 0) # Make sure we have the right keys for key in frameBuffer.buffer.keys(): self.assertTrue(key in (283685766952000000, )) # Make sure the buffer keys have the right sizes self.assertEqual(len(frameBuffer.buffer[283685766952000000]), 5)
def test_tbf_errors(self): """Test TBF reading errors.""" fh = open(tbfFile, 'rb') # Frames 1 through 5 for i in range(1, 6): frame = tbf.read_frame(fh) # Last frame should be an error (errors.EOFError) self.assertRaises(errors.EOFError, tbf.read_frame, fh) fh.close() # If we offset in the file by 1 byte, we should be a # sync error (errors.SyncError). fh = open(tbfFile, 'rb') fh.seek(1) self.assertRaises(errors.SyncError, tbf.read_frame, fh) fh.close()
def test_tbf_buffer_reorder(self): """Test the reorder function of the TBF ring buffer.""" fh = open(tbfFile, 'rb') nFpO = tbf.get_frames_per_obs(fh) # Create the FrameBuffer instance frameBuffer = buffer.TBFFrameBuffer( chans=[2348, 2360, 2372, 2384, 2396], reorder=True) # Go while True: try: cFrame = tbf.read_frame(fh) except errors.EOFError: break except errors.SyncError: continue frameBuffer.append(cFrame) cFrames = frameBuffer.get() if cFrames is None: continue # Make sure the dump has one of the expected time tags self.assertTrue( cFrames[0].payload.timetag in (283685766952000000, )) # Make sure it has the right number of frames self.assertEqual(len(cFrames), nFpO) # Check the order for i in range(1, len(cFrames)): pC = cFrames[i - 1].header.first_chan cC = cFrames[i].header.first_chan self.assertTrue(cC > pC) fh.close()
def test_tbf_math(self): """Test mathematical operations on TBF frame data via frames.""" fh = open(tbfFile, 'rb') # Frames 1 through 3 frames = [] for i in range(1, 4): frames.append(tbf.read_frame(fh)) fh.close() # Multiplication frameT = frames[0] * 2.0 numpy.testing.assert_allclose(frameT.payload.data, 2 * frames[0].payload.data, atol=1e-6) frameT *= 2.0 numpy.testing.assert_allclose(frameT.payload.data, 4 * frames[0].payload.data, atol=1e-6) frameT = frames[0] * frames[1] numpy.testing.assert_allclose(frameT.payload.data, frames[0].payload.data * frames[1].payload.data, atol=1e-6) # Addition frameA = frames[0] + 2.0 numpy.testing.assert_allclose(frameA.payload.data, 2 + frames[0].payload.data, atol=1e-6) frameA += 2.0 numpy.testing.assert_allclose(frameA.payload.data, 4 + frames[0].payload.data, atol=1e-6) frameA = frames[0] + frames[1] numpy.testing.assert_allclose(frameA.payload.data, frames[0].payload.data + frames[1].payload.data, atol=1e-6)
def main(args): # Setup the LWA station information if args.metadata is not None: try: station = stations.parse_ssmif(args.metadata) except ValueError: station = metabundleADP.get_station(args.metadata, apply_sdm=True) else: station = stations.lwasv antennas = station.antennas fh = open(args.filename, 'rb') nFrames = os.path.getsize(args.filename) / tbf.FRAME_SIZE antpols = len(antennas) # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbf.read_frame(fh) fh.seek(0) beginDate = junkFrame.time.datetime # Figure out how many frames there are per observation and the number of # channels that are in the file nFramesPerObs = tbf.get_frames_per_obs(fh) nchannels = tbf.get_channel_count(fh) nSamples = 7840 # Figure out how many chunks we need to work with nChunks = nFrames / nFramesPerObs # Pre-load the channel mapper mapper = [] for i in range(2*nFramesPerObs): cFrame = tbf.read_frame(fh) if cFrame.header.first_chan not in mapper: mapper.append( cFrame.header.first_chan ) fh.seek(-2*nFramesPerObs*tbf.FRAME_SIZE, 1) mapper.sort() # Calculate the frequencies freq = numpy.zeros(nchannels) for i,c in enumerate(mapper): freq[i*12:i*12+12] = c + numpy.arange(12) freq *= 25e3 # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Frames per Observation: %i" % nFramesPerObs) print("Channel Count: %i" % nchannels) print("Frames: %i" % nFrames) print("===") print("Chunks: %i" % nChunks) spec = numpy.zeros((nchannels,256,2)) norm = numpy.zeros_like(spec) for i in range(nChunks): # Inner loop that actually reads the frames into the data array for j in range(nFramesPerObs): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbf.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbf.FRAME_SIZE-1)) continue if not cFrame.header.is_tbf: continue first_chan = cFrame.header.first_chan # Figure out where to map the channel sequence to try: aStand = mapper.index(first_chan) except ValueError: mapper.append(first_chan) aStand = mapper.index(first_chan) # Actually load the data. spec[aStand*12:aStand*12+12,:,:] += numpy.abs(cFrame.payload.data)**2 norm[aStand*12:aStand*12+12,:,:] += 1 spec /= norm fh.close() # Reshape and transpose to get it in to a "normal" order spec.shape = (spec.shape[0], spec.shape[1]*spec.shape[2]) spec = spec.T # Apply the cable loss corrections, if requested if False: for s in range(spec.shape[0]): currGain = antennas[s].cable.gain(freq) spec[s,:] /= currGain # Put the frequencies in the best units possible freq, units = _best_freq_units(freq) # Deal with the `keep` options if args.keep == 'all': antpolsDisp = int(numpy.ceil(antpols/20)) js = [i for i in range(antpols)] else: antpolsDisp = int(numpy.ceil(len(args.keep)*2/20)) if antpolsDisp < 1: antpolsDisp = 1 js = [] for k in args.keep: for i,ant in enumerate(antennas): if ant.stand.id == k: js.append(i) nPlot = len(js) if nPlot < 16: if nPlot % 4 == 0 and nPlot != 4: figsY = 4 else: figsY = 2 figsX = int(numpy.ceil(1.0*nPlot/figsY)) else: figsY = 4 figsX = 4 figsN = figsX*figsY for i in range(antpolsDisp): # Normal plotting fig = plt.figure() for k in range(i*figsN, i*figsN+figsN): try: j = js[k] currSpectra = numpy.squeeze( numpy.log10(spec[j,:])*10.0 ) except IndexError: break ax = fig.add_subplot(figsX, figsY, (k%figsN)+1) ax.plot(freq, currSpectra, label='Stand: %i, Pol: %i (Dig: %i)' % (antennas[j].stand.id, antennas[j].pol, antennas[j].digitizer)) ax.set_title('Stand: %i (%i); Dig: %i [%i]' % (antennas[j].stand.id, antennas[j].pol, antennas[j].digitizer, antennas[j].combined_status)) ax.set_xlabel('Frequency [%s]' % units) ax.set_ylabel('P.S.D. [dB/RBW]') ax.set_ylim([-10, 30]) # Save spectra image if requested if args.output is not None: base, ext = os.path.splitext(args.output) outFigure = "%s-%02i%s" % (base, i+1, ext) fig.savefig(outFigure) plt.draw() print("RBW: %.4f %s" % ((freq[1]-freq[0]), units)) plt.show()
def main(args): # Set the station if args.metadata is not None: station = stations.parse_ssmif(args.metadata) ssmifContents = open(args.metadata).readlines() else: station = stations.lwasv ssmifContents = open(os.path.join(dataPath, 'lwa1-ssmif.txt')).readlines() antennas = station.antennas antpols = len(antennas) fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) / tbf.FRAME_SIZE # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbf.read_frame(fh) fh.seek(0) beginTime = junkFrame.time beginDate = junkFrame.time.datetime # Figure out how many frames there are per observation and the number of # channels that are in the file nFramesPerObs = tbf.get_frames_per_obs(fh) nchannels = tbf.get_channel_count(fh) # Figure out how many chunks we need to work with nChunks = nFrames / nFramesPerObs # Pre-load the channel mapper mapper = [] freq = [] for i in xrange(2*nFramesPerObs): cFrame = tbf.read_frame(fh) mapper.append( cFrame.header.first_chan ) freq.extend( list(cFrame.header.channel_freqs) ) fh.seek(-2*nFramesPerObs*tbf.FRAME_SIZE, 1) mapper.sort() freq.sort() # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Frames per Observation: %i" % nFramesPerObs) print("Channel Count: %i" % nchannels) print("Frames: %i" % nFrames) print("===") print("Chunks: %i" % nChunks) outfile = os.path.split(args.filename)[1] outfile = os.path.splitext(outfile)[0] outfile = "%s.npz" % outfile if (not os.path.exists(outfile)) or args.force: # Master loop over all of the file chunks masterSpectra = numpy.zeros((nChunks, 512, nchannels), numpy.float32) for i in range(nChunks): # Inner loop that actually reads the frames into the data array for j in range(nFramesPerObs): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbf.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbf.FRAME_SIZE-1)) continue if not cFrame.header.is_tbf: continue aStand = mapper.index(cFrame.header.first_chan) # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array if cFrame.header.frame_count % 10000 == 0 and args.verbose: print("%4i -> %3i %6.3f %5i %i" % (cFrame.header.first_chan, aStand, cFrame.time, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers if i == 0 and j == 0: refCount = cFrame.header.frame_count count = cFrame.header.frame_count - refCount masterSpectra[count,0::2,aStand*12:(aStand+1)*12] = numpy.abs( numpy.rollaxis(cFrame.payload.data[:,:,0], 1) )**2 masterSpectra[count,1::2,aStand*12:(aStand+1)*12] = numpy.abs( numpy.rollaxis(cFrame.payload.data[:,:,1], 1) )**2 # Compute the 1 ms average power and the data range within each 1 ms window subSize = 1960 nsegments = masterSpectra.shape[1] / subSize print("Computing average power and data range in %i-sample intervals, ADC histogram" % subSize) pb = ProgressBar(max=masterSpectra.shape[0]) avgPower = numpy.zeros((antpols, nsegments), dtype=numpy.float32) dataRange = numpy.zeros((antpols, nsegments, 3), dtype=numpy.int16) adcHistogram = numpy.zeros((antpols, 4096), dtype=numpy.int32) histBins = range(-2048, 2049) # Apply the cable loss corrections, if requested if True: for s in xrange(masterSpectra.shape[1]): currGain = antennas[s].cable.gain(freq) for c in xrange(masterSpectra.shape[0]): masterSpectra[c,s,:] /= currGain # Now that we have read through all of the chunks, perform the final averaging by # dividing by all of the chunks spec = masterSpectra.mean(axis=0) # Estimate the dipole resonance frequencies print("Computing dipole resonance frequencies") pb = ProgressBar(max=spec.shape[0]) resFreq = numpy.zeros(spec.shape[0]) toCompare = numpy.where( (freq>31e6) & (freq<70e6) )[0] for i in xrange(spec.shape[0]): bestOrder = 0 bestRMS = 1e34 for j in xrange(3, 12): coeff = numpy.polyfit(freq[toCompare]/1e6, numpy.log10(spec[i,toCompare])*10, j) fit = numpy.polyval(coeff, freq[toCompare]/1e6) rms = ((fit - numpy.log10(spec[i,toCompare])*10)**2).sum() if rms < bestRMS: bestOrder = j bestRMS = rms coeff = numpy.polyfit(freq[toCompare]/1e6, numpy.log10(spec[i,toCompare])*10, bestOrder) fit = numpy.polyval(coeff, freq[toCompare]/1e6) try: resFreq[i] = freq[toCompare[numpy.where( fit == fit.max() )[0][0]]] / 1e6 except: pass pb.inc(amount=1) if pb.amount != 0 and pb.amount % 10 == 0: sys.stdout.write(pb.show()+'\r') sys.stdout.flush() sys.stdout.write(pb.show()+'\r') sys.stdout.write('\n') sys.stdout.flush() numpy.savez(outfile, date=str(beginDate), freq=freq, masterSpectra=masterSpectra, resFreq=resFreq, avgPower=avgPower, dataRange=dataRange, adcHistogram=adcHistogram, ssmifContents=ssmifContents) else: dataDict = numpy.load(outfile) freq = dataDict['freq'] masterSpectra = dataDict['masterSpectra'] resFreq = dataDict['resFreq'] # Now that we have read through all of the chunks, perform the final averaging by # dividing by all of the chunks spec = masterSpectra.mean(axis=0) # Create a good template spectra specTemplate = numpy.median(spec, axis=0) specDiff = numpy.zeros(spec.shape[0]) toCompare = numpy.where( (freq>32e6) & (freq<50e6) )[0] print(len(toCompare)) for i in xrange(spec.shape[0]): specDiff[i] = (spec[i,toCompare] / specTemplate[toCompare]).mean() specDiff = numpy.where( specDiff < 2, specDiff, 2) # Get the station standPos = numpy.array([[ant.stand.x, ant.stand.y, ant.stand.z] for ant in antennas if ant.pol == 0]) # Plots if args.verbose: fig = plt.figure() ax1 = fig.add_subplot(1, 2, 1) ax1.scatter(standPos[:,0], standPos[:,1], c=specDiff[0::2], s=40.0, alpha=0.50) ## Set the limits to just zoom in on the main stations ax1.set_xlim([-65, 65]) ax1.set_ylim([-65, 65]) ax2 = fig.add_subplot(1, 2, 2) ax2.plot(freq/1e6, numpy.log10(specTemplate)*10, alpha=0.50) print("RBW: %.1f Hz" % (freq[1]-freq[0])) plt.show()
def main(args): fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) / tbf.FRAME_SIZE # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbf.read_frame(fh) fh.seek(0) beginDate = junkFrame.time.datetime # Figure out how many frames there are per observation and the number of # channels that are in the file nFramesPerObs = tbf.get_frames_per_obs(fh) nchannels = tbf.get_channel_count(fh) nSamples = 7840 # Figure out how many chunks we need to work with nChunks = nFrames / nFramesPerObs # Pre-load the channel mapper mapper = [] for i in xrange(2 * nFramesPerObs): cFrame = tbf.read_frame(fh) if cFrame.header.first_chan not in mapper: mapper.append(cFrame.header.first_chan) fh.seek(-2 * nFramesPerObs * tbf.FRAME_SIZE, 1) mapper.sort() # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Frames per Observation: %i" % nFramesPerObs) print("Channel Count: %i" % nchannels) print("Frames: %i" % nFrames) print("===") print("Chunks: %i" % nChunks) # Master loop over all of the file chunks timetags = numpy.zeros((nFramesPerObs, nChunks), dtype=numpy.int64) - 1 for i in xrange(nChunks): # Inner loop that actually reads the frames into the data array for j in xrange(nFramesPerObs): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbf.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell()) / tbf.FRAME_SIZE - 1)) continue if not cFrame.header.is_tbf: continue first_chan = cFrame.header.first_chan # Figure out where to map the channel sequence to try: aStand = mapper.index(first_chan) except ValueError: mapper.append(first_chan) aStand = mapper.index(first_chan) if cFrame.header.frame_count % 10000 == 0: print("%4i -> %4i %7i %i" % (first_chan, aStand, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers if i == 0 and j == 0: refCount = cFrame.header.frame_count count = cFrame.header.frame_count - refCount timetags[aStand, count] = cFrame.payload.timetag # Check for missing frames missing = numpy.where(timetags < 0) if len(missing) != 0: print("Found %i missing frames. Missing data from:" % len(missing[0])) for i, f in zip(missing[0], missing[1]): print(" channel set %4i @ frame %5i" % (mapper[i], f + 1)) # Check time tags to make sure every ant/pol as the same time as each frame for f in xrange(timetags.shape[1]): ## For each frame count value, get the median time tag and use this for comparison. ## If things are really bad, we will get a lot of errors. frameTime = numpy.median(timetags[:, f]) ## Compare all of the antpols at a particular frame count, ignoring the ones that ## are missing. missing = numpy.where((timetags[:, f] != frameTime) & (timetags[:, f] >= 0))[0] ## Report any errors for m in missing: print("ERROR: t.t. %i @ frame %i != frame median of %i" % (timetags[m, f], f + 1, frameTime)) print(" -> difference: %i" % (timetags[m, f] - frameTime, )) # Check time tags to make sure the times increment correctly between frames for i in xrange(timetags.shape[0]): for f in xrange(1, timetags.shape[1]): ## Skip missing frames since they always fail if timetags[i, f] < 0 or timetags[i, f - 1] < 0: continue ## Compare the current time tag with previous and report an error if there ## is a discrepancy between the two modulo the expected skip. if timetags[i, f] > (timetags[i, f - 1] + nSamples): ## Too far into the future print("ERROR: t.t. %i @ frame %i > t.t. %i @ frame %i + skip" % (timetags[i, f], f + 1, timetags[i, f - 1], f)) print(" -> difference: %i" % (timetags[i, f] - timetags[i, f - 1], )) elif timetags[i, f] < (timetags[i, f - 1] + nSamples): ## Not far enough into the future print("ERROR: t.t. %i @ frame %i < t.t. %i @ frame %i + skip" % (timetags[i, f], f + 1, timetags[i, f - 1], f)) print(" -> difference: %i" % (timetags[i, f] - timetags[i, f - 1], )) else: ## Everything is good if we make it here pass