def test_basic_tbn(self): """Test building a basic TBN signal""" testFile = os.path.join(self.testPath, 'tbn.dat') fh = open(testFile, 'wb') dp.basic_signal(fh, numpy.array([1,2,3,4]), 2000, mode='TBN', filter=7, start_time=1000) fh.close() # Check the file size fileSize = os.path.getsize(testFile) nSamples = fileSize // tbn.FRAME_SIZE self.assertEqual(nSamples, 2000*4*2) # Check the time of the first frame fh = open(testFile, 'rb') frame = tbn.read_frame(fh) fh.close() self.assertEqual(frame.payload.timetag, 1000*dp_common.fS)
def __getTBN(self, vanilla=False): """Private function to load in the test TBN data and get the frames. If the keyword 'vanilla' is set to True, gain, sample rate, and frequency meta- data are not added to the frames.""" fh = open(tbnFile, 'rb') # Frames 1 through 8 frames = [] for i in range(1, 9): frames.append(tbn.read_frame(fh)) if not vanilla: # Set some values for the other meta-data for frame in frames: frame.sample_rate = 100000 fh.close() return frames
def test_point_tbn(self): """Test building a point source TBN signal""" testFile = os.path.join(self.testPath, 'tbn.dat') station = lwa_common.lwa1 antennas = station.antennas fh = open(testFile, 'wb') dp.point_source(fh, antennas[:8:2], self.src, 4, mode='TBN', filter=7, start_time=1000) fh.close() # Check the file size fileSize = os.path.getsize(testFile) nSamples = fileSize // tbn.FRAME_SIZE self.assertEqual(nSamples, 4*4*2) # Check the time of the first frame fh = open(testFile, 'rb') frame = tbn.read_frame(fh) fh.close() self.assertEqual(frame.payload.timetag, 1000*dp_common.fS)
def test_tbn_math(self): """Test mathematical operations on TBN frame data via frames.""" fh = open(tbnFile, 'rb') # Frames 1 through 29 frames = [] for i in range(1, 30): frames.append(tbn.read_frame(fh)) fh.close() # Multiplication frameT = frames[0] * 2.0 numpy.testing.assert_allclose(frameT.payload.data, 2 * frames[0].payload.data, atol=1e-6) frameT *= 2.0 numpy.testing.assert_allclose(frameT.payload.data, 4 * frames[0].payload.data, atol=1e-6) frameT = frames[0] * frames[1] numpy.testing.assert_allclose(frameT.payload.data, frames[0].payload.data * frames[1].payload.data, atol=1e-6) # Addition frameA = frames[0] + 2.0 numpy.testing.assert_allclose(frameA.payload.data, 2 + frames[0].payload.data, atol=1e-6) frameA += 2.0 numpy.testing.assert_allclose(frameA.payload.data, 4 + frames[0].payload.data, atol=1e-6) frameA = frames[0] + frames[1] numpy.testing.assert_allclose(frameA.payload.data, frames[0].payload.data + frames[1].payload.data, atol=1e-6)
def test_tbn_math(self): """Test mathematical operations on TBN frame data via frames.""" fh = open(tbnFile, 'rb') # Frames 1 through 29 frames = [] for i in range(1, 30): frames.append(tbn.read_frame(fh)) fh.close() # Multiplication frameT = frames[0] * 2.0 for i in range(512): self.assertAlmostEqual(frameT.payload.data[i], 2 * frames[0].payload.data[i], 2) frameT *= 2.0 for i in range(512): self.assertAlmostEqual(frameT.payload.data[i], 4 * frames[0].payload.data[i], 2) frameT = frames[0] * frames[1] for i in range(512): self.assertAlmostEqual( frameT.payload.data[i], frames[0].payload.data[i] * frames[1].payload.data[i], 2) # Addition frameA = frames[0] + 2.0 for i in range(512): self.assertAlmostEqual(frameA.payload.data[i], 2 + frames[0].payload.data[i], 2) frameA += 2.0 for i in range(512): self.assertAlmostEqual(frameA.payload.data[i], 4 + frames[0].payload.data[i], 2) frameA = frames[0] + frames[1] for i in range(512): self.assertAlmostEqual( frameA.payload.data[i], frames[0].payload.data[i] + frames[1].payload.data[i], 2)
def test_frame_data_errors(self): """Test the data error scenarios when validating a TBN SimFrame.""" # Read in a TBN frame from the test file fh = open(tbnFile, 'rb') origFrame = tbnReader.read_frame(fh) fh.close() # Try to validate frame with the wrong data type fakeFrame = tbnWriter.SimFrame() fakeFrame.load_frame(copy.deepcopy(origFrame)) fakeFrame.data = fakeFrame.payload.data.real self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True) # Try to validate frame with the wrong data size fakeFrame = tbnWriter.SimFrame() fakeFrame.load_frame(copy.deepcopy(origFrame)) fakeFrame.data = None self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True) fakeFrame = tbnWriter.SimFrame() fakeFrame.load_frame(copy.deepcopy(origFrame)) fakeFrame.data = fakeFrame.payload.data[0:50] self.assertRaises(ValueError, fakeFrame.is_valid, raise_errors=True)
def test_tbn_buffer_flush(self): """Test the TBN ring buffer's flush() function.""" fh = open(tbnFile, 'rb') nFpO = tbn.get_frames_per_obs(fh) nFpO = nFpO[0] + nFpO[1] # Create the FrameBuffer instance frameBuffer = buffer.TBNFrameBuffer(stands=range(1, nFpO // 2 + 1), pols=[0, 1]) # Go while True: try: cFrame = tbn.read_frame(fh) except errors.EOFError: break except errors.SyncError: continue frameBuffer.append(cFrame) cFrames = frameBuffer.get() if cFrames is None: continue fh.close() # Flush the buffer for cFrames in frameBuffer.flush(): # Make sure the dump has one of the expected time tags self.assertTrue(cFrames[0].payload.timetag in (119196674956800, 119196675960320)) # Make sure it has the right number of frames self.assertEqual(len(cFrames), nFpO)
def main(args): filename = args.filename sizeB = os.path.getsize(filename) # Open the file and get some basic info about the data contained fh = open(filename, 'rb') sample_rate = tbn.get_sample_rate(fh) nFramesX, nFramesY = tbn.get_frames_per_obs(fh) nCaptures = sizeB // tbn.FRAME_SIZE // (nFramesX + nFramesY) print("Filename: %s" % filename) print("Size: %.1f MB" % (float(sizeB) / 1024 / 1024)) print("Captures: %i (%.2f seconds)" % (nCaptures, nCaptures * 512 / sample_rate)) print("Stands: %i (%i x pol., %i y pol.)" % ((nFramesX + nFramesY), nFramesX, nFramesY)) print("Sample Rate: %.2f kHz" % (sample_rate / 1000.0)) print("===") if args.count > 0: nCaptures = args.count * sample_rate // 512 else: nCaptures -= args.offset * sample_rate // 512 args.count = nCaptures * 512 // sample_rate nSkip = int(args.offset * sample_rate / 512) print("Seconds to Skip: %.2f (%i captures)" % (args.offset, nSkip)) print("Seconds to Split: %.2f (%i captures)" % (args.count, nCaptures)) # Make sure that the first frame in the file is the first frame if a capture # (stand 1, pol 0). If not, read in as many frames as necessary to get to # the beginning of a complete capture. frame = tbn.read_frame(fh) stand, pol = frame.id skip = 0 while (2 * (stand - 1) + pol) != 0: frame = tbn.read_frame(fh) stand, pol = frame.id skip += 1 fh.seek(fh.tell() - tbn.FRAME_SIZE) if skip != 0: print("Skipped %i frames at the beginning of the file" % skip) for c in list(range(nSkip)): if c < nSkip: fh.seek(fh.tell() + tbn.FRAME_SIZE * (nFramesX + nFramesY)) continue nFramesRemaining = (sizeB - fh.tell()) // tbn.FRAME_SIZE nRecursions = int(nFramesRemaining // (nCaptures * (nFramesX + nFramesY))) if not args.recursive: nRecursions = 1 scale = int(math.log10(nRecursions)) + 1 ifString = "Working on #%%%ii of %i (%%s)" % (scale, nRecursions) for r in range(nRecursions): if args.date: filePos = fh.tell() junkFrame = tbn.read_frame(fh) fh.seek(filePos) dt = junkFrame.time.datetime captFilename = "%s_%s.dat" % (os.path.splitext( os.path.basename(filename))[0], dt.isoformat()) else: captFilename = "%s_s%04i_p%%0%ii.dat" % (os.path.splitext( os.path.basename(filename))[0], args.count, scale) captFilename = captFilename % r if not args.recursive: captFilename = "%s_s%04i.dat" % (os.path.splitext( os.path.basename(filename))[0], args.count) print(ifString % (r + 1, captFilename)) t0 = time.time() fhOut = open(captFilename, 'wb') split_file(fh, fhOut, nCaptures, nFramesX + nFramesY) fhOut.close() t1 = time.time() print(" Copied %i bytes in %.3f s (%.3f MB/s)" % (os.path.getsize(captFilename), t1 - t0, os.path.getsize(captFilename) / 1024.0**2 / (t1 - t0))) fh.close()
def main(args): # Set the station station = stations.lwa1 antennas = station.antennas fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion maxFrames = 30000 * 260 antpols = len(antennas) nChunks = int(math.ceil(1.0 * nFrames / maxFrames)) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("Chunks: %i" % nChunks) print("===") nChunks = 1 # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2 * tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Master loop over all of the file chunks timetags = numpy.zeros((antpols, 30000), dtype=numpy.int64) - 1 for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = nFrames print("Working on chunk %i, %i frames remaining" % ((i + 1), framesRemaining)) # Inner loop that actually reads the frames into the data array for j in range(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell()) / tbw.FRAME_SIZE - 1)) continue if not cFrame.header.is_tbw: continue stand = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) if cFrame.header.frame_count % 10000 == 0: print("%3i -> %3i %5i %i" % (stand, aStand, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers count = cFrame.header.frame_count - 1 timetags[aStand, count] = cFrame.payload.timetag timetags[aStand + 1, count] = cFrame.payload.timetag # Check for missing frames missing = numpy.where(timetags < 0) if len(missing) != 0: dp1Boards = {} print( "Found %i missing frames (%i missing time tags). Missing data from:" % (len(missing[0]) / 2, len(missing[0]))) for i, f in zip(missing[0], missing[1]): try: dp1Boards[antennas[i].board] += 1 except KeyError: dp1Boards[antennas[i].board] = 1 print(" stand %3i, pol. %1i (dig. %3i) @ frame %5i" % (antennas[i].stand.id, antennas[i].pol, antennas[i].digitizer, f + 1)) print("-> DP1 boards with missing frames:") for k in dp1Boards.keys(): v = dp1Boards[k] print(" %2i %6i (%7.3f%%)" % (k, v, 100.0 * v / (30000 * 10))) # Check time tags to make sure every ant/pol as the same time as each frame for f in xrange(timetags.shape[1]): ## For each frame count value, get the median time tag and use this for comparison. ## If things are really bad, we will get a lot of errors. frameTime = numpy.median(timetags[:, f]) ## Compare all of the antpols at a particular frame count, ignoring the ones that ## are missing. missing = numpy.where((timetags[:, f] != frameTime) & (timetags[:, f] >= 0))[0] ## Report any errors for m in missing: print("ERROR: t.t. %i @ frame %i != frame median of %i" % (timetags[m, f], f + 1, frameTime)) print(" -> difference: %i" % (timetags[m, f] - frameTime, )) # Check time tags to make sure the times increment correctly between frames for i in xrange(timetags.shape[0]): for f in xrange(1, timetags.shape[1]): ## Skip missing frames since they always fail if timetags[i, f] < 0 or timetags[i, f - 1] < 0: continue ## Compare the current time tag with previous and report an error if there ## is a discrepancy between the two modulo the expected skip. if timetags[i, f] > (timetags[i, f - 1] + nSamples): ## Too far into the future print("ERROR: t.t. %i @ frame %i > t.t. %i @ frame %i + skip" % (timetags[i, f], f + 1, timetags[i, f - 1], f)) print(" -> difference: %i" % (timetags[i, f] - timetags[i, f - 1], )) elif timetags[i, f] < (timetags[i, f - 1] + nSamples): ## Not far enough into the future print("ERROR: t.t. %i @ frame %i < t.t. %i @ frame %i + skip" % (timetags[i, f], f + 1, timetags[i, f - 1], f)) print(" -> difference: %i" % (timetags[i, f] - timetags[i, f - 1], )) else: ## Everything is good if we make it here pass
def main(args): # Set the station if args.metadata is not None: station = stations.parse_ssmif(args.metadata) ssmifContents = open(args.metadata).readlines() else: station = stations.lwa1 ssmifContents = open(os.path.join(dataPath, 'lwa1-ssmif.txt')).readlines() antennas = station.antennas toKeep = [] for g in (1, 10, 54, 248, 251, 258): for i, ant in enumerate(antennas): if ant.stand.id == g and ant.pol == 0: toKeep.append(i) for i, j in enumerate(toKeep): print(i, j, antennas[j].stand.id) # Length of the FFT LFFT = args.fft_length # Make sure that the file chunk size contains is an integer multiple # of the FFT length so that no data gets dropped maxFrames = int((30000 * 260) / float(LFFT)) * LFFT # It seems like that would be a good idea, however... TBW data comes one # capture at a time so doing something like this actually truncates data # from the last set of stands for the first integration. So, we really # should stick with maxFrames = (30000 * 260) fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion antpols = len(antennas) nChunks = int(math.ceil(1.0 * nFrames / maxFrames)) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginTime = junkFrame.time beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("Chunks: %i" % nChunks) print("===") nChunks = 1 # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2 * tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Master loop over all of the file chunks masterSpectra = numpy.zeros((nChunks, antpols, LFFT)) for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining print("Working on chunk %i, %i frames remaining" % ((i + 1), framesRemaining)) data = numpy.zeros((12, 12000000), dtype=numpy.int16) # If there are fewer frames than we need to fill an FFT, skip this chunk if data.shape[1] < 2 * LFFT: break # Inner loop that actually reads the frames into the data array for j in range(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbw.FRAME_SIZE-1)) continue if not cFrame.header.is_tbw: continue stand = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) #if cFrame.header.frame_count % 10000 == 0 and config['verbose']: #print("%3i -> %3i %6.3f %5i %i" % (stand, aStand, cFrame.time, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers count = cFrame.header.frame_count - 1 if aStand not in toKeep: continue # Convert to reduced index aStand = 2 * toKeep.index(aStand) data[aStand, count * nSamples:(count + 1) * nSamples] = cFrame.payload.data[0, :] data[aStand + 1, count * nSamples:(count + 1) * nSamples] = cFrame.payload.data[1, :] # Time series analysis - mean, std. dev, saturation count tsMean = data.mean(axis=1) tsStd = data.std(axis=1) tsSat = numpy.where((data == 2047) | (data == -2047), 1, 0).sum(axis=1) # Time series analysis - percentiles p = [50, 75, 90, 95, 99] tsPct = numpy.zeros((data.shape[0], len(p))) for i in xrange(len(p)): for j in xrange(data.shape[0]): tsPct[j, i] = percentile(numpy.abs(data[j, :]), p[i]) # Frequency domain analysis - spectra freq = numpy.fft.fftfreq(2 * args.fft_length, d=1.0 / 196e6) freq = freq[:args.fft_length] delays = numpy.zeros((data.shape[0], freq.size)) signalsF, validF = FEngine(data, freq, delays, LFFT=args.fft_length, Overlap=1, sample_rate=196e6, clip_level=0) # Cleanup to save memory del validF, data print(signalsF.shape) # SK control values skM = signalsF.shape[2] skN = 1 # Frequency domain analysis - spectral kurtosis k = numpy.zeros((signalsF.shape[0], signalsF.shape[1])) for l in xrange(signalsF.shape[0]): for m in xrange(freq.size): k[l, m] = kurtosis.spectral_fft(signalsF[l, m, :]) kl, kh = kurtosis.get_limits(4, skM, skN) print(kl, kh) # Integrate the spectra for as long as we can masterSpectra = (numpy.abs(signalsF)**2).mean(axis=2) del signalsF # Mask out bad values (high spectral kurtosis) for the plot mask = numpy.where((k < kl) | (k > kh), 1, 0) mask = expandMask(mask, radius=4, merge=True) masterSpectra = numpy.ma.array(masterSpectra, mask=mask) # Save the data to an HDF5 file outname = os.path.splitext(args.filename)[0] outname = "%s-RFI.hdf5" % outname f = h5py.File(outname, 'w') f.attrs['filename'] = args.filename f.attrs['mode'] = 'TBW' f.attrs['station'] = 'LWA-1' f.attrs['dataBits'] = dataBits f.attrs['startTime'] = beginTime f.attrs['startTime_units'] = 's' f.attrs['startTime_sys'] = 'unix' f.attrs['sample_rate'] = 196e6 f.attrs['sample_rate_units'] = 'Hz' f.attrs['RBW'] = freq[1] - freq[0] f.attrs['RBW_Units'] = 'Hz' f.attrs['SK-M'] = skM f.attrs['SK-N'] = skN for l in xrange(len(toKeep)): antX = antennas[toKeep[l]] antY = antennas[toKeep[l] + 1] stand = f.create_group('Stand%03i' % antX.stand.id) stand['freq'] = freq stand['freq'].attrs['Units'] = 'Hz' polX = stand.create_group('X') polY = stand.create_group('Y') polX.attrs['tsMean'] = tsMean[2 * l] polY.attrs['tsMean'] = tsMean[2 * l + 1] polX.attrs['tsStd'] = tsStd[2 * l] polY.attrs['tsStd'] = tsStd[2 * l + 1] polX.attrs['tsSat'] = tsSat[2 * l] polY.attrs['tsSat'] = tsSat[2 * l + 1] for i, v in enumerate(p): polX.attrs['ts%02i' % v] = tsPct[2 * l][i] polY.attrs['ts%02i' % v] = tsPct[2 * l + 1][i] polX['spectrum'] = masterSpectra[2 * l, :] polX['spectrum'].attrs['axis0'] = 'frequency' polY['spectrum'] = masterSpectra[2 * l + 1, :] polY['spectrum'].attrs['axis0'] = 'frequency' polX['kurtosis'] = k[2 * l, :] polX['kurtosis'].attrs['axis0'] = 'frequency' polY['kurtosis'] = k[2 * l + 1, :] polY['kurtosis'].attrs['axis0'] = 'frequency' # The plot fig = plt.figure() ax1 = fig.add_subplot(2, 1, 1) ax2 = fig.add_subplot(2, 1, 2) for l in xrange(k.shape[0]): ant = antennas[toKeep[l / 2]] ax1.plot(freq / 1e6, numpy.log10(masterSpectra[l, :]) * 10, label='Stand %i, Pol %i' % (ant.stand.id, ant.pol + l % 2)) ax2.plot(freq / 1e6, k[l, :], label='Stand %i, Pol %i' % (ant.stand.id, ant.pol + l % 2)) ax2.hlines(kl, freq[0] / 1e6, freq[-1] / 1e6, linestyle=':', label='Kurtosis Limit 4$\sigma$') ax2.hlines(kh, freq[0] / 1e6, freq[-1] / 1e6, linestyle=':', label='Kurtosis Limit 4$\sigma$') ax1.set_xlabel('Frequency [MHz]') ax1.set_ylabel('PSD [arb. dB/RBW]') ax1.legend(loc=0) ax2.set_ylim((kl / 2, kh * 2)) ax2.set_xlabel('Frequency [MHz]') ax2.set_ylabel('Spectral Kurtosis') ax2.legend(loc=0) plt.show()
def main(args): filename = args[0] stands = [int(i) for i in args[1:]] antennas = lwa1.antennas fh = open(filename, "rb") nFrames = os.path.getsize(filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion antpols = len(antennas) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginTime = junkFrame.time beginDate = junkFrame.time.datetime # Figure out which digitizers to keep toKeep = [] for a in antennas: if a.stand.id in stands: toKeep.append( a.digitizer ) # File summary print("Filename: %s" % filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("===") print("Keeping Stands:") for a in toKeep: print(" Stand #%3i, pol %i (digitizer %3i)" % (antennas[a-1].stand.id, antennas[a-1].pol, antennas[a-1].digitizer)) # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2*tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Create the HDF5 file outname = os.path.splitext(filename)[0] outname = "%shdf5" % outname f = h5py.File(outname, 'w') f.attrs['filename'] = filename f.attrs['mode'] = 'TBW' f.attrs['station'] = 'LWA-1' f.attrs['dataBits'] = dataBits f.attrs['startTime'] = beginTime f.attrs['startTime_units'] = 's' f.attrs['startTime_sys'] = 'unix' f.attrs['sample_rate'] = 196e6 f.attrs['sample_rate_units'] = 'Hz' ## Create the digitizer to dataset lookup table and the standLookup = {} standData = [] i = 0 for a in toKeep: if a % 2 == 0: continue s = antennas[a-1].stand.id standLookup[a] = i temp = f.create_group('Stand%03i' % s) ### Combined status code temp.attrs['statusCode'] = antennas[a-1].combined_status ### Antenna number temp.attrs['antennaID'] = antennas[a-1].id ### Cable information temp.attrs['cableID'] = antennas[a-1].cable.id temp.attrs['cableLength'] = antennas[a-1].cable.length temp.attrs['cableLength_units'] = 'm' ### Stand location information temp.attrs['posX'] = antennas[a-1].stand.x temp.attrs['posX_units'] = 'm' temp.attrs['posY'] = antennas[a-1].stand.y temp.attrs['posY_units'] = 'm' temp.attrs['posZ'] = antennas[a-1].stand.z temp.attrs['posZ_units'] = 'm' ### Time series data sets temp.attrs['axis0'] = 'time' xpol = temp.create_dataset('X', (12000000,), 'i2', chunks=True) ypol = temp.create_dataset('Y', (12000000,), 'i2', chunks=True) standData.append( (xpol, ypol, temp) ) i += 1 # Go! while True: # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbw.FRAME_SIZE-1)) continue if not cFrame.header.is_tbw: continue # Get the DP "stand" ID and the digitizer number stand = cFrame.header.id aStand = 2*(stand-1) digitizer = aStand + 1 # If we don't need it, skip it if digitizer not in toKeep: continue # Actually load the data. ## Frame count count = cFrame.header.frame_count - 1 ## Which data set dataset = standLookup[digitizer] ## Load standData[dataset][0][count*nSamples:(count+1)*nSamples] = cFrame.payload.data[0,:] standData[dataset][1][count*nSamples:(count+1)*nSamples] = cFrame.payload.data[1,:] fh.close() f.close()
def main(args): # The task at hand clnfile = args[0] az = float(args[1]) el = float(args[2]) filename = args[3] # The station observer = lwa1.get_observer() antennas = lwa1.antennas # The file's parameters fh = open(filename, 'rb') nFramesFile = os.path.getsize(filename) / tbn.FRAME_SIZE srate = tbn.get_sample_rate(fh) antpols = len(antennas) # Reference antenna ref = 258 for a in antennas: if a.stand.id == ref and a.pol == 0: refX = a.digitizer elif a.stand.id == ref and a.pol == 1: refY = a.digitizer else: pass # Integration time (seconds and frames) tInt = 5.0 nFrames = int(round(tInt * srate / 512 * antpols)) tInt = nFrames / antpols * 512 / srate # Total run length #nChunks = int(round(1.0*nFramesFile / nFrames)) nChunks = 240 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbn.read_frame(fh) fh.seek(-tbn.FRAME_SIZE, 1) startFC = junkFrame.header.frame_count central_freq = junkFrame.central_freq beginDate = junkFrame.time.datetime observer.date = beginDate srcs = [] for line in _srcs: srcs.append(ephem.readdb(line)) srcs[-1].compute(observer) if srcs[-1].alt > 0: print("source %s: alt %.1f degrees, az %.1f degrees" % (srcs[-1].name, srcs[-1].alt * 180 / numpy.pi, srcs[-1].az * 180 / numpy.pi)) # File summary print("Filename: %s" % filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz" % central_freq) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate)) print("---") print("Integration: %.3f s (%i frames; %i frames per stand/pol)" % (tInt, nFrames, nFrames / antpols)) print("Chunks: %i" % nChunks) junkFrame = tbn.read_frame(fh) while junkFrame.header.frame_count < startFC + 3: junkFrame = tbn.read_frame(fh) fh.seek(-tbn.FRAME_SIZE, 1) # Get the beamformer coefficients - three sets: # (1) at the requested az, el # (2) at az, el - 15 degrees # (3) at the transit location of Cyg A dataDict = numpy.load(clnfile) cln = dataDict['cln'] aln1 = [] aln2 = [] aln3 = [] for i in xrange(cln.shape[1]): gd = getGeoDelay(antennas[i], az, el, central_freq, Degrees=True) aln1.append(numpy.exp(2j * numpy.pi * central_freq * gd)) gd = getGeoDelay(antennas[i], az, el - 15, central_freq, Degrees=True) aln2.append(numpy.exp(2j * numpy.pi * central_freq * gd)) gd = getGeoDelay(antennas[i], 0.5, 83.3, central_freq, Degrees=True) aln3.append(numpy.exp(2j * numpy.pi * central_freq * gd)) aln1 = numpy.array(aln1) aln2 = numpy.array(aln2) aln3 = numpy.array(aln3) bln1 = (cln * aln1).conj() / numpy.abs(cln * aln1) bln2 = (cln * aln2).conj() / numpy.abs(cln * aln2) bln3 = (cln * aln3).conj() / numpy.abs(cln * aln3) for i in xrange(cln.shape[1]): if antennas[i].combined_status != 33 or antennas[i].stand.id == ref: bln1[:, i] = 0.0 bln2[:, i] = 0.0 bln3[:, i] = 0.0 # Create the FrameBuffer instance buffer = TBNFrameBuffer(stands=range(1, antpols / 2 + 1), pols=[0, 1], reorder=False) # Create the beam times = numpy.zeros(nChunks, dtype=numpy.float64) beam1 = numpy.zeros((nChunks, 2), dtype=numpy.float64) beam2 = numpy.zeros((nChunks, 2), dtype=numpy.float64) beam3 = numpy.zeros((nChunks, 2), dtype=numpy.float64) # Go! k = 0 for i in xrange(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFramesFile - k if framesRemaining > nFrames: framesWork = nFrames data = numpy.zeros((antpols, framesWork / antpols * 512), dtype=numpy.complex64) else: framesWork = framesRemaining + antpols * buffer.nsegments data = numpy.zeros((antpols, framesWork / antpols * 512), dtype=numpy.complex64) print("Working on chunk %i, %i frames remaining" % (i + 1, framesRemaining)) count = [0 for a in xrange(antpols)] j = 0 fillsWork = framesWork / antpols # Inner loop that actually reads the frames into the data array while j < fillsWork: try: cFrame = tbn.read_frame(fh) k = k + 1 except errors.EOFError: break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbn.FRAME_SIZE-1)) continue buffer.append(cFrame) cFrames = buffer.get() if cFrames is None: continue valid = sum(lambda x, y: x + int(y.valid), cFrames, 0) if valid != antpols: print( "WARNING: frame count %i at %i missing %.2f%% of frames" % (cFrames[0].header.frame_count, cFrames[0].payload.timetag, float(antpols - valid) / antpols * 100)) continue for cFrame in cFrames: stand, pol = cFrame.header.id # In the current configuration, stands start at 1 and go up to 260. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) + pol # Save the time if j == 0 and aStand == 0: times[i] = cFrame.time data[aStand, count[aStand] * 512:(count[aStand] + 1) * 512] = cFrame.payload.data # Update the counters so that we can average properly later on count[aStand] = count[aStand] + 1 j += 1 # Mask bad = numpy.where(numpy.abs(data) >= 90) data[bad] = 0.0 # Beam forming taskPool = Pool(processes=6) taskList = [] taskList.append( (i, 1, 0, taskPool.apply_async(form_beam, args=(data[0::2, :], bln1[0, 0::2])))) taskList.append( (i, 1, 1, taskPool.apply_async(form_beam, args=(data[1::2, :], bln1[0, 1::2])))) taskList.append( (i, 2, 0, taskPool.apply_async(form_beam, args=(data[0::2, :], bln2[0, 0::2])))) taskList.append( (i, 2, 1, taskPool.apply_async(form_beam, args=(data[1::2, :], bln2[0, 1::2])))) taskList.append( (i, 3, 0, taskPool.apply_async(form_beam, args=(data[0::2, :], bln3[0, 0::2])))) taskList.append( (i, 3, 1, taskPool.apply_async(form_beam, args=(data[1::2, :], bln3[0, 1::2])))) taskPool.close() taskPool.join() for i, b, p, task in taskList: if b == 1: beam1[i, p] = task.get() elif b == 2: beam2[i, p] = task.get() else: beam3[i, p] = task.get() print('1', beam1[i, 0], '2', beam2[i, 0], '3', beam3[i, 0], '1/2', beam1[i, 0] / beam2[i, 0], '3/2', beam3[i, 0] / beam2[i, 0]) del data # Plot the data print('CygA :', beam1[:, 0]) print('Pointing 2:', beam2[:, 0]) print('Pointing 1:', beam3[:, 0]) fig = plt.figure() ax1 = fig.add_subplot(1, 2, 1) ax2 = fig.add_subplot(1, 2, 2) ax1.plot(times - times[0], beam1[:, 0]) ax1.plot(times - times[0], beam2[:, 0]) ax1.plot(times - times[0], beam3[:, 0]) ax2.plot(times - times[0], beam1[:, 1]) ax2.plot(times - times[0], beam2[:, 1]) ax2.plot(times - times[0], beam3[:, 1]) plt.show()
def main(args): # Set the station if args.lwasv: station = stations.lwasv else: station = stations.lwa1 antennas = station.antennas fh = open(args.filename, "rb") nFramesFile = os.path.getsize(args.filename) // tbn.FRAME_SIZE srate = tbn.get_sample_rate(fh) #antpols = tbn.get_frames_per_obs(fh) antpols = len(antennas) # Offset in frames for beampols beam/tuning/pol. sets offset = int(args.skip * srate / 512 * antpols) offset = int(1.0 * offset / antpols) * antpols args.skip = 1.0 * offset / antpols * 512 / srate fh.seek(offset * tbn.FRAME_SIZE) # Number of frames to integrate over nFrames = int(args.average * srate / 512 * antpols) args.average = 1.0 * nFrames / antpols * 512 / srate # Number of remaining chunks nChunks = int(math.ceil(1.0 * (nFrames) / (200 * 520))) # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbn.read_frame(fh) fh.seek(0) beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Rate: %i Hz" % srate) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Integration: %.3f s (%i frames; %i frames per stand/pol)" % (args.average, nFrames, nFrames / antpols)) print("Chunks: %i" % nChunks) # Sanity check if offset > nFramesFile: raise RuntimeError("Requested offset is greater than file length") if nFrames > (nFramesFile - offset): raise RuntimeError( "Requested integration time+offset is greater than file length") # Create the FrameBuffer instance buffer = TBNFrameBuffer(stands=range(1, antpols // 2 + 1), pols=[0, 1]) # Master loop over all of the file chunks masterCount = [0 for a in xrange(len(antennas))] # Missing packet control variables missingPackets = numpy.ones((antpols, 2048), dtype=numpy.int8) pc = 0 missing = 0 missingList = [] # Figure fig = plt.figure() ax1 = fig.add_subplot(1, 2, 1) ax2 = fig.add_subplot(1, 2, 2) k = 0 for i in xrange(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time ((200*520)), # only deal with that chunk framesRemaining = nFrames - k if framesRemaining > (200 * 520): framesWork = (200 * 520) else: framesWork = framesRemaining count = [0 for a in xrange(len(antennas))] j = 0 fillsWork = framesWork // antpols # Inner loop that actually reads the frames into the data array while j < fillsWork: try: cFrame = tbn.read_frame(fh) k = k + 1 except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell()) / tbn.FRAME_SIZE - 1)) continue #print(cFrame.header.frame_count, cFrame.payload.timetag, cFrame.id) buffer.append(cFrame) cFrames = buffer.get() if cFrames is None: continue valid = sum(lambda x, y: x + int(y.valid), cFrames, 0) print("Frame #%5i: %.4f seconds with %i valid ant/pols%s" % (cFrames[0].header.frame_count, cFrames[0].time, valid, '!' if valid != antpols else '')) if valid != antpols: bad = [] for cFrame in cFrames: if not cFrame.valid: bad.append(cFrame.id) missingPackets[2 * (bad[-1][0] - 1) + bad[-1][1], pc] = 0 bad.sort() pc += 1 if pc == missingPackets.shape[1]: plotMissing(ax1, ax2, missingPackets, missingList, antpols) plt.show() sys.exit(0) missing += (antpols - valid) total = (buffer.full + buffer.partial) * antpols #print(j, valid, antpols-valid, cFrames[0].header.frame_count, 1.0*missing / total* 100, bad[0], bad[-1], buffer.dropped) #print(buffer.status()) missingList.append(antpols - valid) else: total = (buffer.full + buffer.partial) * antpols missingList.append(0) times = numpy.array([f.payload.timetag for f in cFrames], dtype=numpy.int64) #print(cFrames[0].header.frame_count, times.min(), times.max(), times.max()-times.min(), "%6.3f%%" % (1.0*missing/total*100,)) for cFrame in cFrames: stand, pol = cFrame.header.id # In the current configuration, stands start at 1 and go up to 260. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) + pol # Update the counters so that we can average properly later on count[aStand] = count[aStand] + 1 masterCount[aStand] = masterCount[aStand] + 1 j += 1 # Empty the remaining portion of the buffer and integrate what's left for cFrames in buffer.flush(): valid = sum(lambda x, y: x + int(y.valid), cFrames, 0) print("Frame #%5i: %.4f seconds with %i valid ant/pols" % (cFrames[0].header.frame_count, cFrames[0].time, valid)) if valid != antpols: bad = [] for cFrame in cFrames: if not cFrame.valid: bad.append(cFrame.id) missingPackets[2 * (bad[-1][0] - 1) + bad[-1][1], pc] = 0 bad.sort() pc += 1 if pc == missingPackets.shape[1]: plotMissing(ax1, ax2, missingPackets, missingList, antpols) plt.show() sys.exit(0) missing += (antpols - valid) total = (buffer.full + buffer.partial) * antpols #print(j, valid, antpols-valid, cFrames[0].header.frame_count, 1.0*missing / total* 100, bad[0], bad[-1], buffer.dropped) #print(buffer.status()) missingList.append(antpols - valid) else: total = (buffer.full + buffer.partial) * antpols missingList.append(0) # Inner loop that actually reads the frames into the data array for cFrame in cFrames: stand, pol = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) + pol # Update the counters so that we can average properly later on count[aStand] = count[aStand] + 1 masterCount[aStand] = masterCount[aStand] + 1 j += 1 plotMissing(ax1, ax2, missingPackets, missingList, antpols) plt.show() sys.exit(0)
def main(args): filename = args.filename # Set the station if args.lwasv: station = stations.lwasv else: station = stations.lwa1 antennas = station.antennas fh = open(filename, "rb") nFramesFile = os.path.getsize(filename) // tbn.FRAME_SIZE srate = tbn.get_sample_rate(fh) antpols = len(antennas) # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbn.read_frame(fh) fh.seek(-tbn.FRAME_SIZE, 1) central_freq = junkFrame.central_freq beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz" % central_freq) print(" ") # Convert chunk length to total frame count chunkLength = int(args.length * srate / 512 * antpols) chunkLength = int(1.0 * chunkLength / antpols) * antpols # Convert chunk skip to total frame count chunkSkip = int(args.skip * srate / 512 * antpols) chunkSkip = int(1.0 * chunkSkip / antpols) * antpols # Create the FrameBuffer instance buffer = TBNFrameBuffer(stands=range(1,antpols//2+1), pols=[0, 1]) # Output arrays clipFraction = [] meanPower = [] # Find stands #10 toUse = [] for i in xrange(antpols): ant = antennas[i] if ant.stand.id == 10: toUse.append(i) # Go! i = 1 done = False print(" | Clipping | Power |") print(" | 10X 10Y | 10X 10Y |") print("---+-----------------+-------------------+") while True: count = [0 for j in xrange(antpols)] data = numpy.zeros((antpols, chunkLength*512//antpols), dtype=numpy.csingle) for j in xrange(chunkLength): try: cFrame = tbn.read_frame(fh) except errors.EOFError: done = True break except errors.SyncError: continue buffer.append(cFrame) cFrames = buffer.get() if cFrames is None: continue for cFrame in cFrames: stand,pol = cFrame.header.id # In the current configuration, stands start at 1 and go up to 260. So, we # can use this little trick to populate the data array aStand = 2*(stand-1)+pol try: data[aStand, count[aStand]*512:(count[aStand]+1)*512] = cFrame.payload.data # Update the counters so that we can average properly later on count[aStand] = count[aStand] + 1 except ValueError: pass # Empty the remaining portion of the buffer for cFrames in buffer.flush(): # Inner loop that actually reads the frames into the data array for cFrame in cFrames: stand,pol = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2*(stand-1)+pol try: data[aStand, count[aStand]*512:(count[aStand]+1)*512] = cFrame.payload.data # Update the counters so that we can average properly later on count[aStand] = count[aStand] + 1 except ValueError: pass if done: break else: data = numpy.abs(data)**2 data = data.astype(numpy.int32) clipFraction.append( numpy.zeros(antpols) ) meanPower.append( data.mean(axis=1) ) for j in xrange(antpols): bad = numpy.nonzero(data[j,:] > args.trim_level)[0] clipFraction[-1][j] = 1.0*len(bad) / data.shape[1] clip = clipFraction[-1] power = meanPower[-1] print("%2i | %6.2f%% %6.2f%% | %8.2f %8.2f |" % (i, clip[toUse[0]]*100.0, clip[toUse[1]]*100.0, power[toUse[0]], power[toUse[1]])) i += 1 fh.seek(tbn.FRAME_SIZE*chunkSkip, 1) clipFraction = numpy.array(clipFraction) meanPower = numpy.array(meanPower) clip = clipFraction.mean(axis=0) power = meanPower.mean(axis=0) print("---+-----------------+-------------------+") print("%2s | %6.2f%% %6.2f%% | %8.2f %8.2f |" % ('M', clip[toUse[0]]*100.0, clip[toUse[1]]*100.0, power[toUse[0]], power[toUse[1]]))
def main(args): # Set the station if args.metadata is not None: station = stations.parse_ssmif(args.metadata) else: station = stations.lwa1 antennas = station.antennas # Make sure that the file chunk size contains is an integer multiple # of the FFT length so that no data gets dropped maxFrames = int((30000 * 260)) # It seems like that would be a good idea, however... TBW data comes one # capture at a time so doing something like this actually truncates data # from the last set of stands for the first integration. So, we really # should stick with maxFrames = (30000 * 260) fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion antpols = len(antennas) nChunks = int(math.ceil(1.0 * nFrames / maxFrames)) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("Chunks: %i" % nChunks) print("===") nChunks = 1 # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2 * tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Master loop over all of the file chunks for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining print("Working on chunk %i, %i frames remaining" % ((i + 1), framesRemaining)) # # NOTE # Major change here from tbwSpectra.py/stationMaster.py. We are only keeping # the first 30,000 frames of the TBW file since we don't really need to read # in all of it to find bursts # data = numpy.zeros((antpols, 30000 * nSamples), dtype=numpy.int16) # Inner loop that actually reads the frames into the data array for j in range(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell()) // tbw.FRAME_SIZE - 1)) continue if not cFrame.header.is_tbw: continue # Skip frames over 30,000 on all stands if cFrame.header.frame_count > 30000: continue stand = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) if cFrame.header.frame_count % 5000 == 0 and args.verbose: print("%3i -> %3i %6.3f %5i %i" % (stand, aStand, cFrame.time, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers count = cFrame.header.frame_count - 1 data[aStand, count * nSamples:(count + 1) * nSamples] = cFrame.payload.data[0, :] data[aStand + 1, count * nSamples:(count + 1) * nSamples] = cFrame.payload.data[1, :] # Compute the power data = numpy.abs(data) # We need to various time series data to be aligned so we need to do a # correction for the cable delays. Using the various Antenna instances, # we create a array of delays (in seconds) and do everything relative to # the longest delay. # # After this, we can align the data from all of the antennas. delays = numpy.array([a.cable.delay(30e6) for a in antennas]) delays = numpy.round(delays * fS).astype(numpy.int32) delays = delays.max() - delays alignedData = numpy.zeros( (data.shape[0], data.shape[1] - delays.max()), dtype=data.dtype) for s in xrange(data.shape[0]): alignedData[s, :] = data[s, delays[s]:(delays[s] + alignedData.shape[1])] del (data) # Using the good antennas (Antenna.combined_status == 33), we need to find the # start of the RFI pulses by looking for "large" data values. To make sure # that we aren't getting stuck on a first partial burst, skip the first one # and use the second set of "large" data values found. # # I was using "large" as saturation/clipping (>= 2047), but the new lower # value for the ARX gain makes me want to lower to something more like # inOne = False first = 0 status = numpy.array([ant.combined_status for ant in antennas]) good = numpy.where(status == 33)[0] while first < alignedData.shape[1]: mv = alignedData[good, first].max() if mv >= args.threshold: if not inOne: first += 5000 inOne = True else: break else: first += 1 print("Second burst at %i" % first) # Keep only what would be interesting (200 samples before and 2,800 samples # afterward) around the burst. This corresponds to a time range from 1 # microsecond before the start of the pulse to 14 microseconds later. Save # the aligned data snippet to a NPZ file. alignedData = alignedData[:, first - 200:first + 2800] standPos = numpy.array([[ant.stand.x, ant.stand.y, ant.stand.z] for ant in antennas]) junk, basename = os.path.split(args.filename) shortname, ext = os.path.splitext(basename) numpy.savez('%s-burst.npz' % shortname, data=alignedData, ssmif=args.metadata) # Make the movie (if needed) if args.movie: if args.verbose: print("Creating movie frames") pb = ProgressBar(max=alignedData.shape[1] / 2) else: pb = None fig = plt.figure(figsize=(12, 6)) for i in xrange(0, alignedData.shape[1], 2): fig.clf() axX = fig.add_subplot(1, 2, 1) axY = fig.add_subplot(1, 2, 2) colorsX = 1.0 * (alignedData[0::2, i] + alignedData[0::2, i + 1]) / 2 colorsY = 1.0 * (alignedData[1::2, i] + alignedData[1::2, i + 1]) / 2 axX.scatter(standPos[0::2, 0], standPos[0::2, 1], c=colorsX, s=40.0, vmin=0, vmax=args.threshold) axY.scatter(standPos[1::2, 0], standPos[1::2, 1], c=colorsY, s=40.0, vmin=0, vmax=args.threshold) ## Add the fence as a dashed line axX.plot([-59.827, 59.771, 60.148, -59.700, -59.827], [59.752, 59.864, -59.618, -59.948, 59.752], linestyle='--', color='k') axY.plot([-59.827, 59.771, 60.148, -59.700, -59.827], [59.752, 59.864, -59.618, -59.948, 59.752], linestyle='--', color='k') ## Add the shelter axX.plot([55.863, 58.144, 58.062, 55.791, 55.863], [45.946, 45.999, 51.849, 51.838, 45.946], linestyle='-', color='k') axY.plot([55.863, 58.144, 58.062, 55.791, 55.863], [45.946, 45.999, 51.849, 51.838, 45.946], linestyle='-', color='k') axX.set_xlim([-65, 65]) axX.set_ylim([-65, 65]) axX.set_title('X pol. $\Delta$t = %.1f ns' % (1.0 * i / fS * 1e9, )) axX.set_xlabel('$\Delta$ X [m]') axX.set_ylabel('$\Delta$ Y [m]') axY.set_xlim([-65, 65]) axY.set_ylim([-65, 65]) axY.set_title('Y pol.') axY.set_xlabel('$\Delta$ X [m]') #axY.set_ylabel('$\Delta$ Y [m]') fig.savefig('burst-%05i.png' % (i / 2, )) if pb is not None: pb.inc(amount=1) if pb.amount != 0 and pb.amount % 10 == 0: sys.stdout.write(pb.show() + '\r') sys.stdout.flush() del (fig) if pb is not None: sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() if args.verbose: print("Creating movie") os.system( "mencoder mf://burst-*.png -mf fps=20:type=png -ovc lavc -lavcopts vcodec=mpeg4:aspect=2/1 -o %s-burst.avi -ffourcc DX50 -vf scale=600:1200,expand=600:1200" % shortname)
def main(args): # The task at hand filename = args.filename # The station if args.metadata is not None: site = parse_ssmif(args.metadata) ssmifContents = open(args.metadata).readlines() else: site = lwa1 ssmifContents = open(os.path.join(dataPath, 'lwa1-ssmif.txt')).readlines() observer = site.get_observer() antennas = site.antennas # The file's parameters fh = open(filename, 'rb') nFramesFile = os.path.getsize(filename) // tbn.FRAME_SIZE srate = tbn.get_sample_rate(fh) antpols = len(antennas) fh.seek(0) if srate < 1000: fh.seek(len(antennas) * 4 * tbn.FRAME_SIZE) srate = tbn.get_sample_rate(fh) antpols = len(antennas) fh.seek(len(antennas) * 4 * tbn.FRAME_SIZE) # Reference antenna ref = args.reference foundRef = False for i, a in enumerate(antennas): if a.stand.id == ref and a.pol == 0: refX = i foundRef = True elif a.stand.id == ref and a.pol == 1: refY = i else: pass if not foundRef: raise RuntimeError("Cannot file Stand #%i" % ref) # Integration time (seconds and frames) tInt = args.average nFrames = int(round(tInt * srate / 512 * antpols)) tInt = nFrames / antpols * 512 / srate # Total run length nChunks = int(1.0 * nFramesFile / antpols * 512 / srate / tInt) # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbn.read_frame(fh) fh.seek(-tbn.FRAME_SIZE, 1) startFC = junkFrame.header.frame_count try: central_freq = junkFrame.central_freq except AttributeError: from lsl.common.dp import fS central_freq = fS * junkFrame.header.second_count / 2**32 beginDate = junkFrame.time.datetime observer.date = beginDate srcs = [ ephem.Sun(), ] for line in _srcs: srcs.append(ephem.readdb(line)) for i in xrange(len(srcs)): srcs[i].compute(observer) if srcs[i].alt > 0: print("source %s: alt %.1f degrees, az %.1f degrees" % (srcs[i].name, srcs[i].alt * 180 / numpy.pi, srcs[i].az * 180 / numpy.pi)) # File summary print("Filename: %s" % filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz" % central_freq) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate)) print("---") print("Integration: %.3f s (%i frames; %i frames per stand/pol)" % (tInt, nFrames, nFrames // antpols)) print("Chunks: %i" % nChunks) # Create the FrameBuffer instance buffer = TBNFrameBuffer(stands=range(1, antpols // 2 + 1), pols=[0, 1]) # Create the phase average and times LFFT = 512 times = numpy.zeros(nChunks, dtype=numpy.float64) simpleVis = numpy.zeros((nChunks, antpols), dtype=numpy.complex64) central_freqs = numpy.zeros(nChunks, dtype=numpy.float64) # Go! k = 0 for i in xrange(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFramesFile - k if framesRemaining > nFrames: framesWork = nFrames data = numpy.zeros((antpols, framesWork // antpols * 512), dtype=numpy.complex64) else: framesWork = framesRemaining + antpols * buffer.nsegments data = numpy.zeros((antpols, framesWork // antpols * 512), dtype=numpy.complex64) print("Working on chunk %i, %i frames remaining" % (i + 1, framesRemaining)) count = [0 for a in xrange(len(antennas))] j = 0 fillsWork = framesWork // antpols # Inner loop that actually reads the frames into the data array done = False while j < fillsWork: cFrames = deque() for l in xrange(len(antennas)): try: cFrames.append(tbn.read_frame(fh)) k = k + 1 except errors.EOFError: ## Exit at the EOF done = True break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbn.FRAME_SIZE-1)) ## Exit at the first sync error done = True break buffer.append(cFrames) cFrames = buffer.get() if cFrames is None: continue for cFrame in cFrames: stand, pol = cFrame.header.id # In the current configuration, stands start at 1 and go up to 260. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) + pol # Save the time if j == 0 and aStand == 0: times[i] = cFrame.time try: central_freqs[i] = cFrame.central_freq except AttributeError: central_freqs[ i] = fS * cFrame.header.second_count / 2**32 if i > 0: if central_freqs[i] != central_freqs[i - 1]: print( "Frequency change from %.3f to %.3f MHz at chunk %i" % (central_freqs[i - 1] / 1e6, central_freqs[i] / 1e6, i + 1)) data[aStand, count[aStand] * 512:(count[aStand] + 1) * 512] = cFrame.payload.data # Update the counters so that we can average properly later on count[aStand] = count[aStand] + 1 j += 1 if done: break if done: break # Time-domain blanking and cross-correlation with the outlier simpleVis[i, :] = fringe.Simple(data, refX, refY, args.clip) fh.close() # Save the data outname = os.path.split(filename)[1] outname = os.path.splitext(outname)[0] outname = "%s-ref%03i-multi-vis.npz" % (outname, args.reference) numpy.savez(outname, ref=ref, refX=refX, refY=refY, tInt=tInt, central_freqs=central_freqs, times=times, simpleVis=simpleVis, ssmifContents=ssmifContents)
def main(args): # Set the station if args.metadata is not None: station = stations.parse_ssmif(args.metadata) ssmifContents = open(args.metadata).readlines() else: station = stations.lwa1 ssmifContents = open(os.path.join(dataPath, 'lwa1-ssmif.txt')).readlines() antennas = station.antennas # Length of the FFT LFFT = args.fft_length # Make sure that the file chunk size contains is an integer multiple # of the FFT length so that no data gets dropped maxFrames = int((30000 * 260) / float(LFFT)) * LFFT # It seems like that would be a good idea, however... TBW data comes one # capture at a time so doing something like this actually truncates data # from the last set of stands for the first integration. So, we really # should stick with maxFrames = (30000 * 260) fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion antpols = len(antennas) nChunks = int(math.ceil(1.0 * nFrames / maxFrames)) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("Chunks: %i" % nChunks) print("===") nChunks = 1 # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2 * tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Setup the window function to use if args.pfb: window = fxc.null_window elif args.bartlett: window = numpy.bartlett elif args.blackman: window = numpy.blackman elif args.hanning: window = numpy.hanning else: window = fxc.null_window base, ext = os.path.splitext(args.filename) base = os.path.basename(base) if (not os.path.exists("%s.npz" % base)) or args.force: # Master loop over all of the file chunks masterSpectra = numpy.zeros((nChunks, antpols, LFFT)) for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining print("Working on chunk %i, %i frames remaining" % ((i + 1), framesRemaining)) data = numpy.memmap('temp.mmap', dtype=numpy.int16, mode='w+', shape=(antpols, 2 * 30000 * 260 * nSamples // antpols)) # If there are fewer frames than we need to fill an FFT, skip this chunk if data.shape[1] < 2 * LFFT: break # Inner loop that actually reads the frames into the data array for j in range(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell()) // tbw.FRAME_SIZE - 1)) continue if not cFrame.header.is_tbw: continue stand = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) if cFrame.header.frame_count % 10000 == 0 and args.verbose: print("%3i -> %3i %6.3f %5i %i" % (stand, aStand, cFrame.time, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers count = cFrame.header.frame_count - 1 data[aStand, count * nSamples:(count + 1) * nSamples] = 1 * cFrame.payload.data[0, :] data[aStand + 1, count * nSamples:(count + 1) * nSamples] = 1 * cFrame.payload.data[1, :] del cFrame # Calculate the spectra for this block of data and then weight the results by # the total number of frames read. This is needed to keep the averages correct. # NB: The weighting is the same for the x and y polarizations because of how # the data are packed in TBW for j in xrange(0, masterSpectra.shape[1], 4): tempData = numpy.zeros((4, data.shape[1]), dtype=data.dtype) tempData = data[j:j + 4, :] freq, tempSpec = fxc.SpecMaster(tempData, LFFT=LFFT, window=window, verbose=args.verbose, clip_level=args.clip_level) masterSpectra[i, j:j + 4, :] = tempSpec # Compute the 1 ms average power and the data range within each 1 ms window subSize = 1960 nsegments = data.shape[1] // subSize print( "Computing average power and data range in %i-sample intervals" % subSize) pb = ProgressBar(max=data.shape[0]) avgPower = numpy.zeros((antpols, nsegments), dtype=numpy.float32) dataRange = numpy.zeros((antpols, nsegments, 3), dtype=numpy.int16) for s in xrange(data.shape[0]): for p in xrange(nsegments): subData = data[s, (p * subSize):((p + 1) * subSize)] avgPower[s, p] = numpy.mean(numpy.abs(subData)) dataRange[s, p, 0] = subData.min() dataRange[s, p, 1] = subData.mean() dataRange[s, p, 2] = subData.max() ### This little block here looks for likely saturation events and save ### the raw time series around them into individual NPZ files for stand ### number 14. #if (dataRange[s,p,0] < -1000 or dataRange[s,p,0] > 1000) and antennas[s].stand.id == 14: #subData = data[s,((p-1)*1960):((p+2)*1960)] #satFileName = 'stand-14-pol-%i-%i.npz' % (antennas[s].pol, (p-1)*1960) #print(satFileName) #numpy.savez(satFileName, start=(p-1)*1960, data=subData) pb.inc(amount=1) if pb.amount != 0 and pb.amount % 10 == 0: sys.stdout.write(pb.show() + '\r') sys.stdout.flush() sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() # We don't really need the data array anymore, so delete it del (data) os.unlink('temp.mmap') # Apply the cable loss corrections, if requested if True: for s in xrange(masterSpectra.shape[1]): currGain = antennas[s].cable.gain(freq) for c in xrange(masterSpectra.shape[0]): masterSpectra[c, s, :] /= currGain # Now that we have read through all of the chunks, perform the final averaging by # dividing by all of the chunks spec = masterSpectra.mean(axis=0) # Estimate the dipole resonance frequencies print("Computing dipole resonance frequencies") pb = ProgressBar(max=spec.shape[0]) resFreq = numpy.zeros(spec.shape[0]) toCompare = numpy.where((freq > 31e6) & (freq < 70e6))[0] for i in xrange(spec.shape[0]): bestOrder = 0 bestRMS = 1e34 for j in xrange(3, 12): coeff = numpy.polyfit(freq[toCompare] / 1e6, numpy.log10(spec[i, toCompare]) * 10, j) fit = numpy.polyval(coeff, freq[toCompare] / 1e6) rms = ((fit - numpy.log10(spec[i, toCompare]) * 10)**2).sum() if rms < bestRMS: bestOrder = j bestRMS = rms coeff = numpy.polyfit(freq[toCompare] / 1e6, numpy.log10(spec[i, toCompare]) * 10, bestOrder) fit = numpy.polyval(coeff, freq[toCompare] / 1e6) try: resFreq[i] = freq[toCompare[numpy.where( fit == fit.max())[0][0]]] / 1e6 except: pass pb.inc(amount=1) if pb.amount != 0 and pb.amount % 10 == 0: sys.stdout.write(pb.show() + '\r') sys.stdout.flush() sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() numpy.savez("%s.npz" % base, date=str(beginDate), freq=freq, masterSpectra=masterSpectra, resFreq=resFreq, avgPower=avgPower, dataRange=dataRange, ssmifContents=ssmifContents) else: dataDict = numpy.load("%s.npz" % base) freq = dataDict['freq'] masterSpectra = dataDict['masterSpectra'] resFreq = dataDict['resFreq'] # Now that we have read through all of the chunks, perform the final averaging by # dividing by all of the chunks spec = masterSpectra.mean(axis=0) # Create a good template spectra specTemplate = numpy.median(spec, axis=0) specDiff = numpy.zeros(spec.shape[0]) toCompare = numpy.where((freq > 32e6) & (freq < 50e6))[0] print(len(toCompare)) for i in xrange(spec.shape[0]): specDiff[i] = (spec[i, toCompare] / specTemplate[toCompare]).mean() specDiff = numpy.where(specDiff < 2, specDiff, 2) # Get the station standPos = numpy.array([[ant.stand.x, ant.stand.y, ant.stand.z] for ant in antennas if ant.pol == 0]) # Plots if args.verbose: fig = plt.figure() ax1 = fig.add_subplot(1, 2, 1) ax1.scatter(standPos[:, 0], standPos[:, 1], c=specDiff[0::2], s=40.0, alpha=0.50) ## Add the fence as a dashed line ax1.plot([-59.827, 59.771, 60.148, -59.700, -59.827], [59.752, 59.864, -59.618, -59.948, 59.752], linestyle='--', color='k') ## Add the shelter ax1.plot([55.863, 58.144, 58.062, 55.791, 55.863], [45.946, 45.999, 51.849, 51.838, 45.946], linestyle='-', color='k') ## Set the limits to just zoom in on the main stations ax1.set_xlim([-65, 65]) ax1.set_ylim([-65, 65]) ax2 = fig.add_subplot(1, 2, 2) ax2.plot(freq / 1e6, numpy.log10(specTemplate) * 10, alpha=0.50) print("RBW: %.1f Hz" % (freq[1] - freq[0])) plt.show()
def main(args): # Set the station if args.lwasv: station = stations.lwasv else: station = stations.lwa1 antennas = station.antennas fh = open(args.filename, "rb", buffering=tbn.FRAME_SIZE*10000) # Get the first frame and find out what the firt time tag is, which the # first frame number is, and what the sample rate it. From the sample # rate, estimate how the time tag should advance between frames. junkFrame = tbn.read_frame(fh) sample_rate = tbn.get_sample_rate(fh) antpols = len(antennas) tagSkip = fS // sample_rate * junkFrame.payload.data.shape[0] fh.seek(0) # Store the information about the first frame. prevTime = junkFrame.payload.timetag prevDate = junkFrame.time.datetime prevFrame = junkFrame.header.frame_count # Report on the file print("Filename: %s" % os.path.basename(args.filename)) print("Date of first frame: %i -> %s" % (prevTime, str(prevDate))) print("Sample rate: %i Hz" % sample_rate) print("Time tag skip per frame: %i" % tagSkip) # Create the FrameBuffer instance buffer = TBNFrameBuffer(stands=range(1,antpols//2+1), pols=[0, 1]) j = 0 k = 0 while True: try: cFrame = tbn.read_frame(fh) k += 1 except errors.EOFError: break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbn.FRAME_SIZE-1)) continue buffer.append(cFrame) cFrames = buffer.get() if cFrames is None: continue valid = reduce(lambda x,y: x+int(y.valid), cFrames, 0) if valid != antpols: print("WARNING: frame count %i at %i missing %.2f%% of frames" % (cFrames[0].header.frame_count, cFrames[0].payload.timetag, float(antpols - valid)/antpols*100)) timetags = numpy.zeros(len(cFrames), dtype=numpy.int64) - 1 for cFrame in cFrames: stand,pol = cFrame.id timetags[2*(stand-1)+pol] = cFrame.payload.timetag if j == 0: prevTime = numpy.median(timetags) prevDate = cFrames[0].time.datetime prevFrame = cFrames[0].header.frame_count j += 1 continue else: currTime = numpy.median(timetags) currDate = cFrames[0].time.datetime currFrame = cFrames[0].header.frame_count if currFrame % 1000 == 0: print("At frame %i t.t. is %i -> %s" % (currFrame, currTime, currDate)) if currTime < prevTime: print("ERROR: t.t. %i @ frame %i < t.t. %i @ frame %i" % (currTime, currFrame, prevTime, prevFrame)) print(" -> difference: %i (%.5f seconds); %s" % (currTime-prevTime, float(currTime-prevTime)/fS, str(currDate))) if (currTime-prevTime) > tagSkip: print("ERROR: t.t. %i @ frame %i > t.t. %i @ frame %i + skip" % (currTime, currFrame, prevTime, prevFrame)) print(" -> difference: %i (%.5f seconds); %s" % (currTime-prevTime, float(currTime-prevTime)/fS, str(currDate))) for i in xrange(timetags.size): if timetags[i] != currTime: print("ERROR: t.t. of dig. %i != frame set median of %i" % (i, currTime)) print(" -> difference: %i" % (currTime-timetags[i],)) prevTime = currTime prevData = currDate prevFrame = currFrame j += 1