def test_tbw_bits(self): """Test getting the data bits from a TBW file.""" fh = open(tbwFile, 'rb') # File contains 12-bit data, two ways self.assertEqual(tbw.get_data_bits(fh), 12) frame1 = tbw.read_frame(fh) self.assertEqual(frame1.data_bits, 12) fh.close()
def main(args): # Set the station if args.metadata is not None: station = stations.parse_ssmif(args.metadata) ssmifContents = open(args.metadata).readlines() else: station = stations.lwa1 ssmifContents = open(os.path.join(dataPath, 'lwa1-ssmif.txt')).readlines() antennas = station.antennas # Length of the FFT LFFT = args.fft_length # Make sure that the file chunk size contains is an integer multiple # of the FFT length so that no data gets dropped maxFrames = int((30000 * 260) / float(LFFT)) * LFFT # It seems like that would be a good idea, however... TBW data comes one # capture at a time so doing something like this actually truncates data # from the last set of stands for the first integration. So, we really # should stick with maxFrames = (30000 * 260) fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion antpols = len(antennas) nChunks = int(math.ceil(1.0 * nFrames / maxFrames)) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("Chunks: %i" % nChunks) print("===") nChunks = 1 # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2 * tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Setup the window function to use if args.pfb: window = fxc.null_window elif args.bartlett: window = numpy.bartlett elif args.blackman: window = numpy.blackman elif args.hanning: window = numpy.hanning else: window = fxc.null_window base, ext = os.path.splitext(args.filename) base = os.path.basename(base) if (not os.path.exists("%s.npz" % base)) or args.force: # Master loop over all of the file chunks masterSpectra = numpy.zeros((nChunks, antpols, LFFT)) for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining print("Working on chunk %i, %i frames remaining" % ((i + 1), framesRemaining)) data = numpy.memmap('temp.mmap', dtype=numpy.int16, mode='w+', shape=(antpols, 2 * 30000 * 260 * nSamples // antpols)) # If there are fewer frames than we need to fill an FFT, skip this chunk if data.shape[1] < 2 * LFFT: break # Inner loop that actually reads the frames into the data array for j in range(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell()) // tbw.FRAME_SIZE - 1)) continue if not cFrame.header.is_tbw: continue stand = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) if cFrame.header.frame_count % 10000 == 0 and args.verbose: print("%3i -> %3i %6.3f %5i %i" % (stand, aStand, cFrame.time, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers count = cFrame.header.frame_count - 1 data[aStand, count * nSamples:(count + 1) * nSamples] = 1 * cFrame.payload.data[0, :] data[aStand + 1, count * nSamples:(count + 1) * nSamples] = 1 * cFrame.payload.data[1, :] del cFrame # Calculate the spectra for this block of data and then weight the results by # the total number of frames read. This is needed to keep the averages correct. # NB: The weighting is the same for the x and y polarizations because of how # the data are packed in TBW for j in xrange(0, masterSpectra.shape[1], 4): tempData = numpy.zeros((4, data.shape[1]), dtype=data.dtype) tempData = data[j:j + 4, :] freq, tempSpec = fxc.SpecMaster(tempData, LFFT=LFFT, window=window, verbose=args.verbose, clip_level=args.clip_level) masterSpectra[i, j:j + 4, :] = tempSpec # Compute the 1 ms average power and the data range within each 1 ms window subSize = 1960 nsegments = data.shape[1] // subSize print( "Computing average power and data range in %i-sample intervals" % subSize) pb = ProgressBar(max=data.shape[0]) avgPower = numpy.zeros((antpols, nsegments), dtype=numpy.float32) dataRange = numpy.zeros((antpols, nsegments, 3), dtype=numpy.int16) for s in xrange(data.shape[0]): for p in xrange(nsegments): subData = data[s, (p * subSize):((p + 1) * subSize)] avgPower[s, p] = numpy.mean(numpy.abs(subData)) dataRange[s, p, 0] = subData.min() dataRange[s, p, 1] = subData.mean() dataRange[s, p, 2] = subData.max() ### This little block here looks for likely saturation events and save ### the raw time series around them into individual NPZ files for stand ### number 14. #if (dataRange[s,p,0] < -1000 or dataRange[s,p,0] > 1000) and antennas[s].stand.id == 14: #subData = data[s,((p-1)*1960):((p+2)*1960)] #satFileName = 'stand-14-pol-%i-%i.npz' % (antennas[s].pol, (p-1)*1960) #print(satFileName) #numpy.savez(satFileName, start=(p-1)*1960, data=subData) pb.inc(amount=1) if pb.amount != 0 and pb.amount % 10 == 0: sys.stdout.write(pb.show() + '\r') sys.stdout.flush() sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() # We don't really need the data array anymore, so delete it del (data) os.unlink('temp.mmap') # Apply the cable loss corrections, if requested if True: for s in xrange(masterSpectra.shape[1]): currGain = antennas[s].cable.gain(freq) for c in xrange(masterSpectra.shape[0]): masterSpectra[c, s, :] /= currGain # Now that we have read through all of the chunks, perform the final averaging by # dividing by all of the chunks spec = masterSpectra.mean(axis=0) # Estimate the dipole resonance frequencies print("Computing dipole resonance frequencies") pb = ProgressBar(max=spec.shape[0]) resFreq = numpy.zeros(spec.shape[0]) toCompare = numpy.where((freq > 31e6) & (freq < 70e6))[0] for i in xrange(spec.shape[0]): bestOrder = 0 bestRMS = 1e34 for j in xrange(3, 12): coeff = numpy.polyfit(freq[toCompare] / 1e6, numpy.log10(spec[i, toCompare]) * 10, j) fit = numpy.polyval(coeff, freq[toCompare] / 1e6) rms = ((fit - numpy.log10(spec[i, toCompare]) * 10)**2).sum() if rms < bestRMS: bestOrder = j bestRMS = rms coeff = numpy.polyfit(freq[toCompare] / 1e6, numpy.log10(spec[i, toCompare]) * 10, bestOrder) fit = numpy.polyval(coeff, freq[toCompare] / 1e6) try: resFreq[i] = freq[toCompare[numpy.where( fit == fit.max())[0][0]]] / 1e6 except: pass pb.inc(amount=1) if pb.amount != 0 and pb.amount % 10 == 0: sys.stdout.write(pb.show() + '\r') sys.stdout.flush() sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() numpy.savez("%s.npz" % base, date=str(beginDate), freq=freq, masterSpectra=masterSpectra, resFreq=resFreq, avgPower=avgPower, dataRange=dataRange, ssmifContents=ssmifContents) else: dataDict = numpy.load("%s.npz" % base) freq = dataDict['freq'] masterSpectra = dataDict['masterSpectra'] resFreq = dataDict['resFreq'] # Now that we have read through all of the chunks, perform the final averaging by # dividing by all of the chunks spec = masterSpectra.mean(axis=0) # Create a good template spectra specTemplate = numpy.median(spec, axis=0) specDiff = numpy.zeros(spec.shape[0]) toCompare = numpy.where((freq > 32e6) & (freq < 50e6))[0] print(len(toCompare)) for i in xrange(spec.shape[0]): specDiff[i] = (spec[i, toCompare] / specTemplate[toCompare]).mean() specDiff = numpy.where(specDiff < 2, specDiff, 2) # Get the station standPos = numpy.array([[ant.stand.x, ant.stand.y, ant.stand.z] for ant in antennas if ant.pol == 0]) # Plots if args.verbose: fig = plt.figure() ax1 = fig.add_subplot(1, 2, 1) ax1.scatter(standPos[:, 0], standPos[:, 1], c=specDiff[0::2], s=40.0, alpha=0.50) ## Add the fence as a dashed line ax1.plot([-59.827, 59.771, 60.148, -59.700, -59.827], [59.752, 59.864, -59.618, -59.948, 59.752], linestyle='--', color='k') ## Add the shelter ax1.plot([55.863, 58.144, 58.062, 55.791, 55.863], [45.946, 45.999, 51.849, 51.838, 45.946], linestyle='-', color='k') ## Set the limits to just zoom in on the main stations ax1.set_xlim([-65, 65]) ax1.set_ylim([-65, 65]) ax2 = fig.add_subplot(1, 2, 2) ax2.plot(freq / 1e6, numpy.log10(specTemplate) * 10, alpha=0.50) print("RBW: %.1f Hz" % (freq[1] - freq[0])) plt.show()
def main(args): # Set the station if args.metadata is not None: station = stations.parse_ssmif(args.metadata) ssmifContents = open(args.metadata).readlines() else: station = stations.lwa1 ssmifContents = open(os.path.join(dataPath, 'lwa1-ssmif.txt')).readlines() antennas = station.antennas toKeep = [] for g in (1, 10, 54, 248, 251, 258): for i, ant in enumerate(antennas): if ant.stand.id == g and ant.pol == 0: toKeep.append(i) for i, j in enumerate(toKeep): print(i, j, antennas[j].stand.id) # Length of the FFT LFFT = args.fft_length # Make sure that the file chunk size contains is an integer multiple # of the FFT length so that no data gets dropped maxFrames = int((30000 * 260) / float(LFFT)) * LFFT # It seems like that would be a good idea, however... TBW data comes one # capture at a time so doing something like this actually truncates data # from the last set of stands for the first integration. So, we really # should stick with maxFrames = (30000 * 260) fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion antpols = len(antennas) nChunks = int(math.ceil(1.0 * nFrames / maxFrames)) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginTime = junkFrame.time beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("Chunks: %i" % nChunks) print("===") nChunks = 1 # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2 * tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Master loop over all of the file chunks masterSpectra = numpy.zeros((nChunks, antpols, LFFT)) for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining print("Working on chunk %i, %i frames remaining" % ((i + 1), framesRemaining)) data = numpy.zeros((12, 12000000), dtype=numpy.int16) # If there are fewer frames than we need to fill an FFT, skip this chunk if data.shape[1] < 2 * LFFT: break # Inner loop that actually reads the frames into the data array for j in range(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbw.FRAME_SIZE-1)) continue if not cFrame.header.is_tbw: continue stand = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) #if cFrame.header.frame_count % 10000 == 0 and config['verbose']: #print("%3i -> %3i %6.3f %5i %i" % (stand, aStand, cFrame.time, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers count = cFrame.header.frame_count - 1 if aStand not in toKeep: continue # Convert to reduced index aStand = 2 * toKeep.index(aStand) data[aStand, count * nSamples:(count + 1) * nSamples] = cFrame.payload.data[0, :] data[aStand + 1, count * nSamples:(count + 1) * nSamples] = cFrame.payload.data[1, :] # Time series analysis - mean, std. dev, saturation count tsMean = data.mean(axis=1) tsStd = data.std(axis=1) tsSat = numpy.where((data == 2047) | (data == -2047), 1, 0).sum(axis=1) # Time series analysis - percentiles p = [50, 75, 90, 95, 99] tsPct = numpy.zeros((data.shape[0], len(p))) for i in xrange(len(p)): for j in xrange(data.shape[0]): tsPct[j, i] = percentile(numpy.abs(data[j, :]), p[i]) # Frequency domain analysis - spectra freq = numpy.fft.fftfreq(2 * args.fft_length, d=1.0 / 196e6) freq = freq[:args.fft_length] delays = numpy.zeros((data.shape[0], freq.size)) signalsF, validF = FEngine(data, freq, delays, LFFT=args.fft_length, Overlap=1, sample_rate=196e6, clip_level=0) # Cleanup to save memory del validF, data print(signalsF.shape) # SK control values skM = signalsF.shape[2] skN = 1 # Frequency domain analysis - spectral kurtosis k = numpy.zeros((signalsF.shape[0], signalsF.shape[1])) for l in xrange(signalsF.shape[0]): for m in xrange(freq.size): k[l, m] = kurtosis.spectral_fft(signalsF[l, m, :]) kl, kh = kurtosis.get_limits(4, skM, skN) print(kl, kh) # Integrate the spectra for as long as we can masterSpectra = (numpy.abs(signalsF)**2).mean(axis=2) del signalsF # Mask out bad values (high spectral kurtosis) for the plot mask = numpy.where((k < kl) | (k > kh), 1, 0) mask = expandMask(mask, radius=4, merge=True) masterSpectra = numpy.ma.array(masterSpectra, mask=mask) # Save the data to an HDF5 file outname = os.path.splitext(args.filename)[0] outname = "%s-RFI.hdf5" % outname f = h5py.File(outname, 'w') f.attrs['filename'] = args.filename f.attrs['mode'] = 'TBW' f.attrs['station'] = 'LWA-1' f.attrs['dataBits'] = dataBits f.attrs['startTime'] = beginTime f.attrs['startTime_units'] = 's' f.attrs['startTime_sys'] = 'unix' f.attrs['sample_rate'] = 196e6 f.attrs['sample_rate_units'] = 'Hz' f.attrs['RBW'] = freq[1] - freq[0] f.attrs['RBW_Units'] = 'Hz' f.attrs['SK-M'] = skM f.attrs['SK-N'] = skN for l in xrange(len(toKeep)): antX = antennas[toKeep[l]] antY = antennas[toKeep[l] + 1] stand = f.create_group('Stand%03i' % antX.stand.id) stand['freq'] = freq stand['freq'].attrs['Units'] = 'Hz' polX = stand.create_group('X') polY = stand.create_group('Y') polX.attrs['tsMean'] = tsMean[2 * l] polY.attrs['tsMean'] = tsMean[2 * l + 1] polX.attrs['tsStd'] = tsStd[2 * l] polY.attrs['tsStd'] = tsStd[2 * l + 1] polX.attrs['tsSat'] = tsSat[2 * l] polY.attrs['tsSat'] = tsSat[2 * l + 1] for i, v in enumerate(p): polX.attrs['ts%02i' % v] = tsPct[2 * l][i] polY.attrs['ts%02i' % v] = tsPct[2 * l + 1][i] polX['spectrum'] = masterSpectra[2 * l, :] polX['spectrum'].attrs['axis0'] = 'frequency' polY['spectrum'] = masterSpectra[2 * l + 1, :] polY['spectrum'].attrs['axis0'] = 'frequency' polX['kurtosis'] = k[2 * l, :] polX['kurtosis'].attrs['axis0'] = 'frequency' polY['kurtosis'] = k[2 * l + 1, :] polY['kurtosis'].attrs['axis0'] = 'frequency' # The plot fig = plt.figure() ax1 = fig.add_subplot(2, 1, 1) ax2 = fig.add_subplot(2, 1, 2) for l in xrange(k.shape[0]): ant = antennas[toKeep[l / 2]] ax1.plot(freq / 1e6, numpy.log10(masterSpectra[l, :]) * 10, label='Stand %i, Pol %i' % (ant.stand.id, ant.pol + l % 2)) ax2.plot(freq / 1e6, k[l, :], label='Stand %i, Pol %i' % (ant.stand.id, ant.pol + l % 2)) ax2.hlines(kl, freq[0] / 1e6, freq[-1] / 1e6, linestyle=':', label='Kurtosis Limit 4$\sigma$') ax2.hlines(kh, freq[0] / 1e6, freq[-1] / 1e6, linestyle=':', label='Kurtosis Limit 4$\sigma$') ax1.set_xlabel('Frequency [MHz]') ax1.set_ylabel('PSD [arb. dB/RBW]') ax1.legend(loc=0) ax2.set_ylim((kl / 2, kh * 2)) ax2.set_xlabel('Frequency [MHz]') ax2.set_ylabel('Spectral Kurtosis') ax2.legend(loc=0) plt.show()
def main(args): filename = args[0] stands = [int(i) for i in args[1:]] antennas = lwa1.antennas fh = open(filename, "rb") nFrames = os.path.getsize(filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion antpols = len(antennas) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginTime = junkFrame.time beginDate = junkFrame.time.datetime # Figure out which digitizers to keep toKeep = [] for a in antennas: if a.stand.id in stands: toKeep.append( a.digitizer ) # File summary print("Filename: %s" % filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("===") print("Keeping Stands:") for a in toKeep: print(" Stand #%3i, pol %i (digitizer %3i)" % (antennas[a-1].stand.id, antennas[a-1].pol, antennas[a-1].digitizer)) # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2*tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Create the HDF5 file outname = os.path.splitext(filename)[0] outname = "%shdf5" % outname f = h5py.File(outname, 'w') f.attrs['filename'] = filename f.attrs['mode'] = 'TBW' f.attrs['station'] = 'LWA-1' f.attrs['dataBits'] = dataBits f.attrs['startTime'] = beginTime f.attrs['startTime_units'] = 's' f.attrs['startTime_sys'] = 'unix' f.attrs['sample_rate'] = 196e6 f.attrs['sample_rate_units'] = 'Hz' ## Create the digitizer to dataset lookup table and the standLookup = {} standData = [] i = 0 for a in toKeep: if a % 2 == 0: continue s = antennas[a-1].stand.id standLookup[a] = i temp = f.create_group('Stand%03i' % s) ### Combined status code temp.attrs['statusCode'] = antennas[a-1].combined_status ### Antenna number temp.attrs['antennaID'] = antennas[a-1].id ### Cable information temp.attrs['cableID'] = antennas[a-1].cable.id temp.attrs['cableLength'] = antennas[a-1].cable.length temp.attrs['cableLength_units'] = 'm' ### Stand location information temp.attrs['posX'] = antennas[a-1].stand.x temp.attrs['posX_units'] = 'm' temp.attrs['posY'] = antennas[a-1].stand.y temp.attrs['posY_units'] = 'm' temp.attrs['posZ'] = antennas[a-1].stand.z temp.attrs['posZ_units'] = 'm' ### Time series data sets temp.attrs['axis0'] = 'time' xpol = temp.create_dataset('X', (12000000,), 'i2', chunks=True) ypol = temp.create_dataset('Y', (12000000,), 'i2', chunks=True) standData.append( (xpol, ypol, temp) ) i += 1 # Go! while True: # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/tbw.FRAME_SIZE-1)) continue if not cFrame.header.is_tbw: continue # Get the DP "stand" ID and the digitizer number stand = cFrame.header.id aStand = 2*(stand-1) digitizer = aStand + 1 # If we don't need it, skip it if digitizer not in toKeep: continue # Actually load the data. ## Frame count count = cFrame.header.frame_count - 1 ## Which data set dataset = standLookup[digitizer] ## Load standData[dataset][0][count*nSamples:(count+1)*nSamples] = cFrame.payload.data[0,:] standData[dataset][1][count*nSamples:(count+1)*nSamples] = cFrame.payload.data[1,:] fh.close() f.close()
def main(args): # Set the station if args.metadata is not None: station = stations.parse_ssmif(args.metadata) else: station = stations.lwa1 antennas = station.antennas # Make sure that the file chunk size contains is an integer multiple # of the FFT length so that no data gets dropped maxFrames = int((30000 * 260)) # It seems like that would be a good idea, however... TBW data comes one # capture at a time so doing something like this actually truncates data # from the last set of stands for the first integration. So, we really # should stick with maxFrames = (30000 * 260) fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion antpols = len(antennas) nChunks = int(math.ceil(1.0 * nFrames / maxFrames)) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("Chunks: %i" % nChunks) print("===") nChunks = 1 # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2 * tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Master loop over all of the file chunks for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining print("Working on chunk %i, %i frames remaining" % ((i + 1), framesRemaining)) # # NOTE # Major change here from tbwSpectra.py/stationMaster.py. We are only keeping # the first 30,000 frames of the TBW file since we don't really need to read # in all of it to find bursts # data = numpy.zeros((antpols, 30000 * nSamples), dtype=numpy.int16) # Inner loop that actually reads the frames into the data array for j in range(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell()) // tbw.FRAME_SIZE - 1)) continue if not cFrame.header.is_tbw: continue # Skip frames over 30,000 on all stands if cFrame.header.frame_count > 30000: continue stand = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) if cFrame.header.frame_count % 5000 == 0 and args.verbose: print("%3i -> %3i %6.3f %5i %i" % (stand, aStand, cFrame.time, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers count = cFrame.header.frame_count - 1 data[aStand, count * nSamples:(count + 1) * nSamples] = cFrame.payload.data[0, :] data[aStand + 1, count * nSamples:(count + 1) * nSamples] = cFrame.payload.data[1, :] # Compute the power data = numpy.abs(data) # We need to various time series data to be aligned so we need to do a # correction for the cable delays. Using the various Antenna instances, # we create a array of delays (in seconds) and do everything relative to # the longest delay. # # After this, we can align the data from all of the antennas. delays = numpy.array([a.cable.delay(30e6) for a in antennas]) delays = numpy.round(delays * fS).astype(numpy.int32) delays = delays.max() - delays alignedData = numpy.zeros( (data.shape[0], data.shape[1] - delays.max()), dtype=data.dtype) for s in xrange(data.shape[0]): alignedData[s, :] = data[s, delays[s]:(delays[s] + alignedData.shape[1])] del (data) # Using the good antennas (Antenna.combined_status == 33), we need to find the # start of the RFI pulses by looking for "large" data values. To make sure # that we aren't getting stuck on a first partial burst, skip the first one # and use the second set of "large" data values found. # # I was using "large" as saturation/clipping (>= 2047), but the new lower # value for the ARX gain makes me want to lower to something more like # inOne = False first = 0 status = numpy.array([ant.combined_status for ant in antennas]) good = numpy.where(status == 33)[0] while first < alignedData.shape[1]: mv = alignedData[good, first].max() if mv >= args.threshold: if not inOne: first += 5000 inOne = True else: break else: first += 1 print("Second burst at %i" % first) # Keep only what would be interesting (200 samples before and 2,800 samples # afterward) around the burst. This corresponds to a time range from 1 # microsecond before the start of the pulse to 14 microseconds later. Save # the aligned data snippet to a NPZ file. alignedData = alignedData[:, first - 200:first + 2800] standPos = numpy.array([[ant.stand.x, ant.stand.y, ant.stand.z] for ant in antennas]) junk, basename = os.path.split(args.filename) shortname, ext = os.path.splitext(basename) numpy.savez('%s-burst.npz' % shortname, data=alignedData, ssmif=args.metadata) # Make the movie (if needed) if args.movie: if args.verbose: print("Creating movie frames") pb = ProgressBar(max=alignedData.shape[1] / 2) else: pb = None fig = plt.figure(figsize=(12, 6)) for i in xrange(0, alignedData.shape[1], 2): fig.clf() axX = fig.add_subplot(1, 2, 1) axY = fig.add_subplot(1, 2, 2) colorsX = 1.0 * (alignedData[0::2, i] + alignedData[0::2, i + 1]) / 2 colorsY = 1.0 * (alignedData[1::2, i] + alignedData[1::2, i + 1]) / 2 axX.scatter(standPos[0::2, 0], standPos[0::2, 1], c=colorsX, s=40.0, vmin=0, vmax=args.threshold) axY.scatter(standPos[1::2, 0], standPos[1::2, 1], c=colorsY, s=40.0, vmin=0, vmax=args.threshold) ## Add the fence as a dashed line axX.plot([-59.827, 59.771, 60.148, -59.700, -59.827], [59.752, 59.864, -59.618, -59.948, 59.752], linestyle='--', color='k') axY.plot([-59.827, 59.771, 60.148, -59.700, -59.827], [59.752, 59.864, -59.618, -59.948, 59.752], linestyle='--', color='k') ## Add the shelter axX.plot([55.863, 58.144, 58.062, 55.791, 55.863], [45.946, 45.999, 51.849, 51.838, 45.946], linestyle='-', color='k') axY.plot([55.863, 58.144, 58.062, 55.791, 55.863], [45.946, 45.999, 51.849, 51.838, 45.946], linestyle='-', color='k') axX.set_xlim([-65, 65]) axX.set_ylim([-65, 65]) axX.set_title('X pol. $\Delta$t = %.1f ns' % (1.0 * i / fS * 1e9, )) axX.set_xlabel('$\Delta$ X [m]') axX.set_ylabel('$\Delta$ Y [m]') axY.set_xlim([-65, 65]) axY.set_ylim([-65, 65]) axY.set_title('Y pol.') axY.set_xlabel('$\Delta$ X [m]') #axY.set_ylabel('$\Delta$ Y [m]') fig.savefig('burst-%05i.png' % (i / 2, )) if pb is not None: pb.inc(amount=1) if pb.amount != 0 and pb.amount % 10 == 0: sys.stdout.write(pb.show() + '\r') sys.stdout.flush() del (fig) if pb is not None: sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() if args.verbose: print("Creating movie") os.system( "mencoder mf://burst-*.png -mf fps=20:type=png -ovc lavc -lavcopts vcodec=mpeg4:aspect=2/1 -o %s-burst.avi -ffourcc DX50 -vf scale=600:1200,expand=600:1200" % shortname)
def main(args): # Set the station station = stations.lwa1 antennas = station.antennas fh = open(args.filename, "rb") nFrames = os.path.getsize(args.filename) // tbw.FRAME_SIZE dataBits = tbw.get_data_bits(fh) # The number of ant/pols in the file is hard coded because I cannot figure out # a way to get this number in a systematic fashion maxFrames = 30000 * 260 antpols = len(antennas) nChunks = int(math.ceil(1.0 * nFrames / maxFrames)) if dataBits == 12: nSamples = 400 else: nSamples = 1200 # Read in the first frame and get the date/time of the first sample # of the frame. This is needed to get the list of stands. junkFrame = tbw.read_frame(fh) fh.seek(0) beginDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Ant/Pols: %i" % antpols) print("Sample Length: %i-bit" % dataBits) print("Frames: %i" % nFrames) print("Chunks: %i" % nChunks) print("===") nChunks = 1 # Skip over any non-TBW frames at the beginning of the file i = 0 junkFrame = tbw.read_frame(fh) while not junkFrame.header.is_tbw: try: junkFrame = tbw.read_frame(fh) except errors.SyncError: fh.seek(0) while True: try: junkFrame = tbn.read_frame(fh) i += 1 except errors.SyncError: break fh.seek(-2 * tbn.FRAME_SIZE, 1) junkFrame = tbw.read_frame(fh) i += 1 fh.seek(-tbw.FRAME_SIZE, 1) print("Skipped %i non-TBW frames at the beginning of the file" % i) # Master loop over all of the file chunks timetags = numpy.zeros((antpols, 30000), dtype=numpy.int64) - 1 for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = nFrames print("Working on chunk %i, %i frames remaining" % ((i + 1), framesRemaining)) # Inner loop that actually reads the frames into the data array for j in range(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = tbw.read_frame(fh) except errors.EOFError: break except errors.SyncError: print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell()) / tbw.FRAME_SIZE - 1)) continue if not cFrame.header.is_tbw: continue stand = cFrame.header.id # In the current configuration, stands start at 1 and go up to 10. So, we # can use this little trick to populate the data array aStand = 2 * (stand - 1) if cFrame.header.frame_count % 10000 == 0: print("%3i -> %3i %5i %i" % (stand, aStand, cFrame.header.frame_count, cFrame.payload.timetag)) # Actually load the data. x pol goes into the even numbers, y pol into the # odd numbers count = cFrame.header.frame_count - 1 timetags[aStand, count] = cFrame.payload.timetag timetags[aStand + 1, count] = cFrame.payload.timetag # Check for missing frames missing = numpy.where(timetags < 0) if len(missing) != 0: dp1Boards = {} print( "Found %i missing frames (%i missing time tags). Missing data from:" % (len(missing[0]) / 2, len(missing[0]))) for i, f in zip(missing[0], missing[1]): try: dp1Boards[antennas[i].board] += 1 except KeyError: dp1Boards[antennas[i].board] = 1 print(" stand %3i, pol. %1i (dig. %3i) @ frame %5i" % (antennas[i].stand.id, antennas[i].pol, antennas[i].digitizer, f + 1)) print("-> DP1 boards with missing frames:") for k in dp1Boards.keys(): v = dp1Boards[k] print(" %2i %6i (%7.3f%%)" % (k, v, 100.0 * v / (30000 * 10))) # Check time tags to make sure every ant/pol as the same time as each frame for f in xrange(timetags.shape[1]): ## For each frame count value, get the median time tag and use this for comparison. ## If things are really bad, we will get a lot of errors. frameTime = numpy.median(timetags[:, f]) ## Compare all of the antpols at a particular frame count, ignoring the ones that ## are missing. missing = numpy.where((timetags[:, f] != frameTime) & (timetags[:, f] >= 0))[0] ## Report any errors for m in missing: print("ERROR: t.t. %i @ frame %i != frame median of %i" % (timetags[m, f], f + 1, frameTime)) print(" -> difference: %i" % (timetags[m, f] - frameTime, )) # Check time tags to make sure the times increment correctly between frames for i in xrange(timetags.shape[0]): for f in xrange(1, timetags.shape[1]): ## Skip missing frames since they always fail if timetags[i, f] < 0 or timetags[i, f - 1] < 0: continue ## Compare the current time tag with previous and report an error if there ## is a discrepancy between the two modulo the expected skip. if timetags[i, f] > (timetags[i, f - 1] + nSamples): ## Too far into the future print("ERROR: t.t. %i @ frame %i > t.t. %i @ frame %i + skip" % (timetags[i, f], f + 1, timetags[i, f - 1], f)) print(" -> difference: %i" % (timetags[i, f] - timetags[i, f - 1], )) elif timetags[i, f] < (timetags[i, f - 1] + nSamples): ## Not far enough into the future print("ERROR: t.t. %i @ frame %i < t.t. %i @ frame %i + skip" % (timetags[i, f], f + 1, timetags[i, f - 1], f)) print(" -> difference: %i" % (timetags[i, f] - timetags[i, f - 1], )) else: ## Everything is good if we make it here pass