def test_drx_block(self): """Test finding out how many tunings/pols. per beam are in a DRX file.""" fh = open(drxFile, 'rb') b1, b2, b3, b4 = drx.get_frames_per_obs(fh) self.assertEqual(b1, 0) self.assertEqual(b2, 0) self.assertEqual(b3, 0) self.assertEqual(b4, 4) fh.close()
def main(args): fh = open(args.filename, "rb") nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE while True: junkFrame = drx.read_frame(fh) try: srate = junkFrame.sample_rate break except ZeroDivisionError: pass fh.seek(-drx.FRAME_SIZE, 1) beams = drx.get_beam_count(fh) tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol # Offset in frames for beampols beam/tuning/pol. sets offset = int(round(args.offset * srate / 4096 * beampols)) offset = int(1.0 * offset / beampols) * beampols args.offset = 1.0 * offset / beampols * 4096 / srate fh.seek(offset * drx.FRAME_SIZE) # Make sure that the file chunk size contains is an intger multiple # of the beampols. maxFrames = int((19144 * 4) / beampols) * beampols # Setup the statistics data set if args.stats: if args.plot_range < 0.1: args.plot_range = 0.5 # Number of frames to integrate over nFrames = int(args.plot_range * srate / 4096 * beampols) nFrames = int(1.0 * nFrames / beampols) * beampols args.plot_range = 1.0 * nFrames / beampols * 4096 / srate # Number of remaining chunks nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames)) # File summary print("Filename: %s" % args.filename) print("Beams: %i" % beams) print("Tune/Pols: %i %i %i %i" % tunepols) print("Sample Rate: %i Hz" % srate) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate)) print("---") print("Offset: %.3f s (%i frames)" % (args.offset, offset)) print("Plot time: %.3f s (%i frames; %i frames per beam/tune/pol)" % (args.plot_range, nFrames, nFrames // beampols)) print("Chunks: %i" % nChunks) # Sanity check if offset > nFramesFile: raise RuntimeError("Requested offset is greater than file length") if nFrames > (nFramesFile - offset): raise RuntimeError( "Requested integration time+offset is greater than file length") # Align the file handle so that the first frame read in the # main analysis loop is from tuning 1, polarization 0 junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id while 2 * (t - 1) + p != 0: junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id fh.seek(-drx.FRAME_SIZE, 1) # Master loop over all of the file chuncks standMapper = [] for i in xrange(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining print("Working on chunk %i, %i frames remaining" % (i, framesRemaining)) count = {0: 0, 1: 0, 2: 0, 3: 0} data = numpy.zeros((beampols, framesWork * 4096 // beampols), dtype=numpy.csingle) # Inner loop that actually reads the frames into the data array print("Working on %.1f ms of data" % ((framesWork * 4096 / beampols / srate) * 1000.0)) t0 = time.time() for j in xrange(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = drx.read_frame(fh, verbose=False) except errors.EOFError: break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1)) continue beam, tune, pol = cFrame.id aStand = 2 * (tune - 1) + pol data[aStand, count[aStand] * 4096:(count[aStand] + 1) * 4096] = numpy.abs(cFrame.payload.data)**2 # Update the counters so that we can average properly later on count[aStand] += 1 # Statistics print("Running robust statistics") means = [robust.mean(data[i, :]) for i in xrange(data.shape[0])] stds = [robust.std(data[i, :]) for i in xrange(data.shape[0])] if args.stats: ## Report statistics print("Mean: %s" % ' '.join(["%.3f" % m for m in means])) print("StdD: %s" % ' '.join(["%.3f" % s for s in stds])) print("Levels:") ## Count'em up j = 0 counts = [ 1, ] * data.shape[0] while (means[i] + j * stds[i] <= 98) and max(counts) != 0: counts = [ len( numpy.where( numpy.abs(data[i, :] - means[i]) >= j * stds[i])[0]) for i in xrange(data.shape[0]) ] print(" %2isigma (%5.1f%%): %s" % (j, 100.0 * (1 - erf(j / numpy.sqrt(2))), ' '.join([ "%7i (%5.1f%%)" % (c, 100.0 * c / data.shape[1]) for c in counts ]))) j += 1 ## Why j-2? Well, j is 1 more than the last iteration. So, that last iteration ## is j-1, which is always filled with 0s by construction. So, the last crazy ## bin is j-2. jP = j - 2 if jP > 20: counts = [ len( numpy.where( numpy.abs(data[i, :] - means[i]) >= jP * stds[i])[0]) for i in xrange(data.shape[0]) ] for i in xrange(data.shape[0]): if counts[i] > 0: break if counts[i] == 1: print( " -> Clip-o-rama likely occuring with %i %i-sigma detection on tuning %i, pol %i" % (counts[i], jP, i // 2 + 1, i % 2)) else: print( " -> Clip-o-rama likely occuring with %i %i-sigma detections on tuning %i, pol %i" % (counts[i], jP, i // 2 + 1, i % 2)) else: outfile = os.path.splitext(args.filename)[0] outfile = '%s.txt' % outfile fh = open(outfile, 'w') # Plot possible clip-o-rama and flag it print("Computing power derivatives w.r.t. time") deriv = numpy.zeros_like(data) for i in xrange(data.shape[0]): deriv[i, :] = numpy.roll(data[i, :], -1) - data[i, :] # The plots: This is setup for the current configuration of 20 beampols print("Plotting") fig = plt.figure() figsX = int(round(math.sqrt(beampols))) figsY = beampols // figsX for i in xrange(data.shape[0]): ax = fig.add_subplot(figsX, figsY, i + 1) ax.plot(args.offset + numpy.arange(0, data.shape[1]) / srate, data[i, :]) ## Mark areas of crazy derivatives bad = numpy.where( deriv[i, :] > 20 * stds[i] * numpy.sqrt(2))[0] for j in bad: fh.write( "Clip-o-rama on tuning %i, pol. %i at %.6f seconds\n" % (i // 2 + 1, i % 2, args.offset + j / srate)) print("Clip-o-rama on tuning %i, pol. %i at %.6f seconds" % (i // 2 + 1, i % 2, args.offset + j / srate)) ax.vlines(args.offset + j / srate, -10, 100, linestyle='--', color='red', linewidth=2.0) ## Mark areas of crazy power levels bad = numpy.where(data[i, :] == 98)[0] for j in bad: fh.write( "Saturation on tuning %i, pol. %i at %.6f seconds\n" % (i // 2 + 1, i % 2, args.offset + j / srate)) print("Saturation on tuning %i, pol. %i at %.6f seconds" % (i // 2 + 1, i % 2, args.offset + j / srate)) ax.vlines(args.offset + j / srate, -10, 100, linestyle='-.', color='red') ax.set_ylim([-10, 100]) ax.set_title('Beam %i, Tune. %i, Pol. %i' % (beam, i // 2 + 1, i % 2)) ax.set_xlabel('Time [seconds]') ax.set_ylabel('I$^2$ + Q$^2$') fh.close() if args.do_plot: plt.show()
def main(args): fh = open(args.filename, "rb") nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE while True: junkFrame = drx.read_frame(fh) try: srate = junkFrame.sample_rate break except ZeroDivisionError: pass fh.seek(-drx.FRAME_SIZE, 1) print(junkFrame.header.time_offset) beams = drx.get_beam_count(fh) tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol # Offset in frames for beampols beam/tuning/pol. sets offset = int(round(args.skip * srate / 4096 * beampols)) offset = int(1.0 * offset / beampols) * beampols args.skip = 1.0 * offset / beampols * 4096 / srate fh.seek(offset * drx.FRAME_SIZE) # Make sure that the file chunk size contains is an intger multiple # of the beampols. maxFrames = int(19144 / beampols) * beampols # Number of frames to integrate over toClip = False oldAverage = args.plot_range if args.plot_range < 4096 / srate: toClip = True args.plot_range = 4096 / srate nFrames = int(args.plot_range * srate / 4096 * beampols) nFrames = int(1.0 * nFrames / beampols) * beampols args.plot_range = 1.0 * nFrames / beampols * 4096 / srate # Number of remaining chunks nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames)) # File summary print("Filename: %s" % args.filename) print("Beams: %i" % beams) print("Tune/Pols: %i %i %i %i" % tunepols) print("Sample Rate: %i Hz" % srate) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Plot time: %.3f s (%i frames; %i frames per beam/tune/pol)" % (args.plot_range, nFrames, nFrames // beampols)) print("Chunks: %i" % nChunks) # Sanity check if offset > nFramesFile: raise RuntimeError("Requested offset is greater than file length") if nFrames > (nFramesFile - offset): raise RuntimeError( "Requested integration time+offset is greater than file length") junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id while 2 * (t - 1) + p != 0: junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id print(b, t, p) print(fh.tell()) fh.seek(-drx.FRAME_SIZE, 1) # Master loop over all of the file chuncks standMapper = [] for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining print("Working on chunk %i, %i frames remaining" % (i, framesRemaining)) count = {} data = numpy.zeros((beampols, framesWork * 4096 // beampols), dtype=numpy.csingle) # Inner loop that actually reads the frames into the data array print("Working on %.1f ms of data" % ((framesWork * 4096 / beampols / srate) * 1000.0)) t0 = time.time() for j in xrange(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = drx.read_frame(fh, verbose=False) except errors.EOFError: break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1)) continue beam, tune, pol = cFrame.id aStand = 4 * (beam - 1) + 2 * (tune - 1) + pol #print(aStand, beam, tune, pol) if aStand not in standMapper: standMapper.append(aStand) oStand = 1 * aStand aStand = standMapper.index(aStand) print( "Mapping beam %i, tune. %1i, pol. %1i (%2i) to array index %3i" % (beam, tune, pol, oStand, aStand)) else: aStand = standMapper.index(aStand) if aStand not in count.keys(): count[aStand] = 0 #if cFrame.header.frame_count % 10000 == 0 and args.verbose: # print("%2i,%1i,%1i -> %2i %5i %i" % (beam, tune, pol, aStand, cFrame.header.frame_count, cFrame.payload.timetag)) #print(data.shape, count[aStand]*4096, (count[aStand]+1)*4096, cFrame.payload.data.shape) data[aStand, count[aStand] * 4096:(count[aStand] + 1) * 4096] = cFrame.payload.data # Update the counters so that we can average properly later on count[aStand] += 1 # The plots: This is setup for the current configuration of 20 beampols fig = plt.figure() figsX = int(round(math.sqrt(beampols))) figsY = beampols // figsX t1X = 1 t1Y = 1 offset = 0 samples = 65536 for sec in xrange(data.shape[1] // samples): if toClip: print("Plotting only the first %i samples (%.3f ms) of data" % (samples, oldAverage * 1000.0)) sortedMapper = sorted(standMapper) for k, aStand in enumerate(sortedMapper): i = standMapper.index(aStand) if standMapper[i] % 2 == 0: ref = data[0, :] t1R = t1X else: ref = data[1, :] t1R = t1Y (lag, cc), junkI, junkQ = crossCorrelate( data[i, sec * samples:(sec + 1) * samples], ref[offset + sec * samples:offset + (sec + 1) * samples]) best = numpy.where(cc == cc.max())[0][0] if args.verbose: print('tune %i pol. %s' % (standMapper[i] % 4 // 2 + 1, standMapper[i] % 2)) print(' -> best peak of %.0f at a lag of %i samples' % (cc.max(), lag[best])) print(' -> NCM with tuning 1 of %.3f' % (cc.max() / t1R)) # Plot ax = fig.add_subplot(figsX, figsY, k + 1) ax.plot(lag, cc, label='Same', color='blue') # Center on the peak best = numpy.where(cc == cc.max())[0][0] ax.set_xlim([lag[best - 50], lag[best + 50]]) ax.set_title('Beam %i, Tune. %i, Pol. %i' % (standMapper[i] // 4 + 1, standMapper[i] % 4 // 2 + 1, standMapper[i] % 2)) ax.set_xlabel('Lag [samples]') ax.set_ylabel('Analysis Sets') # Save the tuning 1 values for the peak of the CC function if standMapper[i] % 4 / 2 + 1 == 1: if standMapper[i] % 2 == 0: t1X = cc.max() else: t1Y = cc.max() plt.show() # Save image if requested if args.output is not None: fig.savefig(args.output)
def main(args): LFFT = args.fft_length stand1 = 0 stand2 = int(args.dipole_id_y) filenames = args.filename # Build up the station if args.lwasv: site = stations.lwasv else: site = stations.lwa1 # Get the antennas we need (and a fake one for the beam) rawAntennas = site.antennas antennas = [] dipole = None xyz = numpy.zeros((len(rawAntennas), 3)) i = 0 for ant in rawAntennas: if ant.stand.id == stand2 and ant.pol == 0: dipole = ant xyz[i, 0] = ant.stand.x xyz[i, 1] = ant.stand.y xyz[i, 2] = ant.stand.z i += 1 arrayX = xyz[:, 0].mean() arrayY = xyz[:, 1].mean() arrayZ = xyz[:, 2].mean() ## Fake one down here... beamStand = stations.Stand(0, arrayX, arrayY, arrayZ) beamFEE = stations.FEE('Beam', 0, gain1=0, gain2=0, status=3) beamCable = stations.Cable('Beam', 0, vf=1.0) beamAntenna = stations.Antenna(0, stand=beamStand, pol=0, theta=0, phi=0, status=3) beamAntenna.fee = beamFEE beamAntenna.feePort = 1 beamAntenna.cable = beamCable antennas.append(beamAntenna) ## Dipole down here... ### NOTE ### Here we zero out the cable length for the dipole since the delay ### setup that is used for these observations already takes the ### cable/geometric delays into account. We shouldn't need anything ### else to get good fringes. dipole.cable.length = 0 antennas.append(dipole) # Loop over the input files... for filename in filenames: fh = open(filename, "rb") nFramesFile = os.path.getsize(filename) // drx.FRAME_SIZE #junkFrame = drx.read_frame(fh) #fh.seek(0) while True: try: junkFrame = drx.read_frame(fh) try: srate = junkFrame.sample_rate t0 = junkFrame.time break except ZeroDivisionError: pass except errors.SyncError: fh.seek(-drx.FRAME_SIZE + 1, 1) fh.seek(-drx.FRAME_SIZE, 1) beam, tune, pol = junkFrame.id srate = junkFrame.sample_rate tunepols = drx.get_frames_per_obs(fh) tunepols = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepols # Offset in frames for beampols beam/tuning/pol. sets offset = int(args.skip * srate / 4096 * beampols) offset = int(1.0 * offset / beampols) * beampols fh.seek(offset * drx.FRAME_SIZE, 1) # Iterate on the offsets until we reach the right point in the file. This # is needed to deal with files that start with only one tuning and/or a # different sample rate. while True: ## Figure out where in the file we are and what the current tuning/sample ## rate is junkFrame = drx.read_frame(fh) srate = junkFrame.sample_rate t1 = junkFrame.time tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol fh.seek(-drx.FRAME_SIZE, 1) ## See how far off the current frame is from the target tDiff = t1 - (t0 + args.skip) ## Half that to come up with a new seek parameter tCorr = -tDiff / 8.0 cOffset = int(tCorr * srate / 4096 * beampols) cOffset = int(1.0 * cOffset / beampols) * beampols offset += cOffset ## If the offset is zero, we are done. Otherwise, apply the offset ## and check the location in the file again/ if cOffset is 0: break fh.seek(cOffset * drx.FRAME_SIZE, 1) # Update the offset actually used args.skip = t1 - t0 offset = int(round(args.skip * srate / 4096 * beampols)) offset = int(1.0 * offset / beampols) * beampols tnom = junkFrame.header.time_offset tStart = junkFrame.time # Get the DRX frequencies cFreq1 = 0.0 cFreq2 = 0.0 for i in xrange(32): junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id if p == 0 and t == 1: cFreq1 = junkFrame.central_freq elif p == 0 and t == 2: cFreq2 = junkFrame.central_freq else: pass fh.seek(-32 * drx.FRAME_SIZE, 1) # Align the files as close as possible by the time tags and then make sure that # the first frame processed is from tuning 1, pol 0. junkFrame = drx.read_frame(fh) beam, tune, pol = junkFrame.id pair = 2 * (tune - 1) + pol j = 0 while pair != 0: junkFrame = drx.read_frame(fh) beam, tune, pol = junkFrame.id pair = 2 * (tune - 1) + pol j += 1 fh.seek(-drx.FRAME_SIZE, 1) print("Shifted beam %i data by %i frames (%.4f s)" % (beam, j, j * 4096 / srate / 4)) # Set integration time tInt = args.avg_time nFrames = int(round(tInt * srate / 4096)) tInt = nFrames * 4096 / srate # Read in some data tFile = nFramesFile / 4 * 4096 / srate # Report print("Filename: %s" % filename) print(" Sample Rate: %i Hz" % srate) print(" Tuning 1: %.1f Hz" % cFreq1) print(" Tuning 2: %.1f Hz" % cFreq2) print(" ===") print(" Integration Time: %.3f s" % tInt) print(" Integrations in File: %i" % int(tFile / tInt)) print(" Duration of File: %f" % tFile) print(" Offset: %f s" % offset) if args.duration != 0: nChunks = int(round(args.duration / tInt)) else: nChunks = int(tFile / tInt) print("Processing: %i integrations" % nChunks) # Here we start the HDF5 file outname = os.path.split(filename)[1] outname = os.path.splitext(outname)[0] outname = "%s.hdf5" % outname outfile = h5py.File(outname) group1 = outfile.create_group("Time") group2 = outfile.create_group("Frequencies") group3 = outfile.create_group("Visibilities") out = raw_input("Target Name: ") outfile.attrs["OBJECT"] = out out = raw_input("Polarization (X/Y): ") outfile.attrs["POLARIZATION"] = out dset1 = group1.create_dataset("Timesteps", (nChunks, 3), numpy.float64, maxshape=(nChunks, 3)) dset2 = group2.create_dataset("Tuning1", (LFFT, ), numpy.float64, maxshape=(LFFT, )) dset3 = group2.create_dataset("Tuning2", (LFFT, ), numpy.float64, maxshape=(LFFT, )) dset4 = group3.create_dataset("Tuning1", (nChunks, 3, LFFT), numpy.complex64, maxshape=(nChunks, 3, LFFT)) dset5 = group3.create_dataset("Tuning2", (nChunks, 3, LFFT), numpy.complex64, maxshape=(nChunks, 3, LFFT)) drxBuffer = buffer.DRXFrameBuffer(beams=[ beam, ], tunes=[1, 2], pols=[0, 1]) data = numpy.zeros((2, 2, 4096 * nFrames), dtype=numpy.complex64) pb = ProgressBarPlus(max=nChunks) tsec = numpy.zeros(1, dtype=numpy.float64) for i in xrange(nChunks): j = 0 while j < nFrames: for k in xrange(4): try: cFrame = drx.read_frame(fh) drxBuffer.append(cFrame) except errors.SyncError: pass cFrames = drxBuffer.get() if cFrames is None: continue for cFrame in cFrames: if j == 0: tStart = cFrame.time beam, tune, pol = cFrame.id pair = 2 * (tune - 1) + pol if tune == 1: data[0, pol, j * 4096:(j + 1) * 4096] = cFrame.payload.data else: data[1, pol, j * 4096:(j + 1) * 4096] = cFrame.payload.data j += 1 # Correlate blList1, freq1, vis1 = fxc.FXMaster(data[0, :, :], antennas, LFFT=LFFT, overlap=1, include_auto=True, verbose=False, sample_rate=srate, central_freq=cFreq1, pol='XX', return_baselines=True, gain_correct=False, clip_level=0) blList2, freq2, vis2 = fxc.FXMaster(data[1, :, :], antennas, LFFT=LFFT, overlap=1, include_auto=True, verbose=False, sample_rate=srate, central_freq=cFreq2, pol='XX', return_baselines=True, gain_correct=False, clip_level=0) if i == 0: tsec = tInt / 2 outfile.attrs["STANDS"] = numpy.array([stand1, stand2]) outfile.attrs["SRATE"] = srate date = datetime.fromtimestamp(tStart).date() outfile.attrs["DATE"] = str(date) dset2.write_direct(freq1) dset3.write_direct(freq2) else: tsec += tInt temp = numpy.zeros(3, dtype=numpy.float64) temp[0] = tStart temp[1] = tInt temp[2] = tsec dset1.write_direct(temp, dest_sel=numpy.s_[i]) dset4.write_direct(vis1, dest_sel=numpy.s_[i]) dset5.write_direct(vis2, dest_sel=numpy.s_[i]) pb.inc(amount=1) sys.stdout.write(pb.show() + '\r') sys.stdout.flush() sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() outfile.close() # Plot fig = plt.figure() i = 0 for bl, vi in zip(blList1, vis1): ax = fig.add_subplot(4, 3, i + 1) ax.plot(freq1 / 1e6, numpy.unwrap(numpy.angle(vi))) ax.set_title('Stand %i - Stand %i' % (bl[0].stand.id, bl[1].stand.id)) ax = fig.add_subplot(4, 3, i + 4) ax.plot(freq1 / 1e6, numpy.abs(vi)) i += 1 coeff = numpy.polyfit(freq1, numpy.unwrap(numpy.angle(vi)), 1) #print(coeff[0]/2/numpy.pi*1e9, coeff[1]*180/numpy.pi) i = 6 for bl, vi in zip(blList2, vis2): ax = fig.add_subplot(4, 3, i + 1) ax.plot(freq2 / 1e6, numpy.unwrap(numpy.angle(vi))) ax.set_title('Stand %i - Stand %i' % (bl[0].stand.id, bl[1].stand.id)) ax = fig.add_subplot(4, 3, i + 4) ax.plot(freq2 / 1e6, numpy.abs(vi)) i += 1 coeff = numpy.polyfit(freq2, numpy.unwrap(numpy.angle(vi)), 1)
def main(args): filename = args[0] fh = open(filename, "rb") nFramesFile = os.path.getsize(filename) / drx.FRAME_SIZE junkFrame = drx.read_frame(fh) beam, tune, pol = junkFrame.id while 2 * (tune - 1) + pol != 0: junkFrame = drx.read_frame(fh) beam, tune, pol = junkFrame.id fh.seek(fh.tell() - drx.FRAME_SIZE) srate = junkFrame.sample_rate beams = drx.get_beam_count(fh) tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol # File summary out = "Filename: %s" % filename out += "\nBeams: %i" % beams out += "\nTune/Pols: %i %i %i %i" % tunepols out += "\nSample Rate: %i Hz" % srate out += "\nFrames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate) out += "\n===" print(out) tuningOffset = numpy.zeros(nFramesFile / 8, dtype=numpy.int64) try: screen = curses.initscr() curses.noecho() curses.cbreak() screen.nodelay(1) strdict = {'preamble': out} for i in xrange(tuningOffset.size): screen.clear() beamIDs = [0, 0, 0, 0] timetags = numpy.zeros(4, dtype=numpy.int64) - 1 time_offsets = numpy.zeros(4, dtype=numpy.int64) - 1 timeValues = numpy.zeros(4, dtype=numpy.float64) for j in xrange(4): # Read in the next frame and anticipate any problems that could occur try: cFrame = drx.read_frame(fh, verbose=False) except errors.EOFError: break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1)) continue ## Save the time time, time offset, and computed time values beam, tune, pol = cFrame.id aStand = 2 * (tune - 1) + pol if timetags[aStand] == -1: beamIDs[aStand] = (beam, tune, pol) timetags[aStand] = cFrame.payload.timetag time_offsets[aStand] = cFrame.header.time_offset timeValues[aStand] = cFrame.time k = 0 for id, tt, to, tv in zip(beamIDs, timetags, time_offsets, timeValues): strdict['b%i' % k] = id[0] strdict['t%i' % k] = id[1] strdict['p%i' % k] = id[2] strdict['tt%i' % k] = tt strdict['os%i' % k] = to strdict['tv%i' % k] = tv k += 1 t1t = timetags[0] - time_offsets[0] t2t = timetags[3] - time_offsets[3] tuningOffset[i] = t2t - t1t strdict['ttd'] = t2t - t1t strdict['tvd'] = (t2t - t1t) / fS screen.addstr(0, 0, display.safe_substitute(strdict)) screen.refresh() # Check for keypress and exit if Q c = screen.getch() if (c > 0): if chr(c) == 'q': break if chr(c) == 'Q': break curses.nocbreak() curses.echo() curses.endwin() except KeyboardInterrupt: curses.nocbreak() curses.echo() curses.endwin() tuningOffset = tuningOffset[0:i] print(display.safe_substitute(strdict)) print( "T2-T1 time tag offset range: %i to %i (based on %i sets of frames)" % (tuningOffset.min(), tuningOffset.max(), len(tuningOffset)))
def main(args): LFFT = args.fft_length stand1 = int(args.dipole_id_x) stand2 = int(args.dipole_id_y) filenames = args.filename # Build up the station if args.lwasv: site = stations.lwasv else: site = stations.lwa1 # Figure out which antennas we need antennas = [] for ant in site.antennas: if ant.stand.id == stand1 and ant.pol == 0: antennas.append(ant) for ant in site.antennas: if ant.stand.id == stand2 and ant.pol == 0: antennas.append(ant) # Loop through the input files... for filename in filenames: fh = open(filename, "rb") nFramesFile = os.path.getsize(filename) // drx.FRAME_SIZE #junkFrame = drx.read_frame(fh) #fh.seek(0) while True: try: junkFrame = drx.read_frame(fh) try: srate = junkFrame.sample_rate t0 = junkFrame.time break except ZeroDivisionError: pass except errors.SyncError: fh.seek(-drx.FRAME_SIZE + 1, 1) fh.seek(-drx.FRAME_SIZE, 1) beam, tune, pol = junkFrame.id srate = junkFrame.sample_rate tunepols = drx.get_frames_per_obs(fh) tunepols = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepols # Offset in frames for beampols beam/tuning/pol. sets offset = int(args.skip * srate / 4096 * beampols) offset = int(1.0 * offset / beampols) * beampols fh.seek(offset * drx.FRAME_SIZE, 1) # Iterate on the offsets until we reach the right point in the file. This # is needed to deal with files that start with only one tuning and/or a # different sample rate. while True: ## Figure out where in the file we are and what the current tuning/sample ## rate is junkFrame = drx.read_frame(fh) srate = junkFrame.sample_rate t1 = junkFrame.time tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol fh.seek(-drx.FRAME_SIZE, 1) ## See how far off the current frame is from the target tDiff = t1 - (t0 + args.skip) ## Half that to come up with a new seek parameter tCorr = -tDiff / 8.0 cOffset = int(tCorr * srate / 4096 * beampols) cOffset = int(1.0 * cOffset / beampols) * beampols offset += cOffset ## If the offset is zero, we are done. Otherwise, apply the offset ## and check the location in the file again/ if cOffset is 0: break fh.seek(cOffset * drx.FRAME_SIZE, 1) # Update the offset actually used args.skip = t1 - t0 offset = int(round(args.skip * srate / 4096 * beampols)) offset = int(1.0 * offset / beampols) * beampols tnom = junkFrame.header.time_offset tStart = junkFrame.time # Get the DRX frequencies cFreq1 = 0.0 cFreq2 = 0.0 for i in xrange(4): junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id if p == 0 and t == 1: cFreq1 = junkFrame.central_freq elif p == 0 and t == 2: cFreq2 = junkFrame.central_freq else: pass fh.seek(-4 * drx.FRAME_SIZE, 1) # Align the files as close as possible by the time tags and then make sure that # the first frame processed is from tuning 1, pol 0. junkFrame = drx.read_frame(fh) beam, tune, pol = junkFrame.id pair = 2 * (tune - 1) + pol j = 0 while pair != 0: junkFrame = drx.read_frame(fh) beam, tune, pol = junkFrame.id pair = 2 * (tune - 1) + pol j += 1 fh.seek(-drx.FRAME_SIZE, 1) print("Shifted beam %i data by %i frames (%.4f s)" % (beam, j, j * 4096 / srate / 4)) # Set integration time tInt = args.avg_time nFrames = int(round(tInt * srate / 4096)) tInt = nFrames * 4096 / srate # Read in some data tFile = nFramesFile / 4 * 4096 / srate # Report print("Filename: %s" % filename) print(" Sample Rate: %i Hz" % srate) print(" Tuning 1: %.1f Hz" % cFreq1) print(" Tuning 2: %.1f Hz" % cFreq2) print(" ===") print(" Integration Time: %.3f s" % tInt) print(" Integrations in File: %i" % int(tFile / tInt)) nChunks = int(tFile / tInt) pb = ProgressBar(max=nChunks) for i in xrange(nChunks): junkFrame = drx.read_frame(fh) tStart = junkFrame.time fh.seek(-drx.FRAME_SIZE, 1) count1 = [0, 0] data1 = numpy.zeros((2, 4096 * nFrames), dtype=numpy.complex64) count2 = [0, 0] data2 = numpy.zeros((2, 4096 * nFrames), dtype=numpy.complex64) for j in xrange(nFrames): for k in xrange(4): cFrame = drx.read_frame(fh) beam, tune, pol = cFrame.id pair = 2 * (tune - 1) + pol if tune == 1: data1[pol, count1[pol] * 4096:(count1[pol] + 1) * 4096] = cFrame.payload.data count1[pol] += 1 else: data2[pol, count2[pol] * 4096:(count2[pol] + 1) * 4096] = cFrame.payload.data count2[pol] += 1 # Correlate blList1, freq1, vis1 = fxc.FXMaster(data1, antennas, LFFT=LFFT, overlap=1, include_auto=True, verbose=False, sample_rate=srate, central_freq=cFreq1, pol='XX', return_baselines=True, gain_correct=False, clip_level=0) blList2, freq2, vis2 = fxc.FXMaster(data2, antennas, LFFT=LFFT, overlap=1, include_auto=True, verbose=False, sample_rate=srate, central_freq=cFreq2, pol='XX', return_baselines=True, gain_correct=False, clip_level=0) if nChunks != 1: outfile = os.path.split(filename)[1] outfile = os.path.splitext(outfile)[0] outfile = "%s-vis-%04i.npz" % (outfile, i + 1) else: outfile = os.path.split(filename)[1] outfile = os.path.splitext(outfile)[0] outfile = "%s-vis.npz" % outfile numpy.savez(outfile, srate=srate, freq1=freq1, vis1=vis1, freq2=freq2, vis2=vis2, tStart=tStart, tInt=tInt, stands=numpy.array([stand1, stand2])) del data1 del data2 pb.inc(amount=1) sys.stdout.write(pb.show() + '\r') sys.stdout.flush() sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() # Plot fig = plt.figure() i = 0 for bl, vi in zip(blList1, vis1): ax = fig.add_subplot(4, 3, i + 1) ax.plot(freq1 / 1e6, numpy.unwrap(numpy.angle(vi))) ax.set_title('Stand %i - Stand %i' % (bl[0].stand.id, bl[1].stand.id)) ax = fig.add_subplot(4, 3, i + 4) ax.plot(freq1 / 1e6, numpy.abs(vi)) i += 1 coeff = numpy.polyfit(freq1, numpy.unwrap(numpy.angle(vi)), 1) #print(coeff[0]/2/numpy.pi*1e9, coeff[1]*180/numpy.pi) i = 6 for bl, vi in zip(blList2, vis2): ax = fig.add_subplot(4, 3, i + 1) ax.plot(freq2 / 1e6, numpy.unwrap(numpy.angle(vi))) ax.set_title('Stand %i - Stand %i' % (bl[0].stand.id, bl[1].stand.id)) ax = fig.add_subplot(4, 3, i + 4) ax.plot(freq2 / 1e6, numpy.abs(vi)) i += 1 coeff = numpy.polyfit(freq2, numpy.unwrap(numpy.angle(vi)), 1)
def main(args): filename = args.filename fh = open(filename, "rb") nFramesFile = os.path.getsize(filename) // drx.FRAME_SIZE while True: try: junkFrame = drx.read_frame(fh) try: srate = junkFrame.sample_rate break except ZeroDivisionError: pass except errors.SyncError: fh.seek(-drx.FRAME_SIZE + 1, 1) fh.seek(-drx.FRAME_SIZE, 1) beam, tune, pol = junkFrame.id tunepols = max(drx.get_frames_per_obs(fh)) # Date & Central Frequnecy beginDate = junkFrame.time.datetime central_freq1 = 0.0 central_freq2 = 0.0 for i in xrange(32): junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id if p == 0 and t == 1: central_freq1 = junkFrame.central_freq elif p == 0 and t == 2: central_freq2 = junkFrame.central_freq else: pass fh.seek(-32 * drx.FRAME_SIZE, 1) # Report on the file print("Filename: %s" % filename) print("Date of First Frame: %s" % str(beginDate)) print("Beam: %i" % beam) print("Tune/Pols: %i" % tunepols) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2)) print(" ") # Convert chunk length to total frame count chunkLength = int(args.length * srate / 4096 * tunepols) chunkLength = int(1.0 * chunkLength / tunepols) * tunepols # Convert chunk skip to total frame count chunkSkip = int(args.skip * srate / 4096 * tunepols) chunkSkip = int(1.0 * chunkSkip / tunepols) * tunepols # Output arrays clipFraction = [] meanPower = [] # Go! i = 1 done = False print(" | Clipping | Power |") print(" | 1X 1Y 2X 2Y | 1X 1Y 2X 2Y |") print("---+---------------------------------+-------------------------+") while True: count = {0: 0, 1: 0, 2: 0, 3: 0} data = numpy.empty((4, chunkLength * 4096 // tunepols), dtype=numpy.csingle) for j in xrange(chunkLength): # Read in the next frame and anticipate any problems that could occur try: cFrame = drx.read_frame(fh, verbose=False) except errors.EOFError: done = True break except errors.SyncError: continue beam, tune, pol = cFrame.id aStand = 2 * (tune - 1) + pol try: data[aStand, count[aStand] * 4096:(count[aStand] + 1) * 4096] = cFrame.payload.data # Update the counters so that we can average properly later on count[aStand] += 1 except ValueError: pass if done: break else: data = numpy.abs(data)**2 data = data.astype(numpy.int32) clipFraction.append(numpy.zeros(4)) meanPower.append(data.mean(axis=1)) for j in xrange(4): bad = numpy.nonzero(data[j, :] > args.trim_level)[0] clipFraction[-1][j] = 1.0 * len(bad) / data.shape[1] clip = clipFraction[-1] power = meanPower[-1] print( "%2i | %6.2f%% %6.2f%% %6.2f%% %6.2f%% | %5.2f %5.2f %5.2f %5.2f |" % (i, clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0, clip[3] * 100.0, power[0], power[1], power[2], power[3])) i += 1 fh.seek(drx.FRAME_SIZE * chunkSkip, 1) clipFraction = numpy.array(clipFraction) meanPower = numpy.array(meanPower) clip = clipFraction.mean(axis=0) power = meanPower.mean(axis=0) print("---+---------------------------------+-------------------------+") print("%2s | %6.2f%% %6.2f%% %6.2f%% %6.2f%% | %5.2f %5.2f %5.2f %5.2f |" % ('M', clip[0] * 100.0, clip[1] * 100.0, clip[2] * 100.0, clip[3] * 100.0, power[0], power[1], power[2], power[3]))
def main(args): filename = args.filename sizeB = os.path.getsize(filename) # Open the file and get some basic info about the data contained fh = open(filename, 'rb') nFramesFile = sizeB / drx.FRAME_SIZE while True: try: junkFrame = drx.read_frame(fh) try: srate = junkFrame.sample_rate t0 = junkFrame.time break except ZeroDivisionError: pass except errors.SyncError: fh.seek(-drx.FRAME_SIZE + 1, 1) fh.seek(-drx.FRAME_SIZE, 1) beams = drx.get_beam_count(fh) tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol # Offset in frames for beampols beam/tuning/pol. sets offset = int(round(args.offset * srate / 4096 * beampols)) offset = int(1.0 * offset / beampols) * beampols fh.seek(offset * drx.FRAME_SIZE, 1) # Iterate on the offsets until we reach the right point in the file. This # is needed to deal with files that start with only one tuning and/or a # different sample rate. while True: ## Figure out where in the file we are and what the current tuning/sample ## rate is junkFrame = drx.read_frame(fh) srate = junkFrame.sample_rate t1 = junkFrame.time tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol fh.seek(-drx.FRAME_SIZE, 1) ## See how far off the current frame is from the target tDiff = t1 - (t0 + args.offset) ## Half that to come up with a new seek parameter tCorr = -tDiff / 2.0 cOffset = int(tCorr * srate / 4096 * beampols) cOffset = int(1.0 * cOffset / beampols) * beampols offset += cOffset ## If the offset is zero, we are done. Otherwise, apply the offset ## and check the location in the file again/ if cOffset is 0: break fh.seek(cOffset * drx.FRAME_SIZE, 1) # Update the offset actually used args.offset = t1 - t0 nCaptures = nFramesFile / beampols print("Filename: %s" % filename) print("Size: %.1f MB" % (float(sizeB) / 1024 / 1024)) print("Captures: %i (%.2f seconds)" % (nCaptures, nCaptures * 4096 / srate)) print("Tuning/Pols.: %i " % tunepol) print("Sample Rate: %.2f MHz" % (srate / 1e6)) print("===") if args.count > 0: nCaptures = args.count * srate // 4096 else: nCaptures -= args.offset * srate // 4096 args.count = nCaptures * 4096 // srate print("Seconds to Skip: %.2f (%i captures)" % (args.offset, offset / beampols)) print("Seconds to Split: %.2f (%i captures)" % (args.count, nCaptures)) # Make sure that the first frame in the file is the first frame of a capture # (tuning 1, pol 0). If not, read in as many frames as necessary to get to # the beginning of a complete capture. beam, tune, pol = junkFrame.id skip = 0 while (2 * (tune - 1) + pol) != 0: frame = drx.read_frame(fh) beam, tune, pol = frame.id skip += 1 nFramesRemaining = (sizeB - fh.tell()) // drx.FRAME_SIZE nRecursions = int(nFramesRemaining // (nCaptures * beampols)) if not args.recursive: nRecursions = 1 scale = int(math.log10(nRecursions)) + 1 ifString = "Working on #%%%ii of %i (%%s)" % (scale, nRecursions) for r in range(nRecursions): if args.date: filePos = fh.tell() junkFrame = drx.read_frame(fh) fh.seek(filePos) dt = junkFrame.time.datetime captFilename = "%s_%s.dat" % (os.path.splitext( os.path.basename(filename))[0], dt.isoformat()) else: captFilename = "%s_s%04i_p%%0%ii.dat" % (os.path.splitext( os.path.basename(filename))[0], args.count, scale) captFilename = captFilename % r if not args.recursive: captFilename = "%s_s%04i.dat" % (os.path.splitext( os.path.basename(filename))[0], args.count) print(ifString % (r + 1, captFilename)) t0 = time.time() fhOut = open(captFilename, 'wb') split_file(fh, fhOut, nCaptures, beampols) fhOut.close() t1 = time.time() print(" Copied %i bytes in %.3f s (%.3f MB/s)" % (os.path.getsize(captFilename), t1 - t0, os.path.getsize(captFilename) / 1024.0**2 / (t1 - t0))) fh.close()
def main(args): fh = open(args.filename, "rb") nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE while True: try: junkFrame = drx.read_frame(fh) try: srate = junkFrame.sample_rate t0i, t0f = junkFrame.time break except ZeroDivisionError: pass except errors.SyncError: fh.seek(-drx.FRAME_SIZE + 1, 1) fh.seek(-drx.FRAME_SIZE, 1) beams = drx.get_beam_count(fh) tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol # Offset in frames for beampols beam/tuning/pol. sets offset = int(round(args.skip * srate / 4096 * beampols)) offset = int(1.0 * offset / beampols) * beampols fh.seek(offset * drx.FRAME_SIZE, 1) # Iterate on the offsets until we reach the right point in the file. This # is needed to deal with files that start with only one tuning and/or a # different sample rate. while True: ## Figure out where in the file we are and what the current tuning/sample ## rate is junkFrame = drx.read_frame(fh) srate = junkFrame.sample_rate t1i, t1f = junkFrame.time tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol fh.seek(-drx.FRAME_SIZE, 1) ## See how far off the current frame is from the target tDiff = t1i - (t0i + args.skip) + (t1f - t0f) ## Half that to come up with a new seek parameter tCorr = -tDiff / 2.0 cOffset = int(tCorr * srate / 4096 * beampols) cOffset = int(1.0 * cOffset / beampols) * beampols offset += cOffset ## If the offset is zero, we are done. Otherwise, apply the offset ## and check the location in the file again/ if cOffset is 0: break fh.seek(cOffset * drx.FRAME_SIZE, 1) # Update the offset actually used args.skip = t1i - t0i + t1f - t0f offset = int(round(args.skip * srate / 4096 * beampols)) offset = int(1.0 * offset / beampols) * beampols # Make sure that the file chunk size contains is an intger multiple # of the beampols. maxFrames = int(19144 / beampols) * beampols # Number of frames to integrate over toClip = False oldAverage = args.plot_range if args.plot_range < 4096 / srate: toClip = True args.plot_range = 4096 / srate nFrames = int(args.plot_range * srate / 4096 * beampols) nFrames = int(1.0 * nFrames / beampols) * beampols args.plot_range = 1.0 * nFrames / beampols * 4096 / srate # Number of remaining chunks nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames)) # File summary print("Filename: %s" % args.filename) print("Beams: %i" % beams) print("Tune/Pols: %i %i %i %i" % tunepols) print("Sample Rate: %i Hz" % srate) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Plot time: %.3f s (%i frames; %i frames per beam/tune/pol)" % (args.plot_range, nFrames, nFrames // beampols)) print("Chunks: %i" % nChunks) # Sanity check if nFrames > (nFramesFile - offset): raise RuntimeError( "Requested integration time+offset is greater than file length") # Align the file handle so that the first frame read in the # main analysis loop is from tuning 1, polarization 0 junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id while 2 * (t - 1) + p != 0: junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id fh.seek(-drx.FRAME_SIZE, 1) # Master loop over all of the file chuncks standMapper = [] for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining print("Working on chunk %i, %i frames remaining" % (i, framesRemaining)) count = {0: 0, 1: 0, 2: 0, 3: 0} tt = numpy.zeros( (beampols, framesWork // beampols), dtype=numpy.int64) - 1 data = numpy.zeros((beampols, framesWork * 4096 // beampols), dtype=numpy.csingle) # Inner loop that actually reads the frames into the data array print("Working on %.1f ms of data" % ((framesWork * 4096 / beampols / srate) * 1000.0)) t0 = time.time() for j in xrange(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = drx.read_frame(fh, verbose=False) except errors.EOFError: break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())//drx.FRAME_SIZE-1)) continue beam, tune, pol = cFrame.id aStand = 2 * (tune - 1) + pol tt[aStand, count[aStand]] = cFrame.payload.timetag if args.instantaneous_power: data[aStand, count[aStand] * 4096:(count[aStand] + 1) * 4096] = numpy.abs(cFrame.payload.data)**2 else: data[aStand, count[aStand] * 4096:(count[aStand] + 1) * 4096] = cFrame.payload.data # Update the counters so that we can average properly later on count[aStand] += 1 # The plots: This is setup for the current configuration of 20 beampols fig = plt.figure() figsX = int(round(math.sqrt(beampols))) figsY = beampols // figsX samples = int(oldAverage * srate) if toClip: print("Plotting only the first %i samples (%.3f ms) of data" % (samples, oldAverage * 1000.0)) sortedMapper = sorted(standMapper) for i in xrange(data.shape[0]): ax = fig.add_subplot(figsX, figsY, i + 1) if args.instantaneous_power: limits = (-10, 100) if toClip: ax.plot(args.skip + numpy.arange(0, samples) / srate, data[i, 0:samples]) else: ax.plot(args.skip + numpy.arange(0, data.shape[1]) / srate, data[i, :]) else: limits = (-8, 8) if toClip: ax.plot(args.skip + numpy.arange(0, samples) / srate, data[i, 0:samples].real, label='I') ax.plot(args.skip + numpy.arange(0, samples) / srate, data[i, 0:samples].imag, label='Q') else: ax.plot(args.skip + numpy.arange(0, data.shape[1]) / srate, data[i, :].real, label='I') ax.plot(args.skip + numpy.arange(0, data.shape[1]) / srate, data[i, :].imag, label='Q') ax.legend(loc=0) if args.mark_frames: for j in xrange(0, samples - 4096, 4096): ax.vlines(float(j) / srate, limits[0], limits[1], color='k', label='%i' % tt[i, j // 4096]) ax.set_ylim(limits) ax.set_title('Beam %i, Tune. %i, Pol. %i' % (beam, i // 2 + 1, i % 2)) ax.set_xlabel('Time [seconds]') if args.instantaneous_power: ax.set_ylabel('I$^2$ + Q$^2$') else: ax.set_ylabel('Output Level') plt.show() # Save image if requested if args.output is not None: fig.savefig(args.output)
def main(args): fh = open(args.filename, "rb") nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE while True: junkFrame = drx.read_frame(fh) try: srate = junkFrame.sample_rate t0i, t0f = junkFrame.time break except ZeroDivisionError: pass fh.seek(-drx.FRAME_SIZE, 1) beams = drx.get_beam_count(fh) tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol # Offset in frames for beampols beam/tuning/pol. sets offset = int(round(args.skip * srate / 4096 * beampols)) offset = int(1.0 * offset / beampols) * beampols fh.seek(offset * drx.FRAME_SIZE, 1) # Iterate on the offsets until we reach the right point in the file. This # is needed to deal with files that start with only one tuning and/or a # different sample rate. while True: ## Figure out where in the file we are and what the current tuning/sample ## rate is junkFrame = drx.read_frame(fh) srate = junkFrame.sample_rate t1i, t1f = junkFrame.time tunepols = drx.get_frames_per_obs(fh) tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3] beampols = tunepol fh.seek(-drx.FRAME_SIZE, 1) ## See how far off the current frame is from the target tDiff = t1i - (t0i + args.skip) + (t1f - t0f) ## Half that to come up with a new seek parameter tCorr = -tDiff / 2.0 cOffset = int(tCorr * srate / 4096 * beampols) cOffset = int(1.0 * cOffset / beampols) * beampols offset += cOffset ## If the offset is zero, we are done. Otherwise, apply the offset ## and check the location in the file again/ if cOffset is 0: break fh.seek(cOffset * drx.FRAME_SIZE, 1) # Update the offset actually used args.skip = t1i - t0i + t1f - t0f offset = int(round(args.skip * srate / 4096 * beampols)) offset = int(1.0 * offset / beampols) * beampols # Make sure that the file chunk size contains is an intger multiple # of the beampols. maxFrames = int(round(args.average * srate / 4096)) * beampols if maxFrames < beampols: maxFrames = beampols args.average = 1.0 * maxFrames / beampols * 4096 / srate # Number of remaining chunks nChunks = int(round(args.duration / args.average)) nFrames = maxFrames * nChunks # Store the information about the first frame. prevTime = junkFrame.payload.timetag prevDate = junkFrame.time.datetime # File summary print("Filename: %s" % args.filename) print("Beams: %i" % beams) print("Tune/Pols: %i %i %i %i" % tunepols) print("Sample Rate: %i Hz" % srate) print("Date of first frame: %i -> %s" % (prevTime, str(prevDate))) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Integration: %.4f s (%i frames; %i frames per beam/tune/pol)" % (args.average, maxFrames, maxFrames // beampols)) print("Duration: %.4f s (%i frames; %i frames per beam/tune/pol)" % (args.average * nChunks, nFrames, nFrames // beampols)) print(" ") # Sanity check if nFrames > (nFramesFile - offset): raise RuntimeError( "Requested integration time+offset is greater than file length") # Align the file handle so that the first frame read in the # main analysis loop is from tuning 1, polarization 0 junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id while 2 * (t - 1) + p != 0: junkFrame = drx.read_frame(fh) b, t, p = junkFrame.id fh.seek(-drx.FRAME_SIZE, 1) # Master loop over all of the file chuncks masterTimes = numpy.zeros((nChunks, 4), dtype=numpy.float64) masterData = numpy.zeros((nChunks, 4), dtype=numpy.float32) masterData2 = numpy.zeros((nChunks, 4), dtype=numpy.float32) masterProcessed = [0, 0, 0, 0] masterRemoved = [0, 0, 0, 0] for i in range(nChunks): # Find out how many frames remain in the file. If this number is larger # than the maximum of frames we can work with at a time (maxFrames), # only deal with that chunk framesRemaining = nFrames - i * maxFrames if framesRemaining > maxFrames: framesWork = maxFrames else: framesWork = framesRemaining #print("Working on chunk %i, %i frames remaining" % (i, framesRemaining)) count = {0: 0, 1: 0, 2: 0, 3: 0} data = numpy.zeros((4, framesWork * 4096 // beampols), dtype=numpy.float32) data2 = numpy.zeros((4, framesWork * 4096 // beampols), dtype=numpy.float32) ## Inner loop that actually reads the frames into the data array #print("Working on %.2f ms of data" % ((framesWork*4096/beampols/srate)*1000.0)) t0 = time.time() for j in xrange(framesWork): # Read in the next frame and anticipate any problems that could occur try: cFrame = drx.read_frame(fh, verbose=False) except errors.EOFError: break except errors.SyncError: #print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1)) continue beam, tune, pol = cFrame.id aStand = 2 * (tune - 1) + pol if j < 4: masterTimes[i, aStand] = cFrame.time try: framePower = numpy.abs(cFrame.payload.data)**2 # Calculate the clipping mask = numpy.where(framePower <= args.trim_level, 1, 0) data[aStand, count[aStand] * 4096:(count[aStand] + 1) * 4096] = framePower data2[aStand, count[aStand] * 4096:(count[aStand] + 1) * 4096] = framePower * mask masterProcessed[aStand] += mask.size masterRemoved[aStand] += (mask.size - mask.sum()) # Update the counters so that we can average properly later on count[aStand] += 1 except ValueError: pass # Save the data masterData[i, :] = data.sum(axis=1) masterData2[i, :] = data2.sum(axis=1) # Really save the data to a NPZ file if args.write_npz: outfile = os.path.split(args.filename)[1] outfile = os.path.splitext(outfile)[0] outfile = '%s-power.npz' % outfile numpy.savez(outfile, beam=beam, avgPowerFull=masterData, avgPowerTrim=masterData2, times=masterTimes, trimLevel=args.trim_level) # Report on the clipping print("Summary:") for i in xrange(4): print(" Tuning %i, Pol. %s:" % (i / 2 + 1, 'X' if i % 2 else 'Y')) print(" Processed: %i samples" % masterProcessed[i]) print(" Clipped: %i samples" % masterRemoved[i]) print(" -> %.1f%% blanked" % (100.0 * masterRemoved[i] / masterProcessed[i], )) # The plots: This is setup for the current configuration of 20 beampols fig = plt.figure() figsX = int(round(math.sqrt(4))) figsY = 4 // figsX for i in xrange(masterData.shape[1]): ax = fig.add_subplot(figsX, figsY, i + 1) ax.plot(numpy.arange(0, masterData.shape[0]) * args.average, masterData[:, i], label='Full') ax.plot(numpy.arange(0, masterData2.shape[0]) * args.average, masterData2[:, i], label='Trimmed') ax.set_ylim([0, masterData.max()]) ax.set_title('Beam %i, Tune. %i, Pol. %i' % (beam, i // 2 + 1, i % 2)) ax.set_xlabel('Time [seconds]') ax.set_ylabel('Output Power Level') ax.legend(loc=0) # Part 2, polarization stuff fig2 = plt.figure() ax = fig2.add_subplot(3, 2, 1) ax.plot( numpy.arange(0, masterData.shape[0]) * args.average, numpy.sqrt(masterData[:, 0]**2 + masterData[:, 1]**2)) ax.set_title('$\\sqrt{X1^2 + Y1^2}$') ax.set_xlabel('Time [seconds]') ax = fig2.add_subplot(3, 2, 2) ax.plot( numpy.arange(0, masterData.shape[0]) * args.average, masterData[:, 1] / masterData[:, 0]) ax.set_title('$Y1 / X1$') ax.set_xlabel('Time [seconds]') ax = fig2.add_subplot(3, 2, 3) ax.plot( numpy.arange(0, masterData.shape[0]) * args.average, numpy.sqrt(masterData[:, 2]**2 + masterData[:, 3]**2)) ax.set_title('$\\sqrt{X2^2 + Y2^2}$') ax.set_xlabel('Time [seconds]') ax = fig2.add_subplot(3, 2, 4) ax.plot( numpy.arange(0, masterData.shape[0]) * args.average, masterData[:, 3] / masterData[:, 2]) ax.set_title('$Y2 / X2$') ax.set_xlabel('Time [seconds]') ax = fig2.add_subplot(3, 2, 5) ax.plot( numpy.arange(0, masterData.shape[0]) * args.average, numpy.sqrt(masterData[:, 2]**2 + masterData[:, 3]**2) / numpy.sqrt(masterData[:, 0]**2 + masterData[:, 1]**2)) ax.set_title('$\\sqrt{X2^2 + Y2^2} / \\sqrt{X1^2 + Y1^2}$') ax.set_xlabel('Time [seconds]') ax = fig2.add_subplot(3, 2, 6) ax.plot( numpy.arange(0, masterData.shape[0]) * args.average, (masterData[:, 3] / masterData[:, 2]) / (masterData[:, 1] / masterData[:, 0])) ax.set_title('$(Y2 / X2) / (Y1 / X1)$') ax.set_xlabel('Time [seconds]') plt.show() # Save image if requested if args.output is not None: fig.savefig(args.output)
def main(args): files = args fh = [] nFramesFile = [] srate = [] beams = [] tunepols = [] beampols = [] tnom = [] tStart = [] # Open the various files and get basic data information: beam, number of # frames per obs, T_NOM, time tag of first frame for filename in files: fh.append( open(filename, "rb") ) nFramesFile.append( os.path.getsize(filename) / drx.FRAME_SIZE ) while True: junkFrame = drx.read_frame(fh[-1]) try: srate = junkFrame.sample_rate break except ZeroDivisionError: pass fh[-1].seek(-drx.FRAME_SIZE, 1) beam, tune, pol = junkFrame.id beams.append( beam ) tunepols.append( drx.get_frames_per_obs(fh[-1]) ) tunepols.append( tunepols[-1][0] + tunepols[-1][1] + tunepols[-1][2] + tunepols[-1][3] ) beampols.append( tunepols[-1] ) tnom.append( junkFrame.header.time_offset ) tStart.append( junkFrame.payload.timetag ) # Align the files as close as possible by the time tags and then make sure that # the first frame processed is from tuning 1, pol 0. for i in xrange(len(files)): junkFrame = drx.read_frame(fh[i]) beam, tune, pol = junkFrame.id pair = 2*(tune-1) + pol j = 0 while junkFrame.payload.timetag < max(tStart): junkFrame = drx.read_frame(fh[i]) beam, tune, pol = junkFrame.id pair = 2*(tune-1) + pol j += 1 while pair != 0: junkFrame = drx.read_frame(fh[i]) beam, tune, pol = junkFrame.id pair = 2*(tune-1) + pol j += 1 fh[i].seek(-drx.FRAME_SIZE, 1) print("Shifted beam %i data by %i frames (%.4f s)" % (i, j, j*4096/srate[i]/4)) # Date beginDate = junkFrame.time.datetime # File summary print(" ") print("Filenames: %s" % ' '.join([os.path.split(f)[1] for f in files])) print("Date of First Frame: ~%s" % str(beginDate)) print("Beams: %s" % ' '.join([str(b) for b in beams])) print("Sample Rate: %s Hz" % ' '.join([str(s) for s in srate])) print(" ") # Main reader loop - save the data to the `data` list and the raw time tag values # to the `times` list. nFrames = 2000 data = numpy.zeros((len(files), 4, 4096*nFrames), dtype=numpy.csingle) times = numpy.zeros((len(files), 4, nFrames), dtype=numpy.int64) for i in xrange(len(files)): for j in xrange(nFrames): for k in xrange(4): frame = drx.read_frame(fh[i]) beam, tune, pol = frame.id pair = 2*(tune-1) + pol data[i,pair,j*4096:(j+1)*4096] = frame.payload.data times[i,pair,j] = frame.payload.timetag #print(i, j, k, beam, tune, pol, frame.payload.timetag) # Cross-correlate refs = [0,0,0,0] for i in xrange(len(files)): for j in xrange(i, len(files)): if i != j: fig = plt.figure() for k in xrange(4): lag, cc = crossCorrelate(data[j,k,:], data[i,k,:]) best = numpy.where( cc == cc.max() )[0][0] if i == j: refs[k] = cc.max() else: ccOffset = lag[best]*fS/srate[i] rtOffset = times[i,k,0] - times[j,k,0] ctOffset = (times[i,k,0] - tnom[i]) - (times[j,k,0] - tnom[j]) print("Beams %i & %i, Tuning %i, Pol. %i" % (beams[i], beams[j], k/2+1, k%2)) print(" T_NOM%i: %i ticks" % (beams[i], tnom[i])) print(" T_NOM%i: %i ticks" % (beams[j], tnom[j])) print(" -> T_NOM%i - T_NOM%i: %i ticks" % (beams[j], beams[i], tnom[j] - tnom[i])) print(" NCM: %.3f" % (cc.max() / refs[k])) print(" CC Offset: %i ticks" % ccOffset) print(" raw time tag Offset: %i ticks" % rtOffset) print(" cor time tag Offset: %i ticks" % ctOffset) print(" -> CC - raw: %i ticks" % (ccOffset - rtOffset)) print(" -> CC - cor: %i ticks" % (ccOffset - ctOffset)) ax = fig.add_subplot(2, 2, k+1) ax.plot(lag[best-50:best+50], cc[best-50:best+50]) ax.set_title('Beams %i & %i, Tuning %i, Pol. %i' % (beams[i], beams[j], k/2+1, k%2)) ax.set_xlabel('Lag [samples]') ax.set_ylabel('Analysis Sets') if i != j: plt.draw() plt.show() for f in fh: f.close()