def main(): fn = args.infile if args.inplace: fil = filterbank.FilterbankFile(fn, mode='readwrite') else: fil = filterbank.FilterbankFile(fn, mode='read') if args.inprof is not None: warnings.warn("Saved profiles already may be tuned to a particular " \ "DM, period and filterbank file (freq, nchans, " \ "tsamp, etc).") prof = load_profile(args.inprof) else: prof = make_profile(args.vonmises) prof = apply_dm(prof, args.period, args.dm, \ fil.foff, fil.frequencies, fil.tsamp) scale_profile(prof, args.scale_name, args.scale_cfgstrs, fil) if args.outprof is not None: save_profile(prof, args.outprof) outfn = args.outname % fil.header print "Showing plot of profile to be injected..." plt.figure() plt.clf() prof.plot(dedisp=True) plt.xlim(0,1) plt.savefig(outfn+".ps") if args.dryrun: sys.exit() inject(fil, outfn, prof, args.period, args.dm, \ nbitsout=args.output_nbits, block_size=args.block_size, \ pulsar_only=args.pulsar_only, inplace=args.inplace)
def main(): fn = args[0] if fn.endswith(".fil"): # Filterbank file filetype = "filterbank" rawdatafile = filterbank.FilterbankFile(fn) elif fn.endswith(".fits"): # PSRFITS file filetype = "psrfits" rawdatafile = psrfits.PsrfitsFile(fn) else: raise ValueError("Cannot recognize data file type from " "extension. (Only '.fits' and '.fil' " "are supported.)") data, bins, nbins, start, source_name = waterfall(rawdatafile, options.start, \ options.duration, dm=options.dm,\ nbins=options.nbins, nsub=options.nsub,\ subdm=options.subdm, zerodm=options.zerodm, \ downsamp=options.downsamp, \ scaleindep=options.scaleindep, \ width_bins=options.width_bins, mask=options.mask, \ maskfn=options.maskfile, \ bandpass_corr=options.bandpass_corr) plot_waterfall(data, start, source_name, options.duration, \ dm=options.dm,ofile=options.ofile, integrate_ts=options.integrate_ts, \ integrate_spec=options.integrate_spec, show_cb=options.show_cb, cmap_str=options.cmap, sweep_dms=options.sweep_dms, \ sweep_posns=options.sweep_posns, downsamp=options.downsamp,width=options.width,snr=options.snr)
def read_fil_data(fn, start=0, stop=1): """ Read in filterbank data object starting at sample start and reading in chunksize stop. """ print("Reading filterbank file %s \n" % fn) fil_obj = filterbank_.FilterbankFile(fn) header = fil_obj.header header_size = fil_obj.header_size header['hdr'] = header_size delta_t = fil_obj.header['tsamp'] # delta_t in seconds fch1 = header['fch1'] nchans = header['nchans'] foff = header['foff'] fch_f = fch1 + nchans * foff freq = np.linspace(fch1, fch_f, nchans) try: data = fil_obj.get_spectra(start, stop) except (ValueError): data = 0 # turn array into time-major, for preprocess # data = data.transpose() return data, freq, delta_t, header
def h5_to_fil(fnh5, fn_fil_out, nchunk='all', rficlean=False): if rficlean: print("RFI Cleaning") f = h5py.File(fnh5,'r') chunksize = int(1e4) if nchunk=='all': nchunk=int(1e6) for ii in range(int(nchunk)): startsample, endsample = ii*chunksize, (ii+1)*chunksize data = f["/SUB_ARRAY_POINTING_000/BEAM_000/STOKES_0"][startsample:endsample,:] if rficlean: data = data.T data, ind_use, mask = dumb_clean(data, plot_clean=False) data[np.isnan(data)] = 0.0 data = data.T data = data.astype('f4') if len(data)==0: continue if ii==0: header = read_fil_data(fn_fil_out, start=0, stop=1)[-1] fn_rfi_clean = write_to_fil(data, header, fn_fil_out) elif ii>0: print("Writing chunk %d" % ii) fil_obj = filterbank.FilterbankFile(fn_fil_out, mode='readwrite') fil_obj.append_spectra(data)
def main(): fn = args[0] if fn.endswith(".fil"): # Filterbank file filetype = "filterbank" rawdatafile = filterbank.FilterbankFile(fn) elif fn.endswith(".fits"): # PSRFITS file filetype = "psrfits" rawdatafile = psrfits.PsrfitsFile(fn) else: raise ValueError("Cannot recognize data file type from " "extension. (Only '.fits' and '.fil' " "are supported.)") data, bins, nbins, start, source_name = waterfall(rawdatafile, options.start, \ options.duration, dm=options.dm,\ nbins=options.nbins, nsub=options.nsub,\ subdm=options.subdm, zerodm=options.zerodm, \ downsamp=options.downsamp, \ scaleindep=options.scaleindep, \ width_bins=options.width_bins, mask=options.mask, \ maskfn=options.maskfile, \ csv_file=options.csv_file,\ bandpass_corr=options.bandpass_corr) ofile = plot_waterfall(data, start, source_name, options.duration, \ dm=options.dm,ofile=options.ofile, integrate_ts=options.integrate_ts, \ integrate_spec=options.integrate_spec, show_cb=options.show_cb, cmap_str=options.cmap, sweep_dms=options.sweep_dms, \ sweep_posns=options.sweep_posns, downsamp=options.downsamp,width=options.width,snr=options.snr,csv_file=options.csv_file) # Update CSV file if file is provided if csv_file: sourcename=rawdatafile.header['source_name'] src_ra=rawdatafile.header['src_raj'] src_dec=rawdatafile.header['src_dej'] tstart=rawdatafile.header['tstart'] fch1=rawdatafile.header['fch1'] nchans=rawdatafile.header['nchans'] bw=int(rawdatafile.header['nchans'])*rawdatafile.header['foff'] cat=ofile.split("_")[0] snr=options.snr width=options.width dm=options.dm df = pd.DataFrame({'PNGFILE':[ofile],'Category':[cat],'SNR':[snr],'WIDTH':[width],'DM':[dm],'SourceName':[sourcename],'RA':[src_ra],'DEC':[src_dec],'MJD':[tstart],'Hfreq':[fch1],'NCHANS':[nchans],'BANDWIDTH':[bw],'filename':[fn]}) #Column order coming out irregular, so fixing it here col=['PNGFILE','Category','SNR','WIDTH','DM','SourceName','RA','DEC','MJD','Hfreq','NCHANS','BANDWIDTH','filename'] df = df.reindex(columns=col) if os.path.exists(csv_file) is False: with open(csv_file,'w') as f: df.to_csv(f,header=True,index=False) else: with open(csv_file,'a') as f: df.to_csv(f,header=False,index=False)
def read_fil_data(fn, start=0, stop=1e7): print "Reading filterbank file %s \n" % fn fil_obj = filterbank.FilterbankFile(fn) header = fil_obj.header delta_t = fil_obj.header['tsamp'] # delta_t in milliseconds freq = get_freqs(fil_obj) data = fil_obj.get_spectra(start, stop) # turn array into time-major, for preprocess # data = data.transpose() return data, freq, delta_t, header
def main(): fn = args[0] if fn.endswith(".fil"): # Filterbank file filetype = "filterbank" print_debug("Reading filterbank file..") rawdatafile = filterbank.FilterbankFile(fn) basename = fn[:-4] elif fn.endswith(".fits"): # PSRFITS file filetype = "psrfits" print_debug("Reading PSRFITS file..") rawdatafile = psrfits.PsrfitsFile(fn) basename = fn[:-5] else: raise ValueError("Cannot recognize data file type from " "extension. (Only '.fits' and '.fil' " "are supported.)") if options.outbasenm: basename = options.outbasenm spdcand = spcand.params() if not options.man_params: print_debug('Maximum number of candidates to plot: %i' % options.maxnumcands) make_spd_from_file(spdcand, rawdatafile, \ options.txtfile, options.maskfile, \ options.min_rank, options.group_rank, \ options.plot, options.just_waterfall, \ options.integrate_ts, options.integrate_spec, options.disp_pulse, \ options.loc_pulse, options.nsub, \ options.maxnumcands, \ basename, \ mask=options.mask, barytime=options.barytime, \ bandpass_corr=options.bandpass_corr) else: print_debug("Making spd files based on mannual parameters. I suggest" \ "reading in parameters from the groups.txt file.") make_spd_from_man_params(spdcand, rawdatafile, \ options.txtfile, options.maskfile, \ options.plot, options.just_waterfall, \ options.subdm, options.dm, options.sweep_dms, \ options.sigma, \ options.start, options.duration, \ options.width_bins, options.nbins, options.downsamp, \ options.nsub, \ options.scaleindep, \ options.spec_width, options.loc_pulse, \ options.integrate_ts, options.integrate_spec, options.disp_pulse, \ basename, \ options.mask, options.bandpass_corr, options.barytime, \ options.man_params)
def read_fil_data(fn, start=0, stop=1e7): print("Reading filterbank file %s \n" % fn) fil_obj = filterbank.FilterbankFile(fn) header = fil_obj.header delta_t = fil_obj.header['tsamp'] # delta_t in milliseconds fch1 = header['fch1'] nchans = header['nchans'] foff = header['foff'] fch_f = fch1 + nchans * foff freq = np.linspace(fch1, fch_f, nchans) data = fil_obj.get_spectra(start, stop) # turn array into time-major, for preprocess # data = data.transpose() return data, freq, delta_t, header
def process_full_fil(fn): ii = 0 while True: start, stop = 5e5 * ii, 5e5 * (ii + 1) data, freq, delta_t, header = read_fil_data(fn, start=start, stop=stop) if len(data[0]) == 0: break print data.shape, start # data = dataapply_rfi_filters(data, delta_t) if ii == 0: fn_rfi_clean = write_to_fil(data.transpose(), header, fn) elif ii > 0: fil_obj = filterbank.FilterbankFile(fn_rfi_clean, mode='readwrite') fil_obj.append_spectra(data.transpose()) ii += 1
def main(): fn = args[0] if fn.endswith(".fil"): # Filterbank file filetype = "filterbank" rawdatafile = filterbank.FilterbankFile(fn) elif fn.endswith(".fits"): # PSRFITS file filetype = "psrfits" rawdatafile = psrfits.PsrfitsFile(fn) else: raise ValueError("Cannot recognize data file type from " "extension. (Only '.fits' and '.fil' " "are supported.)") data, bins, nbins, start = waterfall(rawdatafile, options.start, \ options.duration, dm=options.dm,\ nbins=options.nbins, nsub=options.nsub,\ subdm=options.subdm, zerodm=options.zerodm, \ downsamp=options.downsamp, \ scaleindep=options.scaleindep, \ width_bins=options.width_bins, mask=options.mask, \ maskfn=options.maskfile, \ bandpass_corr=options.bandpass_corr) #### PuMA edition starts: #Get the data: nbinlim = np.int(options.duration / data.dt) data_out = np.array(data.data[..., :nbinlim]) Dedisp_ts = data_out.sum(axis=0) times = (np.arange(data.numspectra) * data.dt + start)[..., :nbinlim] # we write the original time and intensity vectors into csv files np.savetxt('times.csv', times, delimiter=',') np.savetxt('original.csv', Dedisp_ts, delimiter=',')
def inject(infile, outfn, prof, period, dm, nbitsout=None, block_size=BLOCKSIZE, pulsar_only=False, inplace=False): if isinstance(infile, filterbank.FilterbankFile): fil = infile elif inplace: fil = filterbank.FilterbankFile(infile, 'readwrite') else: fil = filterbank.FilterbankFile(infile, 'read') print "Injecting pulsar signal into: %s" % fil.filename if False: delays = psr_utils.delay_from_DM(dm, fil.frequencies) delays -= delays[np.argmax(fil.frequencies)] get_phases = lambda times: (times-delays)/period % 1 else: get_phases = lambda times: times/period % 1 # Create the output filterbank file if nbitsout is None: nbitsout = fil.nbits if inplace: warnings.warn("Injecting pulsar signal *in-place*") outfil = fil else: # Start an output file print "Creating out file: %s" % outfn outfil = filterbank.create_filterbank_file(outfn, fil.header, \ nbits=nbitsout, mode='append') if outfil.nbits == 8: raise NotImplementedError("This code is out of date. 'delays' is not " \ "done in this way anymore..") # Read the first second of data to get the global scaling to use onesec = fil.get_timeslice(0, 1).copy() onesec_nspec = onesec.shape[0] times = np.atleast_2d(np.arange(onesec_nspec)*fil.tsamp).T+delays phases = times/period % 1 onesec += prof(phases) minimum = np.min(onesec) median = np.median(onesec) # Set median to 1/3 of dynamic range global_scale = (256.0/3.0) / median del onesec else: # No scaling to be performed # These values will cause scaling to keep data unchanged minimum = 0 global_scale = 1 sys.stdout.write(" %3.0f %%\r" % 0) sys.stdout.flush() oldprogress = -1 # Loop over data lobin = 0 spectra = fil.get_spectra(0, block_size) numread = spectra.shape[0] while numread: if pulsar_only: # Do not write out data from input file # zero it out spectra *= 0 hibin = lobin+numread # Sample at middle of time bin times = (np.arange(lobin, hibin, 1.0/NINTEG_PER_BIN)+0.5/NINTEG_PER_BIN)*fil.dt #times = (np.arange(lobin, hibin)+0.5)*fil.dt phases = get_phases(times) profvals = prof(phases) shape = list(profvals.shape) shape[1:1] = [NINTEG_PER_BIN] shape[0] /= NINTEG_PER_BIN profvals.shape = shape toinject = profvals.mean(axis=1) #toinject = profvals if np.ndim(toinject) > 1: injected = spectra+toinject else: injected = spectra+toinject[:,np.newaxis] scaled = (injected-minimum)*global_scale if inplace: outfil.write_spectra(scaled, lobin) else: outfil.append_spectra(scaled) # Print progress to screen progress = int(100.0*hibin/fil.nspec) if progress > oldprogress: sys.stdout.write(" %3.0f %%\r" % progress) sys.stdout.flush() oldprogress = progress # Prepare for next iteration lobin = hibin spectra = fil.get_spectra(lobin, block_size) numread = spectra.shape[0] sys.stdout.write("Done \n") sys.stdout.flush()
lrows, 100, 25, 4, 7) hdu.close() bank_freq_index += 1 print "writing rows : ", nrow_list[rows], "to", nrow_list[rows + 1] # write the 7 beams # band flip if needed # ignore chan in needed if ignorechannels: if last_pass_flag: last_pass[:, :, ignorechan, :, :] = 0 else: band_pass[:, :, ignorechan, :, :] = 0 for file_num in range(7): file = fb.FilterbankFile(out_file_names[file_num], mode='append') if last_pass_flag: if flip: file.append_spectra((last_pass[:, :, :, 1, file_num] + last_pass[:, :, :, 0, file_num]).reshape( 100 * lrows, 500)[:, ::-1]) else: file.append_spectra((last_pass[:, :, :, 1, file_num] + last_pass[:, :, :, 0, file_num]).reshape( 100 * lrows, 500)) else: if flip: file.append_spectra((band_pass[:, :, :, 1, file_num] + band_pass[:, :, :, 0, file_num]).reshape( 100 * nrows, 500)[:, ::-1]) else:
import copy import filterbank import imp import numpy as np from matplotlib import pyplot as plt """Takes in a filterbank file, dedisperses the Spectra object inside, and displays its dedispersed image.""" wt = imp.load_source('waterfaller', '/home/vgajjar/SpS/sps/src/waterfaller/waterfaller.py') rawdatafile = filterbank.FilterbankFile("/datax/scratch/vgajjar/Test_pipeline/fake.fil") spectra_data, bins, nbins, start_time = wt.waterfall(rawdatafile, 4.4, 0.3, 100, 1024, 128) plt.imshow(spectra_data.data, aspect='auto') plt.show() print(spectra_data.data) lodm, hidm, dmstep = 600, 800, 1 dmvstm_array = [] print(nbins) datacopy = copy.deepcopy(spectra_data) for ii in np.arange(lodm,hidm,dmstep): spectra_data.dedisperse(0,padval='rotate') spectra_data.dedisperse(ii,padval='rotate') Data = np.array(spectra_data.data[..., :nbins]) Dedisp_ts = Data.sum(axis=0) dmvstm_array.append(Dedisp_ts) dmvstm_array=np.array(dmvstm_array) plt.imshow(dmvstm_array, aspect='auto') plt.show() plt.imshow(spectra_data.data, aspect='auto') plt.show()
def main(): infn = args[0] print("Reading filterbank file (%s)" % infn) fil = filterbank.FilterbankFile(infn) if options.start_time is None: startbin = 0 else: startbin = int(np.round(options.start_time / fil.tsamp)) if options.end_time is None: endbin = fil.nspec else: endbin = int(np.round(options.end_time / fil.tsamp)) + 1 new_nspec = endbin - startbin if new_nspec <= 0: raise ValueError("Bad number of spectra to be written (%d). " \ "Check start/end times." % new_nspec) # Determine lo/hi channels to write to file # If high frequencies come first in spectra 'hichan' refers to # the lo-freq cutoff and 'lochan' refers to the hi-freq cutoff. if options.lo_freq is None: if fil.foff > 0: lochan = 0 else: hichan = fil.nchans else: ichan = int(np.round((options.lo_freq - fil.fch1) / fil.foff)) if fil.foff > 0: lochan = ichan else: hichan = ichan + 1 if options.hi_freq is None: if fil.foff > 0: hichan = fil.nchans else: lochan = 0 else: ichan = int(np.round((options.hi_freq - fil.fch1) / fil.foff)) if fil.foff > 0: hichan = ichan + 1 else: lochan = ichan new_nchans = hichan - lochan if new_nchans <= 0: raise ValueError("Bad number of channels to be written (%d). " \ "Check lo/hi frequencies." % new_nchans) print("Will extract") print(" %d bins (%d to %d incl.)" % (new_nspec, startbin, endbin - 1)) print(" (Original num bins: %d)" % fil.nspec) print(" %d channels (%d to %d incl.)" % (new_nchans, lochan, hichan - 1)) print(" (Original num chans: %d)" % fil.nchans) # Create output file outfn = options.outname % fil.header print("Creating out file: %s" % outfn) outhdr = copy.deepcopy(fil.header) outhdr['nchans'] = new_nchans outhdr['fch1'] = fil.frequencies[lochan] filterbank.create_filterbank_file(outfn, outhdr, nbits=fil.nbits) outfil = filterbank.FilterbankFile(outfn, mode='write') # Write data sys.stdout.write(" %3.0f %%\r" % 0) sys.stdout.flush() nblocks = int(new_nspec / options.block_size) remainder = new_nspec % options.block_size oldprogress = -1 for iblock in np.arange(nblocks): lobin = iblock * options.block_size + startbin hibin = lobin + options.block_size spectra = fil.get_spectra(lobin, hibin) spectra = spectra[:, lochan:hichan] # restrict channels outfil.append_spectra(spectra) progress = int(100.0 * ((hibin - startbin) / new_nspec)) if progress > oldprogress: sys.stdout.write(" %3.0f %%\r" % progress) sys.stdout.flush() oldprogress = progress # Read all remaining spectra if remainder: spectra = fil.get_spectra(endbin - remainder, endbin) spectra = spectra[:, lochan:hichan] # restrict channels outfil.append_spectra(spectra) sys.stdout.write("Done \n") sys.stdout.flush()
def readFilterbank(inputFilename, logFile=""): if (logFile == ""): print("Reading filterbank file (%s)...\n" % inputFilename) else: logFile.write("Reading filterbank file (%s)...\n\n" % inputFilename) fb = filterbank.FilterbankFile(inputFilename) inputHeader = copy.deepcopy(fb.header) inputNbits = fb.nbits totalChans = fb.nchans nchans = np.arange(0, fb.nchans - 1, 1) # Top of the band is index 0. freqs = fb.frequencies startbin = 0 endbin = fb.nspec nspec = np.subtract(endbin, startbin) nblocks = int(np.divide(nspec, BLOCKSIZE)) remainder = nspec % BLOCKSIZE totalBlocks = nblocks if (remainder): totalBlocks = nblocks + 1 h5pyFile = h5py.File("%s.hdf5" % inputFilename, "w") spectraData = h5pyFile.create_dataset("data", (totalChans, nspec), dtype="float32") iblock = 0 for iblock in np.arange(0, nblocks, 1): progress = np.multiply(np.divide(iblock + 1.0, totalBlocks), 100.0) if (logFile == ""): sys.stdout.write("Reading... [%3.2f%%]\r" % progress) sys.stdout.flush() else: logFile.write("Reading... [%3.2f%%]\n" % progress) lobin = int(np.add(np.multiply(iblock, BLOCKSIZE), startbin)) hibin = int(np.add(lobin, BLOCKSIZE)) spectra = fb.get_spectra(lobin, hibin) for ichan in np.arange(0, totalChans, 1): spectraData[ichan, lobin:hibin] = spectra[:, ichan] if (remainder): progress = np.multiply(np.divide(iblock + 2.0, totalBlocks), 100.0) if (logFile == ""): sys.stdout.write("Reading... [%3.2f%%]\r" % progress) sys.stdout.flush() else: logFile.write("Reading... [%3.2f%%]\n" % progress) lobin = int(np.subtract(endbin, remainder)) hibin = int(endbin) spectra = fb.get_spectra(lobin, hibin) for ichan in np.arange(0, totalChans, 1): spectraData[ichan, lobin:hibin] = spectra[:, ichan] if (logFile == ""): print("\n") else: logFile.write("\n") return spectraData, inputHeader, inputNbits, h5pyFile
def writeFilterbank(outputFilename, spectraData, inputHeader, inputNbits, logFile=""): if (logFile == ""): print("Writing filterbank file (%s)...\n" % outputFilename) else: logFile.write("Writing filterbank file (%s)...\n\n" % outputFilename) filterbank.create_filterbank_file(outputFilename, inputHeader, nbits=inputNbits) outfil = filterbank.FilterbankFile(outputFilename, mode='write') startbin = 0 endbin = np.shape(spectraData)[1] nblocks = int(np.divide(endbin, BLOCKSIZE)) remainder = endbin % BLOCKSIZE totalBlocks = nblocks if (remainder): totalBlocks = nblocks + 1 iblock = 0 for iblock in np.arange(0, nblocks, 1): progress = np.multiply(np.divide(iblock + 1.0, totalBlocks), 100.0) if (logFile == ""): sys.stdout.write("Writing... [%3.2f%%]\r" % progress) sys.stdout.flush() else: logFile.write("Writing... [%3.2f%%]\n" % progress) lobin = int(np.add(np.multiply(iblock, BLOCKSIZE), startbin)) hibin = int(np.add(lobin, BLOCKSIZE)) spectra = spectraData[:, lobin:hibin].T outfil.append_spectra(spectra) if (remainder): progress = np.multiply(np.divide(iblock + 2.0, totalBlocks), 100.0) if (logFile == ""): sys.stdout.write("Writing... [%3.2f%%]\r" % progress) sys.stdout.flush() else: logFile.write("Writing... [%3.2f%%]\n" % progress) lobin = int(np.subtract(endbin, remainder)) hibin = int(endbin) spectra = spectraData[:, lobin:hibin].T outfil.append_spectra(spectra) if (logFile == ""): print("\n") else: logFile.write("\n") return