def test_invertFreq(self, filfile): myFil = FilReader(filfile) data = myFil.readBlock(0, 100) outfile = myFil.invertFreq() mynewFil = FilReader(outfile) newdata = mynewFil.readBlock(0, 100) np.testing.assert_equal(mynewFil.header.dtype, myFil.header.dtype) np.testing.assert_equal(mynewFil.header.nsamples, myFil.header.nsamples) np.testing.assert_equal(mynewFil.header.foff, -1 * myFil.header.foff) np.testing.assert_array_equal(data, np.flip(newdata, axis=0)) os.remove(outfile)
def main(): out_name = 'fake_pulsar_train' png_path = os.path.join('./fake_images', out_name) if not os.path.exists(png_path): create_folder(png_path) fil_path = os.path.join('./fake_fils', out_name) if not os.path.exists(fil_path): create_folder(fil_path) ann_path = './annotations' if not os.path.exists(ann_path): create_folder(ann_path) nopulse_files = glob.glob('./nopulse_fils/*.fil') fake_anns = multiprocessing.Manager().dict() count = 0 pool = multiprocessing.Pool(12) # option: 使用多线程加速 for f in tqdm.tqdm(nopulse_files): fil = FilReader(f) creator = PulseCreator(fil) for i in range(20): #每个背景数据生成20个不同参数的fake pulsar文件 pool.apply_async(run, args=(creator, fil_path, png_path, fake_anns)) # run(creator, fil_path, png_path, fake_anns) # count += 1 # if count == 10: # break pool.close() pool.join() # 保存标注文件 fake_anns = dict(fake_anns) with open('./annotations/{}_ann.json'.format(out_name), 'w') as jf: json.dump(fake_anns, jf, indent=4)
def get_spectral_stats(fname, chunksize=10000, show=False): log.info("Getting spectral statistics...") fil = FilReader(fname) header = fil.header nsamples = header['nsamples'] spectrum = 0. spectrsq = 0. for istart in tqdm.tqdm(range(0, nsamples, chunksize)): size = min(chunksize, nsamples - istart) array = fil.readBlock(istart, size, as_filterbankBlock=False) local_spec = array.astype(float).sum(1) local_sq = ((array).astype(float)**2).sum(1) spectrum += local_spec spectrsq += local_sq if show: plt.figure("Spectrum") plt.plot(local_spec, lw=0.5, alpha=0.5, color='k') plt.figure("Var spectrum") plt.plot(np.sqrt(local_sq / size - (local_spec / size)**2), lw=0.5, alpha=0.5, color='k') mean_spec = spectrum / nsamples mean_spectrsq = spectrsq / nsamples std_spec = np.sqrt(mean_spectrsq - mean_spec**2) if show: plt.show() return mean_spec, std_spec
def fb_avg(fname, tgoal=1000, fgoal=128, fscr=None): """ Downsample a filterbank file to appropriate resolution for plotting. """ fr = FilReader(fname) if fscr is None: fscr = int(fr.header['nchans'] / fgoal) tscr = 2**int(np.log2(fr.header['nsamples'] / tgoal)) gulp = tscr nchan = fr.header['nchans'] nblock = fr.header['nsamples'] / tscr nsub = nchan / fscr output = np.empty((nsub, nblock)) for nsamps, i, data in fr.readPlan(gulp, verbose=False): if nsamps != gulp: #print 'breaking at %d'%i break t = data.reshape( (gulp, nchan)).transpose().astype(np.float32).mean(axis=1) output[:, i] = np.average(t.reshape(nchan / fscr, fscr), axis=1) freqs = np.arange( fr.header['nchans']) * fr.header['foff'] + fr.header['fch1'] freqs = freqs.reshape(len(freqs) / fscr, fscr).mean(axis=1) times = np.arange(nblock) * fr.header['tsamp'] * gulp return output, freqs, times
def main(): parser = argparse.ArgumentParser(description='Compute SEFD using the '+ 'input filterbank files') parser.add_argument('-on', dest='on_files', required=True, help = 'on-source filterbank files', nargs='+', type=str) parser.add_argument('-off', dest='off_files', required=True, help = 'off-source filterbank files', nargs='+', type=str) parser.add_argument('-s', dest='source_name', type=str, required=True, help = 'source name') parser.add_argument('-p', dest='plot', action='store_true', help = 'plot on/off power') parser.add_argument('-v', dest='verbose', action='store_true', help = 'be verbose') args = parser.parse_args() # make sure we have the same number of filterbank files assert len(args.on_files) == len(args.off_files),\ "Expecting same number of on and off files" # prepare to read data on_fils = [FilReader(eachFil) for eachFil in args.on_files] off_fils = [FilReader(eachFil) for eachFil in args.off_files] # some sanity checks assert len(set([fil.header.fch1 for fil in on_fils+off_fils])) == 1,\ "Filterbanks do not have the same start frequency" assert len(set([fil.header.bandwidth for fil in on_fils+off_fils])) == 1,\ "Filterbanks do not have the same bandwidth" assert len(set([fil.header.tsamp for fil in on_fils+off_fils])) == 1,\ "Filterbanks do not have the same sampling time" SEFD, SEFD_var, powOn, powOff, maskedArrays, flxSEFD =\ calcSEFDFils(on_fils, off_fils, args.source_name, args.verbose) print ("-"*79) print ("SEFD: ", SEFD) print ("var: ", SEFD_var) print ("-"*79) if args.plot: for i in range(len(powOn)): plt.figure(i) plt.plot(powOn[i], label="On") plt.plot(powOff[i], label="Off") plt.legend() plt.show()
def openfile(filename): files = FilReader(filename) nsamples = files.header['nsamples'] data_arr = files.readBlock(0, nsamples) chans = np.array(range(files.header['nchans'])) bandwidth = files.header['fch1'] - files.header['fbottom'] freqs = files.header['fbottom'] + chans / float( files.header['nchans']) * bandwidth
def get_scaling(dest_value): f = FilReader('gain.fil') bandpass = f.bandpass() / f.header.nsamples maxsample = np.amax(bandpass[np.nonzero(bandpass)]) avgsample = np.average(bandpass[np.nonzero(bandpass)]) print "Maximum sample value: {:.2f}".format(maxsample) print "Average sample value: {:.2f}".format(avgsample) scale = float(dest_value) / maxsample return scale
def plot_sigproc(cand): utc = utc_input.value beam_str = str(int(cand['beam'])).zfill(BeamZfill) fil_path = DataPathTemplate.substitute(BEAM=beam_str, UTC=utc) #fil_path = os.path.join(data_path,".fil") sys.stderr.write("%s\n" % fil_path) fil = FilReader(fil_path) disp = fil.readBlock(int(cand['sample']), 5000) dedisp = disp.dedisperse(cand['dm']) sigproc_source.data = dict(disp=[disp], dedisp=[dedisp], conv=[dedisp])
def split_filterbank(opts, info): filename = info['filename'] segments = info["segments"] output_location = info["basename"] segment_lengths = map(float, segments.split(',')) for segment_length in segment_lengths: obs_length_req = segment_length * 60.0 input_file = FilReader(filename) number_samples = input_file.header.nsamples obs_time_seconds = input_file.header.tobs if obs_length_req > obs_time_seconds: log.info( "Segment length exceeds observation file length. Skipping the split.." ) continue log.info("calculate acceleration range for %d min" % segment_length) pb = 10.0 * segment_length / (60.0) # 10 percent rule amax = calculate_amax(pb, info["mc"]) info['start_acc'] = -1 * amax info['end_acc'] = amax sampling_time = input_file.header.tsamp NTOREAD = int(obs_length_req / sampling_time) number_of_fils = int(obs_time_seconds / obs_length_req) for i in range(number_of_fils + 1): START = i * NTOREAD path = output_location + '_segment_%d_length_%dmin' % ( i, segment_length) + '.fil' output_name = os.path.basename(path)[:-4] if i == number_of_fils: log.info("Last split") input_file.split(number_samples - NTOREAD, NTOREAD, filename=path) info["output_name"] = output_name info["input_name"] = path call_peasoup(info) log.info("Segmented search done!") else: log.info("Split %d" % i) log.info('START at sample: %d' % START) input_file.split(START, NTOREAD, filename=path) info["output_name"] = output_name info["input_name"] = path call_peasoup(info)
def get_power(utc, ant): filfile = glob.glob(os.path.join(OBSDIR, utc, ant, "*x.fil"))[0] fil = FilReader(filfile) chans = np.linspace(fil.header.ftop, fil.header.fbottom, fil.header.nchans) block = fil.readBlock(0, fil.header.nsamples) bp = block.mean(axis=1) args = (chans > 1678) & (chans < 1696) power_inband = (bp[args]).sum() return power_inband
def find_0329(): fnames = glob.glob('/data/kerrm/vlite_fb/*.fil') good_fnames = [] for fname in fnames: try: fr = FilReader(fname) #if '0329' in fr.header.source_name: if fr.header.source_name == 'J0303+4716': good_fnames.append(fname) except: print 'failed on %s.'%fname continue return sorted(good_fnames)
def fil_splitter(fil_fns, outfn, nsub, tqdm_desc=None, verbose=True): """ Split filterbank data to different subband files """ start_time = timeit.default_timer() if isinstance(fil_fns, str): fil_fns = [fil_fns] if len(fil_fns) > 1: raise TypeError(f'Input files: "{len(fil_fns)}". Not supported yet!.') if nsub == 1: raise TypeError(f'Nsub: {nsub}. No need for conversion!.') myFil = FilReader(fil_fns[0]) out_files = [] hdr_changes = {} chanpersub = myFil.header.nchans // nsub for isub in range(nsub): hdr_changes["nchans"] = chanpersub hdr_changes[ "fch1"] = myFil.header.fch1 + isub * chanpersub * myFil.header.foff outfil = f'{outfn.split(".fil")[0]}_sub{str(isub).zfill(2)}.fil' out_files.append( myFil.header.prepOutfile(outfil, hdr_changes, nbits=myFil.header.nbits)) if verbose: print("Splitting Filterbank:") print("---------------------------------") gulpsize = 32768 for nsamps, ii, data in tqdm(myFil.readPlan(gulp=gulpsize, verbose=False), total=np.ceil(myFil.header.nsamples / gulpsize).astype('int'), desc=tqdm_desc): for isub, out_file in enumerate(out_files): data = data.reshape(nsamps, myFil.header.nchans) subint_tofil = data[:, chanpersub * isub:chanpersub * (isub + 1)] out_file.cwrite(subint_tofil.ravel()) # Now close each of the filterbank file. for out_file in out_files: out_file.close() end_time = timeit.default_timer() if verbose: print(f'Done') print(f'Execution time: {(end_time-start_time):.3f} seconds\n')
def fb_avg(fname,tbingoal=1000,fgoal=128,fscr=None,last_minute=False): """ Downsample a filterbank file to appropriate resolution for plotting. """ fr = FilReader(fname) if fscr is None: fscr = int(fr.header['nchans']/fgoal) nsamp = fr.header['nsamples'] if (nsamp*fr.header['tsamp']) < 60: last_minute = False if last_minute: nsamp = int(60./fr.header['tsamp'])+1 try: tscr = 2**int(np.log2(nsamp/tbingoal)) except OverflowError: tscr = 1 gulp = tscr nchan = fr.header['nchans'] nblock = nsamp/tscr nsub = nchan/fscr output = np.empty( (nsub, nblock) ) start = 0 if last_minute: start = max(0,int(fr.header['nsamples']-nsamp)-1) for nsamps,i,data in fr.readPlan(gulp,start=start,verbose=False): if nsamps != gulp: #print 'breaking at %d'%i break t = data.reshape((gulp,nchan)).transpose().astype(np.float32).mean(axis=1) output[:,i] = np.average(t.reshape(nchan/fscr,fscr),axis=1) freqs = np.arange(fr.header['nchans'])*fr.header['foff']+fr.header['fch1'] freqs = freqs.reshape(len(freqs)/fscr,fscr).mean(axis=1) times = (start + np.arange(nblock)*gulp)*fr.header['tsamp'] return output,freqs,times
#!/usr/bin/python from sigpyproc.Readers import FilReader import sigpyproc.Filterbank import matplotlib.pyplot as plt import numpy as np file = FilReader("new.fil") file.downsample(tfactor=500, ffactor=1, filename="test500.fil")
options, args = parser.parse_args() if (not options.fil_file): print('Input file required.') print(parser.print_help()) sys.exit(1) nodsamp = options.nodsamp fil_file = os.path.abspath(options.fil_file) if options.csv_file: csv_file = os.path.abspath(options.csv_file) else: csv_file = "" if options.model: ml_model = options.model else: ml_model = "" origf = FilReader(fil_file) inchans = origf.header['nchans'] inbits = origf.header['nbits'] # For 32-bit BL, downsample to 8-bit and create new file if (nodsamp is not True): if (inchans > 8192 or inbits > 8): print("Running sum_fil") fil_file = downsample(fil_file, inbits, inchans) fname = fil_file.split("/")[-1] print(fil_file) time = options.time timesig = options.timesig freqsig = options.freqsig chanfrac = options.chanfrac
hd_cands = np.genfromtxt('/home/cbochenek/heimdall.cand', dtype=str) boxcar_filters = np.array([1, 2, 3, 5, 10]) SNR = hd_cands[:, 0].astype(float) inds = hd_cands[:, 1].astype(int) cand_time = hd_cands[:, 2].astype(float) cand_sample_width = boxcar_filters[hd_cands[:, 3].astype(int)] cand_DM_trial_n = hd_cands[:, 4].astype(int) cand_DM = hd_cands[:, 5].astype(float) cand_members = hd_cands[:, 6].astype(int) cand_data = hd_cands[:, 7] good_inds = np.where(SNR >= 8.7)[0] with PdfPages("candidates.pdf") as pdf: for ind in good_inds: files = FilReader("%s" % (cand_data[ind])) nsamples = files.header['nsamples'] data_arr = files.readBlock(inds[ind] - 150, 300) tsamp = files.header['tsamp'] chans = np.array(range(files.header['nchans'])) bandwidth = files.header['fch1'] - files.header['fbottom'] freqs = files.header['fbottom'] + chans / float( files.header['nchans']) * bandwidth smooth_bins = int(0.066 / tsamp) plt.figure(ind, figsize=(20, 10)) gs = gridspec.GridSpec(nrows=6, ncols=1) ax0 = plt.subplot(gs[0]) ax1 = plt.subplot(gs[1]) ax2 = plt.subplot(gs[2]) ax3 = plt.subplot(gs[3]) ax4 = plt.subplot(gs[4])
int(num_file * (BeamID - 1) + n_file)] #"*M%02d_%00004d.fil"%(BeamID,n_file) filename_last = pathlist[ index_last] #"*M%02d_%00004d.fil"%(BeamID,n_file-1) filename_next = pathlist[ index_next] #"*M%02d_%00004d.fil"%(BeamID,n_file+1) filename = filename.split('/')[-1] filename_next = filename_next.split('/')[-1] filename_last = filename_last.split('/')[-1] #print "next filterbank:\n",filename_next #print "Last filterbank:\n",filename_last #print "Right Now:\n",filename print "File Path:", fil_path + project + date_f + filename f = FilReader(fil_path + project + date_f + '/' + filename) MJD_Start = f.header.tstart t_rsl = f.header.tsamp # Unit (s) f_rsl = abs(f.header.foff) f_top = f.header.ftop f_bot = f.header.fbottom mjd_burst = sample_ID * t_rsl / 3600 / 24. + MJD_Start sample_delay = np.int( ((f_bot**-2 - f_top**-2) * C * DM) / (t_rsl * 1000)) if sample_delay <= 15000: sample_delay = 15000 filterbank_new_dir = "../Data/Pick_Filterbank_Data/" + project + date_f + cand_f.split( '/')[-1][:-5] + '/' # filterbank_new_dir = cand_f[:-5]+"/Pick_Filterbank_Data/" if not os.path.exists(filterbank_new_dir):
import numpy as np import scipy as sp import sys from sigpyproc.Readers import FilReader import matplotlib.pyplot as plt if __name__ == "__main__": if (len(sys.argv) < 4): print "USAGE : <.fil file> <Start samp> <Total samp> " print "Plot Filterbank file starting from the Start samples with total samples." sys.exit(0) f = FilReader(sys.argv[1]) f_top = f.header['ftop'] f_bott = f.header['fbottom'] t_rsl = f.header['tsamp'] d = f.readBlock(int(sys.argv[2]), int(sys.argv[3])) freq = np.linspace(f_top, f_bott, len(d)) np.save(sys.argv[1][:-4], d) np.save(sys.argv[1][:-4] + '_freq', freq) plt.pcolormesh(d) plt.colorbar() # plt.imshow(d,aspect='auto') plt.savefig('DM_sweep.png') plt.show()
#bandpass=downsampled_file.bandpass() #bandpass.toFile(" from sigpyproc.Readers import FilReader import matplotlib.pyplot as plt import numpy as np #compute the bandpass and frequency of each channel my_bpass = FilReader("100D.32.fil").bandpass() freqs = np.linspace(my_bpass.header.ftop, my_bpass.header.fbottom, my_bpass.size) #plot the result plt.plot(freqs, my_bpass) plt.title("Bandpass") plt.xlabel("Observing frequency (MHz)") plt.ylabel("Power (Arbitrary units)") plt.show() plt.savefig("new_bandpass.png")
import numpy as np import sys import glob from sigpyproc.Readers import FilReader listx = [] listy = [] obsdir = "2021*" #for i, filename in enumerate(sys.argv[1:]): for filename in glob.glob(obsdir): #print(filename) filx = FilReader(filename + "/" + "2kB/" + filename + "_x.fil") #print(filename+"/"+"2kB/"+filename+"_x.fil") blockx = filx.readBlock(0, filx.header.nsamples)[1660:1740] fily = FilReader(filename + "/" + "2kB/" + filename + "_y.fil") blocky = fily.readBlock(0, filx.header.nsamples)[1660:1740] xmean = blockx.mean() ymean = blocky.mean() #print(xmean) listx.append(xmean) listy.append(ymean)
from sigpyproc.Readers import FilReader import numpy as np import matplotlib.pyplot as plt f = FilReader('../data/2011-02-20-01:52:19.fil') data = f.readBlock(2 * 1024 * 20, 1024 * 20) d_l = [] fft_l = [] for i in range(10): d_l.append(data[:, i * 2048:(i + 1) * 2048]) d_l = np.array(d_l) for i in range(10): fft = np.fft.fft2(d_l[i]) fft = np.fft.fftshift(fft) / fft.size fft = fft[1:fft.shape[0] / 2 + 1, fft.shape[1] / 2:fft.shape[1] / 2 + fft.shape[0] / 2] fft[-1, :] = 0 fft[:, 0] = 0 fft_l.append(abs(fft)) for i in range(10): plt.pcolormesh(fft_l[i]) plt.colorbar() plt.show()
def cleanup_data(fname, outname): mask = get_bad_chans(fname) fil = FilReader(fname)
def search_by_chunks(fname, chunk_length=None, new_sample_time=None, tmin=0, dmmin=200, dmmax=800, surelybad=[]): log.info(f"Opening file {fname}") fname_root = os.path.basename(fname).split('.')[0] mask = get_bad_chans(fname) for bad_chan in surelybad: mask[bad_chan] = True fil = FilReader(fname) header = fil.header nsamples = header['nsamples'] sample_time = header['tsamp'] start_freq = header['fbottom'] stop_freq = header['ftop'] bandwidth = header['bandwidth'] sample_time = header['tsamp'] nchan = header['nchans'] foff = header['foff'] date = header['tstart'] delta = delta_delay(dmmax, start_freq, stop_freq) log.info(f"Expected delay at DM {dmmax} between {start_freq}--{stop_freq} MHz: {delta:2} s") if chunk_length is None: chunk_length = delta step = max(int(chunk_length / sample_time) * 2, 128) # Get the _minimum_ DM broadening to set the automatic rebinning dm_dt = dm_broadening(dmmin, start_freq, np.abs(foff)) if new_sample_time is None: new_sample_time = max(dm_dt / 10, sample_time) log.info(f"Expected broadening in spectral bin at DM {dmmin} at {start_freq} MHz: {dm_dt * 1e6:2} us") sampl_ratio = new_sample_time / sample_time N = 1 if sampl_ratio >= 2: N = int(np.rint(sampl_ratio)) new_sample_time = N * sample_time log.info(f"Data will be resampled to {new_sample_time} s") for istart in tqdm.tqdm(range(0, nsamples, step // 2)): chunk_size = min(step, nsamples - istart) t0 = istart * sample_time if t0 < tmin: continue info = PulseInfo() if chunk_size < step // 2: continue iend = istart + chunk_size array = fil.readBlock(istart, chunk_size, as_filterbankBlock=False) array = renormalize_data(array, badchans_mask=mask) print(not np.any(array[mask, :])) # array = (array - array.min()) / np.std(array) if foff < 0: array = array[::-1] if N > 1: array = quick_resample(array, N) info.allprofs = array info.start_freq = start_freq info.bandwidth = bandwidth info.nbin = array.shape[1] info.nchan = array.shape[0] info.date = date info.pulse_freq = 1 / (info.nbin * new_sample_time) table = fast_dedispersion_search(info.allprofs, dmmin, dmmax, start_freq, bandwidth, new_sample_time) plot_diagnostics(info, outname=f'{fname_root}_{istart}-{iend}.jpg', dmmin=dmmin, dmmax=dmmax, t0=t0, show=True) if np.any(table['snr'] > 6): plot_diagnostics(info, outname=f'{fname_root}_{istart}-{iend}.jpg', dmmin=dmmin, dmmax=dmmax, t0=t0) pickle.dump(info, open(f'{fname_root}_{istart}-{iend}.pkl', 'wb'))
import numpy as np import matplotlib.pyplot as plt from sigpyproc.Readers import FilReader import time t1 = time.time() f_dir = '../data/' f_name = 'out.fil' f = FilReader(f_dir + f_name) #dm = np.linspace(1,2000,1000) DM = 100 snr = [] dd = f.dedisperse(DM) snr = (dd.max() - dd.mean()) / dd.std() t2 = time.time() t_tol = t2 - t1 print "Time Total:", t_tol, " s = ", t_tol / 60., " m" plt.title("SNR:" + str(snr)) plt.xlabel("SRC:" + f.header['source_name']) plt.plot(dd) plt.show()
def main(args): inputlist = args.inlist outdir = args.outdir neig = args.neig nbeams = len(inputlist) outnames = [ os.path.join(outdir, "cleaned_" + os.path.basename(i)) for i in inputlist ] if len(set(outnames)) < nbeams: sys.stderr.write( "ERR: please do not use same basename for input filterbank files\n" ) sys.exit() inputfils = [] for beam in inputlist: inputfils.append(FilReader(beam)) nsamples = inputfils[0].header.nsamples nchans = inputfils[0].header.nchans samp_per_block = args.samps work = np.zeros((nbeams, nchans, samp_per_block), dtype=np.float32) if inputfils[0].header.nbits == 8: rescale = True for inputfil in inputfils: inputfil.header.nbits = 32 else: rescale = False outfiles = [ inputfils[ifile].header.prepOutfile(i) for ifile, i in enumerate(outnames) ] total_n_blocks = nsamples / samp_per_block statfile = args.statfile if not statfile.endswith(".eig"): statfile += ".eig" eig_file = open(args.statfile, "w") atexit.register(eig_file.close) if USEPBAR: pbar = Bar('Processing filterbanks', max=total_n_blocks, suffix='%(percent)d%%') atexit.register(pbar.finish) for iblock in range(total_n_blocks): nzaps = [] # Extract data for ibeam in range(nbeams): work[ibeam] = inputfils[ibeam].readBlock(iblock * samp_per_block, samp_per_block) if rescale: work[ibeam] = (work[ibeam].T - np.median(work[ibeam], axis=1)).T std = 1.4826 * get_mad(work[ibeam]) mask = np.abs(std) > 1e-12 work[ibeam][mask] = (work[ibeam][mask].T / std[mask]).T # Do SVD for each data block for ichan in range(nchans): try: U, W, Vt = np.linalg.svd(work[:, ichan, :], full_matrices=False) except Exception as e: np.save("debug.npy", work[:, ichan, :]) sys.stderr.write("Exception caught: saving ./debug.npy\n") raise e neig = neigs_to_zap(W) nzaps.append(neig) if neig > 0: Y = np.dot(work[:, ichan, :], Vt.T) Vt.T[:, :neig] = 0.0 work[:, ichan, :] = np.dot(Y, Vt) # Write output nzap file nzaps = np.array(nzaps, dtype=np.int32) nzaps.tofile(eig_file) # Write output for ibeam, outfile in enumerate(outfiles): if args.resc_out: work[ibeam] = (work[ibeam].T - np.median(work[ibeam], axis=1)).T std = 1.4826 * get_mad(work[ibeam]) mask = ((np.abs(std) > 1e-12) & (nzaps > 0)) work[ibeam][mask] = (work[ibeam][mask].T / std[mask]).T work[ibeam].T.astype("float32").tofile(outfile) if USEPBAR: pbar.next() if USEPBAR: pbar.finish()
cands_snr = cands[:, 0] cands_dm = cands[:, 5] cands_ind = cands[:, 1] cands_width = cands[:, 3] cands2 = np.genfromtxt("/home/user/cand_times_sync_od/heimdall_2.cand") cands2_snr = cands2[:, 0] cands2_dm = cands2[:, 5] cands2_ind = cands2[:, 1] cands2_width = cands2[:, 3] #ovro_int = 20355632#112793376 #gdscc_int = 103012504 #20355632 #ovro = FilReader("/home/user/coincidences/candidate_%s.fil" %ovro_int) #gdscc = FilReader("/home/user/coincidences/candidate_%s.fil" %gdscc_int) ovro = FilReader("candidate_ovro_200428.fil") gdscc = FilReader("candidate_gdscc_200428.fil") gdscc_arr = gdscc.readBlock(0, gdscc.header['nsamples']) ovro_arr = ovro.readBlock(0, ovro.header['nsamples']) ovro_ind = np.where(cands_ind == ovro_int)[0][0].astype(int) gdscc_ind = np.where(cands2_ind == gdscc_int)[0][0].astype(int) av_window = int(np.min([2**cands2_width[gdscc_ind], 2**cands_width[ovro_ind]])) #av_window = 26 dm = 333.3 #cands_dm[ovro_ind]/2. + cands2_dm[gdscc_ind]/2. gdscc_arr_conv = snd.uniform_filter1d(gdscc_arr, av_window, axis=1) ovro_arr_conv = snd.uniform_filter1d(ovro_arr, av_window, axis=1) fs = np.linspace(1280, 1530, 2048) gdscc_dedisp = dedisperse(gdscc_arr_conv, dm, 0.000065536, fs) ovro_dedisp = dedisperse(ovro_arr_conv, dm, 0.000065536, fs)
'names': ('snr', 'time', 'samp_idx', 'dm', 'filter', 'prim_beam'), 'formats': ('f4', 'f4', 'i4', 'f4', 'i4', 'i4') }) #uGMRT #fl = 300 #fh = 500 #FAST fl = 1100 fh = 1500 noplot = 0 tint = 0.000163 Ttot = 80 # Total length of the file kill_time_range = [] kill_chans = [] nchan = 2048 source_name = "Fake" f = FilReader(fil_file) nchan = f.header['nchans'] fch1 = f.header['fch1'] foff = f.header['foff'] tint = f.header['tsamp'] Ttot = f.header['tobs'] fh = fch1 fl = fch1 + (foff * nchan) source_name = f.header['source_name'] extractPlotCand(fil_file, frb_cands, noplot, fl, fh, tint, Ttot, kill_time_range, kill_chans, source_name, nchan)
#PWR = [-3, 0, 3, 6, 9 ,12] PWR = [10] BASEDIR = "/mnt/buf0/obs" outdir = "/home/sonata/RF_signal_generator/june24/same_RF_freq/" for iutc, utc in enumerate(UTCs): print (utc) for ant in os.listdir(os.path.join(BASEDIR, utc)): if len(ant) != 2: continue print(ant) filx = glob.glob(os.path.join(BASEDIR, utc, ant, "*_x.fil"))[0] fil = FilReader(filx) block = fil.readBlock(0, fil.header.nsamples) N = 200000 fig, ax = plt.subplots(nrows=3, sharex=True, figsize=(13,10)) tsx = 10*np.log10(block[:-1].sum(axis=0)) m = np.median(tsx) up = np.median(tsx[tsx > m]) down = np.median(tsx[tsx < m]) ax[0].plot(np.arange(block.shape[1])*fil.header.tsamp, tsx, label="XX") ax[0].hlines([up,down], 0, (block.shape[1])*fil.header.tsamp, ls="--", label="XX compression: %.1f dB" %(up - down))
def get_readers(fil_files, nbeams=16): """Load blimpy Waterfall objects for filterbank file reading""" fils = [] for f in sorted(fil_files)[:nbeams]: fils.append(FilReader(f)) return fils
def main(): parser = argparse.ArgumentParser(description='Sum filterbank'\ 'files') parser.add_argument('-c', dest='candfile', type=str, help="heimdall's candidate file") parser.add_argument('-f', dest='filfile', type=str, help="filterbank file") parser.add_argument('-s', dest='show', action='store_true', help='iteratively show, rather than save, plots') parser.add_argument('-u', dest='utc', type=str, default="", help='utc name (to append to png name') parser.add_argument( '-n', dest='maxcand', type=int, help='Maximum number of candidates to plot [default: %i]' % (MAXN), default=MAXN) parser.add_argument('-z', '--zap_chans', dest='zap_chans', action='append', nargs=2, type=int, help='channels to zap') args = parser.parse_args() all_candidates = pd.read_table(args.candfile, index_col=False, names=COL_NAMES) # hack to get the sampling time tsamp = (all_candidates.tsec / all_candidates.tsample).mean() box_thresh = int(np.log2(W_THRESH / tsamp)) all_candidates = all_candidates[(all_candidates.SNR > SNR_THRESH) & (all_candidates.boxcar <= box_thresh) & (all_candidates.DM > DM_THRESH)] basedir = os.path.dirname(args.candfile) candsdir = os.path.join(basedir, "candidates") if not args.show: if not os.path.isdir(candsdir): os.mkdir(candsdir) fil = FilReader(args.filfile) nskip = int(0.3 / fil.header.tsamp) print("ncands: %i" % all_candidates.shape[0]) pbar = tqdm(total=all_candidates.shape[0]) if all_candidates.shape[0] > args.maxcand: print ("WARNING: number of candidates (%i) is greater than"\ "the maximum allowed (%i)" %(all_candidates.shape[0], args.maxcand)) if not os.path.exists("cands.warning"): open("cands.warning", 'a').close() icand = 0 for ind, cand in all_candidates.iterrows(): dm_smear_sec = (8.3 * fil.header.bandwidth*1e-3 *\ cand.DM * (fil.header.fcenter*1e-3)**(-3))*1e-3 dm_smear_nsamp = int(dm_smear_sec / fil.header.tsamp) istart = int(cand.tsample) - nskip istart = 0 if istart < 0 else istart block = fil.readBlock(istart, dm_smear_nsamp + 2 * nskip) # zap unwanted channels if args.zap_chans: for low_freq, high_freq in args.zap_chans: block[low_freq:high_freq] = 0. #np.nan disp_org = block.dedisperse(cand.DM) if cand.SNR < 10: ffactor = 32 elif cand.SNR < 15: ffactor = 16 elif cand.SNR < 20: ffactor = 8 else: ffactor = 1 tfactor = min(8, int(2**cand.boxcar)) disp = block_reduce(disp_org, (ffactor, tfactor), func=np.sum, cval=np.mean(disp_org)) matplotlib.use("Agg") fig = plt.figure(0) fig.suptitle("SNR = %.2f, DM = %.2f, boxcar width = %i" % (cand.SNR, cand.DM, cand.boxcar)) gs = gridspec.GridSpec(2, 1) gs.update(hspace=0) ax1 = fig.add_subplot(gs[0]) ax2 = fig.add_subplot(gs[1], sharex=ax1) ts = disp.sum(axis=0) x = np.arange( int(cand.tsample) - nskip, int(cand.tsample) - nskip + len(ts) * tfactor, tfactor) ax1.plot(x * fil.header.tsamp, ts) ax1.get_xaxis().set_visible(False) ylim = ax1.get_ylim() ax1.vlines(np.array([cand.tsample - (2**cand.boxcar/2), cand.tsample + (2**cand.boxcar/2)])*fil.header.tsamp -\ tfactor/2*fil.header.tsamp, ymin=ylim[0], ymax=ylim[1], lw=1, ls='--') if args.zap_chans: for low_freq, high_freq in args.zap_chans: disp_org[low_freq:high_freq] = np.nan disp_org = block_reduce(disp_org, (ffactor, tfactor), func=np.sum, cval=np.mean(disp_org)) ax2.imshow(disp_org, interpolation='nearest', aspect='auto', extent=[ x[0] * fil.header.tsamp, x[-1] * fil.header.tsamp, fil.header.fbottom, fil.header.ftop ]) ax2.set_ylabel("Frequency (MHz)") ax2.set_xlabel("Time since obs_start") if args.show: plt.show() else: basefigname = "tsamp_%i_dm_%.2f.png" % (cand.tsample, cand.DM) if args.utc: basefigname = args.utc + "_" + basefigname figname = os.path.join(candsdir, basefigname) plt.savefig(figname, fmt='png', bbox_inches='tight') plt.close(fig) pbar.update(1) if icand > args.maxcand: sys.exit(-1) icand += 1 pbar.close()