def accelsift(filenm, zmax, wmax): """ This function is a translation of the PRESTO code ACCEL_sift.py so that it can be more easily incorporated into our code. It sifts through the ACCEL cands, making cuts on various parameters and removing duplicates and harmonics. """ # Set a bunch of parameters from our params.py file min_num_DMs = params.min_num_DMs low_DM_cutoff = params.low_DM_cutoff sifting.sigma_threshold = params.sigma_threshold sifting.c_pow_threshold = params.c_pow_threshold sifting.known_birds_p = params.known_birds_p sifting.known_birds_f = params.known_birds_f sifting.r_err = params.r_err sifting.short_period = params.short_period sifting.long_period = params.long_period sifting.harm_pow_cutoff = params.harm_pow_cutoff # Try to read the .inf files first, as _if_ they are present, all of # them should be there. (if no candidates are found by accelsearch # we get no ACCEL files... inffiles = glob("*.inf") # Check to see if this is from a short search if len(re.findall("_[0-9][0-9][0-9]M_", inffiles[0])): dmstrs = [x.split("DM")[-1].split("_")[0] for x in candfiles] else: dmstrs = [x.split("DM")[-1].split(".inf")[0] for x in inffiles] dms = map(float, dmstrs) dms.sort() dmstrs = ["%.2f" % x for x in dms] candfiles = glob("*ACCEL_%d_JERK_%d" % (zmax, wmax)) # Read in candfiles cands = sifting.read_candidates(candfiles) # Remove candidates that are duplicated in other ACCEL files if len(cands): cands = sifting.remove_duplicate_candidates(cands) # Remove candidates with DM problems if len(cands): cands = sifting.remove_DM_problems(cands, min_num_DMs, dmstrs, low_DM_cutoff) # Remove candidates that are harmonically related to each other # Note: this includes only a small set of harmonics if len(cands): cands = sifting.remove_harmonics(cands) # Write candidates to STDOUT if len(cands): cands.sort(sifting.cmp_snr) sifting.write_candlist(cands, candfilenm=filenm) Ncands = len(cands) return Ncands
def sift_periodicity(job, dmstrs): # Sift through the candidates to choose the best to fold job.sifting_time = time.time() lo_accel_cands = sifting.read_candidates(glob.glob( "*ACCEL_%d" % config.searching.lo_accel_zmax), track=True) if len(lo_accel_cands): lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands) if len(lo_accel_cands): lo_accel_cands = sifting.remove_DM_problems( lo_accel_cands, config.searching.numhits_to_fold, dmstrs, config.searching.low_DM_cutoff) hi_accel_cands = sifting.read_candidates(glob.glob( "*ACCEL_%d" % config.searching.hi_accel_zmax), track=True) if len(hi_accel_cands): hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands) if len(hi_accel_cands): hi_accel_cands = sifting.remove_DM_problems( hi_accel_cands, config.searching.numhits_to_fold, dmstrs, config.searching.low_DM_cutoff) all_accel_cands = lo_accel_cands + hi_accel_cands if len(all_accel_cands): all_accel_cands = sifting.remove_harmonics(all_accel_cands) # Note: the candidates will be sorted in _sigma_ order, not _SNR_! all_accel_cands.sort(sifting.cmp_sigma) print "Sending candlist to stdout before writing to file" sifting.write_candlist(all_accel_cands) sys.stdout.flush() sifting.write_candlist(all_accel_cands, job.basefilenm + ".accelcands") # Make sifting summary plots all_accel_cands.plot_rejects() plt.title("%s Rejected Cands" % job.basefilenm) plt.savefig(job.basefilenm + ".accelcands.rejects.png") all_accel_cands.plot_summary() plt.title("%s Periodicity Summary" % job.basefilenm) plt.savefig(job.basefilenm + ".accelcands.summary.png") # Write out sifting candidate summary all_accel_cands.print_cand_summary(job.basefilenm + ".accelcands.summary") # Write out sifting comprehensive report of bad candidates all_accel_cands.write_cand_report(job.basefilenm + ".accelcands.report") timed_execute("gzip --best %s" % job.basefilenm + ".accelcands.report") # Moving of results to resultsdir now happens in clean_up(...) # shutil.copy(job.basefilenm+".accelcands", job.outputdir) job.sifting_time = time.time() - job.sifting_time return all_accel_cands
def sift_process_bon(job): # #cpu_idx = int(cpu_id) # Change to the specified working directory os.chdir(job.workdir) # Make the single-pulse plot cmd = "single_pulse_search.py -t %f %s/*.singlepulse"%(singlepulse_plot_SNR,job.workdir) job.singlepulse_time += timed_execute(cmd) # Sort dmstrs job.dmstrs.sort() # Sift through the candidates to choose the best to fold job.sifting_time = time.time() lo_accel_cands = sifting.read_candidates(glob.glob("%s/*ACCEL_%d"%(job.workdir,lo_accel_zmax))) if len(lo_accel_cands): lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands) if len(lo_accel_cands): lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, numhits_to_fold, job.dmstrs, low_DM_cutoff) # # hi_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d"%hi_accel_zmax)) # if len(hi_accel_cands): # hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands) # if len(hi_accel_cands): # hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, numhits_to_fold, # dmstrs, low_DM_cutoff) job.all_accel_cands = lo_accel_cands #+ hi_accel_cands if len(job.all_accel_cands): job.all_accel_cands = sifting.remove_harmonics(job.all_accel_cands) # Note: the candidates will be sorted in _sigma_ order, not _SNR_! job.all_accel_cands.sort(sifting.cmp_sigma) sifting.write_candlist(job.all_accel_cands, job.basefilenm+".accelcands") try: cmd = "cp %s/*.accelcands "%job.workdir +job.outputdir os.system(cmd) except: pass job.sifting_time = time.time() - job.sifting_time
def ACCEL_sift(zmax): ''' The following code come from PRESTO's ACCEL_sift.py ''' globaccel = "*ACCEL_%d" % zmax globinf = "*DM*.inf" # In how many DMs must a candidate be detected to be considered "good" min_num_DMs = 2 # Lowest DM to consider as a "real" pulsar low_DM_cutoff = 2.0 # Ignore candidates with a sigma (from incoherent power summation) less than this sifting.sigma_threshold = 4.0 # Ignore candidates with a coherent power less than this sifting.c_pow_threshold = 100.0 # If the birds file works well, the following shouldn't # be needed at all... If they are, add tuples with the bad # values and their errors. # (ms, err) sifting.known_birds_p = [] # (Hz, err) sifting.known_birds_f = [] # The following are all defined in the sifting module. # But if we want to override them, uncomment and do it here. # You shouldn't need to adjust them for most searches, though. # How close a candidate has to be to another candidate to # consider it the same candidate (in Fourier bins) sifting.r_err = 1.1 # Shortest period candidates to consider (s) sifting.short_period = 0.0005 # Longest period candidates to consider (s) sifting.long_period = 15.0 # Ignore any candidates where at least one harmonic does exceed this power sifting.harm_pow_cutoff = 8.0 # -------------------------------------------------------------- # Try to read the .inf files first, as _if_ they are present, all of # them should be there. (if no candidates are found by accelsearch # we get no ACCEL files... inffiles = glob.glob(globinf) candfiles = glob.glob(globaccel) # Check to see if this is from a short search if len(re.findall("_[0-9][0-9][0-9]M_", inffiles[0])): dmstrs = [x.split("DM")[-1].split("_")[0] for x in candfiles] else: dmstrs = [x.split("DM")[-1].split(".inf")[0] for x in inffiles] dms = map(float, dmstrs) dms.sort() dmstrs = ["%.2f" % x for x in dms] # Read in all the candidates cands = sifting.read_candidates(candfiles) # Remove candidates that are duplicated in other ACCEL files if len(cands): cands = sifting.remove_duplicate_candidates(cands) # Remove candidates with DM problems if len(cands): cands = sifting.remove_DM_problems(cands, min_num_DMs, dmstrs, low_DM_cutoff) # Remove candidates that are harmonically related to each other # Note: this includes only a small set of harmonics if len(cands): cands = sifting.remove_harmonics(cands) # Write candidates to STDOUT if len(cands): cands.sort(sifting.cmp_sigma) # for cand in cands[:1]: # print cand.filename, cand.candnum, cand.p, cand.DMstr # sifting.write_candlist(cands) return cands
def search_job(job): """Search the observation defined in the obs_info instance 'job'. """ # Use whatever .zaplist is found in the current directory zaplist = glob.glob("*.zaplist")[0] print "Using %s as zaplist" % zaplist if config.searching.use_subbands and config.searching.fold_rawdata: # make a directory to keep subbands so they can be used to fold later try: os.makedirs(os.path.join(job.workdir, 'subbands')) except: pass # rfifind the data file cmd = "rfifind %s -time %.17g -o %s %s" % \ (config.searching.datatype_flag, config.searching.rfifind_chunk_time, job.basefilenm, job.filenmstr) job.rfifind_time += timed_execute(cmd, stdout="%s_rfifind.out" % job.basefilenm) maskfilenm = job.basefilenm + "_rfifind.mask" # Find the fraction that was suggested to be masked # Note: Should we stop processing if the fraction is # above some large value? Maybe 30%? job.masked_fraction = find_masked_fraction(job) # Iterate over the stages of the overall de-dispersion plan dmstrs = [] for ddplan in job.ddplans: # Iterate over the individual passes through the data file for passnum in range(ddplan.numpasses): subbasenm = "%s_DM%s"%(job.basefilenm, ddplan.subdmlist[passnum]) if config.searching.use_subbands: try: os.makedirs(os.path.join(job.tempdir, 'subbands')) except: pass # Create a set of subbands cmd = "prepsubband %s -sub -subdm %s -downsamp %d -nsub %d -mask %s " \ "-o %s/subbands/%s %s" % \ (config.searching.datatype_flag, ddplan.subdmlist[passnum], ddplan.sub_downsamp, ddplan.numsub, maskfilenm, job.tempdir, job.basefilenm, job.filenmstr) job.subbanding_time += timed_execute(cmd, stdout="%s.subout" % subbasenm) # Now de-disperse using the subbands cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \ "-nsub %d -numout %d -o %s/%s %s/subbands/%s.sub[0-9]*" % \ (ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, ddplan.dd_downsamp, ddplan.numsub, psr_utils.choose_N(job.orig_N/ddplan.downsamp), job.tempdir, job.basefilenm, job.tempdir, subbasenm) job.dedispersing_time += timed_execute(cmd, stdout="%s.prepout" % subbasenm) else: # Not using subbands cmd = "prepsubband -mask %s -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \ "-numout %d -o %s/%s %s"%\ (maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, ddplan.dd_downsamp*ddplan.sub_downsamp, psr_utils.choose_N(job.orig_N/ddplan.downsamp), job.tempdir, job.basefilenm, job.filenmstr) job.dedispersing_time += timed_execute(cmd) # Iterate over all the new DMs for dmstr in ddplan.dmlist[passnum]: dmstrs.append(dmstr) basenm = os.path.join(job.tempdir, job.basefilenm+"_DM"+dmstr) datnm = basenm+".dat" fftnm = basenm+".fft" infnm = basenm+".inf" # Do the single-pulse search cmd = "single_pulse_search.py -p -m %f -t %f %s"%\ (config.searching.singlepulse_maxwidth, \ config.searching.singlepulse_threshold, datnm) job.singlepulse_time += timed_execute(cmd) try: shutil.move(basenm+".singlepulse", job.workdir) except: pass # FFT, zap, and de-redden cmd = "realfft %s"%datnm job.FFT_time += timed_execute(cmd) cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\ (zaplist, job.baryv, fftnm) job.FFT_time += timed_execute(cmd) cmd = "rednoise %s"%fftnm job.FFT_time += timed_execute(cmd) try: os.rename(basenm+"_red.fft", fftnm) except: pass # Do the low-acceleration search cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \ "-zmax %d -flo %f %s"%\ (config.searching.lo_accel_numharm, \ config.searching.lo_accel_sigma, \ config.searching.lo_accel_zmax, \ config.searching.lo_accel_flo, fftnm) job.lo_accelsearch_time += timed_execute(cmd) try: os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.lo_accel_zmax) except: pass try: # This prevents errors if there are no cand files to copy shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.lo_accel_zmax, \ job.workdir) shutil.move(basenm+"_ACCEL_%d" % config.searching.lo_accel_zmax, \ job.workdir) except: pass # Do the high-acceleration search cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \ "-zmax %d -flo %f %s"%\ (config.searching.hi_accel_numharm, \ config.searching.hi_accel_sigma, \ config.searching.hi_accel_zmax, \ config.searching.hi_accel_flo, fftnm) job.hi_accelsearch_time += timed_execute(cmd) try: os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.hi_accel_zmax) except: pass try: # This prevents errors if there are no cand files to copy shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.hi_accel_zmax, \ job.workdir) shutil.move(basenm+"_ACCEL_%d" % config.searching.hi_accel_zmax, \ job.workdir) except: pass # Move the .inf files try: shutil.move(infnm, job.workdir) except: pass # Remove the .dat and .fft files try: os.remove(datnm) except: pass try: os.remove(fftnm) except: pass if config.searching.use_subbands: if config.searching.fold_rawdata: # Subband files are no longer needed shutil.rmtree(os.path.join(job.tempdir, 'subbands')) else: # Move subbands to workdir for sub in glob.glob(os.path.join(job.tempdir, 'subbands', "*")): shutil.move(sub, os.path.join(job.workdir, 'subbands')) # Make the single-pulse plots basedmb = job.basefilenm+"_DM" basedme = ".singlepulse " # The following will make plots for DM ranges: # 0-110, 100-310, 300-1000+ dmglobs = [basedmb+"[0-9].[0-9][0-9]"+basedme + basedmb+"[0-9][0-9].[0-9][0-9]"+basedme + basedmb+"10[0-9].[0-9][0-9]"+basedme, basedmb+"[12][0-9][0-9].[0-9][0-9]"+basedme + basedmb+"30[0-9].[0-9][0-9]"+basedme, basedmb+"[3-9][0-9][0-9].[0-9][0-9]"+basedme + basedmb+"1[0-9][0-9][0-9].[0-9][0-9]"+basedme] dmrangestrs = ["0-110", "100-310", "300-1000+"] psname = job.basefilenm+"_singlepulse.ps" for dmglob, dmrangestr in zip(dmglobs, dmrangestrs): dmfiles = [] for dmg in dmglob.split(): dmfiles += glob.glob(dmg.strip()) # Check that there are matching files and they are not all empty if dmfiles and sum([os.path.getsize(f) for f in dmfiles]): cmd = 'single_pulse_search.py -t %f -g "%s"' % \ (config.searching.singlepulse_plot_SNR, dmglob) job.singlepulse_time += timed_execute(cmd) os.rename(psname, job.basefilenm+"_DMs%s_singlepulse.ps" % dmrangestr) # Sift through the candidates to choose the best to fold job.sifting_time = time.time() lo_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.lo_accel_zmax)) if len(lo_accel_cands): lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands) if len(lo_accel_cands): lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, config.searching.numhits_to_fold, dmstrs, config.searching.low_DM_cutoff) hi_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.hi_accel_zmax)) if len(hi_accel_cands): hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands) if len(hi_accel_cands): hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, config.searching.numhits_to_fold, dmstrs, config.searching.low_DM_cutoff) all_accel_cands = lo_accel_cands + hi_accel_cands if len(all_accel_cands): all_accel_cands = sifting.remove_harmonics(all_accel_cands) # Note: the candidates will be sorted in _sigma_ order, not _SNR_! all_accel_cands.sort(sifting.cmp_sigma) sifting.write_candlist(all_accel_cands, job.basefilenm+".accelcands") # Moving of results to resultsdir now happens in clean_up(...) # shutil.copy(job.basefilenm+".accelcands", job.outputdir) job.sifting_time = time.time() - job.sifting_time # Fold the best candidates cands_folded = 0 for cand in all_accel_cands: if cands_folded == config.searching.max_cands_to_fold: break if cand.sigma >= config.searching.to_prepfold_sigma: job.folding_time += timed_execute(get_folding_command(cand, job)) cands_folded += 1 job.num_cands_folded = cands_folded # Now step through the .ps files and convert them to .png and gzip them psfiles = glob.glob("*.ps") for psfile in psfiles: # The '[0]' appeneded to the end of psfile is to convert only the 1st page timed_execute("convert -quality 90 %s -background white -flatten -rotate 90 +matte %s" % \ (psfile+"[0]", psfile[:-3]+".png")) timed_execute("gzip "+psfile)
def search_job(job): """Search the observation defined in the obs_info instance 'job'. """ # Use whatever .zaplist is found in the current directory zaplist = glob.glob("*.zaplist")[0] print "Using %s as zaplist" % zaplist if config.searching.use_subbands and config.searching.fold_rawdata: # make a directory to keep subbands so they can be used to fold later try: os.makedirs(os.path.join(job.workdir, 'subbands')) except: pass # rfifind the data file cmd = "rfifind %s -time %.17g -o %s %s" % \ (config.searching.datatype_flag, config.searching.rfifind_chunk_time, job.basefilenm, job.filenmstr) job.rfifind_time += timed_execute(cmd, stdout="%s_rfifind.out" % job.basefilenm) maskfilenm = job.basefilenm + "_rfifind.mask" # Find the fraction that was suggested to be masked # Note: Should we stop processing if the fraction is # above some large value? Maybe 30%? job.masked_fraction = find_masked_fraction(job) # Iterate over the stages of the overall de-dispersion plan dmstrs = [] for ddplan in job.ddplans: # Make a downsampled filterbank file if we are not using subbands if not config.searching.use_subbands: if ddplan.downsamp > 1: cmd = "downsample_psrfits.py %d %s"%(ddplan.downsamp, job.filenmstr) job.downsample_time += timed_execute(cmd) dsfiles = [] for f in job.filenames: fbase = f.rstrip(".fits") dsfiles.append(fbase+"_DS%d.fits"%ddplan.downsamp) filenmstr = ' '.join(dsfiles) else: filenmstr = job.filenmstr # Iterate over the individual passes through the data file for passnum in range(ddplan.numpasses): subbasenm = "%s_DM%s"%(job.basefilenm, ddplan.subdmlist[passnum]) if config.searching.use_subbands: try: os.makedirs(os.path.join(job.tempdir, 'subbands')) except: pass # Create a set of subbands cmd = "prepsubband %s -sub -subdm %s -downsamp %d -nsub %d -mask %s " \ "-o %s/subbands/%s %s" % \ (config.searching.datatype_flag, ddplan.subdmlist[passnum], ddplan.sub_downsamp, ddplan.numsub, maskfilenm, job.tempdir, job.basefilenm, job.filenmstr) job.subbanding_time += timed_execute(cmd, stdout="%s.subout" % subbasenm) # Now de-disperse using the subbands cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \ "-numout %d -o %s/%s %s/subbands/%s.sub[0-9]*" % \ (ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, ddplan.dd_downsamp, psr_utils.choose_N(job.orig_N/ddplan.downsamp), job.tempdir, job.basefilenm, job.tempdir, subbasenm) job.dedispersing_time += timed_execute(cmd, stdout="%s.prepout" % subbasenm) else: # Not using subbands cmd = "prepsubband -mask %s -lodm %.2f -dmstep %.2f -numdms %d " \ "-numout %d -o %s/%s %s"%\ (maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, psr_utils.choose_N(job.orig_N/ddplan.downsamp), job.tempdir, job.basefilenm, filenmstr) job.dedispersing_time += timed_execute(cmd) # Iterate over all the new DMs for dmstr in ddplan.dmlist[passnum]: dmstrs.append(dmstr) basenm = os.path.join(job.tempdir, job.basefilenm+"_DM"+dmstr) datnm = basenm+".dat" fftnm = basenm+".fft" infnm = basenm+".inf" # Do the single-pulse search cmd = "single_pulse_search.py -p -m %f -t %f %s"%\ (config.searching.singlepulse_maxwidth, \ config.searching.singlepulse_threshold, datnm) job.singlepulse_time += timed_execute(cmd) try: shutil.move(basenm+".singlepulse", job.workdir) except: pass # FFT, zap, and de-redden cmd = "realfft %s"%datnm job.FFT_time += timed_execute(cmd) cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\ (zaplist, job.baryv, fftnm) job.FFT_time += timed_execute(cmd) cmd = "rednoise %s"%fftnm job.FFT_time += timed_execute(cmd) try: os.rename(basenm+"_red.fft", fftnm) except: pass # Do the low-acceleration search cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \ "-zmax %d -flo %f %s"%\ (config.searching.lo_accel_numharm, \ config.searching.lo_accel_sigma, \ config.searching.lo_accel_zmax, \ config.searching.lo_accel_flo, fftnm) job.lo_accelsearch_time += timed_execute(cmd) try: os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.lo_accel_zmax) except: pass try: # This prevents errors if there are no cand files to copy shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.lo_accel_zmax, \ job.workdir) shutil.move(basenm+"_ACCEL_%d" % config.searching.lo_accel_zmax, \ job.workdir) except: pass # Do the high-acceleration search cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \ "-zmax %d -flo %f %s"%\ (config.searching.hi_accel_numharm, \ config.searching.hi_accel_sigma, \ config.searching.hi_accel_zmax, \ config.searching.hi_accel_flo, fftnm) job.hi_accelsearch_time += timed_execute(cmd) try: os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.hi_accel_zmax) except: pass try: # This prevents errors if there are no cand files to copy shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.hi_accel_zmax, \ job.workdir) shutil.move(basenm+"_ACCEL_%d" % config.searching.hi_accel_zmax, \ job.workdir) except: pass # Move the .inf files try: shutil.move(infnm, job.workdir) except: pass # Remove the .dat and .fft files try: os.remove(datnm) except: pass try: os.remove(fftnm) except: pass if config.searching.use_subbands: if config.searching.fold_rawdata: # Subband files are no longer needed shutil.rmtree(os.path.join(job.tempdir, 'subbands')) else: # Move subbands to workdir for sub in glob.glob(os.path.join(job.tempdir, 'subbands', "*")): shutil.move(sub, os.path.join(job.workdir, 'subbands')) # Make the single-pulse plots basedmb = job.basefilenm+"_DM" basedme = ".singlepulse " # The following will make plots for DM ranges: # 0-110, 100-310, 300-1000+ dmglobs = [basedmb+"[0-9].[0-9][0-9]"+basedme + basedmb+"[0-9][0-9].[0-9][0-9]"+basedme + basedmb+"10[0-9].[0-9][0-9]"+basedme, basedmb+"[12][0-9][0-9].[0-9][0-9]"+basedme + basedmb+"30[0-9].[0-9][0-9]"+basedme, basedmb+"[3-9][0-9][0-9].[0-9][0-9]"+basedme + basedmb+"1[0-9][0-9][0-9].[0-9][0-9]"+basedme] dmrangestrs = ["0-110", "100-310", "300-1000+"] psname = job.basefilenm+"_singlepulse.ps" for dmglob, dmrangestr in zip(dmglobs, dmrangestrs): dmfiles = [] for dmg in dmglob.split(): dmfiles += glob.glob(dmg.strip()) # Check that there are matching files and they are not all empty if dmfiles and sum([os.path.getsize(f) for f in dmfiles]): cmd = 'single_pulse_search.py -t %f -g "%s"' % \ (config.searching.singlepulse_plot_SNR, dmglob) job.singlepulse_time += timed_execute(cmd) os.rename(psname, job.basefilenm+"_DMs%s_singlepulse.ps" % dmrangestr) # Sift through the candidates to choose the best to fold job.sifting_time = time.time() lo_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.lo_accel_zmax)) if len(lo_accel_cands): lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands) if len(lo_accel_cands): lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, config.searching.numhits_to_fold, dmstrs, config.searching.low_DM_cutoff) hi_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.hi_accel_zmax)) if len(hi_accel_cands): hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands) if len(hi_accel_cands): hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, config.searching.numhits_to_fold, dmstrs, config.searching.low_DM_cutoff) all_accel_cands = lo_accel_cands + hi_accel_cands if len(all_accel_cands): all_accel_cands = sifting.remove_harmonics(all_accel_cands) # Note: the candidates will be sorted in _sigma_ order, not _SNR_! all_accel_cands.sort(sifting.cmp_sigma) sifting.write_candlist(all_accel_cands, job.basefilenm+".accelcands") # Moving of results to resultsdir now happens in clean_up(...) # shutil.copy(job.basefilenm+".accelcands", job.outputdir) job.sifting_time = time.time() - job.sifting_time # Fold the best candidates cands_folded = 0 for cand in all_accel_cands: if cands_folded == config.searching.max_cands_to_fold: break if cand.sigma >= config.searching.to_prepfold_sigma: job.folding_time += timed_execute(get_folding_command(cand, job)) cands_folded += 1 job.num_cands_folded = cands_folded # Now step through the .ps files and convert them to .png and gzip them psfiles = glob.glob("*.ps") for psfile in psfiles: # The '[0]' appeneded to the end of psfile is to convert only the 1st page timed_execute("convert -quality 90 %s -background white -flatten -rotate 90 +matte %s" % \ (psfile+"[0]", psfile[:-3]+".png")) timed_execute("gzip "+psfile)
dms = map(float, dmstrs) dms.sort() dmstrs = ["%.2f" % x for x in dms] # Read in all the candidates cands = sifting.read_candidates(candfiles) # Remove candidates that are duplicated in other ACCEL files if len(cands): cands = sifting.remove_duplicate_candidates(cands) # Remove candidates with DM problems if len(cands): cands = sifting.remove_DM_problems(cands, min_num_DMs, dmstrs, low_DM_cutoff) # Remove candidates that are harmonically related to each other # Note: this includes only a small set of harmonics if len(cands) > 1: cands = sifting.remove_harmonics(cands) print("Number of candidates remaining {}".format(len(cands))) # Write candidates to STDOUT cands_file_name = 'cand_files/cands_' + d.replace("/", "_") + '.txt' if len(cands): cands.sort(sifting.cmp_sigma) sifting.write_candlist(cands, cands_file_name) print(cands_file_name) else: print("No candidates left, created empty file") open(cands_file_name, 'a').close()
print 'Number of candidates %d' % len(accel_cands) if accel_cands: accel_cands = sifting.remove_DM_problems(accel_cands, 2, dm_strs, minimum_dm_cutoff) print 'Number of candidates %d' % len(accel_cands) if accel_cands: # The PRESTO sifting.py module can crash if there are not enough # pulsar candidates, below I test for those symptoms and because # the PRESTO sifting.remove_harmonics overwrites its inputs I # keep a temporary copy around in case there is a crash. # TODO : check whether this fix does not create problems further # down in the sifting process. If there are more problems it is # probably better to discard the candidates and write that to the # log. tmp_accel_cands = copy.deepcopy(accel_cands) try: accel_cands = sifting.remove_harmonics(accel_cands) except IndexError, e: accel_cands = tmp_accel_cands print 'Number of candidates %d' % len(accel_cands) if accel_cands: bpc = knownpulsar.load_bright_pulsar_catalog() accel_cands = knownpulsar.remove_bright_pulsars(accel_cands, bpc) print 'Number of candidates %d' % len(accel_cands) if accel_cands: accel_cands = duplicates.remove_duplicates(accel_cands) print 'Number of candidates %d' % len(accel_cands) sifted_accelcands.extend(accel_cands) sifted_accelcands = duplicates.remove_duplicates(sifted_accelcands) if sifted_accelcands: sifted_accelcands.sort(sifting.cmp_sigma)
def main(fil_filenm, workdir): # Change to the specified working directory os.chdir(workdir) # Get information on the observation and the jbo job = obs_info(fil_filenm) if job.T < low_T_to_search: print "The observation is too short (%.2f s) to search."%job.T sys.exit() job.total_time = time.time() # Use whatever .zaplist is found in the current directory default_zaplist = glob.glob("*.zaplist")[0] # Make sure the output directory (and parent directories) exist try: os.makedirs(job.outputdir) except: pass # Create a directory to hold all the subbands if use_subbands: try: os.makedirs("subbands") except: pass print "\nBeginning PALFA search of '%s'"%job.fil_filenm print "UTC time is: %s"%(time.asctime(time.gmtime())) # rfifind the filterbank file cmd = "rfifind -time %.17g -o %s %s > %s_rfifind.out"%\ (rfifind_chunk_time, job.basefilenm, job.fil_filenm, job.basefilenm) job.rfifind_time += timed_execute(cmd) maskfilenm = job.basefilenm + "_rfifind.mask" # Find the fraction that was suggested to be masked # Note: Should we stop processing if the fraction is # above some large value? Maybe 30%? job.masked_fraction = find_masked_fraction(job) # Iterate over the stages of the overall de-dispersion plan dmstrs = [] for ddplan in ddplans: # Make a downsampled filterbank file if we are not using subbands if not use_subbands: if ddplan.downsamp > 1: cmd = "downsample_filterbank.py %d %s"%(ddplan.downsamp, job.fil_filenm) job.downsample_time += timed_execute(cmd) fil_filenm = job.fil_filenm[:job.fil_filenm.find(".fil")] + \ "_DS%d.fil"%ddplan.downsamp else: fil_filenm = job.fil_filenm # Iterate over the individual passes through the .fil file for passnum in range(ddplan.numpasses): subbasenm = "%s_DM%s"%(job.basefilenm, ddplan.subdmlist[passnum]) if use_subbands: # Create a set of subbands cmd = "prepsubband -sub -subdm %s -downsamp %d -nsub %d -mask %s -o subbands/%s %s > %s.subout"%\ (ddplan.subdmlist[passnum], ddplan.sub_downsamp, ddplan.numsub, maskfilenm, job.basefilenm, job.fil_filenm, subbasenm) job.subbanding_time += timed_execute(cmd) # Now de-disperse using the subbands cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d -numout %d -o %s subbands/%s.sub[0-9]* > %s.prepout"%\ (ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, ddplan.dd_downsamp, job.N/ddplan.downsamp, job.basefilenm, subbasenm, subbasenm) job.dedispersing_time += timed_execute(cmd) else: # Not using subbands cmd = "prepsubband -mask %s -lodm %.2f -dmstep %.2f -numdms %d -numout %d -o %s %s"%\ (maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, job.N/ddplan.downsamp, job.basefilenm, fil_filenm) job.dedispersing_time += timed_execute(cmd) # Iterate over all the new DMs for dmstr in ddplan.dmlist[passnum]: dmstrs.append(dmstr) basenm = job.basefilenm+"_DM"+dmstr datnm = basenm+".dat" fftnm = basenm+".fft" infnm = basenm+".inf" # Do the single-pulse search cmd = "single_pulse_search.py -p -m %f -t %f %s"%\ (singlepulse_maxwidth, singlepulse_threshold, datnm) job.singlepulse_time += timed_execute(cmd) # FFT, zap, and de-redden cmd = "realfft %s"%datnm job.FFT_time += timed_execute(cmd) cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\ (default_zaplist, job.baryv, fftnm) job.FFT_time += timed_execute(cmd) cmd = "rednoise %s"%fftnm job.FFT_time += timed_execute(cmd) try: os.rename(basenm+"_red.fft", fftnm) except: pass # Do the low-acceleration search cmd = "accelsearch -locpow -harmpolish -numharm %d -sigma %f -zmax %d -flo %f %s"%\ (lo_accel_numharm, lo_accel_sigma, lo_accel_zmax, lo_accel_flo, fftnm) job.lo_accelsearch_time += timed_execute(cmd) try: os.remove(basenm+"_ACCEL_%d.txtcand"%lo_accel_zmax) except: pass # Do the high-acceleration search cmd = "accelsearch -locpow -harmpolish -numharm %d -sigma %f -zmax %d -flo %f %s"%\ (hi_accel_numharm, hi_accel_sigma, hi_accel_zmax, hi_accel_flo, fftnm) job.hi_accelsearch_time += timed_execute(cmd) try: os.remove(basenm+"_ACCEL_%d.txtcand"%hi_accel_zmax) except: pass # Remove the .dat and .fft files try: os.remove(datnm) os.remove(fftnm) except: pass # Make the single-pulse plots basedmb = job.basefilenm+"_DM" basedme = ".singlepulse " # The following will make plots for DM ranges: # 0-110, 100-310, 300-1000+ dmglobs = [basedmb+"[0-9].[0-9][0-9]"+basedme + basedmb+"[0-9][0-9].[0-9][0-9]"+basedme + basedmb+"10[0-9].[0-9][0-9]"+basedme, basedmb+"[12][0-9][0-9].[0-9][0-9]"+basedme + basedmb+"30[0-9].[0-9][0-9]"+basedme, basedmb+"[3-9][0-9][0-9].[0-9][0-9]"+basedme + basedmb+"1[0-9][0-9][0-9].[0-9][0-9]"+basedme] dmrangestrs = ["0-110", "100-310", "300-1000+"] psname = job.basefilenm+"_singlepulse.ps" for dmglob, dmrangestr in zip(dmglobs, dmrangestrs): cmd = 'single_pulse_search.py -t %f -g "%s"' % \ (singlepulse_plot_SNR, dmglob) job.singlepulse_time += timed_execute(cmd) try: os.rename(psname, job.basefilenm+"_DMs%s_singlepulse.ps"%dmrangestr) except: pass # Sift through the candidates to choose the best to fold job.sifting_time = time.time() lo_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d"%lo_accel_zmax)) if len(lo_accel_cands): lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands) if len(lo_accel_cands): lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, numhits_to_fold, dmstrs, low_DM_cutoff) hi_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d"%hi_accel_zmax)) if len(hi_accel_cands): hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands) if len(hi_accel_cands): hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, numhits_to_fold, dmstrs, low_DM_cutoff) all_accel_cands = lo_accel_cands + hi_accel_cands if len(all_accel_cands): all_accel_cands = sifting.remove_harmonics(all_accel_cands) # Note: the candidates will be sorted in _sigma_ order, not _SNR_! all_accel_cands.sort(sifting.cmp_sigma) sifting.write_candlist(all_accel_cands, job.basefilenm+".accelcands") try: cmd = "cp *.accelcands "+job.outputdir os.system(cmd) except: pass job.sifting_time = time.time() - job.sifting_time # Fold the best candidates cands_folded = 0 for cand in all_accel_cands: if cands_folded == max_cands_to_fold: break if cand.sigma > to_prepfold_sigma: job.folding_time += timed_execute(get_folding_command(cand, job, ddplans)) cands_folded += 1 # Now step through the .ps files and convert them to .png and gzip them psfiles = glob.glob("*.ps") for psfile in psfiles: if "singlepulse" in psfile: # For some reason the singlepulse files don't transform nicely... epsfile = psfile.replace(".ps", ".eps") os.system("eps2eps "+psfile+" "+epsfile) os.system("pstoimg -density 100 -crop a "+epsfile) try: os.remove(epsfile) except: pass else: os.system("pstoimg -density 100 -flip cw "+psfile) os.system("gzip "+psfile) # NOTE: need to add database commands # Tar up the results files tar_suffixes = ["_ACCEL_%d.tgz"%lo_accel_zmax, "_ACCEL_%d.tgz"%hi_accel_zmax, "_ACCEL_%d.cand.tgz"%lo_accel_zmax, "_ACCEL_%d.cand.tgz"%hi_accel_zmax, "_singlepulse.tgz", "_inf.tgz", "_pfd.tgz", "_bestprof.tgz"] tar_globs = ["*_ACCEL_%d"%lo_accel_zmax, "*_ACCEL_%d"%hi_accel_zmax, "*_ACCEL_%d.cand"%lo_accel_zmax, "*_ACCEL_%d.cand"%hi_accel_zmax, "*.singlepulse", "*_DM[0-9]*.inf", "*.pfd", "*.pfd.bestprof"] for (tar_suffix, tar_glob) in zip(tar_suffixes, tar_globs): tf = tarfile.open(job.basefilenm+tar_suffix, "w:gz") for infile in glob.glob(tar_glob): tf.add(infile) os.remove(infile) tf.close() # And finish up job.total_time = time.time() - job.total_time print "\nFinished" print "UTC time is: %s"%(time.asctime(time.gmtime())) # Write the job report job.write_report(job.basefilenm+".report") job.write_report(os.path.join(job.outputdir, job.basefilenm+".report")) # Copy all the important stuff to the output directory try: cmd = "cp *rfifind.[bimors]* *.ps.gz *.tgz *.png "+job.outputdir os.system(cmd) except: pass
def main(fil_filenm, workdir): # Change to the specified working directory os.chdir(workdir) # Get information on the observation and the jbo job = obs_info(fil_filenm) if job.T < low_T_to_search: print "The observation is too short (%.2f s) to search." % job.T sys.exit() job.total_time = time.time() # Use whatever .zaplist is found in the current directory default_zaplist = glob.glob("*.zaplist")[0] # Make sure the output directory (and parent directories) exist try: os.makedirs(job.outputdir) except: pass # Create a directory to hold all the subbands if use_subbands: try: os.makedirs("subbands") except: pass print "\nBeginning PALFA search of '%s'" % job.fil_filenm print "UTC time is: %s" % (time.asctime(time.gmtime())) # rfifind the filterbank file cmd = "rfifind -time %.17g -o %s %s > %s_rfifind.out"%\ (rfifind_chunk_time, job.basefilenm, job.fil_filenm, job.basefilenm) job.rfifind_time += timed_execute(cmd) maskfilenm = job.basefilenm + "_rfifind.mask" # Find the fraction that was suggested to be masked # Note: Should we stop processing if the fraction is # above some large value? Maybe 30%? job.masked_fraction = find_masked_fraction(job) # Iterate over the stages of the overall de-dispersion plan dmstrs = [] for ddplan in ddplans: # Make a downsampled filterbank file if we are not using subbands if not use_subbands: if ddplan.downsamp > 1: cmd = "downsample_filterbank.py %d %s" % (ddplan.downsamp, job.fil_filenm) job.downsample_time += timed_execute(cmd) fil_filenm = job.fil_filenm[:job.fil_filenm.find(".fil")] + \ "_DS%d.fil"%ddplan.downsamp else: fil_filenm = job.fil_filenm # Iterate over the individual passes through the .fil file for passnum in range(ddplan.numpasses): subbasenm = "%s_DM%s" % (job.basefilenm, ddplan.subdmlist[passnum]) if use_subbands: # Create a set of subbands cmd = "prepsubband -sub -subdm %s -downsamp %d -nsub %d -mask %s -o subbands/%s %s > %s.subout"%\ (ddplan.subdmlist[passnum], ddplan.sub_downsamp, ddplan.numsub, maskfilenm, job.basefilenm, job.fil_filenm, subbasenm) job.subbanding_time += timed_execute(cmd) # Now de-disperse using the subbands cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d -numout %d -o %s subbands/%s.sub[0-9]* > %s.prepout"%\ (ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, ddplan.dd_downsamp, job.N/ddplan.downsamp, job.basefilenm, subbasenm, subbasenm) job.dedispersing_time += timed_execute(cmd) else: # Not using subbands cmd = "prepsubband -mask %s -lodm %.2f -dmstep %.2f -numdms %d -numout %d -o %s %s"%\ (maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, job.N/ddplan.downsamp, job.basefilenm, fil_filenm) job.dedispersing_time += timed_execute(cmd) # Iterate over all the new DMs for dmstr in ddplan.dmlist[passnum]: dmstrs.append(dmstr) basenm = job.basefilenm + "_DM" + dmstr datnm = basenm + ".dat" fftnm = basenm + ".fft" infnm = basenm + ".inf" # Do the single-pulse search cmd = "single_pulse_search.py -p -m %f -t %f %s"%\ (singlepulse_maxwidth, singlepulse_threshold, datnm) job.singlepulse_time += timed_execute(cmd) # FFT, zap, and de-redden cmd = "realfft %s" % datnm job.FFT_time += timed_execute(cmd) cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\ (default_zaplist, job.baryv, fftnm) job.FFT_time += timed_execute(cmd) cmd = "rednoise %s" % fftnm job.FFT_time += timed_execute(cmd) try: os.rename(basenm + "_red.fft", fftnm) except: pass # Do the low-acceleration search cmd = "accelsearch -locpow -harmpolish -numharm %d -sigma %f -zmax %d -flo %f %s"%\ (lo_accel_numharm, lo_accel_sigma, lo_accel_zmax, lo_accel_flo, fftnm) job.lo_accelsearch_time += timed_execute(cmd) try: os.remove(basenm + "_ACCEL_%d.txtcand" % lo_accel_zmax) except: pass # Do the high-acceleration search cmd = "accelsearch -locpow -harmpolish -numharm %d -sigma %f -zmax %d -flo %f %s"%\ (hi_accel_numharm, hi_accel_sigma, hi_accel_zmax, hi_accel_flo, fftnm) job.hi_accelsearch_time += timed_execute(cmd) try: os.remove(basenm + "_ACCEL_%d.txtcand" % hi_accel_zmax) except: pass # Remove the .dat and .fft files try: os.remove(datnm) os.remove(fftnm) except: pass # Make the single-pulse plots basedmb = job.basefilenm + "_DM" basedme = ".singlepulse " # The following will make plots for DM ranges: # 0-110, 100-310, 300-1000+ dmglobs = [ basedmb + "[0-9].[0-9][0-9]" + basedme + basedmb + "[0-9][0-9].[0-9][0-9]" + basedme + basedmb + "10[0-9].[0-9][0-9]" + basedme, basedmb + "[12][0-9][0-9].[0-9][0-9]" + basedme + basedmb + "30[0-9].[0-9][0-9]" + basedme, basedmb + "[3-9][0-9][0-9].[0-9][0-9]" + basedme + basedmb + "1[0-9][0-9][0-9].[0-9][0-9]" + basedme ] dmrangestrs = ["0-110", "100-310", "300-1000+"] psname = job.basefilenm + "_singlepulse.ps" for dmglob, dmrangestr in zip(dmglobs, dmrangestrs): cmd = 'single_pulse_search.py -t %f -g "%s"' % \ (singlepulse_plot_SNR, dmglob) job.singlepulse_time += timed_execute(cmd) try: os.rename(psname, job.basefilenm + "_DMs%s_singlepulse.ps" % dmrangestr) except: pass # Sift through the candidates to choose the best to fold job.sifting_time = time.time() lo_accel_cands = sifting.read_candidates( glob.glob("*ACCEL_%d" % lo_accel_zmax)) if len(lo_accel_cands): lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands) if len(lo_accel_cands): lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, numhits_to_fold, dmstrs, low_DM_cutoff) hi_accel_cands = sifting.read_candidates( glob.glob("*ACCEL_%d" % hi_accel_zmax)) if len(hi_accel_cands): hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands) if len(hi_accel_cands): hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, numhits_to_fold, dmstrs, low_DM_cutoff) all_accel_cands = lo_accel_cands + hi_accel_cands if len(all_accel_cands): all_accel_cands = sifting.remove_harmonics(all_accel_cands) # Note: the candidates will be sorted in _sigma_ order, not _SNR_! all_accel_cands.sort(sifting.cmp_sigma) sifting.write_candlist(all_accel_cands, job.basefilenm + ".accelcands") try: cmd = "cp *.accelcands " + job.outputdir os.system(cmd) except: pass job.sifting_time = time.time() - job.sifting_time # Fold the best candidates cands_folded = 0 for cand in all_accel_cands: if cands_folded == max_cands_to_fold: break if cand.sigma > to_prepfold_sigma: job.folding_time += timed_execute( get_folding_command(cand, job, ddplans)) cands_folded += 1 # Now step through the .ps files and convert them to .png and gzip them psfiles = glob.glob("*.ps") for psfile in psfiles: if "singlepulse" in psfile: # For some reason the singlepulse files don't transform nicely... epsfile = psfile.replace(".ps", ".eps") os.system("eps2eps " + psfile + " " + epsfile) os.system("pstoimg -density 100 -crop a " + epsfile) try: os.remove(epsfile) except: pass else: os.system("pstoimg -density 100 -flip cw " + psfile) os.system("gzip " + psfile) # NOTE: need to add database commands # Tar up the results files tar_suffixes = [ "_ACCEL_%d.tgz" % lo_accel_zmax, "_ACCEL_%d.tgz" % hi_accel_zmax, "_ACCEL_%d.cand.tgz" % lo_accel_zmax, "_ACCEL_%d.cand.tgz" % hi_accel_zmax, "_singlepulse.tgz", "_inf.tgz", "_pfd.tgz", "_bestprof.tgz" ] tar_globs = [ "*_ACCEL_%d" % lo_accel_zmax, "*_ACCEL_%d" % hi_accel_zmax, "*_ACCEL_%d.cand" % lo_accel_zmax, "*_ACCEL_%d.cand" % hi_accel_zmax, "*.singlepulse", "*_DM[0-9]*.inf", "*.pfd", "*.pfd.bestprof" ] for (tar_suffix, tar_glob) in zip(tar_suffixes, tar_globs): tf = tarfile.open(job.basefilenm + tar_suffix, "w:gz") for infile in glob.glob(tar_glob): tf.add(infile) os.remove(infile) tf.close() # And finish up job.total_time = time.time() - job.total_time print "\nFinished" print "UTC time is: %s" % (time.asctime(time.gmtime())) # Write the job report job.write_report(job.basefilenm + ".report") job.write_report(os.path.join(job.outputdir, job.basefilenm + ".report")) # Copy all the important stuff to the output directory try: cmd = "cp *rfifind.[bimors]* *.ps.gz *.tgz *.png " + job.outputdir os.system(cmd) except: pass
candfiles = glob.glob(globaccel) # Check to see if this is from a short search if len(re.findall("_[0-9][0-9][0-9]M_" , inffiles[0])): dmstrs = [x.split("DM")[-1].split("_")[0] for x in candfiles] else: dmstrs = [x.split("DM")[-1].split(".inf")[0] for x in inffiles] dms = map(float, dmstrs) dms.sort() dmstrs = ["%.2f"%x for x in dms] # Read in all the candidates cands = sifting.read_candidates(candfiles) # Remove candidates that are duplicated in other ACCEL files if len(cands): cands = sifting.remove_duplicate_candidates(cands) # Remove candidates with DM problems if len(cands): cands = sifting.remove_DM_problems(cands, min_num_DMs, dmstrs, low_DM_cutoff) # Remove candidates that are harmonically related to each other # Note: this includes only a small set of harmonics if len(cands): cands = sifting.remove_harmonics(cands) # Write candidates to STDOUT if len(cands): cands.sort(sifting.cmp_sigma) sifting.write_candlist(cands)
def main_process_bon(bon_filenm_long, cpu_id, box): # cpu_idx = int(cpu_id) c = bon_filenm_long.rindex("/") bon_filenm = bon_filenm_long[c+1:] print bon_filenm workdir="/data/BON/work/%s"%bon_filenm.rstrip(".fbk") # Make sure the temp directory exist try: os.makedirs(workdir) except: pass # Change to the specified working directory os.chdir(workdir) # Copy data cmd = "scp -p root@clairvaux:/data?/%s_*.fbk ."%(bon_filenm_long.rstrip(".fbk")) print cmd os.system(cmd) # Convert BON file to sigproc format - need .i file print "Join BON .fbk files\n" cmd = "taskset -c %d joinfbk %s_*.fbk"%(cpu_idx,bon_filenm.rstrip(".fbk")) print cmd os.system(cmd) fil_filenm = bon_filenm #fil_filenm = fil_filenm.rstrip(".fb")+".sig" # Get information on the observation and the jbo job = obs_info(fil_filenm, box) if job.T < low_T_to_search: print "The observation is too short (%.2f s) to search."%job.T sys.exit() ddplans = [] if (job.dt<0.000048 and job.nchans==512): # BON 32us and 0.25 Mhz or 512 channels over 128MHz band # The values here are: lodm dmstep dms/call #calls #subbands downsamp ddplans.append(dedisp_plan( 0.0, 0.2, 210, 1, 512, 1)) ddplans.append(dedisp_plan( 42.0, 0.2, 220, 1, 512, 1)) ddplans.append(dedisp_plan( 86.0, 0.2, 221, 1, 512, 1)) ddplans.append(dedisp_plan( 130.2, 0.3, 189, 1, 512, 1)) ddplans.append(dedisp_plan( 186.9, 0.3, 190, 1, 512, 1)) ddplans.append(dedisp_plan( 243.9, 0.5, 189, 1, 512, 2)) ddplans.append(dedisp_plan( 338.4, 0.5, 189, 1, 512, 2)) ddplans.append(dedisp_plan( 432.9, 1.0, 211, 1, 512, 4)) ddplans.append(dedisp_plan( 643.9, 1.0, 210, 1, 512, 8)) ddplans.append(dedisp_plan( 853.9, 3.0, 130, 1, 512, 8)) elif (job.dt<0.000048 and job.nchans==256): # BON 32us and 0.25 Mhz or 256 channels over 64MHz band # The values here are: lodm dmstep dms/call #calls #subbands downsamp ddplans.append(dedisp_plan( 0.0, 0.2, 210, 1, 256, 1)) ddplans.append(dedisp_plan( 42.0, 0.2, 220, 1, 256, 1)) ddplans.append(dedisp_plan( 86.0, 0.2, 221, 1, 256, 1)) ddplans.append(dedisp_plan( 130.2, 0.3, 189, 1, 256, 1)) ddplans.append(dedisp_plan( 186.9, 0.3, 190, 1, 256, 1)) ddplans.append(dedisp_plan( 243.9, 0.5, 189, 1, 256, 2)) ddplans.append(dedisp_plan( 338.4, 0.5, 189, 1, 256, 2)) ddplans.append(dedisp_plan( 432.9, 1.0, 211, 1, 256, 4)) ddplans.append(dedisp_plan( 643.9, 1.0, 210, 1, 256, 8)) ddplans.append(dedisp_plan( 853.9, 3.0, 130, 1, 256, 8)) else: # faster option : for 64us and 1MHz or 128 channels # The values here are: lodm dmstep dms/call #calls #subbands downsamp ddplans.append(dedisp_plan( 0.0, 0.3, 212, 1, 128, 1)) ddplans.append(dedisp_plan( 63.7, 0.5, 75, 1, 128, 1)) ddplans.append(dedisp_plan( 101.2, 1.0, 89, 1, 128, 2)) ddplans.append(dedisp_plan( 190.2, 3.0, 81, 1, 128, 4)) ddplans.append(dedisp_plan( 433.2, 5.0, 155, 1, 128, 8)) job.total_time = time.time() # Use whatever .zaplist is found in the current directory #default_zaplist = glob.glob("*.zaplist")[0] default_zaplist = "/data/BON/BON.zaplist" # Make sure the output directory (and parent directories) exist try: os.makedirs(job.outputdir) except: pass # Create a directory to hold all the subbands if use_subbands: try: os.makedirs("subbands") except: pass print "\nBeginning BON search of '%s'"%job.fil_filenm print "UTC time is: %s"%(time.asctime(time.gmtime())) # rfifind the filterbank file cmd = "taskset -c %d rfifind -filterbank -time %.17g -o %s %s > %s_rfifind.out"%\ (cpu_idx, rfifind_chunk_time, job.basefilenm, job.fil_filenm, job.basefilenm) if (job.dt<0.000048 and job.nchans>=256 and job.fch1<1500.0): cmd = "taskset -c %d rfifind -filterbank -zapchan 106,107,108 -time %.17g -o %s %s > %s_rfifind.out"%\ (cpu_idx, rfifind_chunk_time, job.basefilenm, job.fil_filenm, job.basefilenm) job.rfifind_time += timed_execute(cmd) maskfilenm = job.basefilenm + "_rfifind.mask" # Find the fraction that was suggested to be masked # Note: Should we stop processing if the fraction is # above some large value? Maybe 30%? job.masked_fraction = find_masked_fraction(job) # Iterate over the stages of the overall de-dispersion plan dmstrs = [] for ddplan in ddplans: # Iterate over the individual passes through the .fil file for passnum in range(ddplan.numpasses): subbasenm = "%s_DM%s"%(job.basefilenm, ddplan.subdmlist[passnum]) if use_subbands: # Create a set of subbands cmd = "taskset -c %d prepsubband -filterbank -sub -subdm %s -downsamp %d -nsub %d -mask %s -o subbands/%s %s > %s.subout"%\ (cpu_idx, ddplan.subdmlist[passnum], ddplan.sub_downsamp, ddplan.numsub, maskfilenm, job.basefilenm, job.fil_filenm, subbasenm) job.subbanding_time += timed_execute(cmd) # Now de-disperse using the subbands cmd = "taskset -c %d prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d -numout %d -o %s subbands/%s.sub[0-9]* > %s.prepout"%\ (cpu_idx, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, ddplan.dd_downsamp, job.N/ddplan.downsamp, job.basefilenm, subbasenm, subbasenm) job.dedispersing_time += timed_execute(cmd) else: # Not using subbands cmd = "taskset -c %d prepsubband -filterbank -mask %s -lodm %.2f -dmstep %.2f -numdms %d -numout %d -o %s %s -downsamp %d -nsub %d > /dev/null"%\ (cpu_idx, maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, job.N/ddplan.downsamp, job.basefilenm, fil_filenm, ddplan.downsamp, ddplan.numsub) job.dedispersing_time += timed_execute(cmd) # Iterate over all the new DMs for dmstr in ddplan.dmlist[passnum]: dmstrs.append(dmstr) basenm = job.basefilenm+"_DM"+dmstr datnm = basenm+".dat" fftnm = basenm+".fft" infnm = basenm+".inf" # Do the single-pulse search cmd = "taskset -c %d single_pulse_search.py -p -t %f %s > /dev/null"%\ (cpu_idx, singlepulse_threshold, datnm) #cmd = "taskset -c %d single_pulse_search.py -p -m %f -t %f %s > /dev/null"%\ # (cpu_idx, singlepulse_maxwidth, singlepulse_threshold, datnm) job.singlepulse_time += timed_execute(cmd) #try: # shutil.copy(basenm+".singlepulse", job.outputdir) #except: pass job.nb_sp += count_sp(basenm+".singlepulse") # FFT, zap, and de-redden #cmd = "realfft %s"%datnm #job.FFT_time += timed_execute(cmd) #cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\ # (default_zaplist, job.baryv, fftnm) #job.FFT_time += timed_execute(cmd) #cmd = "rednoise %s"%fftnm #job.FFT_time += timed_execute(cmd) #try: # os.rename(basenm+"_red.fft", fftnm) #except: pass # Do the low-acceleration search cmd = "taskset -c %d accelsearch -numharm %d -sigma %f -zmax %d -flo %f %s -zaplist %s -baryv %.6g > /dev/null"%\ (cpu_idx, lo_accel_numharm, lo_accel_sigma, lo_accel_zmax, lo_accel_flo, datnm,default_zaplist, job.baryv) job.lo_accelsearch_time += timed_execute(cmd) try: os.remove(basenm+"_ACCEL_%d.txtcand"%lo_accel_zmax) except: pass #try: # This prevents errors if there are no cand files to copy # shutil.copy(basenm+"_ACCEL_%d.cand"%lo_accel_zmax, job.outputdir) # shutil.copy(basenm+"_ACCEL_%d"%lo_accel_zmax, job.outputdir) #except: pass # Do the high-acceleration search #cmd = "accelsearch -numharm %d -sigma %f -zmax %d -flo %f %s"%\ # (hi_accel_numharm, hi_accel_sigma, hi_accel_zmax, hi_accel_flo, fftnm) #job.hi_accelsearch_time += timed_execute(cmd) #try: # os.remove(basenm+"_ACCEL_%d.txtcand"%hi_accel_zmax) #except: pass #try: # This prevents errors if there are no cand files to copy # shutil.copy(basenm+"_ACCEL_%d.cand"%hi_accel_zmax, job.outputdir) # shutil.copy(basenm+"_ACCEL_%d"%hi_accel_zmax, job.outputdir) #except: pass # Remove the .dat and .fft files try: os.remove(datnm) os.remove(fftnm) except: pass # Make the single-pulse plot cmd = "taskset -c %d single_pulse_search.py -t %f %s/*.singlepulse"%(cpu_idx, singlepulse_plot_SNR,workdir) job.singlepulse_time += timed_execute(cmd) # Sift through the candidates to choose the best to fold job.sifting_time = time.time() lo_accel_cands = sifting.read_candidates(glob.glob("%s/*ACCEL_%d"%(workdir,lo_accel_zmax))) if len(lo_accel_cands): lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands) if len(lo_accel_cands): lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, numhits_to_fold, dmstrs, low_DM_cutoff) # # hi_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d"%hi_accel_zmax)) # if len(hi_accel_cands): # hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands) # if len(hi_accel_cands): # hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, numhits_to_fold, # dmstrs, low_DM_cutoff) all_accel_cands = lo_accel_cands #+ hi_accel_cands if len(all_accel_cands): all_accel_cands = sifting.remove_harmonics(all_accel_cands) # Note: the candidates will be sorted in _sigma_ order, not _SNR_! all_accel_cands.sort(sifting.cmp_sigma) sifting.write_candlist(all_accel_cands, job.basefilenm+".accelcands") try: cmd = "cp %s/*.accelcands "%workdir +job.outputdir os.system(cmd) except: pass job.sifting_time = time.time() - job.sifting_time # Fold the best candidates cands_folded = 0 for cand in all_accel_cands: if cands_folded == max_cands_to_fold: break if cand.sigma > to_prepfold_sigma: job.folding_time += timed_execute("taskset -c %d "%cpu_idx + get_folding_command(cand, job, ddplans, maskfilenm)) cands_folded += 1 # Now step through the .ps files and convert them to .png and gzip them psfiles = glob.glob("%s/*.ps"%(workdir)) for psfile in psfiles: if "singlepulse" in psfile: # For some reason the singlepulse files don't transform nicely... # epsfile = psfile.replace(".ps", ".eps") # os.system("eps2eps "+psfile+" "+epsfile) os.system("taskset -c %d pstoimg -density 300 -flip cw -scale 0.9 -geometry 640x480 "%cpu_idx +psfile) # try: # os.remove(epsfile) # except: pass else: os.system("taskset -c %d pstoimg -density 100 -flip cw -scale 0.9 -geometry 640x480 "%cpu_idx +psfile) #os.system("gzip "+psfile) # Compress Single-pulse data cmd = "tar cfj %s_sp.tar.bz2 *.singlepulse *_singlepulse.ps *.inf "%job.basefilenm os.system(cmd) # Compress Cands data cmd = "tar cfj %s_accel.tar.bz2 *ACCEL_0"%job.basefilenm os.system(cmd) cmd = "tar cfj %s_accel_bin.tar.bz2 *ACCEL_0.cand"%job.basefilenm os.system(cmd) # Compress RFI data cmd = "tar cfj %s_rfifind.tar.bz2 %s_rfifind*"%(job.basefilenm,job.basefilenm) os.system(cmd) # Compress Candidates PostScript cmd = "tar cfj %s_pfd_ps.tar.bz2 *pfd.ps"%job.basefilenm os.system(cmd) # NOTE: need to add database commands # And finish up job.total_time = time.time() - job.total_time print "\nFinished" print "UTC time is: %s"%(time.asctime(time.gmtime())) # Write the job report job.write_report(job.basefilenm+".report") job.write_report(os.path.join(job.outputdir, job.basefilenm+".report")) # Copy all the important stuff to the output directory try: cmd = "cp *.png *.bz2 "+job.outputdir # Not enough space to save .pfd file #cmd = "cp *rfifind.[bimors]* *.pfd *.ps.gz *.png "+job.outputdir os.system(cmd) except: pass # Send results to clairvaux cmd = "scp -rp %s root@clairvaux:/data6/results/%s/."%(job.outputdir,box) print cmd os.system(cmd) # Remove all temporary stuff cmd = "cd; rm -rf %s"%workdir #os.system(cmd) # Return nb of cands found return len(all_accel_cands)
def main(filename, workdir): # Change to the specified working directory os.chdir(workdir) # Get information on the observation and the jbo job = obs_info(filename) if job.T < low_T_to_search: print "The observation is too short (%.2f s) to search."%job.T sys.exit() job.total_time = time.time() # Use whatever .zaplist is found in the current directory default_zaplist = glob.glob("*.zaplist")[0] # Create a directory to hold all the subbands if use_subbands: try: os.makedirs("subbands") except: pass print "\nBeginning NUPPI search of '%s'"%job.filename print "UTC time is: %s"%(time.asctime(time.gmtime())) # rfifind the psrfits file cmd = "rfifind -zapchan %s -psrfits -timesig %.2f -freqsig %.2f -time %.17g -o %s %s > %s_rfifind.out"%\ (bad_chans, rfifind_timesig, rfifind_freqsig, rfifind_chunk_time, job.basefilename, job.filename, job.basefilename) job.rfifind_time += timed_execute(cmd) # Find the fraction that was suggested to be masked # Note: Should we stop processing if the fraction is # above some large value? Maybe 30%? job.masked_fraction = find_masked_fraction(job) # Iterate over the stages of the overall de-dispersion plan dmstrs = [] for ddplan in ddplans: # Iterate over the individual passes through the .fil file for passnum in range(ddplan.numpasses): subbasenm = "%s_DM%s"%(job.basefilename, ddplan.subdmlist[passnum]) if use_subbands: # Create a set of subbands cmd = "prepsubband -psrfits -sub -subdm %s -downsamp %d -nsub %d -mask %s -o subbands/%s %s > %s.subout"%\ (ddplan.subdmlist[passnum], ddplan.sub_downsamp, ddplan.numsub, job.maskfilenm, job.basefilename, job.filename, subbasenm) job.subbanding_time += timed_execute(cmd) # Now de-disperse using the subbands cmd = "prepsubband -psrfits -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d -numout %d -o %s subbands/%s.sub[0-9]* > %s.prepout"%\ (ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, ddplan.dd_downsamp, job.N/ddplan.downsamp, job.basefilename, subbasenm, subbasenm) job.dedispersing_time += timed_execute(cmd) else: # Not using subbands cmd = "prepsubband -psrfits -downsamp %d -mask %s -lodm %.2f -dmstep %.2f -numdms %d -numout %d -o %s %s"%\ (ddplan.downsamp, job.maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep, ddplan.dmsperpass, job.N/ddplan.downsamp, job.basefilename, filename) job.dedispersing_time += timed_execute(cmd) # Iterate over all the new DMs for dmstr in ddplan.dmlist[passnum]: dmstrs.append(dmstr) basenm = job.basefilename+"_DM"+dmstr datnm = basenm+".dat" infnm = basenm+".inf" # Do the single-pulse search cmd = "single_pulse_search.py -p -m %f -t %f %s"%\ (singlepulse_maxwidth, singlepulse_threshold, datnm) #if float(dmstr) < MAX_DM_2_ZAP_SP: # cmd += " -z" if use_fast_SP: cmd += " -f" job.singlepulse_time += timed_execute(cmd) # Do the low-acceleration search cmd = "accelsearch -harmpolish -zaplist %s -baryv %.6g -numharm %d -sigma %f -zmax %d -flo %f %s"%\ (default_zaplist, job.baryv, lo_accel_numharm, lo_accel_sigma, lo_accel_zmax, lo_accel_flo, datnm) job.lo_accelsearch_time += timed_execute(cmd) try: os.remove(basenm+"_ACCEL_%d.txtcand"%lo_accel_zmax) except: pass # Remove the .dat and .fft files try: os.remove(datnm) except: pass # Make the single-pulse plots basedmb = job.basefilename+"_DM" basedme = ".singlepulse " # The following will make plots for DM ranges: # 0-120, 90-340, 280-1000+ dmglobs = [basedmb+"[0-9].[0-9][0-9]"+basedme + basedmb+"[0-9][0-9].[0-9][0-9]"+basedme + basedmb+"10[0-9].[0-9][0-9]"+basedme, basedmb+"[12][0-9][0-9].[0-9][0-9]"+basedme + basedmb+"30[0-9].[0-9][0-9]"+basedme, basedmb+"[3-9][0-9][0-9].[0-9][0-9]"+basedme + basedmb+"1[0-9][0-9][0-9].[0-9][0-9]"+basedme] dmrangestrs = ["0-120", "90-340", "280-1000+"] psname = job.basefilename+"_singlepulse.ps" for dmglob, dmrangestr in zip(dmglobs, dmrangestrs): cmd = 'single_pulse_search.py -t %f -g "%s"' % \ (singlepulse_plot_SNR, dmglob) job.singlepulse_time += timed_execute(cmd) try: os.rename(psname, job.basefilename+"_DMs%s_singlepulse.ps"%dmrangestr) except: pass # Sift through the candidates to choose the best to fold job.sifting_time = time.time() lo_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d"%lo_accel_zmax)) if len(lo_accel_cands): lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands) if len(lo_accel_cands): lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, numhits_to_fold, dmstrs, low_DM_cutoff) all_accel_cands = lo_accel_cands if len(all_accel_cands): all_accel_cands = sifting.remove_harmonics(all_accel_cands) # Note: the candidates will be sorted in _sigma_ order, not _SNR_! all_accel_cands.sort(sifting.cmp_sigma) sifting.write_candlist(all_accel_cands, job.basefilename+".accelcands") job.sifting_time = time.time() - job.sifting_time # Fold the best candidates cands_folded = 0 for cand in all_accel_cands: if cands_folded == max_cands_to_fold: break if cand.sigma > to_prepfold_sigma: print get_folding_command(cand, job, ddplans) job.folding_time += timed_execute(get_folding_command(cand, job, ddplans)) cands_folded += 1 # Now step through the .ps files and convert them to .png and gzip them psfiles = glob.glob("*.ps") for psfile in psfiles: if "singlepulse" in psfile: # For some reason the singlepulse files don't transform nicely... epsfile = psfile.replace(".ps", ".eps") os.system("eps2eps "+psfile+" "+epsfile) os.system("pstoimg -density 100 -crop a "+epsfile) try: os.remove(epsfile) except: pass else: os.system("pstoimg -density 100 -flip cw "+psfile) os.system("gzip "+psfile) # And finish up job.total_time = time.time() - job.total_time print "\nFinished" print "UTC time is: %s"%(time.asctime(time.gmtime())) # Write the job report job.write_report(job.basefilename+".report")
def ACCEL_sift(zmax): globaccel = "*ACCEL_%d" % zmax globinf = "*DM*.inf" # In how many DMs must a candidate be detected to be considered "good" min_num_DMs = 2 # Lowest DM to consider as a "real" pulsar low_DM_cutoff = 2.0 # Ignore candidates with a sigma (from incoherent power summation) less than this sifting.sigma_threshold = 8.0 # Ignore candidates with a coherent power less than this sifting.c_pow_threshold = 100.0 # If the birds file works well, the following shouldn't # be needed at all... If they are, add tuples with the bad # values and their errors. # (ms, err) sifting.known_birds_p = [] # (Hz, err) sifting.known_birds_f = [] # The following are all defined in the sifting module. # But if we want to override them, uncomment and do it here. # You shouldn't need to adjust them for most searches, though. # How close a candidate has to be to another candidate to # consider it the same candidate (in Fourier bins) sifting.r_err = 1.1 # Shortest period candidates to consider (s) sifting.short_period = 0.0005 # Longest period candidates to consider (s) sifting.long_period = 15.0 # Ignore any candidates where at least one harmonic does exceed this power sifting.harm_pow_cutoff = 8.0 #-------------------------------------------------------------- # Try to read the .inf files first, as _if_ they are present, all of # them should be there. (if no candidates are found by accelsearch # we get no ACCEL files... inffiles = glob.glob(globinf) candfiles = glob.glob(globaccel) # Check to see if this is from a short search if len(re.findall("_[0-9][0-9][0-9]M_" , inffiles[0])): dmstrs = [x.split("DM")[-1].split("_")[0] for x in candfiles] else: dmstrs = [x.split("DM")[-1].split(".inf")[0] for x in inffiles] dms = map(float, dmstrs) dms.sort() dmstrs = ["%.2f"%x for x in dms] # Read in all the candidates cands = sifting.read_candidates(candfiles) # Remove candidates that are duplicated in other ACCEL files if len(cands): cands = sifting.remove_duplicate_candidates(cands) # Remove candidates with DM problems if len(cands): cands = sifting.remove_DM_problems(cands, min_num_DMs, dmstrs, low_DM_cutoff) # Remove candidates that are harmonically related to each other # Note: this includes only a small set of harmonics if len(cands): cands = sifting.remove_harmonics(cands) # Write candidates to STDOUT if len(cands): cands.sort(sifting.cmp_sigma) #for cand in cands[:1]: #print cand.filename, cand.candnum, cand.p, cand.DMstr #sifting.write_candlist(cands) return cands