def ACCEL_sift(zmax):
'''
The following code come from PRESTO's ACCEL_sift.py
'''
globaccel = "*ACCEL_%d" % zmax
globinf = "*DM*.inf"
# In how many DMs must a candidate be detected to be considered "good"
min_num_DMs = 2
# Lowest DM to consider as a "real" pulsar
low_DM_cutoff = 2.0
# Ignore candidates with a sigma (from incoherent power summation) less than this
sifting.sigma_threshold = 4.0
# Ignore candidates with a coherent power less than this
sifting.c_pow_threshold = 100.0
# If the birds file works well, the following shouldn't
# be needed at all... If they are, add tuples with the bad
# values and their errors.
# (ms, err)
sifting.known_birds_p = []
# (Hz, err)
sifting.known_birds_f = []
# The following are all defined in the sifting module.
# But if we want to override them, uncomment and do it here.
# You shouldn't need to adjust them for most searches, though.
# How close a candidate has to be to another candidate to
# consider it the same candidate (in Fourier bins)
sifting.r_err = 1.1
# Shortest period candidates to consider (s)
sifting.short_period = 0.0005
# Longest period candidates to consider (s)
sifting.long_period = 15.0
# Ignore any candidates where at least one harmonic does exceed this power
sifting.harm_pow_cutoff = 8.0
#--------------------------------------------------------------
# Try to read the .inf files first, as _if_ they are present, all of
# them should be there. (if no candidates are found by accelsearch
# we get no ACCEL files...
inffiles = glob.glob(globinf)
candfiles = glob.glob(globaccel)
# Check to see if this is from a short search
if len(re.findall("_[0-9][0-9][0-9]M_", inffiles[0])):
dmstrs = [x.split("DM")[-1].split("_")[0] for x in candfiles]
else:
dmstrs = [x.split("DM")[-1].split(".inf")[0] for x in inffiles]
dms = map(float, dmstrs)
dms.sort()
dmstrs = ["%.2f" % x for x in dms]
# Read in all the candidates
cands = sifting.read_candidates(candfiles)
# Remove candidates that are duplicated in other ACCEL files
if len(cands):
cands = sifting.remove_duplicate_candidates(cands)
# Remove candidates with DM problems
if len(cands):
cands = sifting.remove_DM_problems(cands, min_num_DMs, dmstrs,
low_DM_cutoff)
# Remove candidates that are harmonically related to each other
# Note: this includes only a small set of harmonics
if len(cands):
cands = sifting.remove_harmonics(cands)
# Write candidates to STDOUT
if len(cands):
cands.sort(key=attrgetter('sigma'), reverse=True)
#for cand in cands[:1]:
#print cand.filename, cand.candnum, cand.p, cand.DMstr
#sifting.write_candlist(cands)
return cands
# Check to see if this is from a short search
if len(re.findall("_[0-9][0-9][0-9]M_", inffiles[0])):
dmstrs = [x.split("DM")[-1].split("_")[0] for x in candfiles]
else:
dmstrs = [x.split("DM")[-1].split(".inf")[0] for x in inffiles]
dms = list(map(float, dmstrs))
dms.sort()
dmstrs = ["%.2f" % x for x in dms]
# Read in all the candidates
cands = sifting.read_candidates(candfiles)
# Remove candidates that are duplicated in other ACCEL files
if len(cands):
cands = sifting.remove_duplicate_candidates(cands)
# Remove candidates with DM problems
if len(cands):
cands = sifting.remove_DM_problems(cands, min_num_DMs, dmstrs,
low_DM_cutoff)
# Remove candidates that are harmonically related to each other
# Note: this includes only a small set of harmonics
if len(cands):
cands = sifting.remove_harmonics(cands)
# Write candidates to STDOUT
if len(cands):
cands.sort(key=attrgetter('sigma'), reverse=True)
sifting.write_candlist(cands)
def main(fil_filenm, workdir):
# Change to the specified working directory
os.chdir(workdir)
# Get information on the observation and the jbo
job = obs_info(fil_filenm)
if job.T < low_T_to_search:
print("The observation is too short (%.2f s) to search." % job.T)
sys.exit()
job.total_time = time.time()
# Use whatever .zaplist is found in the current directory
default_zaplist = glob.glob("*.zaplist")[0]
# Make sure the output directory (and parent directories) exist
try:
os.makedirs(job.outputdir)
except:
pass
# Create a directory to hold all the subbands
if use_subbands:
try:
os.makedirs("subbands")
except:
pass
print("\nBeginning PALFA search of '%s'" % job.fil_filenm)
print("UTC time is: %s" % (time.asctime(time.gmtime())))
# rfifind the filterbank file
cmd = "rfifind -time %.17g -o %s %s > %s_rfifind.out"%\
(rfifind_chunk_time, job.basefilenm,
job.fil_filenm, job.basefilenm)
job.rfifind_time += timed_execute(cmd)
maskfilenm = job.basefilenm + "_rfifind.mask"
# Find the fraction that was suggested to be masked
# Note: Should we stop processing if the fraction is
# above some large value? Maybe 30%?
job.masked_fraction = find_masked_fraction(job)
# Iterate over the stages of the overall de-dispersion plan
dmstrs = []
for ddplan in ddplans:
# Make a downsampled filterbank file if we are not using subbands
if not use_subbands:
if ddplan.downsamp > 1:
cmd = "downsample_filterbank.py %d %s" % (ddplan.downsamp,
job.fil_filenm)
job.downsample_time += timed_execute(cmd)
fil_filenm = job.fil_filenm[:job.fil_filenm.find(".fil")] + \
"_DS%d.fil"%ddplan.downsamp
else:
fil_filenm = job.fil_filenm
# Iterate over the individual passes through the .fil file
for passnum in range(ddplan.numpasses):
subbasenm = "%s_DM%s" % (job.basefilenm, ddplan.subdmlist[passnum])
if use_subbands:
# Create a set of subbands
cmd = "prepsubband -sub -subdm %s -downsamp %d -nsub %d -mask %s -o subbands/%s %s > %s.subout"%\
(ddplan.subdmlist[passnum], ddplan.sub_downsamp,
ddplan.numsub, maskfilenm, job.basefilenm,
job.fil_filenm, subbasenm)
job.subbanding_time += timed_execute(cmd)
# Now de-disperse using the subbands
cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d -numout %d -o %s subbands/%s.sub[0-9]* > %s.prepout"%\
(ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp, job.N/ddplan.downsamp,
job.basefilenm, subbasenm, subbasenm)
job.dedispersing_time += timed_execute(cmd)
else: # Not using subbands
cmd = "prepsubband -mask %s -lodm %.2f -dmstep %.2f -numdms %d -numout %d -o %s %s"%\
(maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, job.N/ddplan.downsamp,
job.basefilenm, fil_filenm)
job.dedispersing_time += timed_execute(cmd)
# Iterate over all the new DMs
for dmstr in ddplan.dmlist[passnum]:
dmstrs.append(dmstr)
basenm = job.basefilenm + "_DM" + dmstr
datnm = basenm + ".dat"
fftnm = basenm + ".fft"
infnm = basenm + ".inf"
# Do the single-pulse search
cmd = "single_pulse_search.py -p -m %f -t %f %s"%\
(singlepulse_maxwidth, singlepulse_threshold, datnm)
job.singlepulse_time += timed_execute(cmd)
# FFT, zap, and de-redden
cmd = "realfft %s" % datnm
job.FFT_time += timed_execute(cmd)
cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\
(default_zaplist, job.baryv, fftnm)
job.FFT_time += timed_execute(cmd)
cmd = "rednoise %s" % fftnm
job.FFT_time += timed_execute(cmd)
try:
os.rename(basenm + "_red.fft", fftnm)
except:
pass
# Do the low-acceleration search
cmd = "accelsearch -locpow -harmpolish -numharm %d -sigma %f -zmax %d -flo %f %s"%\
(lo_accel_numharm, lo_accel_sigma, lo_accel_zmax, lo_accel_flo, fftnm)
job.lo_accelsearch_time += timed_execute(cmd)
try:
os.remove(basenm + "_ACCEL_%d.txtcand" % lo_accel_zmax)
except:
pass
# Do the high-acceleration search
cmd = "accelsearch -locpow -harmpolish -numharm %d -sigma %f -zmax %d -flo %f %s"%\
(hi_accel_numharm, hi_accel_sigma, hi_accel_zmax, hi_accel_flo, fftnm)
job.hi_accelsearch_time += timed_execute(cmd)
try:
os.remove(basenm + "_ACCEL_%d.txtcand" % hi_accel_zmax)
except:
pass
# Remove the .dat and .fft files
try:
os.remove(datnm)
os.remove(fftnm)
except:
pass
# Make the single-pulse plots
basedmb = job.basefilenm + "_DM"
basedme = ".singlepulse "
# The following will make plots for DM ranges:
# 0-110, 100-310, 300-1000+
dmglobs = [
basedmb + "[0-9].[0-9][0-9]" + basedme + basedmb +
"[0-9][0-9].[0-9][0-9]" + basedme + basedmb + "10[0-9].[0-9][0-9]" +
basedme, basedmb + "[12][0-9][0-9].[0-9][0-9]" + basedme + basedmb +
"30[0-9].[0-9][0-9]" + basedme,
basedmb + "[3-9][0-9][0-9].[0-9][0-9]" + basedme + basedmb +
"1[0-9][0-9][0-9].[0-9][0-9]" + basedme
]
dmrangestrs = ["0-110", "100-310", "300-1000+"]
psname = job.basefilenm + "_singlepulse.ps"
for dmglob, dmrangestr in zip(dmglobs, dmrangestrs):
cmd = 'single_pulse_search.py -t %f -g "%s"' % \
(singlepulse_plot_SNR, dmglob)
job.singlepulse_time += timed_execute(cmd)
try:
os.rename(psname,
job.basefilenm + "_DMs%s_singlepulse.ps" % dmrangestr)
except:
pass
# Sift through the candidates to choose the best to fold
job.sifting_time = time.time()
lo_accel_cands = sifting.read_candidates(
glob.glob("*ACCEL_%d" % lo_accel_zmax))
if len(lo_accel_cands):
lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands)
if len(lo_accel_cands):
lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands,
numhits_to_fold, dmstrs,
low_DM_cutoff)
hi_accel_cands = sifting.read_candidates(
glob.glob("*ACCEL_%d" % hi_accel_zmax))
if len(hi_accel_cands):
hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands)
if len(hi_accel_cands):
hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands,
numhits_to_fold, dmstrs,
low_DM_cutoff)
all_accel_cands = lo_accel_cands + hi_accel_cands
if len(all_accel_cands):
all_accel_cands = sifting.remove_harmonics(all_accel_cands)
# Note: the candidates will be sorted in _sigma_ order, not _SNR_!
all_accel_cands.sort(key=attrgetter('sigma'), reverse=True)
sifting.write_candlist(all_accel_cands, job.basefilenm + ".accelcands")
try:
cmd = "cp *.accelcands " + job.outputdir
os.system(cmd)
except:
pass
job.sifting_time = time.time() - job.sifting_time
# Fold the best candidates
cands_folded = 0
for cand in all_accel_cands:
if cands_folded == max_cands_to_fold:
break
if cand.sigma > to_prepfold_sigma:
job.folding_time += timed_execute(
get_folding_command(cand, job, ddplans))
cands_folded += 1
# Now step through the .ps files and convert them to .png and gzip them
psfiles = glob.glob("*.ps")
for psfile in psfiles:
if "singlepulse" in psfile:
# For some reason the singlepulse files don't transform nicely...
epsfile = psfile.replace(".ps", ".eps")
os.system("eps2eps " + psfile + " " + epsfile)
os.system("pstoimg -density 100 -crop a " + epsfile)
try:
os.remove(epsfile)
except:
pass
else:
os.system("pstoimg -density 100 -flip cw " + psfile)
os.system("gzip " + psfile)
# NOTE: need to add database commands
# Tar up the results files
tar_suffixes = [
"_ACCEL_%d.tgz" % lo_accel_zmax,
"_ACCEL_%d.tgz" % hi_accel_zmax,
"_ACCEL_%d.cand.tgz" % lo_accel_zmax,
"_ACCEL_%d.cand.tgz" % hi_accel_zmax, "_singlepulse.tgz", "_inf.tgz",
"_pfd.tgz", "_bestprof.tgz"
]
tar_globs = [
"*_ACCEL_%d" % lo_accel_zmax,
"*_ACCEL_%d" % hi_accel_zmax,
"*_ACCEL_%d.cand" % lo_accel_zmax,
"*_ACCEL_%d.cand" % hi_accel_zmax, "*.singlepulse", "*_DM[0-9]*.inf",
"*.pfd", "*.pfd.bestprof"
]
for (tar_suffix, tar_glob) in zip(tar_suffixes, tar_globs):
tf = tarfile.open(job.basefilenm + tar_suffix, "w:gz")
for infile in glob.glob(tar_glob):
tf.add(infile)
os.remove(infile)
tf.close()
# And finish up
job.total_time = time.time() - job.total_time
print("\nFinished")
print("UTC time is: %s" % (time.asctime(time.gmtime())))
# Write the job report
job.write_report(job.basefilenm + ".report")
job.write_report(os.path.join(job.outputdir, job.basefilenm + ".report"))
# Copy all the important stuff to the output directory
try:
cmd = "cp *rfifind.[bimors]* *.ps.gz *.tgz *.png " + job.outputdir
os.system(cmd)
except:
pass