def run_prepsubband(basename, maskname, fitslist, dmlow=params.dmlow, \
ddm=params.ddm, ndm=params.dmspercall, \
downsample=params.downsample, nsub=params.nsub):
t_prep_start = time.time()
fitsfiles = ' '.join(fitslist)
print("Dedispersing with 1st batch of DMs")
orig_N = readhdr.get_samples(fitslist, params.dat_type)
numout = psr_utils.choose_N(orig_N)
print(orig_N, numout)
other_flags = params.prep_otherflags
if params.use_mask:
cmd = 'prepsubband -o %s -psrfits -nsub %d -numout %d -lodm %.6f -dmstep %.6f '\
'-numdms %d -downsamp %d %s -mask %s %s' %(basename, nsub, numout/downsample,
dmlow, ddm, ndm, downsample,
other_flags, maskname, fitsfiles)
else:
cmd = 'prepsubband -o %s -psrfits -nsub %d -numout %d -lodm %.6f -dmstep %.6f '\
'-numdms %d -downsamp %d %s %s' %(basename, nsub, numout/downsample,
dmlow, ddm, ndm, downsample,
other_flags, fitsfiles)
try_cmd(cmd)
t_prep_end = time.time()
dt = t_prep_end - t_prep_start
print "De-dispersion took %.2f hours.\n" %(dt/3600.)
return dt
def main(hotpotato):
print("Running PRETO prepsubband.")
t_prep_start = time.time()
params_list = [
'directory', 'rfi_dir', 'prep_dir', 'basename', 'prep_usemask',
'downsample', 'prep_flags', 'prep_otherflags', 'filetype'
]
print_params(params_list)
# get/set file locations
work_dir = get_value(hotpotato, 'directory')
rfi_dir = get_value(hotpotato, 'rfi_dir')
prep_dir = get_value(hotpotato, 'prep_dir') # can change this
basename = get_value(hotpotato, 'basename')
fitslist = glob('%s/%s*.fits' % (rfi_dir, basename))
fitslist.sort()
fitsfiles = ' '.join(fitslist)
# get de-dispersion parameters
prep_usemask = get_value(hotpotato, 'prep_usemask')
downsample = get_value(hotpotato, 'downsample')
prep_flags = get_value(hotpotato, 'prep_flags')
prep_otherflags = get_value(hotpotato, 'prep_otherflags')
# run prepsubband command
print("Dedispersing with 1st batch of DMs")
orig_N = get_samples(fitslist, get_value(hotpotato, 'filetype'))
numout = psr_utils.choose_N(orig_N) / downsample
print(orig_N, numout)
if prep_usemask == True:
# make sure rfi_find has been run previously
try:
rfi_maskname = glob(rfi_dir + '/*.mask')[0]
except IndexError:
raise Exception("Could not access RFI_mask fits file. Please run PRESTO rfifind "\
"before generating masked dynamic spectrum.")
cmd = 'prepsubband -o %s -numout %d %s %s -mask %s %s' % (
basename, numout, prep_flags, prep_otherflags, rfi_maskname,
fitsfiles)
else:
cmd = 'prepsubband -o %s %s %s %s' % (basename, numout, prep_flags,
prep_otherflags, fitsfiles)
try_cmd(cmd)
# move output to prep_dir
mv_cmd1 = 'mv %s %s' % (work_dir + '/*.dat', prep_dir)
mv_cmd2 = 'mv %s %s' % (work_dir + '/*.inf', prep_dir)
try_cmd(mv_cmd1)
try_cmd(mv_cmd2)
t_prep_end = time.time()
prep_time = (t_prep_end - t_prep_start)
print("PRESTO prepsubband took %f seconds." % (prep_time))
return hotpotato
def __init__(self, fil_filenm):
self.fil_filenm = fil_filenm
self.basefilenm = fil_filenm.rstrip(".fil")
self.beam = int(self.basefilenm[-1])
filhdr, self.hdrlen = sigproc.read_header(fil_filenm)
self.orig_filenm = filhdr['rawdatafile']
self.MJD = filhdr['tstart']
self.nchans = filhdr['nchans']
self.ra_rad = sigproc.ra2radians(filhdr['src_raj'])
self.ra_string = psr_utils.coord_to_string(\
*psr_utils.rad_to_hms(self.ra_rad))
self.dec_rad = sigproc.dec2radians(filhdr['src_dej'])
self.dec_string = psr_utils.coord_to_string(\
*psr_utils.rad_to_dms(self.dec_rad))
self.az = filhdr['az_start']
self.el = 90.0-filhdr['za_start']
self.BW = abs(filhdr['foff']) * filhdr['nchans']
self.dt = filhdr['tsamp']
self.orig_N = sigproc.samples_per_file(fil_filenm, filhdr, self.hdrlen)
self.orig_T = self.orig_N * self.dt
self.N = psr_utils.choose_N(self.orig_N)
self.T = self.N * self.dt
# Update the RA and DEC from the database file if required
newposn = read_db_posn(self.orig_filenm, self.beam)
if newposn is not None:
self.ra_string, self.dec_string = newposn
# ... and use them to update the filterbank file
fix_fil_posn(fil_filenm, self.hdrlen,
self.ra_string, self.dec_string)
# Determine the average barycentric velocity of the observation
self.baryv = get_baryv(self.ra_string, self.dec_string,
self.MJD, self.T, obs="AO")
# Where to dump all the results
# Directory structure is under the base_output_directory
# according to base/MJD/filenmbase/beam
self.outputdir = os.path.join(base_output_directory,
str(int(self.MJD)),
self.basefilenm[:-2],
str(self.beam))
# Figure out which host we are processing on
self.hostname = socket.gethostname()
# The fraction of the data recommended to be masked by rfifind
self.masked_fraction = 0.0
# Initialize our timers
self.rfifind_time = 0.0
self.downsample_time = 0.0
self.subbanding_time = 0.0
self.dedispersing_time = 0.0
self.FFT_time = 0.0
self.lo_accelsearch_time = 0.0
self.hi_accelsearch_time = 0.0
self.singlepulse_time = 0.0
self.sifting_time = 0.0
self.folding_time = 0.0
self.total_time = 0.0
# Inialize some candidate counters
self.num_sifted_cands = 0
self.num_folded_cands = 0
self.num_single_cands = 0
def multi_call_prepsubband(basename, maskname, fitslist, dmlow=params.dmlow, \
ddm=params.ddm, downsample=params.downsample, \
dmcalls=params.dmcalls, nsub=params.nsub, \
dsubDM=params.dsubDM, \
dmspercall=params.dmspercall):
t_prep_start = time.time()
fitsfiles = ' '.join(fitslist)
orig_N = readhdr.get_samples(fitslist, params.dat_type)
numout = psr_utils.choose_N(orig_N)
other_flags = params.prep_otherflags
# Downsample organization as in PRESTO dedisp.py (why?)
sub_downsample = downsample / 2
dat_downsample = 2
if downsample < 2: sub_downsample = dat_downsample = 1
print("Dedispersing using %d calls on %d subbands\n" %(dmcalls, nsub))
for ii in xrange(dmcalls):
subDM = dmlow + (ii+0.5)*dsubDM
# Make subband
if params.use_mask:
cmd_sub = "prepsubband -o %s -sub -subdm %.2f -nsub %d -downsamp %d %s -mask %s %s" \
%(basename, subDM, nsub, sub_downsample, other_flags, maskname, fitsfiles)
else:
cmd_sub = "prepsubband -o %s -sub -subdm %.2f -nsub %d -downsamp %d %s %s" \
%(basename, subDM, nsub, sub_downsample, other_flags, fitsfiles)
try_cmd(cmd_sub)
# Get dedispersed time series
sub_dmlow = dmlow + ii*dsubDM
subfiles = basename+"_DM%.2f.sub[0-9]*" %subDM
if params.use_mask:
cmd_dat = "prepsubband -o %s -numout %d -lodm %.2f -dmstep %.2d "\
"-numdms %d -downsamp %d %s -mask %s %s" \
%(basename, numout/downsample, sub_dmlow, ddm, dmspercall, dat_downsample, other_flags, maskname, subfiles)
else:
cmd_dat = "prepsubband -o %s -numout %d -lodm %.2f -dmstep %.2d "\
"-numdms %d -downsamp %d %s %s" \
%(basename, numout/downsample, sub_dmlow, ddm, dmspercall, dat_downsample, other_flags, subfiles)
try_cmd(cmd_dat)
t_prep_end = time.time()
dt = t_prep_end - t_prep_start
print "De-dispersion took %.2f hours.\n" %(dt/3600.)
return dt
def __init__(self, filename):
self.filename = filename
self.basefilename = filename.replace(".fits","")
pfits = psrfits.Psrfits(self.filename)
self.MJD = pfits.get_MJD()
self.nchans = pfits.get_nchan()
self.BW = pfits.get_BW()
self.dt = pfits.get_tsamp()
self.ra_string = pfits.get_RA()
self.dec_string = pfits.get_DEC()
self.orig_N = pfits.get_nsamp()
self.orig_T = pfits.get_obslen()
self.N = psr_utils.choose_N(self.orig_N)
self.T = self.N * self.dt
# Determine the average barycentric velocity of the observation
self.baryv = get_baryv(self.ra_string, self.dec_string,
self.MJD, self.T, obs="NC")
# Figure out which host we are processing on
self.hostname = socket.gethostname()
# The fraction of the data recommended to be masked by rfifind
self.maskfilenm = self.basefilename + "_rfifind.mask"
self.masked_fraction = 0.0
# Initialize our timers
self.rfifind_time = 0.0
self.downsample_time = 0.0
self.subbanding_time = 0.0
self.dedispersing_time = 0.0
self.FFT_time = 0.0
self.lo_accelsearch_time = 0.0
self.hi_accelsearch_time = 0.0
self.singlepulse_time = 0.0
self.sifting_time = 0.0
self.folding_time = 0.0
self.total_time = 0.0
# Inialize some candidate counters
self.num_sifted_cands = 0
self.num_folded_cands = 0
self.num_single_cands = 0
def search_job(job):
"""Search the observation defined in the obs_info
instance 'job'.
"""
# Use whatever .zaplist is found in the current directory
zaplist = glob.glob("*.zaplist")[0]
print "Using %s as zaplist" % zaplist
if config.searching.use_subbands and config.searching.fold_rawdata:
# make a directory to keep subbands so they can be used to fold later
try:
os.makedirs(os.path.join(job.workdir, 'subbands'))
except: pass
# rfifind the data file
cmd = "rfifind %s -time %.17g -o %s %s" % \
(config.searching.datatype_flag, config.searching.rfifind_chunk_time, job.basefilenm,
job.filenmstr)
job.rfifind_time += timed_execute(cmd, stdout="%s_rfifind.out" % job.basefilenm)
maskfilenm = job.basefilenm + "_rfifind.mask"
# Find the fraction that was suggested to be masked
# Note: Should we stop processing if the fraction is
# above some large value? Maybe 30%?
job.masked_fraction = find_masked_fraction(job)
# Iterate over the stages of the overall de-dispersion plan
dmstrs = []
for ddplan in job.ddplans:
# Iterate over the individual passes through the data file
for passnum in range(ddplan.numpasses):
subbasenm = "%s_DM%s"%(job.basefilenm, ddplan.subdmlist[passnum])
if config.searching.use_subbands:
try:
os.makedirs(os.path.join(job.tempdir, 'subbands'))
except: pass
# Create a set of subbands
cmd = "prepsubband %s -sub -subdm %s -downsamp %d -nsub %d -mask %s " \
"-o %s/subbands/%s %s" % \
(config.searching.datatype_flag, ddplan.subdmlist[passnum], ddplan.sub_downsamp,
ddplan.numsub, maskfilenm, job.tempdir, job.basefilenm,
job.filenmstr)
job.subbanding_time += timed_execute(cmd, stdout="%s.subout" % subbasenm)
# Now de-disperse using the subbands
cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-nsub %d -numout %d -o %s/%s %s/subbands/%s.sub[0-9]*" % \
(ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp, ddplan.numsub,
psr_utils.choose_N(job.orig_N/ddplan.downsamp),
job.tempdir, job.basefilenm, job.tempdir, subbasenm)
job.dedispersing_time += timed_execute(cmd, stdout="%s.prepout" % subbasenm)
else: # Not using subbands
cmd = "prepsubband -mask %s -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-numout %d -o %s/%s %s"%\
(maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp*ddplan.sub_downsamp,
psr_utils.choose_N(job.orig_N/ddplan.downsamp),
job.tempdir, job.basefilenm, job.filenmstr)
job.dedispersing_time += timed_execute(cmd)
# Iterate over all the new DMs
for dmstr in ddplan.dmlist[passnum]:
dmstrs.append(dmstr)
basenm = os.path.join(job.tempdir, job.basefilenm+"_DM"+dmstr)
datnm = basenm+".dat"
fftnm = basenm+".fft"
infnm = basenm+".inf"
# Do the single-pulse search
cmd = "single_pulse_search.py -p -m %f -t %f %s"%\
(config.searching.singlepulse_maxwidth, \
config.searching.singlepulse_threshold, datnm)
job.singlepulse_time += timed_execute(cmd)
try:
shutil.move(basenm+".singlepulse", job.workdir)
except: pass
# FFT, zap, and de-redden
cmd = "realfft %s"%datnm
job.FFT_time += timed_execute(cmd)
cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\
(zaplist, job.baryv, fftnm)
job.FFT_time += timed_execute(cmd)
cmd = "rednoise %s"%fftnm
job.FFT_time += timed_execute(cmd)
try:
os.rename(basenm+"_red.fft", fftnm)
except: pass
# Do the low-acceleration search
cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \
"-zmax %d -flo %f %s"%\
(config.searching.lo_accel_numharm, \
config.searching.lo_accel_sigma, \
config.searching.lo_accel_zmax, \
config.searching.lo_accel_flo, fftnm)
job.lo_accelsearch_time += timed_execute(cmd)
try:
os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.lo_accel_zmax)
except: pass
try: # This prevents errors if there are no cand files to copy
shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.lo_accel_zmax, \
job.workdir)
shutil.move(basenm+"_ACCEL_%d" % config.searching.lo_accel_zmax, \
job.workdir)
except: pass
# Do the high-acceleration search
cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \
"-zmax %d -flo %f %s"%\
(config.searching.hi_accel_numharm, \
config.searching.hi_accel_sigma, \
config.searching.hi_accel_zmax, \
config.searching.hi_accel_flo, fftnm)
job.hi_accelsearch_time += timed_execute(cmd)
try:
os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.hi_accel_zmax)
except: pass
try: # This prevents errors if there are no cand files to copy
shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.hi_accel_zmax, \
job.workdir)
shutil.move(basenm+"_ACCEL_%d" % config.searching.hi_accel_zmax, \
job.workdir)
except: pass
# Move the .inf files
try:
shutil.move(infnm, job.workdir)
except: pass
# Remove the .dat and .fft files
try:
os.remove(datnm)
except: pass
try:
os.remove(fftnm)
except: pass
if config.searching.use_subbands:
if config.searching.fold_rawdata:
# Subband files are no longer needed
shutil.rmtree(os.path.join(job.tempdir, 'subbands'))
else:
# Move subbands to workdir
for sub in glob.glob(os.path.join(job.tempdir, 'subbands', "*")):
shutil.move(sub, os.path.join(job.workdir, 'subbands'))
# Make the single-pulse plots
basedmb = job.basefilenm+"_DM"
basedme = ".singlepulse "
# The following will make plots for DM ranges:
# 0-110, 100-310, 300-1000+
dmglobs = [basedmb+"[0-9].[0-9][0-9]"+basedme +
basedmb+"[0-9][0-9].[0-9][0-9]"+basedme +
basedmb+"10[0-9].[0-9][0-9]"+basedme,
basedmb+"[12][0-9][0-9].[0-9][0-9]"+basedme +
basedmb+"30[0-9].[0-9][0-9]"+basedme,
basedmb+"[3-9][0-9][0-9].[0-9][0-9]"+basedme +
basedmb+"1[0-9][0-9][0-9].[0-9][0-9]"+basedme]
dmrangestrs = ["0-110", "100-310", "300-1000+"]
psname = job.basefilenm+"_singlepulse.ps"
for dmglob, dmrangestr in zip(dmglobs, dmrangestrs):
dmfiles = []
for dmg in dmglob.split():
dmfiles += glob.glob(dmg.strip())
# Check that there are matching files and they are not all empty
if dmfiles and sum([os.path.getsize(f) for f in dmfiles]):
cmd = 'single_pulse_search.py -t %f -g "%s"' % \
(config.searching.singlepulse_plot_SNR, dmglob)
job.singlepulse_time += timed_execute(cmd)
os.rename(psname,
job.basefilenm+"_DMs%s_singlepulse.ps" % dmrangestr)
# Sift through the candidates to choose the best to fold
job.sifting_time = time.time()
lo_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.lo_accel_zmax))
if len(lo_accel_cands):
lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands)
if len(lo_accel_cands):
lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, config.searching.numhits_to_fold,
dmstrs, config.searching.low_DM_cutoff)
hi_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.hi_accel_zmax))
if len(hi_accel_cands):
hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands)
if len(hi_accel_cands):
hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, config.searching.numhits_to_fold,
dmstrs, config.searching.low_DM_cutoff)
all_accel_cands = lo_accel_cands + hi_accel_cands
if len(all_accel_cands):
all_accel_cands = sifting.remove_harmonics(all_accel_cands)
# Note: the candidates will be sorted in _sigma_ order, not _SNR_!
all_accel_cands.sort(sifting.cmp_sigma)
sifting.write_candlist(all_accel_cands, job.basefilenm+".accelcands")
# Moving of results to resultsdir now happens in clean_up(...)
# shutil.copy(job.basefilenm+".accelcands", job.outputdir)
job.sifting_time = time.time() - job.sifting_time
# Fold the best candidates
cands_folded = 0
for cand in all_accel_cands:
if cands_folded == config.searching.max_cands_to_fold:
break
if cand.sigma >= config.searching.to_prepfold_sigma:
job.folding_time += timed_execute(get_folding_command(cand, job))
cands_folded += 1
job.num_cands_folded = cands_folded
# Now step through the .ps files and convert them to .png and gzip them
psfiles = glob.glob("*.ps")
for psfile in psfiles:
# The '[0]' appeneded to the end of psfile is to convert only the 1st page
timed_execute("convert -quality 90 %s -background white -flatten -rotate 90 +matte %s" % \
(psfile+"[0]", psfile[:-3]+".png"))
timed_execute("gzip "+psfile)
def search_job(job):
"""Search the observation defined in the obs_info
instance 'job'.
"""
# Use whatever .zaplist is found in the current directory
zaplist = glob.glob("*.zaplist")[0]
print "Using %s as zaplist" % zaplist
if config.searching.use_subbands and config.searching.fold_rawdata:
# make a directory to keep subbands so they can be used to fold later
try:
os.makedirs(os.path.join(job.workdir, 'subbands'))
except: pass
# rfifind the data file
cmd = "rfifind %s -time %.17g -o %s %s" % \
(config.searching.datatype_flag, config.searching.rfifind_chunk_time, job.basefilenm,
job.filenmstr)
job.rfifind_time += timed_execute(cmd, stdout="%s_rfifind.out" % job.basefilenm)
maskfilenm = job.basefilenm + "_rfifind.mask"
# Find the fraction that was suggested to be masked
# Note: Should we stop processing if the fraction is
# above some large value? Maybe 30%?
job.masked_fraction = find_masked_fraction(job)
# Iterate over the stages of the overall de-dispersion plan
dmstrs = []
for ddplan in job.ddplans:
# Make a downsampled filterbank file if we are not using subbands
if not config.searching.use_subbands:
if ddplan.downsamp > 1:
cmd = "downsample_psrfits.py %d %s"%(ddplan.downsamp, job.filenmstr)
job.downsample_time += timed_execute(cmd)
dsfiles = []
for f in job.filenames:
fbase = f.rstrip(".fits")
dsfiles.append(fbase+"_DS%d.fits"%ddplan.downsamp)
filenmstr = ' '.join(dsfiles)
else:
filenmstr = job.filenmstr
# Iterate over the individual passes through the data file
for passnum in range(ddplan.numpasses):
subbasenm = "%s_DM%s"%(job.basefilenm, ddplan.subdmlist[passnum])
if config.searching.use_subbands:
try:
os.makedirs(os.path.join(job.tempdir, 'subbands'))
except: pass
# Create a set of subbands
cmd = "prepsubband %s -sub -subdm %s -downsamp %d -nsub %d -mask %s " \
"-o %s/subbands/%s %s" % \
(config.searching.datatype_flag, ddplan.subdmlist[passnum], ddplan.sub_downsamp,
ddplan.numsub, maskfilenm, job.tempdir, job.basefilenm,
job.filenmstr)
job.subbanding_time += timed_execute(cmd, stdout="%s.subout" % subbasenm)
# Now de-disperse using the subbands
cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-numout %d -o %s/%s %s/subbands/%s.sub[0-9]*" % \
(ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp,
psr_utils.choose_N(job.orig_N/ddplan.downsamp),
job.tempdir, job.basefilenm, job.tempdir, subbasenm)
job.dedispersing_time += timed_execute(cmd, stdout="%s.prepout" % subbasenm)
else: # Not using subbands
cmd = "prepsubband -mask %s -lodm %.2f -dmstep %.2f -numdms %d " \
"-numout %d -o %s/%s %s"%\
(maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, psr_utils.choose_N(job.orig_N/ddplan.downsamp),
job.tempdir, job.basefilenm, filenmstr)
job.dedispersing_time += timed_execute(cmd)
# Iterate over all the new DMs
for dmstr in ddplan.dmlist[passnum]:
dmstrs.append(dmstr)
basenm = os.path.join(job.tempdir, job.basefilenm+"_DM"+dmstr)
datnm = basenm+".dat"
fftnm = basenm+".fft"
infnm = basenm+".inf"
# Do the single-pulse search
cmd = "single_pulse_search.py -p -m %f -t %f %s"%\
(config.searching.singlepulse_maxwidth, \
config.searching.singlepulse_threshold, datnm)
job.singlepulse_time += timed_execute(cmd)
try:
shutil.move(basenm+".singlepulse", job.workdir)
except: pass
# FFT, zap, and de-redden
cmd = "realfft %s"%datnm
job.FFT_time += timed_execute(cmd)
cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\
(zaplist, job.baryv, fftnm)
job.FFT_time += timed_execute(cmd)
cmd = "rednoise %s"%fftnm
job.FFT_time += timed_execute(cmd)
try:
os.rename(basenm+"_red.fft", fftnm)
except: pass
# Do the low-acceleration search
cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \
"-zmax %d -flo %f %s"%\
(config.searching.lo_accel_numharm, \
config.searching.lo_accel_sigma, \
config.searching.lo_accel_zmax, \
config.searching.lo_accel_flo, fftnm)
job.lo_accelsearch_time += timed_execute(cmd)
try:
os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.lo_accel_zmax)
except: pass
try: # This prevents errors if there are no cand files to copy
shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.lo_accel_zmax, \
job.workdir)
shutil.move(basenm+"_ACCEL_%d" % config.searching.lo_accel_zmax, \
job.workdir)
except: pass
# Do the high-acceleration search
cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \
"-zmax %d -flo %f %s"%\
(config.searching.hi_accel_numharm, \
config.searching.hi_accel_sigma, \
config.searching.hi_accel_zmax, \
config.searching.hi_accel_flo, fftnm)
job.hi_accelsearch_time += timed_execute(cmd)
try:
os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.hi_accel_zmax)
except: pass
try: # This prevents errors if there are no cand files to copy
shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.hi_accel_zmax, \
job.workdir)
shutil.move(basenm+"_ACCEL_%d" % config.searching.hi_accel_zmax, \
job.workdir)
except: pass
# Move the .inf files
try:
shutil.move(infnm, job.workdir)
except: pass
# Remove the .dat and .fft files
try:
os.remove(datnm)
except: pass
try:
os.remove(fftnm)
except: pass
if config.searching.use_subbands:
if config.searching.fold_rawdata:
# Subband files are no longer needed
shutil.rmtree(os.path.join(job.tempdir, 'subbands'))
else:
# Move subbands to workdir
for sub in glob.glob(os.path.join(job.tempdir, 'subbands', "*")):
shutil.move(sub, os.path.join(job.workdir, 'subbands'))
# Make the single-pulse plots
basedmb = job.basefilenm+"_DM"
basedme = ".singlepulse "
# The following will make plots for DM ranges:
# 0-110, 100-310, 300-1000+
dmglobs = [basedmb+"[0-9].[0-9][0-9]"+basedme +
basedmb+"[0-9][0-9].[0-9][0-9]"+basedme +
basedmb+"10[0-9].[0-9][0-9]"+basedme,
basedmb+"[12][0-9][0-9].[0-9][0-9]"+basedme +
basedmb+"30[0-9].[0-9][0-9]"+basedme,
basedmb+"[3-9][0-9][0-9].[0-9][0-9]"+basedme +
basedmb+"1[0-9][0-9][0-9].[0-9][0-9]"+basedme]
dmrangestrs = ["0-110", "100-310", "300-1000+"]
psname = job.basefilenm+"_singlepulse.ps"
for dmglob, dmrangestr in zip(dmglobs, dmrangestrs):
dmfiles = []
for dmg in dmglob.split():
dmfiles += glob.glob(dmg.strip())
# Check that there are matching files and they are not all empty
if dmfiles and sum([os.path.getsize(f) for f in dmfiles]):
cmd = 'single_pulse_search.py -t %f -g "%s"' % \
(config.searching.singlepulse_plot_SNR, dmglob)
job.singlepulse_time += timed_execute(cmd)
os.rename(psname,
job.basefilenm+"_DMs%s_singlepulse.ps" % dmrangestr)
# Sift through the candidates to choose the best to fold
job.sifting_time = time.time()
lo_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.lo_accel_zmax))
if len(lo_accel_cands):
lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands)
if len(lo_accel_cands):
lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, config.searching.numhits_to_fold,
dmstrs, config.searching.low_DM_cutoff)
hi_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.hi_accel_zmax))
if len(hi_accel_cands):
hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands)
if len(hi_accel_cands):
hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, config.searching.numhits_to_fold,
dmstrs, config.searching.low_DM_cutoff)
all_accel_cands = lo_accel_cands + hi_accel_cands
if len(all_accel_cands):
all_accel_cands = sifting.remove_harmonics(all_accel_cands)
# Note: the candidates will be sorted in _sigma_ order, not _SNR_!
all_accel_cands.sort(sifting.cmp_sigma)
sifting.write_candlist(all_accel_cands, job.basefilenm+".accelcands")
# Moving of results to resultsdir now happens in clean_up(...)
# shutil.copy(job.basefilenm+".accelcands", job.outputdir)
job.sifting_time = time.time() - job.sifting_time
# Fold the best candidates
cands_folded = 0
for cand in all_accel_cands:
if cands_folded == config.searching.max_cands_to_fold:
break
if cand.sigma >= config.searching.to_prepfold_sigma:
job.folding_time += timed_execute(get_folding_command(cand, job))
cands_folded += 1
job.num_cands_folded = cands_folded
# Now step through the .ps files and convert them to .png and gzip them
psfiles = glob.glob("*.ps")
for psfile in psfiles:
# The '[0]' appeneded to the end of psfile is to convert only the 1st page
timed_execute("convert -quality 90 %s -background white -flatten -rotate 90 +matte %s" % \
(psfile+"[0]", psfile[:-3]+".png"))
timed_execute("gzip "+psfile)
#!/usr/bin/env python
from __future__ import (print_function,division)
import psr_utils as pu
import sys
from infodata import infodata
if len(sys.argv) != 2:
print("chooseN <file.inf|numpoints>")
print(" Prints a good value for fast FFTs to be used for -numout in prepdata/prepsubband")
sys.exit(1)
if (sys.argv[1].endswith('.inf')):
inf = infodata(sys.argv[1])
n = inf.N
else:
try:
n = int(sys.argv[1])
except:
print("chooseN <file.inf|numpoints>")
print(" Prints a good value for fast FFTs to be used for -numout in prepdata/prepsubband")
sys.exit(2)
print(pu.choose_N(n))
def search_job(job):
"""Search the observation defined in the obs_info
instance 'job'.
"""
zerodm_flag = '-zerodm' if job.zerodm else ''
# Use whatever .zaplist is found in the current directory
job.zaplist = glob.glob("*.zaplist")[0]
print "Using %s as zaplist" % job.zaplist
# Use whatever *_radar_samples.txt is found in the current directory
if config.searching.use_radar_clipping:
radar_list = glob.glob("*_radar_samples.txt")[0]
os.putenv('CLIPBINSFILE', os.path.join(job.workdir, radar_list))
print "Using %s as radar samples list" % radar_list
if config.searching.use_subbands and config.searching.fold_rawdata:
# make a directory to keep subbands so they can be used to fold later
try:
os.makedirs(os.path.join(job.workdir, 'subbands'))
except:
pass
# rfifind the data file
cmd = "rfifind %s -time %.17g -o %s %s" % \
(config.searching.datatype_flag, config.searching.rfifind_chunk_time,
job.basefilenm, job.filenmstr)
if config.searching.bad_chans:
cmd += " -zapchan %s" % config.searching.bad_chans
if config.searching.bad_ints:
cmd += " -zapints %s" % config.searching.bad_ints
if config.searching.timesig:
cmd += " -timesig %.2f" % config.searching.timesig
if config.searching.freqsig:
cmd += " -freqsig %.2f" % config.searching.freqsig
if config.searching.intfrac:
cmd += " -intfrac %.2f" % config.searching.intfrac
if config.searching.chanfrac:
cmd += " -chanfrac %.2f" % config.searching.chanfrac
job.rfifind_time += timed_execute(cmd,
stdout="%s_rfifind.out" % job.basefilenm)
maskfilenm = job.basefilenm + "_rfifind.mask"
# Find the fraction that was suggested to be masked
# Note: Should we stop processing if the fraction is
# above some large value? Maybe 30%?
job.masked_fraction = find_masked_fraction(job)
# Iterate over the stages of the overall de-dispersion plan
dmstrs = []
start = time.time()
for ddplan in job.ddplans:
# Iterate over the individual passes through the data file
for passnum in range(ddplan.numpasses):
subbasenm = "%s_DM%s" % (job.basefilenm, ddplan.subdmlist[passnum])
if config.searching.use_subbands:
try:
os.makedirs(os.path.join(job.tempdir, 'subbands'))
except:
pass
# Create a set of subbands
cmd = "prepsubband %s %s -sub -subdm %s -downsamp %d -nsub %d -mask %s " \
"-o %s/subbands/%s %s" % \
(config.searching.datatype_flag, zerodm_flag, ddplan.subdmlist[passnum],
ddplan.sub_downsamp, ddplan.numsub, maskfilenm, job.tempdir,
job.basefilenm, job.filenmstr)
#job.subbanding_time += timed_execute(cmd, stdout="%s.subout" % subbasenm)
# Now de-disperse using the subbands
cmd2 = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-nsub %d -numout %d -o %s/%s %s/subbands/%s.sub[0-9]*" % \
(ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp, ddplan.numsub,
psr_utils.choose_N(job.orig_N/ddplan.downsamp),
job.tempdir, job.basefilenm, job.tempdir, subbasenm)
#job.dedispersing_time += timed_execute(cmd, stdout="%s.prepout" % subbasenm)
queue.put(cmd + ";" + cmd2)
else: # Not using subbands
cmd = "prepsubband %s -mask %s -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-numout %d -nsub %d -o %s/%s %s"%\
(zerodm_flag, maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep,
ddplan.dmstep, ddplan.dmsperpass, ddplan.dd_downsamp*ddplan.sub_downsamp,
psr_utils.choose_N(job.orig_N/ddplan.downsamp), ddplan.numsub,
job.tempdir, job.basefilenm, job.filenmstr)
queue.put(cmd)
#job.dedispersing_time += timed_execute(cmd)
queue.join()
end = time.time()
job.dedispersing_time += (end - start)
for ddplan in job.ddplans:
# Iterate over the individual passes through the data file
for passnum in range(ddplan.numpasses):
# Search all the new DMs
dmlist_forpass = ddplan.dmlist[passnum]
if job.search_pdm:
periodicity_search_pass(job, dmlist_forpass)
if job.search_sp:
singlepulse_search_pass(job, dmlist_forpass)
dmstrs += dmlist_forpass
# Clean up .dat files for pass
for dmstr in dmlist_forpass:
basenm = os.path.join(job.tempdir,
job.basefilenm + "_DM" + dmstr)
try:
os.remove(basenm + ".dat")
except:
pass
# Clean up subbands if using them
if config.searching.use_subbands:
if config.searching.fold_rawdata:
# Subband files are no longer needed
shutil.rmtree(os.path.join(job.tempdir, 'subbands'))
else:
# Move subbands to workdir
for sub in glob.glob(os.path.join(job.tempdir, 'subbands', "*")):
shutil.move(sub, os.path.join(job.workdir, 'subbands'))
if job.search_sp:
sift_singlepulse(job)
if job.search_pdm:
all_accel_cands = sift_periodicity(job, dmstrs)
#####
# Print some info useful for debugging
print "Contents of workdir (%s) before folding: " % job.workdir
for fn in os.listdir(job.workdir):
print " %s" % fn
print "Contents of resultsdir (%s) before folding: " % job.outputdir
for fn in os.listdir(job.outputdir):
print " %s" % fn
print "Contents of job.tempdir (%s) before folding: " % job.tempdir
for fn in os.listdir(job.tempdir):
print " %s" % fn
sys.stdout.flush()
#####
if job.search_pdm:
fold_periodicity_candidates(job, all_accel_cands)
# Print some info useful for debugging
print "Contents of workdir (%s) after folding: " % job.workdir
for fn in os.listdir(job.workdir):
print " %s" % fn
print "Contents of resultsdir (%s) after folding: " % job.outputdir
for fn in os.listdir(job.outputdir):
print " %s" % fn
print "Contents of job.tempdir (%s) after folding: " % job.tempdir
for fn in os.listdir(job.tempdir):
print " %s" % fn
sys.stdout.flush()
#####
# Now step through the .ps files and convert them to .png and gzip them
psfiles = glob.glob("*.ps")
psfiles_rotate = glob.glob("*.pfd.ps") + glob.glob("*_rfifind.ps")
# rotate pfd and rfifind plots but not others
for psfile in psfiles_rotate:
# The '[0]' appeneded to the end of psfile is to convert only the 1st page
cmd = "convert -quality 90 %s -background white -trim -rotate 90 -flatten %s" % \
(psfile+"[0]", psfile[:-3]+".png")
queue.put(cmd)
queue.join()
for psfile in psfiles_rotate:
cmd = "gzip " + psfile
queue.put(cmd)
queue.join()
for psfile in psfiles_rotate:
psfiles.remove(psfile)
for psfile in psfiles:
# The '[0]' appeneded to the end of psfile is to convert only the 1st page
cmd = "convert -quality 90 %s -background white -trim -flatten %s" % \
(psfile+"[0]", psfile[:-3]+".png")
queue.put(cmd)
queue.join()
for psfile in psfiles:
cmd = "gzip " + psfile
queue.put(cmd)
queue.join()
# Print some info useful for debugging
print "Contents of workdir (%s) after conversion: " % job.workdir
for fn in os.listdir(job.workdir):
print " %s" % fn
print "Contents of resultsdir (%s) after conversion: " % job.outputdir
for fn in os.listdir(job.outputdir):
print " %s" % fn
print "Contents of job.tempdir (%s) after conversion: " % job.tempdir
for fn in os.listdir(job.tempdir):
print " %s" % fn
sys.stdout.flush()
def __init__(self, fil_filenm):
self.fil_filenm = fil_filenm
self.basefilenm = fil_filenm.rstrip(".fil")
self.beam = int(self.basefilenm[-1])
filhdr, self.hdrlen = sigproc.read_header(fil_filenm)
self.orig_filenm = filhdr['rawdatafile']
self.MJD = filhdr['tstart']
self.nchans = filhdr['nchans']
self.ra_rad = sigproc.ra2radians(filhdr['src_raj'])
self.ra_string = psr_utils.coord_to_string(\
*psr_utils.rad_to_hms(self.ra_rad))
self.dec_rad = sigproc.dec2radians(filhdr['src_dej'])
self.dec_string = psr_utils.coord_to_string(\
*psr_utils.rad_to_dms(self.dec_rad))
self.az = filhdr['az_start']
self.el = 90.0 - filhdr['za_start']
self.BW = abs(filhdr['foff']) * filhdr['nchans']
self.dt = filhdr['tsamp']
self.orig_N = sigproc.samples_per_file(fil_filenm, filhdr, self.hdrlen)
self.orig_T = self.orig_N * self.dt
self.N = psr_utils.choose_N(self.orig_N)
self.T = self.N * self.dt
# Update the RA and DEC from the database file if required
newposn = read_db_posn(self.orig_filenm, self.beam)
if newposn is not None:
self.ra_string, self.dec_string = newposn
# ... and use them to update the filterbank file
fix_fil_posn(fil_filenm, self.hdrlen, self.ra_string,
self.dec_string)
# Determine the average barycentric velocity of the observation
self.baryv = presto.get_baryv(self.ra_string,
self.dec_string,
self.MJD,
self.T,
obs="AO")
# Where to dump all the results
# Directory structure is under the base_output_directory
# according to base/MJD/filenmbase/beam
self.outputdir = os.path.join(base_output_directory,
str(int(self.MJD)), self.basefilenm[:-2],
str(self.beam))
# Figure out which host we are processing on
self.hostname = socket.gethostname()
# The fraction of the data recommended to be masked by rfifind
self.masked_fraction = 0.0
# Initialize our timers
self.rfifind_time = 0.0
self.downsample_time = 0.0
self.subbanding_time = 0.0
self.dedispersing_time = 0.0
self.FFT_time = 0.0
self.lo_accelsearch_time = 0.0
self.hi_accelsearch_time = 0.0
self.singlepulse_time = 0.0
self.sifting_time = 0.0
self.folding_time = 0.0
self.total_time = 0.0
# Inialize some candidate counters
self.num_sifted_cands = 0
self.num_folded_cands = 0
self.num_single_cands = 0
#!/usr/bin/env python
from __future__ import (print_function, division)
import psr_utils as pu
import sys
from infodata import infodata
if len(sys.argv) != 2:
print("chooseN <file.inf|numpoints>")
print(
" Prints a good value for fast FFTs to be used for -numout in prepdata/prepsubband"
)
sys.exit(1)
if (sys.argv[1].endswith('.inf')):
inf = infodata(sys.argv[1])
n = inf.N
else:
try:
n = int(sys.argv[1])
except:
print("chooseN <file.inf|numpoints>")
print(
" Prints a good value for fast FFTs to be used for -numout in prepdata/prepsubband"
)
sys.exit(2)
print(pu.choose_N(n))
def search_job(job):
"""Search the observation defined in the obs_info
instance 'job'.
"""
# Use whatever .zaplist is found in the current directory
zaplist = glob.glob("*.zaplist")[0]
print "Using %s as zaplist" % zaplist
if config.searching.use_subbands and config.searching.fold_rawdata:
# make a directory to keep subbands so they can be used to fold later
try:
os.makedirs(os.path.join(job.workdir, 'subbands'))
except: pass
# Iterate over the stages of the overall de-dispersion plan
dmstrs = []
for ddplan in job.ddplans:
# Iterate over the individual passes through the data file
for passnum in range(ddplan.numpasses):
subbasenm = "%s_DM%s"%(job.basefilenm, ddplan.subdmlist[passnum])
if config.searching.use_subbands:
try:
os.makedirs(os.path.join(job.tempdir, 'subbands'))
except: pass
# Create a set of subbands
cmd = "prepsubband %s -sub -subdm %s -downsamp %d -nsub %d -mask %s " \
"-o %s/subbands/%s %s" % \
(config.searching.datatype_flag, ddplan.subdmlist[passnum], ddplan.sub_downsamp,
ddplan.numsub, job.maskfilenm, job.tempdir, job.basefilenm,
job.filenmstr)
job.subbanding_time += timed_execute(cmd, stdout="%s.subout" % subbasenm)
# Now de-disperse using the subbands
cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-nsub %d -numout %d -o %s/%s %s/subbands/%s.sub[0-9]*" % \
(ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp, ddplan.numsub,
psr_utils.choose_N(job.orig_N/ddplan.downsamp),
job.tempdir, job.basefilenm, job.tempdir, subbasenm)
job.dedispersing_time += timed_execute(cmd, stdout="%s.prepout" % subbasenm)
if config.searching.use_zerodm_sp or config.searching.use_zerodm_accel:
cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-nsub %d -numout %d -zerodm -o %s/%s_zerodm %s/subbands/%s.sub[0-9]*" % \
(ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp, ddplan.numsub,
psr_utils.choose_N(job.orig_N/ddplan.downsamp),
job.tempdir, job.basefilenm, job.tempdir, subbasenm)
job.dedispersing_time += timed_execute(cmd, stdout="%s.prepout" % subbasenm)
else: # Not using subbands
cmd = "prepsubband -mask %s -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-numout %d -nsub %d -o %s/%s %s"%\
(job.maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp*ddplan.sub_downsamp,
psr_utils.choose_N(job.orig_N/ddplan.downsamp), ddplan.numsub,
job.tempdir, job.basefilenm, job.filenmstr)
job.dedispersing_time += timed_execute(cmd, stdout=os.devnull)
# Iterate over all the new DMs
for dmstr in ddplan.dmlist[passnum]:
dmstrs.append(dmstr)
basenm = os.path.join(job.tempdir, job.basefilenm+"_DM"+dmstr)
basenm_zerodm = os.path.join(job.tempdir, job.basefilenm+"_zerodm_DM"+dmstr)
datnm = basenm+".dat"
datnm_zerodm = basenm_zerodm+".dat"
fftnm = basenm+".fft"
infnm = basenm+".inf"
if not 'part' in job.filenmstr:
# Do the single-pulse search
cmd = "single_pulse_search.py -p -m %f -t %f %s"%\
(config.searching.singlepulse_maxwidth, \
config.searching.singlepulse_threshold, datnm)
job.singlepulse_time += timed_execute(cmd, stdout=os.devnull)
try:
shutil.move(basenm+".singlepulse", job.workdir)
except: pass
if config.searching.use_zerodm_sp:
cmd = "single_pulse_search.py -p -m %f -t %f %s"%\
(config.searching.singlepulse_maxwidth, \
config.searching.singlepulse_threshold, datnm_zerodm)
job.singlepulse_time += timed_execute(cmd, stdout=os.devnull)
try:
shutil.move(basenm_zerodm+".singlepulse", job.workdir)
except: pass
# FFT, zap, and de-redden
cmd = "realfft %s"%datnm
job.FFT_time += timed_execute(cmd, stdout=os.devnull)
cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\
(zaplist, job.baryv, fftnm)
job.FFT_time += timed_execute(cmd, stdout=os.devnull)
cmd = "rednoise %s"%fftnm
job.FFT_time += timed_execute(cmd, stdout=os.devnull)
try:
os.rename(basenm+"_red.fft", fftnm)
except: pass
if 'part' in job.filenmstr:
numharm = config.searching.hi_accel_numharm
sigma = config.searching.hi_accel_sigma
zmax = config.searching.hi_accel_zmax
flo = config.searching.hi_accel_flo
else:
numharm = config.searching.lo_accel_numharm
sigma = config.searching.lo_accel_sigma
zmax = config.searching.lo_accel_zmax
flo = config.searching.lo_accel_flo
# Do the acceleration search
cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \
"-zmax %d -flo %f %s"%\
(numharm, sigma, zmax, flo, fftnm)
# Time it
if 'part' in job.filenmstr:
job.hi_accelsearch_time += timed_execute(cmd, stdout=os.devnull)
else:
job.lo_accelsearch_time += timed_execute(cmd, stdout=os.devnull)
try:
os.remove(basenm+"_ACCEL_%d.txtcand" % zmax)
except: pass
try: # This prevents errors if there are no cand files to copy
shutil.move(basenm+"_ACCEL_%d.cand" % zmax, job.workdir)
shutil.move(basenm+"_ACCEL_%d" % zmax, job.workdir)
except: pass
# Move the .inf files
try:
shutil.move(infnm, job.workdir)
except: pass
# Remove the .dat and .fft files
try:
os.remove(datnm)
except: pass
try:
os.remove(fftnm)
except: pass
if config.searching.use_subbands:
if config.searching.fold_rawdata:
# Subband files are no longer needed
shutil.rmtree(os.path.join(job.tempdir, 'subbands'))
else:
# Move subbands to workdir
for sub in glob.glob(os.path.join(job.tempdir, 'subbands', "*")):
shutil.move(sub, os.path.join(job.workdir, 'subbands'))
def search_job(job):
"""Search the observation defined in the obs_info
instance 'job'.
"""
# Use whatever .zaplist is found in the current directory
zaplist = glob.glob("*.zaplist")[0]
print "Using %s as zaplist" % zaplist
if config.searching.use_subbands and config.searching.fold_rawdata:
# make a directory to keep subbands so they can be used to fold later
try:
os.makedirs(os.path.join(job.workdir, 'subbands'))
except: pass
# rfifind the data file
cmd = "rfifind %s -time %.17g -o %s %s" % \
(config.searching.datatype_flag, config.searching.rfifind_chunk_time, job.basefilenm,
job.filenmstr)
job.rfifind_time += timed_execute(cmd, stdout="%s_rfifind.out" % job.basefilenm)
maskfilenm = job.basefilenm + "_rfifind.mask"
# Find the fraction that was suggested to be masked
# Note: Should we stop processing if the fraction is
# above some large value? Maybe 30%?
job.masked_fraction = find_masked_fraction(job)
# Iterate over the stages of the overall de-dispersion plan
dmstrs = []
for ddplan in job.ddplans:
# Iterate over the individual passes through the data file
for passnum in range(ddplan.numpasses):
subbasenm = "%s_DM%s"%(job.basefilenm, ddplan.subdmlist[passnum])
if config.searching.use_subbands:
try:
os.makedirs(os.path.join(job.tempdir, 'subbands'))
except: pass
# Create a set of subbands
cmd = "prepsubband %s -sub -subdm %s -downsamp %d -nsub %d -mask %s " \
"-o %s/subbands/%s %s" % \
(config.searching.datatype_flag, ddplan.subdmlist[passnum], ddplan.sub_downsamp,
ddplan.numsub, maskfilenm, job.tempdir, job.basefilenm,
job.filenmstr)
job.subbanding_time += timed_execute(cmd, stdout="%s.subout" % subbasenm)
# Now de-disperse using the subbands
cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-nsub %d -numout %d -o %s/%s %s/subbands/%s.sub[0-9]*" % \
(ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp, ddplan.numsub,
psr_utils.choose_N(job.orig_N/ddplan.downsamp),
job.tempdir, job.basefilenm, job.tempdir, subbasenm)
job.dedispersing_time += timed_execute(cmd, stdout="%s.prepout" % subbasenm)
if config.searching.use_zerodm_sp or config.searching.use_zerodm_accel:
cmd = "prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-nsub %d -numout %d -zerodm -o %s/%s_zerodm %s/subbands/%s.sub[0-9]*" % \
(ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp, ddplan.numsub,
psr_utils.choose_N(job.orig_N/ddplan.downsamp),
job.tempdir, job.basefilenm, job.tempdir, subbasenm)
job.dedispersing_time += timed_execute(cmd, stdout="%s.prepout" % subbasenm)
else: # Not using subbands
cmd = "prepsubband -mask %s -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d " \
"-numout %d -nsub %d -o %s/%s %s"%\
(maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,
ddplan.dmsperpass, ddplan.dd_downsamp*ddplan.sub_downsamp,
psr_utils.choose_N(job.orig_N/ddplan.downsamp), ddplan.numsub,
job.tempdir, job.basefilenm, job.filenmstr)
job.dedispersing_time += timed_execute(cmd)
# Iterate over all the new DMs
for dmstr in ddplan.dmlist[passnum]:
dmstrs.append(dmstr)
basenm = os.path.join(job.tempdir, job.basefilenm+"_DM"+dmstr)
basenm_zerodm = os.path.join(job.tempdir, job.basefilenm+"_zerodm_DM"+dmstr)
datnm = basenm+".dat"
datnm_zerodm = basenm_zerodm+".dat"
fftnm = basenm+".fft"
infnm = basenm+".inf"
# Do the single-pulse search
cmd = "single_pulse_search.py -p -m %f -t %f %s"%\
(config.searching.singlepulse_maxwidth, \
config.searching.singlepulse_threshold, datnm)
job.singlepulse_time += timed_execute(cmd)
try:
shutil.move(basenm+".singlepulse", job.workdir)
except: pass
if config.searching.use_zerodm_sp:
cmd = "single_pulse_search.py -p -m %f -t %f %s"%\
(config.searching.singlepulse_maxwidth, \
config.searching.singlepulse_threshold, datnm_zerodm)
job.singlepulse_time += timed_execute(cmd)
try:
shutil.move(basenm_zerodm+".singlepulse", job.workdir)
except: pass
# FFT, zap, and de-redden
cmd = "realfft %s"%datnm
job.FFT_time += timed_execute(cmd)
cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\
(zaplist, job.baryv, fftnm)
job.FFT_time += timed_execute(cmd)
cmd = "rednoise %s"%fftnm
job.FFT_time += timed_execute(cmd)
try:
os.rename(basenm+"_red.fft", fftnm)
except: pass
# Do the low-acceleration search
cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \
"-zmax %d -flo %f %s"%\
(config.searching.lo_accel_numharm, \
config.searching.lo_accel_sigma, \
config.searching.lo_accel_zmax, \
config.searching.lo_accel_flo, fftnm)
job.lo_accelsearch_time += timed_execute(cmd)
try:
os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.lo_accel_zmax)
except: pass
try: # This prevents errors if there are no cand files to copy
shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.lo_accel_zmax, \
job.workdir)
shutil.move(basenm+"_ACCEL_%d" % config.searching.lo_accel_zmax, \
job.workdir)
except: pass
# Do the high-acceleration search
cmd = "accelsearch -harmpolish -numharm %d -sigma %f " \
"-zmax %d -flo %f %s"%\
(config.searching.hi_accel_numharm, \
config.searching.hi_accel_sigma, \
config.searching.hi_accel_zmax, \
config.searching.hi_accel_flo, fftnm)
job.hi_accelsearch_time += timed_execute(cmd)
try:
os.remove(basenm+"_ACCEL_%d.txtcand" % config.searching.hi_accel_zmax)
except: pass
try: # This prevents errors if there are no cand files to copy
shutil.move(basenm+"_ACCEL_%d.cand" % config.searching.hi_accel_zmax, \
job.workdir)
shutil.move(basenm+"_ACCEL_%d" % config.searching.hi_accel_zmax, \
job.workdir)
except: pass
# Move the .inf files
try:
shutil.move(infnm, job.workdir)
except: pass
# Remove the .dat and .fft files
try:
os.remove(datnm)
except: pass
try:
os.remove(fftnm)
except: pass
if config.searching.use_subbands:
if config.searching.fold_rawdata:
# Subband files are no longer needed
shutil.rmtree(os.path.join(job.tempdir, 'subbands'))
else:
# Move subbands to workdir
for sub in glob.glob(os.path.join(job.tempdir, 'subbands', "*")):
shutil.move(sub, os.path.join(job.workdir, 'subbands'))
# Make the single-pulse plots
basedmb = job.basefilenm+"_DM"
basedmb_zerodm = job.basefilenm+"_zerodm_DM"
basedme = ".singlepulse "
# The following will make plots for DM ranges:
# 0-110, 100-310, 300-1000+
dmglobs = [basedmb+"[0-9].[0-9][0-9]"+basedme +
basedmb+"[0-9][0-9].[0-9][0-9]"+basedme +
basedmb+"10[0-9].[0-9][0-9]"+basedme,
basedmb+"[12][0-9][0-9].[0-9][0-9]"+basedme +
basedmb+"30[0-9].[0-9][0-9]"+basedme,
basedmb+"[3-9][0-9][0-9].[0-9][0-9]"+basedme +
basedmb+"1[0-9][0-9][0-9].[0-9][0-9]"+basedme]
dmrangestrs = ["0-110", "100-310", "300-1000+"]
psname = job.basefilenm+"_singlepulse.ps"
psname_zerodm = job.basefilenm+"_zerodm_singlepulse.ps"
if config.searching.use_zerodm_sp:
dmglobs.extend([basedmb_zerodm+"[0-9].[0-9][0-9]"+basedme +
basedmb_zerodm+"[0-9][0-9].[0-9][0-9]"+basedme +
basedmb_zerodm+"10[0-9].[0-9][0-9]"+basedme,
basedmb_zerodm+"[12][0-9][0-9].[0-9][0-9]"+basedme +
basedmb_zerodm+"30[0-9].[0-9][0-9]"+basedme,
basedmb_zerodm+"[3-9][0-9][0-9].[0-9][0-9]"+basedme +
basedmb_zerodm+"1[0-9][0-9][0-9].[0-9][0-9]"+basedme])
dmrangestrs.extend(["0-110_zerodm", "100-310_zerodm", "300-1000+_zerodm"])
for dmglob, dmrangestr in zip(dmglobs, dmrangestrs):
dmfiles = []
for dmg in dmglob.split():
dmfiles += glob.glob(dmg.strip())
# Check that there are matching files and they are not all empty
if dmfiles and sum([os.path.getsize(f) for f in dmfiles]):
cmd = 'single_pulse_search.py -t %f -g "%s"' % \
(config.searching.singlepulse_plot_SNR, dmglob)
job.singlepulse_time += timed_execute(cmd)
if dmrangestr.endswith("zerodm"):
os.rename(psname_zerodm,
job.basefilenm+"_DMs%s_singlepulse.ps" % dmrangestr)
else:
os.rename(psname,
job.basefilenm+"_DMs%s_singlepulse.ps" % dmrangestr)
# Sift through the candidates to choose the best to fold
job.sifting_time = time.time()
lo_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.lo_accel_zmax))
if len(lo_accel_cands):
lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands)
if len(lo_accel_cands):
lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, config.searching.numhits_to_fold,
dmstrs, config.searching.low_DM_cutoff)
hi_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d" % config.searching.hi_accel_zmax))
if len(hi_accel_cands):
hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands)
if len(hi_accel_cands):
hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, config.searching.numhits_to_fold,
dmstrs, config.searching.low_DM_cutoff)
all_accel_cands = lo_accel_cands + hi_accel_cands
if len(all_accel_cands):
all_accel_cands = sifting.remove_harmonics(all_accel_cands)
# Note: the candidates will be sorted in _sigma_ order, not _SNR_!
all_accel_cands.sort(sifting.cmp_sigma)
print "Sending candlist to stdout before writing to file"
sifting.write_candlist(all_accel_cands)
sys.stdout.flush()
sifting.write_candlist(all_accel_cands, job.basefilenm+".accelcands")
# Make sifting summary plots
all_accel_cands.plot_goodbad()
plt.title("%s Rejected Cands" % job.basefilenm)
plt.savefig(job.basefilenm+".accelcands.rejects.png")
all_accel_cands.plot_summary()
plt.title("%s Periodicity Summary" % job.basefilenm)
plt.savefig(job.basefilenm+".accelcands.summary.png")
# Write out sifting candidate summary
all_accel_cands.print_cand_summary(job.basefilenm+".accelcands.summary")
# Write out sifting comprehensive report of bad candidates
all_accel_cands.write_cand_report(job.basefilenm+".accelcands.report")
timed_execute("gzip --best %s" % job.basefilenm+".accelcands.report")
# Moving of results to resultsdir now happens in clean_up(...)
# shutil.copy(job.basefilenm+".accelcands", job.outputdir)
job.sifting_time = time.time() - job.sifting_time
#####
# Print some info useful for debugging
print "Contents of workdir (%s) before folding: " % job.workdir
for fn in os.listdir(job.workdir):
print " %s" % fn
print "Contents of resultsdir (%s) before folding: " % job.outputdir
for fn in os.listdir(job.outputdir):
print " %s" % fn
print "Contents of job.tempdir (%s) before folding: " % job.tempdir
for fn in os.listdir(job.tempdir):
print " %s" % fn
sys.stdout.flush()
#####
# Fold the best candidates
cands_folded = 0
for cand in all_accel_cands:
print "At cand %s" % str(cand)
if cands_folded == config.searching.max_cands_to_fold:
break
if cand.sigma >= config.searching.to_prepfold_sigma:
print "...folding"
job.folding_time += timed_execute(get_folding_command(cand, job))
cands_folded += 1
job.num_cands_folded = cands_folded
# Rate candidates
timed_execute("rate_pfds.py --redirect-warnings --include-all -x pulse_width *.pfd")
sys.stdout.flush()
# Calculate some candidate attributes from pfds
attrib_file = open('candidate_attributes.txt','w')
for pfdfn in glob.glob("*.pfd"):
attribs = {}
pfd = prepfold.pfd(pfdfn)
red_chi2 = pfd.bestprof.chi_sqr
dof = pfd.proflen - 1
attribs['prepfold_sigma'] = \
-scipy.stats.norm.ppf(scipy.stats.chi2.sf(red_chi2*dof, dof))
off_red_chi2 = pfd.estimate_offsignal_redchi2()
new_red_chi2 = red_chi2 / off_red_chi2
# prepfold sigma rescaled to deal with chi-squared suppression
# a problem when strong rfi is present
attribs['rescaled_prepfold_sigma'] = \
-scipy.stats.norm.ppf(scipy.stats.chi2.sf(new_red_chi2*dof, dof))
for key in attribs:
attrib_file.write("%s\t%s\t%.3f\n" % (pfdfn, key, attribs[key]))
attrib_file.close()
# Print some info useful for debugging
print "Contents of workdir (%s) after folding: " % job.workdir
for fn in os.listdir(job.workdir):
print " %s" % fn
print "Contents of resultsdir (%s) after folding: " % job.outputdir
for fn in os.listdir(job.outputdir):
print " %s" % fn
print "Contents of job.tempdir (%s) after folding: " % job.tempdir
for fn in os.listdir(job.tempdir):
print " %s" % fn
sys.stdout.flush()
#####
# Now step through the .ps files and convert them to .png and gzip them
psfiles = glob.glob("*.ps")
for psfile in psfiles:
# The '[0]' appeneded to the end of psfile is to convert only the 1st page
timed_execute("convert -quality 90 %s -background white -flatten -rotate 90 +matte %s" % \
(psfile+"[0]", psfile[:-3]+".png"))
timed_execute("gzip "+psfile)
# Print some info useful for debugging
print "Contents of workdir (%s) after conversion: " % job.workdir
for fn in os.listdir(job.workdir):
print " %s" % fn
print "Contents of resultsdir (%s) after conversion: " % job.outputdir
for fn in os.listdir(job.outputdir):
print " %s" % fn
print "Contents of job.tempdir (%s) after conversion: " % job.tempdir
for fn in os.listdir(job.tempdir):
print " %s" % fn
sys.stdout.flush()