1344, 1408, 1440, 1536, 1568, 1584, 1600, 1680, 1728,

1760, 1792, 1920, 1936, 2000, 2016, 2048, 2112, 2160,

2240, 2304, 2352, 2400, 2464, 2560, 2592, 2640, 2688,

2800, 2816, 2880, 3024, 3072, 3136, 3168, 3200, 3360,

3456, 3520, 3584, 3600, 3696, 3840, 3872, 3888, 3920,

4000, 4032, 4096, 4224, 4320, 4400, 4480, 4608, 4704,

4752, 4800, 4928, 5040, 5120, 5184, 5280, 5376, 5488,

5600, 5632, 5760, 5808, 6000, 6048, 6144, 6160, 6272,

6336, 6400, 6480, 6720, 6912, 7040, 7056, 7168, 7200,

7392, 7680, 7744, 7776, 7840, 7920, 8000, 8064, 8192,

8400, 8448, 8624, 8640, 8800, 8960, 9072, 9216, 9408,

"""

choose_N(orig_N):

Choose a time series length that is larger than

the input value but that is highly factorable.

Note that the returned value must be divisible

by at least the maximum downsample factor * 2.

Currently, this is 8 * 2 = 16.

"""

if orig_N < 10000:

return 0

# Get the number represented by the first 4 digits of orig_N

first4 = int(str(orig_N)[:4])

# Now get the number that is just bigger than orig_N

# that has its first 4 digits equal to "factor"

for factor in goodfactors:

if factor > first4: break

new_N = factor

while new_N < orig_N:

new_N *= 10

# Finally, compare new_N to the closest power_of_two

# greater than orig_N. Take the closest.

two_N = 2

while two_N < orig_N:

two_N *= 2

if two_N < new_N: return two_N

"""

get_baryv(ra, dec, mjd, T):

Determine the average barycentric velocity towards 'ra', 'dec'

during an observation from 'obs'. The RA and DEC are in the

standard string format (i.e. 'hh:mm:ss.ssss' and

'dd:mm:ss.ssss'). 'T' is in sec and 'mjd' is (of course) in MJD.

"""

tts = psr_utils.span(mjd, mjd+T/86400.0, 100)

nn = len(tts)

bts = numpy.zeros(nn, dtype=numpy.float64)

vel = numpy.zeros(nn, dtype=numpy.float64)

presto.barycenter(tts, bts, vel, nn, ra, dec, obs, "DE200")

avgvel = numpy.add.reduce(vel)/nn

"""

fix_fil_posn(fil_filenm, hdrlen, ra, dec):

Modify the filterbank header and update the RA and DEC

fields using the values given as input. ra and dec

should be in 'HH:MM:SS.SSSS' and 'DD:MM:SS.SSSS' format.

hdrlen is the length of the filterbank header as

reported by PRESTO's 'readfile' or SIGPROC's 'header'.

"""

newra = float(ra.replace(":", ""))

newdec = float(dec.replace(":", ""))

header = open(fil_filenm).read(hdrlen)

ra_ptr = header.find("src_raj")+len("src_raj")

dec_ptr = header.find("src_dej")+len("src_dej")

filfile = open(fil_filenm, 'rb+')

filfile.seek(ra_ptr)

filfile.write(struct.pack('d', newra))

filfile.seek(dec_ptr)

filfile.write(struct.pack('d', newdec))

"""

read_db_posn(orig_filenm, beam):

Find the original WAPP filename in the db_pointing_file

and return the sexagesimal position strings for

the choen beam in that file. Return None if not found.

"""

offset = beam % 2

for line in open(db_pointing_file):

sline = line.split()

if sline[0].strip() == orig_filenm:

ra_str = sline[2*offset+1].strip()

dec_str = sline[2*offset+2].strip()

return ra_str, dec_str

"""

find_masked_fraction(obs):

Parse the output file from an rfifind run and return the

fraction of the data that was suggested to be masked.

"""

rfifind_out = obs.basefilenm + "_rfifind.out"

for line in open(rfifind_out):

if "Number of bad intervals" in line:

return float(line.split("(")[1].split("%")[0])/100.0

# If there is a problem reading the file, return 100%

"""

get_all_subdms(ddplans):

Return a sorted array of the subdms from the list of ddplans.

"""

subdmlist = []

for ddplan in ddplans:

subdmlist += [float(x) for x in ddplan.subdmlist]

subdmlist.sort()

subdmlist = numpy.asarray(subdmlist)

"""

find_closest_subbands(obs, subdms, DM):

Return the basename of the closest set of subbands to DM

given an obs_info class and a sorted array of the subdms.

"""

subdm = subdms[numpy.fabs(subdms - DM).argmin()]

"""

timed_execute(cmd):

Execute the command 'cmd' after logging the command

to STDOUT. Return the wall-clock amount of time

the command took to execute.

"""

sys.stdout.write("\n'"+cmd+"'\n")

sys.stdout.flush()

start = time.time()

os.system(cmd)

end = time.time()

pfi = open(filename,'r')

number_sp=0

for line in pfi:

number_sp += 1

pfi.close()

"""

get_folding_command(cand, obs, ddplans):

Return a command for prepfold for folding the subbands using

an obs_info instance, a list of the ddplans, and a candidate

instance that describes the observations and searches.

"""

# Folding rules are based on the facts that we want:

# 1. Between 24 and 200 bins in the profiles

# 2. For most candidates, we want to search length = 101 p/pd/DM cubes

# (The side of the cube is always 2*M*N+1 where M is the "factor",

# either -npfact (for p and pd) or -ndmfact, and N is the number of bins

# in the profile). A search of 101^3 points is pretty fast.

# 3. For slow pulsars (where N=100 or 200), since we'll have to search

# many points, we'll use fewer intervals in time (-npart 30)

# 4. For the slowest pulsars, in order to avoid RFI, we'll

# not search in period-derivative.

zmax = cand.filename.split("_")[-1]

outfilenm = obs.basefilenm+"_DM%s"%(cand.DMstr)

# Note: the following calculations should probably only be done once,

# but in general, these calculation are effectively instantaneous

# compared to the folding itself

if use_subbands: # Fold the subbands

subdms = get_all_subdms(ddplans)

subfiles = find_closest_subbands(obs, subdms, cand.DM)

foldfiles = subfiles

else: # Folding the downsampled filterbank files instead

hidms = [x.lodm for x in ddplans[1:]] + [2000]

dfacts = [x.downsamp for x in ddplans]

for hidm, dfact in zip(hidms, dfacts):

if cand.DM < hidm:

downsamp = dfact

break

#if downsamp==1:

# filfile = obs.fil_filenm

#else:

# filfile = obs.basefilenm+"_DS%d.fil"%downsamp

filfile = obs.fil_filenm

foldfiles = filfile

p = 1.0 / cand.f

if p < 0.002:

Mp, Mdm, N = 2, 2, 24

otheropts = "-npart 50 "

elif p < 0.05:

Mp, Mdm, N = 2, 2, 50

otheropts = "-npart 40 -pstep 1 -dmstep 3"

elif p < 0.5:

Mp, Mdm, N = 1, 1, 100

otheropts = "-npart 30 -pstep 1 -dmstep 1"

else:

Mp, Mdm, N = 1, 1, 100

otheropts = "-npart 30 -pstep 1 -dmstep 1"

return "prepfold -filterbank -noxwin -nopdsearch -mask %s -accelcand %d -accelfile %s.cand -dm %.2f -o %s %s -n %d -npfact %d -ndmfact %d %s > /dev/null" % \

(maskfilenm, cand.candnum, cand.filename, cand.DM, outfilenm,

"""

class obs_info(fil_filenm,box)

A class describing the observation and the analysis.

"""

def __init__(self, fil_filenm, box):

self.fil_filenm = fil_filenm

self.basefilenm = fil_filenm.rstrip(".fbk")

#self.beam = int(self.basefilenm[-1])

self.beam = int(self.basefilenm)

filhdr, self.hdrlen = sigproc.read_header(fil_filenm)

self.orig_filenm = filhdr['rawdatafile']

self.MJD = filhdr['tstart']

self.nchans = filhdr['nchans']

self.ra_rad = sigproc.ra2radians(filhdr['src_raj'])

self.ra_string = psr_utils.coord_to_string(\

*psr_utils.rad_to_hms(self.ra_rad))

self.dec_rad = sigproc.dec2radians(filhdr['src_dej'])

self.dec_string = psr_utils.coord_to_string(\

*psr_utils.rad_to_dms(self.dec_rad))

self.az = filhdr['az_start']

self.el = 90.0-filhdr['za_start']

self.fch1 = filhdr['fch1']

self.BW = abs(filhdr['foff']) * filhdr['nchans']

self.dt = filhdr['tsamp']

self.orig_N = sigproc.samples_per_file(fil_filenm, filhdr, self.hdrlen)

self.orig_T = self.orig_N * self.dt

self.N = choose_N(self.orig_N)

#self.N = 2097152

self.T = self.N * self.dt

# Update the RA and DEC from the database file if required

#newposn = read_db_posn(self.orig_filenm, self.beam)

#if newposn is not None:

# self.ra_string, self.dec_string = newposn

# ... and use them to update the filterbank file

# fix_fil_posn(fil_filenm, self.hdrlen,

# self.ra_string, self.dec_string)

# Determine the average barycentric velocity of the observation

self.baryv = get_baryv(self.ra_string, self.dec_string,

self.MJD, self.T, obs="NC")

# Where to dump all the results

# Directory structure is under the base_output_directory

# according to base/MJD/filenmbase/beam

self.outputdir = os.path.join(base_output_directory,box,

self.basefilenm)

# Figure out which host we are processing on

self.hostname = socket.gethostname()

# The fraction of the data recommended to be masked by rfifind

self.masked_fraction = 0.0

# Initialize our timers

self.rfifind_time = 0.0

self.downsample_time = 0.0

self.subbanding_time = 0.0

self.dedispersing_time = 0.0

self.FFT_time = 0.0

self.lo_accelsearch_time = 0.0

self.hi_accelsearch_time = 0.0

self.singlepulse_time = 0.0

self.sifting_time = 0.0

self.folding_time = 0.0

self.total_time = 0.0

# Inialize some candidate counters

self.num_sifted_cands = 0

self.num_folded_cands = 0

self.num_single_cands = 0

self.nb_sp = 0

def write_report(self, filenm):

report_file = open(filenm, "w")

report_file.write("---------------------------------------------------------\n")

report_file.write("%s was processed on %s\n"%(self.fil_filenm, self.hostname))

report_file.write("Ending UTC time: %s\n"%(time.asctime(time.gmtime())))

report_file.write("Total wall time: %.1f s (%.2f hrs)\n"%\

(self.total_time, self.total_time/3600.0))

report_file.write("Fraction of data masked: %.2f%%\n"%\

(self.masked_fraction*100.0))

report_file.write("Number of SinglePulses detected: %d\n"%self.nb_sp)

report_file.write("---------------------------------------------------------\n")

report_file.write(" rfifind time = %7.1f sec (%5.2f%%)\n"%\

(self.rfifind_time, self.rfifind_time/self.total_time*100.0))

if use_subbands:

report_file.write(" subbanding time = %7.1f sec (%5.2f%%)\n"%\

(self.subbanding_time, self.subbanding_time/self.total_time*100.0))

else:

report_file.write(" downsampling time = %7.1f sec (%5.2f%%)\n"%\

(self.downsample_time, self.downsample_time/self.total_time*100.0))

report_file.write(" dedispersing time = %7.1f sec (%5.2f%%)\n"%\

(self.dedispersing_time, self.dedispersing_time/self.total_time*100.0))

report_file.write(" single-pulse time = %7.1f sec (%5.2f%%)\n"%\

(self.singlepulse_time, self.singlepulse_time/self.total_time*100.0))

report_file.write(" FFT time = %7.1f sec (%5.2f%%)\n"%\

(self.FFT_time, self.FFT_time/self.total_time*100.0))

report_file.write(" lo-accelsearch time = %7.1f sec (%5.2f%%)\n"%\

(self.lo_accelsearch_time, self.lo_accelsearch_time/self.total_time*100.0))

report_file.write(" hi-accelsearch time = %7.1f sec (%5.2f%%)\n"%\

(self.hi_accelsearch_time, self.hi_accelsearch_time/self.total_time*100.0))

report_file.write(" sifting time = %7.1f sec (%5.2f%%)\n"%\

(self.sifting_time, self.sifting_time/self.total_time*100.0))

report_file.write(" folding time = %7.1f sec (%5.2f%%)\n"%\

(self.folding_time, self.folding_time/self.total_time*100.0))

report_file.write("---------------------------------------------------------\n")

"""

class dedisp_plan(lodm, dmstep, dmsperpass, numpasses, numsub, downsamp)

A class describing a de-dispersion plan for prepsubband in detail.

"""

def __init__(self, lodm, dmstep, dmsperpass, numpasses, numsub, downsamp):

self.lodm = float(lodm)

self.dmstep = float(dmstep)

self.dmsperpass = int(dmsperpass)

self.numpasses = int(numpasses)

self.numsub = int(numsub)

self.downsamp = int(downsamp)

# Downsample less for the subbands so that folding

# candidates is more accurate

self.sub_downsamp = self.downsamp / 2

if self.sub_downsamp==0: self.sub_downsamp = 1

# The total downsampling is:

# self.downsamp = self.sub_downsamp * self.dd_downsamp

if self.downsamp==1: self.dd_downsamp = 1

else: self.dd_downsamp = 2

self.sub_dmstep = self.dmsperpass * self.dmstep

self.dmlist = [] # These are strings for comparison with filenames

self.subdmlist = []

for ii in range(self.numpasses):

self.subdmlist.append("%.2f"%(self.lodm + (ii+0.5)*self.sub_dmstep))

lodm = self.lodm + ii * self.sub_dmstep

dmlist = ["%.2f"%dm for dm in \

numpy.arange(self.dmsperpass)*self.dmstep + lodm]

#

cpu_idx = int(cpu_id)

c = bon_filenm_long.rindex("/")

bon_filenm = bon_filenm_long[c+1:]

print bon_filenm

workdir="/data/BON/work/%s"%bon_filenm.rstrip(".fbk")

# Make sure the temp directory exist

try:

os.makedirs(workdir)

except: pass

# Change to the specified working directory

os.chdir(workdir)

# Copy data

cmd = "scp -p root@clairvaux:/data?/%s_*.fbk ."%(bon_filenm_long.rstrip(".fbk"))

print cmd

os.system(cmd)

# Convert BON file to sigproc format - need .i file

print "Join BON .fbk files\n"

cmd = "taskset -c %d joinfbk %s_*.fbk"%(cpu_idx,bon_filenm.rstrip(".fbk"))

print cmd

os.system(cmd)

fil_filenm = bon_filenm

#fil_filenm = fil_filenm.rstrip(".fb")+".sig"

# Get information on the observation and the jbo

job = obs_info(fil_filenm, box)

if job.T < low_T_to_search:

print "The observation is too short (%.2f s) to search."%job.T

sys.exit()

ddplans = []

if (job.dt<0.000048 and job.nchans==512): # BON 32us and 0.25 Mhz or 512 channels over 128MHz band

# The values here are: lodm dmstep dms/call #calls #subbands downsamp

ddplans.append(dedisp_plan( 0.0, 0.2, 210, 1, 512, 1))

ddplans.append(dedisp_plan( 42.0, 0.2, 220, 1, 512, 1))

ddplans.append(dedisp_plan( 86.0, 0.2, 221, 1, 512, 1))

ddplans.append(dedisp_plan( 130.2, 0.3, 189, 1, 512, 1))

ddplans.append(dedisp_plan( 186.9, 0.3, 190, 1, 512, 1))

ddplans.append(dedisp_plan( 243.9, 0.5, 189, 1, 512, 2))

ddplans.append(dedisp_plan( 338.4, 0.5, 189, 1, 512, 2))

ddplans.append(dedisp_plan( 432.9, 1.0, 211, 1, 512, 4))

ddplans.append(dedisp_plan( 643.9, 1.0, 210, 1, 512, 8))

ddplans.append(dedisp_plan( 853.9, 3.0, 130, 1, 512, 8))

elif (job.dt<0.000048 and job.nchans==256): # BON 32us and 0.25 Mhz or 256 channels over 64MHz band

# The values here are: lodm dmstep dms/call #calls #subbands downsamp

ddplans.append(dedisp_plan( 0.0, 0.2, 210, 1, 256, 1))

ddplans.append(dedisp_plan( 42.0, 0.2, 220, 1, 256, 1))

ddplans.append(dedisp_plan( 86.0, 0.2, 221, 1, 256, 1))

ddplans.append(dedisp_plan( 130.2, 0.3, 189, 1, 256, 1))

ddplans.append(dedisp_plan( 186.9, 0.3, 190, 1, 256, 1))

ddplans.append(dedisp_plan( 243.9, 0.5, 189, 1, 256, 2))

ddplans.append(dedisp_plan( 338.4, 0.5, 189, 1, 256, 2))

ddplans.append(dedisp_plan( 432.9, 1.0, 211, 1, 256, 4))

ddplans.append(dedisp_plan( 643.9, 1.0, 210, 1, 256, 8))

ddplans.append(dedisp_plan( 853.9, 3.0, 130, 1, 256, 8))

else: # faster option : for 64us and 1MHz or 128 channels

# The values here are: lodm dmstep dms/call #calls #subbands downsamp

ddplans.append(dedisp_plan( 0.0, 0.3, 212, 1, 128, 1))

ddplans.append(dedisp_plan( 63.7, 0.5, 75, 1, 128, 1))

ddplans.append(dedisp_plan( 101.2, 1.0, 89, 1, 128, 2))

ddplans.append(dedisp_plan( 190.2, 3.0, 81, 1, 128, 4))

ddplans.append(dedisp_plan( 433.2, 5.0, 155, 1, 128, 8))

job.total_time = time.time()

# Use whatever .zaplist is found in the current directory

#default_zaplist = glob.glob("*.zaplist")[0]

default_zaplist = "/data/BON/BON.zaplist"

# Make sure the output directory (and parent directories) exist

try:

os.makedirs(job.outputdir)

except: pass

# Create a directory to hold all the subbands

if use_subbands:

try:

os.makedirs("subbands")

except: pass

print "\nBeginning BON search of '%s'"%job.fil_filenm

print "UTC time is: %s"%(time.asctime(time.gmtime()))

# rfifind the filterbank file

cmd = "taskset -c %d rfifind -filterbank -time %.17g -o %s %s > %s_rfifind.out"%\

(cpu_idx, rfifind_chunk_time, job.basefilenm,

job.fil_filenm, job.basefilenm)

if (job.dt<0.000048 and job.nchans>=256 and job.fch1<1500.0):

cmd = "taskset -c %d rfifind -filterbank -zapchan 106,107,108 -time %.17g -o %s %s > %s_rfifind.out"%\

(cpu_idx, rfifind_chunk_time, job.basefilenm,

job.fil_filenm, job.basefilenm)

job.rfifind_time += timed_execute(cmd)

maskfilenm = job.basefilenm + "_rfifind.mask"

# Find the fraction that was suggested to be masked

# Note: Should we stop processing if the fraction is

# above some large value? Maybe 30%?

job.masked_fraction = find_masked_fraction(job)

# Iterate over the stages of the overall de-dispersion plan

dmstrs = []

for ddplan in ddplans:

# Iterate over the individual passes through the .fil file

for passnum in range(ddplan.numpasses):

subbasenm = "%s_DM%s"%(job.basefilenm, ddplan.subdmlist[passnum])

if use_subbands:

# Create a set of subbands

cmd = "taskset -c %d prepsubband -filterbank -sub -subdm %s -downsamp %d -nsub %d -mask %s -o subbands/%s %s > %s.subout"%\

(cpu_idx, ddplan.subdmlist[passnum], ddplan.sub_downsamp,

ddplan.numsub, maskfilenm, job.basefilenm,

job.fil_filenm, subbasenm)

job.subbanding_time += timed_execute(cmd)

# Now de-disperse using the subbands

cmd = "taskset -c %d prepsubband -lodm %.2f -dmstep %.2f -numdms %d -downsamp %d -numout %d -o %s subbands/%s.sub[0-9]* > %s.prepout"%\

(cpu_idx, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,

ddplan.dmsperpass, ddplan.dd_downsamp, job.N/ddplan.downsamp,

job.basefilenm, subbasenm, subbasenm)

job.dedispersing_time += timed_execute(cmd)

else: # Not using subbands

cmd = "taskset -c %d prepsubband -filterbank -mask %s -lodm %.2f -dmstep %.2f -numdms %d -numout %d -o %s %s -downsamp %d -nsub %d > /dev/null"%\

(cpu_idx, maskfilenm, ddplan.lodm+passnum*ddplan.sub_dmstep, ddplan.dmstep,

ddplan.dmsperpass, job.N/ddplan.downsamp,

job.basefilenm, fil_filenm, ddplan.downsamp, ddplan.numsub)

job.dedispersing_time += timed_execute(cmd)

# Iterate over all the new DMs

for dmstr in ddplan.dmlist[passnum]:

dmstrs.append(dmstr)

basenm = job.basefilenm+"_DM"+dmstr

datnm = basenm+".dat"

fftnm = basenm+".fft"

infnm = basenm+".inf"

# Do the single-pulse search

cmd = "taskset -c %d single_pulse_search.py -p -t %f %s > /dev/null"%\

(cpu_idx, singlepulse_threshold, datnm)

#cmd = "taskset -c %d single_pulse_search.py -p -m %f -t %f %s > /dev/null"%\

# (cpu_idx, singlepulse_maxwidth, singlepulse_threshold, datnm)

job.singlepulse_time += timed_execute(cmd)

#try:

# shutil.copy(basenm+".singlepulse", job.outputdir)

#except: pass

job.nb_sp += count_sp(basenm+".singlepulse")

# FFT, zap, and de-redden

#cmd = "realfft %s"%datnm

#job.FFT_time += timed_execute(cmd)

#cmd = "zapbirds -zap -zapfile %s -baryv %.6g %s"%\

# (default_zaplist, job.baryv, fftnm)

#job.FFT_time += timed_execute(cmd)

#cmd = "rednoise %s"%fftnm

#job.FFT_time += timed_execute(cmd)

#try:

# os.rename(basenm+"_red.fft", fftnm)

#except: pass

# Do the low-acceleration search

cmd = "taskset -c %d accelsearch -numharm %d -sigma %f -zmax %d -flo %f %s -zaplist %s -baryv %.6g > /dev/null"%\

(cpu_idx, lo_accel_numharm, lo_accel_sigma, lo_accel_zmax, lo_accel_flo, datnm,default_zaplist, job.baryv)

job.lo_accelsearch_time += timed_execute(cmd)

try:

os.remove(basenm+"_ACCEL_%d.txtcand"%lo_accel_zmax)

except: pass

#try: # This prevents errors if there are no cand files to copy

# shutil.copy(basenm+"_ACCEL_%d.cand"%lo_accel_zmax, job.outputdir)

# shutil.copy(basenm+"_ACCEL_%d"%lo_accel_zmax, job.outputdir)

#except: pass

# Do the high-acceleration search

#cmd = "accelsearch -numharm %d -sigma %f -zmax %d -flo %f %s"%\

# (hi_accel_numharm, hi_accel_sigma, hi_accel_zmax, hi_accel_flo, fftnm)

#job.hi_accelsearch_time += timed_execute(cmd)

#try:

# os.remove(basenm+"_ACCEL_%d.txtcand"%hi_accel_zmax)

#except: pass

#try: # This prevents errors if there are no cand files to copy

# shutil.copy(basenm+"_ACCEL_%d.cand"%hi_accel_zmax, job.outputdir)

# shutil.copy(basenm+"_ACCEL_%d"%hi_accel_zmax, job.outputdir)

#except: pass

# Remove the .dat and .fft files

try:

os.remove(datnm)

os.remove(fftnm)

except: pass

# Make the single-pulse plot

cmd = "taskset -c %d single_pulse_search.py -t %f %s/*.singlepulse"%(cpu_idx, singlepulse_plot_SNR,workdir)

job.singlepulse_time += timed_execute(cmd)

# Sift through the candidates to choose the best to fold

job.sifting_time = time.time()

lo_accel_cands = sifting.read_candidates(glob.glob("%s/*ACCEL_%d"%(workdir,lo_accel_zmax)))

if len(lo_accel_cands):

lo_accel_cands = sifting.remove_duplicate_candidates(lo_accel_cands)

if len(lo_accel_cands):

lo_accel_cands = sifting.remove_DM_problems(lo_accel_cands, numhits_to_fold,

dmstrs, low_DM_cutoff)

#

# hi_accel_cands = sifting.read_candidates(glob.glob("*ACCEL_%d"%hi_accel_zmax))

# if len(hi_accel_cands):

# hi_accel_cands = sifting.remove_duplicate_candidates(hi_accel_cands)

# if len(hi_accel_cands):

# hi_accel_cands = sifting.remove_DM_problems(hi_accel_cands, numhits_to_fold,

# dmstrs, low_DM_cutoff)

all_accel_cands = lo_accel_cands #+ hi_accel_cands

if len(all_accel_cands):

all_accel_cands = sifting.remove_harmonics(all_accel_cands)

# Note: the candidates will be sorted in _sigma_ order, not _SNR_!

all_accel_cands.sort(sifting.cmp_sigma)

sifting.write_candlist(all_accel_cands, job.basefilenm+".accelcands")

try:

cmd = "cp %s/*.accelcands "%workdir +job.outputdir

os.system(cmd)

except: pass

job.sifting_time = time.time() - job.sifting_time

# Fold the best candidates

cands_folded = 0

for cand in all_accel_cands:

if cands_folded == max_cands_to_fold:

break

if cand.sigma > to_prepfold_sigma:

job.folding_time += timed_execute("taskset -c %d "%cpu_idx + get_folding_command(cand, job, ddplans, maskfilenm))

cands_folded += 1

# Now step through the .ps files and convert them to .png and gzip them

psfiles = glob.glob("%s/*.ps"%(workdir))

for psfile in psfiles:

if "singlepulse" in psfile:

# For some reason the singlepulse files don't transform nicely...

# epsfile = psfile.replace(".ps", ".eps")

# os.system("eps2eps "+psfile+" "+epsfile)

os.system("taskset -c %d pstoimg -density 300 -flip cw -scale 0.9 -geometry 640x480 "%cpu_idx +psfile)

# try:

# os.remove(epsfile)

# except: pass

else:

os.system("taskset -c %d pstoimg -density 100 -flip cw -scale 0.9 -geometry 640x480 "%cpu_idx +psfile)

#os.system("gzip "+psfile)

# Compress Single-pulse data

cmd = "tar cfj %s_sp.tar.bz2 *.singlepulse *_singlepulse.ps *.inf "%job.basefilenm

os.system(cmd)

# Compress Cands data

cmd = "tar cfj %s_accel.tar.bz2 *ACCEL_0"%job.basefilenm

os.system(cmd)

cmd = "tar cfj %s_accel_bin.tar.bz2 *ACCEL_0.cand"%job.basefilenm

os.system(cmd)

# Compress RFI data

cmd = "tar cfj %s_rfifind.tar.bz2 %s_rfifind*"%(job.basefilenm,job.basefilenm)

os.system(cmd)

# Compress Candidates PostScript

cmd = "tar cfj %s_pfd_ps.tar.bz2 *pfd.ps"%job.basefilenm

os.system(cmd)

# NOTE: need to add database commands

# And finish up

job.total_time = time.time() - job.total_time

print "\nFinished"

print "UTC time is: %s"%(time.asctime(time.gmtime()))

# Write the job report

job.write_report(job.basefilenm+".report")

job.write_report(os.path.join(job.outputdir, job.basefilenm+".report"))

# Copy all the important stuff to the output directory

try:

cmd = "cp *.png *.bz2 "+job.outputdir # Not enough space to save .pfd file

#cmd = "cp *rfifind.[bimors]* *.pfd *.ps.gz *.png "+job.outputdir

os.system(cmd)

except: pass

# Send results to clairvaux

cmd = "scp -rp %s root@clairvaux:/data6/results/%s/."%(job.outputdir,box)

print cmd

os.system(cmd)

# Remove all temporary stuff

cmd = "cd; rm -rf %s"%workdir

#os.system(cmd)

# Return nb of cands found