def get_obs_info():
"""Read in an .inf file to extract observation information.
Return observation RA, Dec, duration, and source name.
"""
inffiles = glob.glob('*rfifind.inf')
if len(inffiles) == 0: # no inf files exist
print "No inf files available!"
return None
else:
inffile = inffiles[0] # take the first inf file in current dir
inf = infodata.infodata(inffile)
T = inf.dt * inf.N # total observation time (s)
RA = inf.RA
dec = inf.DEC
src = inf.object
MJD = inf.epoch
telescope = inf.telescope
freq = (inf.numchan / 2 -
0.5) * inf.chan_width + inf.lofreq # center freq
return {
'T': T,
'RA': RA,
'dec': dec,
'src': src,
'MJD': MJD,
'telescope': telescope,
'freq': freq
}
def main():
fn = args.infn
# Read in the raw data.(Probably not necessary anymore. Look into this)
if fn.endswith(".fil"):
filetype = "filterbank"
if not args.outbasenm:
outbasenm = fn[:-4]
else:
outbasenm = args.outbasenm
elif fn.endswith(".fits"):
filetype = "psrfits"
if not args.outbasenm:
outbasenm = fn[:-5]
else:
outbasenm = args.outbasenm
else:
raise ValueError("Cannot recognize data file type from\
extension. (Only '.fits' and '.fil'\
are supported.)")
#Read data
if filetype == 'psrfits':
inffn = fn[:-5]+'_nomask_DM0.00.inf'
else:
inffn = fn[:-4]+'_nomask_DM0.00.inf'
inf = infodata.infodata(inffn)
# Now make the timeseries only containing each of the radar signals individually.
make_timeseries(inf, args.frequenciestomask, args.bandwidth,
args.nchannels, outbasenm)
# Identify intervals contaminated by radar using the original radar removal algorithm.
start = 0
masked_intervals = chans_per_int_with_radar(inf, args.frequenciestomask,
args.bandwidth, args.threshold,
args.winlen, args.nchannels, start, outbasenm)
merge_intervals(masked_intervals, outbasenm)
def create_polycos_from_inf(par, inf):
"""A convenience function to create polycos for the observation
with info in the given *.inf file.
Inputs:
par: parfile's filename, or a parfile object.
inf: inffile's filename, or an infodata object.
Ouput:
new_polycos: polycos object
"""
if type(inf) == types.StringType:
# assume inf is a filename
inf = infodata.infodata(inf)
else:
# assume inf is already an infodata.infodata object
pass
obslength = (inf.dt * inf.N) / psr_utils.SECPERDAY
telescope_id = polycos.telescope_to_id[inf.telescope]
if telescope_id != 'o' and telescope_id != '@':
center_freq = inf.lofreq + (inf.numchan / 2 - 0.5) * inf.chan_width
if inf.bary:
# Data is barycentred, keep max_hour_angle,
# but set telescope_id to barycentre, '@'
telescope_id = '@'
else:
# optical, X-ray, or gamma-ray data
center_freq = 0.0
start_mjd = int(inf.epoch)
end_mjd = int(inf.epoch + obslength) + 1
return polycos.create_polycos(par, telescope_id, center_freq, start_mjd,
end_mjd)
def __init__(self, filename):
self.basename = filename[:filename.find("_rfifind.") + 8]
self.idata = infodata.infodata(self.basename + ".inf")
self.read_stats()
self.read_mask()
self.get_bandpass()
self.get_median_bandpass()
def __init__(self, filename):
self.basename = filename[:filename.find("_rfifind.")+8]
self.idata = infodata.infodata(self.basename+".inf")
self.read_stats()
self.read_mask()
self.get_bandpass()
self.get_median_bandpass()
def read_singlepulse_files(infiles, threshold, T_start, T_end):
DMs = []
candlist = []
num_v_DMstr = {}
for ii, infile in enumerate(infiles):
if infile.endswith(".singlepulse"):
filenmbase = infile[:infile.rfind(".singlepulse")]
else:
filenmbase = infile
info = infodata.infodata(filenmbase + ".inf")
DMstr = "%.2f" % info.DM
DMs.append(info.DM)
num_v_DMstr[DMstr] = 0
if ii == 0:
info0 = info
if os.stat(infile)[6]:
try:
cands = Num.loadtxt(infile)
if len(cands.shape) == 1:
cands = Num.asarray([cands])
for cand in cands:
if cand[2] < T_start: continue
if cand[2] > T_end: break
if cand[1] >= threshold:
candlist.append(candidate(*cand))
num_v_DMstr[DMstr] += 1
except: # No candidates in the file
IndexError
DMs.sort()
return info0, DMs, candlist, num_v_DMstr
def read_singlepulse_files(infiles, threshold, T_start, T_end):
DMs = []
candlist = []
num_v_DMstr = {}
for ii, infile in enumerate(infiles):
if infile.endswith(".singlepulse"):
filenmbase = infile[:infile.rfind(".singlepulse")]
else:
filenmbase = infile
info = infodata.infodata(filenmbase+".inf")
DMstr = "%.2f"%info.DM
DMs.append(info.DM)
num_v_DMstr[DMstr] = 0
if ii==0:
info0 = info
if os.stat(infile)[6]:
try:
cands = Num.loadtxt(infile)
if len(cands.shape)==1:
cands = Num.asarray([cands])
for cand in cands:
if cand[2] < T_start: continue
if cand[2] > T_end: break
if cand[1] >= threshold:
candlist.append(candidate(*cand))
num_v_DMstr[DMstr] += 1
except: # No candidates in the file
IndexError
DMs.sort()
return info0, DMs, candlist, num_v_DMstr
def create_polycos_from_inf(par, inf):
"""A convenience function to create polycos for the observation
with info in the given *.inf file.
Inputs:
par: parfile's filename, or a parfile object.
inf: inffile's filename, or an infodata object.
Ouput:
new_polycos: polycos object
"""
if type(inf) == types.StringType:
# assume inf is a filename
inf = infodata.infodata(inf)
else:
# assume inf is already an infodata.infodata object
pass
obslength = (inf.dt*inf.N) / psr_utils.SECPERDAY
telescope_id = polycos.telescope_to_id[inf.telescope]
if telescope_id != 'o' and telescope_id != '@':
center_freq = inf.lofreq + (inf.numchan/2 - 0.5) * inf.chan_width
if inf.bary:
# Data is barycentred, keep max_hour_angle,
# but set telescope_id to barycentre, '@'
telescope_id = '@'
else:
# optical, X-ray, or gamma-ray data
center_freq = 0.0
start_mjd = int(inf.epoch)
end_mjd = int(inf.epoch+obslength)+1
return polycos.create_polycos(par, telescope_id, center_freq, start_mjd, end_mjd)
def __init__(self, filename):
self.basename = filename[:filename.find("_rfifind.") + 8]
self.idata = infodata.infodata(self.basename + ".inf")
self.read_stats()
self.read_mask()
self.get_bandpass()
if len(self.goodints):
self.get_median_bandpass()
self.determine_padvals()
def __init__(self, filename):
self.basename = filename[:filename.find("_rfifind.")+8]
self.idata = infodata.infodata(self.basename+".inf")
self.read_stats()
self.read_mask()
self.get_bandpass()
if len(self.goodints):
self.get_median_bandpass()
self.determine_padvals()
def get_obs_info(fitsfilenm):
"""Read in an .inf file to extract observation information.
Return observation RA, Dec, duration, and source name.
Function copied from RRATtrap.py
"""
inffile = fitsfilenm[:-5] + "_rfifind.inf"
inf = infodata.infodata(inffile)
lofreq = inf.lofreq
hifreq = (inf.numchan - 1) * inf.chan_width + inf.lofreq
return {'lofreq': lofreq, 'hifreq': hifreq}
def get_obs_info(fitsfilenm):
"""Read in an .inf file to extract observation information.
Return observation RA, Dec, duration, and source name.
Function copied from RRATtrap.py
"""
inffile = fitsfilenm[:-5] + "_rfifind.inf"
inf = infodata.infodata(inffile)
lofreq = inf.lofreq
hifreq = (inf.numchan - 1)*inf.chan_width + inf.lofreq
return {'lofreq': lofreq, 'hifreq': hifreq}
def main():
freqs = []
freqerrs = []
filenums = []
intervals = []
filenum = 0
inffiles = glob.glob("*.inf")
for inffile in inffiles:
accelfile = inffile[:-4]+"_ACCEL_0.cand"
if not os.path.exists(accelfile):
continue
filenum += 1
rzws = presto.read_rzwcands(accelfile)
inf = infodata.infodata(inffile)
T = inf.dt*inf.N
for rzw in rzws:
freq = rzw.r/T
freqerr = rzw.rerr/T
freqs.append(freq)
freqerrs.append(freqerr)
filenums.append(filenum)
fint = FreqInterval(freq, freqerr)
# Traverse list of intervals backwards
for ii in range(len(intervals))[::-1]:
if fint in intervals[ii]:
fint = fint+intervals[ii]
matchfound = True
del intervals[ii]
intervals.append(fint)
freqs = np.array(freqs)
freqerrs = np.array(freqerrs)
filenums = np.array(filenums)
plt.figure(figsize=(11,8.5))
ebax = plt.axes((0.1, 0.1, 0.7, 0.7))
plt.errorbar(freqs, filenums, xerr=freqerrs, fmt=None, zorder=1, ecolor='k')
# Plot intervals worth masking
for i in intervals:
if i.numelements > 7:
r = matplotlib.patches.Rectangle((i.fcent-i.width/2.0,0), i.width, max(filenums), \
fill=True, fc='r', ec='none', \
alpha=0.25, zorder=-1)
plt.gca().add_patch(r)
print i.zaplist_string()
plt.xlabel("Spin Frequency (Hz)")
plt.ylabel("File number (index)")
hax = plt.axes((0.8, 0.1, 0.15, 0.7), sharey=ebax)
plt.hist(filenums, bins=max(filenums), range=(0, max(filenums)), orientation='horizontal', fc='none')
plt.savefig("accelcands.ps", orientation="landscape", papertype="letter")
plt.show()
def read_datfile(fn):
# Get input dat/inf file names
if not fn.endswith(".dat"):
raise ValueError("Input file must be a PRESTO '.dat' file!")
inffn = fn[:-4]+'.inf'
# Read inf/dat files
inf = infodata.infodata(inffn)
rawdata = np.fromfile(fn, dtype=DTYPE, count=inf.N)
#validate_timeseries(inf)
return rawdata, inf
def get_obs_info(inffile):
"""Read in an .inf file to extract observation information.
Return observation RA, Dec, duration, and source name.
"""
inf = infodata.infodata(inffile)
T = inf.dt * inf.N # total observation time (s)
RA = inf.RA
dec = inf.DEC
src = inf.object
MJD = inf.epoch
telescope = inf.telescope
freq = (inf.numchan/2-0.5)*inf.chan_width+inf.lofreq # center freq
return {'T': T, 'RA': RA, 'dec': dec, 'src': src, 'MJD': MJD, 'telescope': telescope, 'freq': freq}
def get_obs_info(inffile):
"""Read in an .inf file to extract observation information.
Return observation RA, Dec, duration, and source name.
"""
inf = infodata.infodata(inffile)
T = inf.dt * inf.N # total observation time (s)
RA = inf.RA
dec = inf.DEC
src = inf.object
MJD = inf.epoch
telescope = inf.telescope
freq = (inf.numchan/2-0.5)*inf.chan_width+inf.lofreq # center freq
return {'T': T, 'RA': RA, 'dec': dec, 'src': src, 'MJD': MJD, 'telescope': telescope, 'freq': freq}
def main():
# Grab *.singlepulse files
spfns = args
for g in options.globs:
spfns += glob.glob(g)
if not len(spfns):
raise singlepulse.errors.SinglePulseError("No *.singlepulses found! " \
"Exiting...\n")
print "Number of *.singlepulse files found: %d" % len(spfns)
# Read *.single pulse files
candlist = singlepulse.cands.CandidateList()
allDMs = []
for ii, spfn in enumerate(spfns):
candlist += singlepulse.cands.read_singlepulses(spfn)
inffn = os.path.splitext(spfn)[0]+".inf"
info = infodata.infodata(inffn)
allDMs.append(info.DM)
sys.stdout.write("\rReading *.singlepulse files (%6.2f%%)" % \
(float(ii+1)/len(spfns)*100))
sys.stdout.flush()
sys.stdout.write("\n")
sys.stdout.flush()
allDMs = np.unique(allDMs)
candlist.trim(dmlim=(options.lodm, options.hidm), \
timelim=(options.start, options.end), \
minsigma=options.threshold)
if not len(candlist):
raise singlepulse.errors.SinglePulseError("No candidates to plot!")
print "Plotting..."
fig = singlepulse.plot.plot(candlist, \
timelim=(options.start, options.end), \
minsigma=options.threshold, allDMs=allDMs)
if options.savefn:
print "Saving to file (%s)..." % options.savefn
plt.savefig(options.savefn, papertype='letter', orientation='landscape')
if options.interactive:
plt.figtext(0.98, 0.02, "Press 'Q' to quit", ha="right", size='x-small')
def quit(event):
if event.key in ('q', 'Q'):
print "Quitting..."
plt.close(fig)
fig.canvas.mpl_connect('key_press_event', quit)
plt.show()
def __init__(self, datfn, dtype=DTYPE):
self.datfn = datfn
self.dtype = np.dtype(dtype)
self.bytes_per_sample = self.dtype.itemsize
if self.datfn.endswith(".dat"):
self.basefn = self.datfn[:-4]
self.datfile = open(datfn, 'r')
self.inffn = "%s.inf" % self.basefn
self.infdata = infodata.infodata(self.inffn)
self.inf = self.infdata
# Corrections need to be applied to data from .inf file
correct_infdata(self.infdata)
else:
raise "Filename (%s) doesn't end with '.dat'"
# initialize file, and current sample, time, and MJD counters
self.rewind()
def __init__(self, datfn, dtype=DTYPE):
self.datfn = datfn
self.dtype = np.dtype(dtype)
self.bytes_per_sample = self.dtype.itemsize
if self.datfn.endswith(".dat"):
self.basefn = self.datfn[:-4]
self.datfile = open(datfn, 'r')
self.inffn = "%s.inf" % self.basefn
self.infdata = infodata.infodata(self.inffn)
self.inf = self.infdata
# Corrections need to be applied to data from .inf file
correct_infdata(self.infdata)
else:
raise "Filename (%s) doesn't end with '.dat'"
# initialize file, and current sample, time, and MJD counters
self.rewind()
def get_obs_info():
"""Read in an .inf file to extract observation information.
Return observation RA, Dec, duration, and source name.
"""
inffiles = glob.glob('*.inf')
if len(inffiles) == 0: # no inf files exist
print "No inf files available!"
return None
else:
inffile = inffiles[0] # take the first inf file in current dir
inf = infodata.infodata(inffile)
T = inf.dt * inf.N # total observation time (s)
RA = inf.RA
dec = inf.DEC
src = inf.object
return {'T': T, 'RA': RA, 'dec': dec, 'src': src}
def read_singlepulses(infile):
"""Read single pulses from *.singlepulse file.
"""
index = infile.rfind(".singlepulse")
if index == -1:
raise errors.SinglePulseError("File is not a *.singlepulse file (%s)!" % infile)
filenmbase = infile[:index]
info = infodata.infodata(filenmbase + ".inf")
candlist = CandidateList()
if os.path.getsize(infile):
canddata = np.loadtxt(
infile, dtype=[("DM", "f8"), ("sigma", "f8"), ("time", "f8"), ("bin", "i8"), ("downfact", "i8")]
)
canddata = np.atleast_1d(canddata)
for (dm, sigma, time, binnum, downfact) in canddata:
cand = Candidate(dm, sigma, time, binnum, downfact, info)
candlist.append(cand)
return candlist
def get_obs_info():
"""Read in an .inf file to extract observation information.
Return observation RA, Dec, duration, and source name.
"""
inffiles = glob.glob('*.inf')
if len(inffiles) == 0: # no inf files exist
print "No inf files available!"
return None
else:
inffile = inffiles[0] # take the first inf file in current dir
inf = infodata.infodata(inffile)
T = inf.dt * inf.N # total observation time (s)
RA = inf.RA
dec = inf.DEC
src = inf.object
MJD = inf.epoch
telescope = inf.telescope
freq = (inf.numchan/2-0.5)*inf.chan_width+inf.lofreq # center freq
return {'T': T, 'RA': RA, 'dec': dec, 'src': src, 'MJD': MJD, 'telescope': telescope, 'freq': freq}
def read_timeseries(datfn):
"""Read a dat file.
Return the timeseries rounded down to the nearest DETRENLEN
and the associated infodata object.
"""
index = datfn.rfind(".dat")
if index == -1:
raise errors.SinglePulseError("File is not a *.dat file (%s)!" % datfn)
filenmbase = datfn[:index]
info = infodata.infodata(filenmbase+".inf")
N = int(info.N)
roundN = N/DETRENDLEN * DETRENDLEN
numchunks = roundN/CHUNKLEN
# Read in the file
print 'Reading "%s"...' % os.path.split(datfn)[-1]
timeseries = np.fromfile(datfn, dtype=np.float32, count=roundN)
return timeseries, info
def __init__(self, fftfn, inffn=None, maxfreq=None):
"""PrestoFFT object creator
'fftfn' is filename of .fft file
'inffn' is filename of .inf file
If None the inffn will default to fftfn with .fft extension replaced by .inf
"""
if not fftfn.endswith(".fft"):
ValueError("FFT filename must end with '.fft'!")
if not os.path.isfile(fftfn):
ValueError("FFT file does not exist!\n\t(%s)" % fftfn)
else:
self.fftfn = fftfn
self.fftfile = open(self.fftfn, "rb")
if inffn is None:
inffn = "%s.inf" % fftfn[:-4]
else:
inffn = inffn
if not os.path.isfile(inffn):
ValueError("Info file does not exist!\n\t(%s)" % inffn)
self.inffn = inffn
self.inf = infodata.infodata(inffn)
freqs = np.fft.fftfreq(self.inf.N, self.inf.dt)
self.freqs = freqs[freqs >= 0]
if maxfreq is not None:
ntoread = np.sum(self.freqs < maxfreq)
self.inf.N = ntoread
self.freqs = self.freqs[:ntoread]
else:
ntoread = -1
self.fft = self.read_fft(count=ntoread)
self.phases = np.angle(self.fft)
self.normalisation = "raw"
self.powers = np.abs(self.fft)**2
self.errs = None
def __init__(self, filename):
self.pfd_filename = filename
infile = open(filename, "rb")
# See if the .bestprof file is around
try:
self.bestprof = bestprof(filename+".bestprof")
except IOError:
self.bestprof = 0
swapchar = '<' # this is little-endian
data = infile.read(5*4)
testswap = struct.unpack(swapchar+"i"*5, data)
# This is a hack to try and test the endianness of the data.
# None of the 5 values should be a large positive number.
if (Num.fabs(Num.asarray(testswap))).max() > 100000:
swapchar = '>' # this is big-endian
(self.numdms, self.numperiods, self.numpdots, self.nsub, self.npart) = \
struct.unpack(swapchar+"i"*5, data)
(self.proflen, self.numchan, self.pstep, self.pdstep, self.dmstep, \
self.ndmfact, self.npfact) = struct.unpack(swapchar+"i"*7, infile.read(7*4))
self.filenm = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
self.candnm = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
self.telescope = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
self.pgdev = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
test = infile.read(16)
has_posn = 1
for ii in range(16):
if test[ii] not in '0123456789:.-\0':
has_posn = 0
break
if has_posn:
self.rastr = test[:test.find('\0')]
test = infile.read(16)
self.decstr = test[:test.find('\0')]
(self.dt, self.startT) = struct.unpack(swapchar+"dd", infile.read(2*8))
else:
self.rastr = "Unknown"
self.decstr = "Unknown"
(self.dt, self.startT) = struct.unpack(swapchar+"dd", test)
(self.endT, self.tepoch, self.bepoch, self.avgvoverc, self.lofreq, \
self.chan_wid, self.bestdm) = struct.unpack(swapchar+"d"*7, infile.read(7*8))
# The following "fixes" (we think) the observing frequency of the Spigot
# based on tests done by Ingrid on 0737 (comparing it to GASP)
# The same sorts of corrections should be made to WAPP data as well...
# The tepoch corrections are empirically determined timing corrections
# Note that epoch is only double precision and so the floating
# point accuracy is ~1 us!
if self.telescope=='GBT':
if (Num.fabs(Num.fmod(self.dt, 8.192e-05) < 1e-12) and \
("spigot" in filename.lower() or "guppi" not in filename.lower()) and \
(self.tepoch < 54832.0)):
sys.stderr.write("Assuming SPIGOT data...\n")
if self.chan_wid==800.0/1024: # Spigot 800 MHz mode 2
self.lofreq -= 0.5 * self.chan_wid
# original values
#if self.tepoch > 0.0: self.tepoch += 0.039334/86400.0
#if self.bestprof: self.bestprof.epochf += 0.039334/86400.0
# values measured with 1713+0747 wrt BCPM2 on 13 Sept 2007
if self.tepoch > 0.0: self.tepoch += 0.039365/86400.0
if self.bestprof: self.bestprof.epochf += 0.039365/86400.0
elif self.chan_wid==800.0/2048:
self.lofreq -= 0.5 * self.chan_wid
if self.tepoch < 53700.0: # Spigot 800 MHz mode 16 (downsampled)
if self.tepoch > 0.0: self.tepoch += 0.039352/86400.0
if self.bestprof: self.bestprof.epochf += 0.039352/86400.0
else: # Spigot 800 MHz mode 14
# values measured with 1713+0747 wrt BCPM2 on 13 Sept 2007
if self.tepoch > 0.0: self.tepoch += 0.039365/86400.0
if self.bestprof: self.bestprof.epochf += 0.039365/86400.0
elif self.chan_wid==50.0/1024 or self.chan_wid==50.0/2048: # Spigot 50 MHz modes
self.lofreq += 0.5 * self.chan_wid
# Note: the offset has _not_ been measured for the 2048-lag mode
if self.tepoch > 0.0: self.tepoch += 0.039450/86400.0
if self.bestprof: self.bestprof.epochf += 0.039450/86400.0
(self.topo_pow, tmp) = struct.unpack(swapchar+"f"*2, infile.read(2*4))
(self.topo_p1, self.topo_p2, self.topo_p3) = struct.unpack(swapchar+"d"*3, \
infile.read(3*8))
(self.bary_pow, tmp) = struct.unpack(swapchar+"f"*2, infile.read(2*4))
(self.bary_p1, self.bary_p2, self.bary_p3) = struct.unpack(swapchar+"d"*3, \
infile.read(3*8))
(self.fold_pow, tmp) = struct.unpack(swapchar+"f"*2, infile.read(2*4))
(self.fold_p1, self.fold_p2, self.fold_p3) = struct.unpack(swapchar+"d"*3, \
infile.read(3*8))
# Save current p, pd, pdd
# NOTE: Fold values are actually frequencies!
self.curr_p1, self.curr_p2, self.curr_p3 = \
psr_utils.p_to_f(self.fold_p1, self.fold_p2, self.fold_p3)
self.pdelays_bins = Num.zeros(self.npart, dtype='d')
(self.orb_p, self.orb_e, self.orb_x, self.orb_w, self.orb_t, self.orb_pd, \
self.orb_wd) = struct.unpack(swapchar+"d"*7, infile.read(7*8))
self.dms = Num.asarray(struct.unpack(swapchar+"d"*self.numdms, \
infile.read(self.numdms*8)))
if self.numdms==1:
self.dms = self.dms[0]
self.periods = Num.asarray(struct.unpack(swapchar+"d"*self.numperiods, \
infile.read(self.numperiods*8)))
self.pdots = Num.asarray(struct.unpack(swapchar+"d"*self.numpdots, \
infile.read(self.numpdots*8)))
self.numprofs = self.nsub*self.npart
if (swapchar=='<'): # little endian
self.profs = Num.zeros((self.npart, self.nsub, self.proflen), dtype='d')
for ii in range(self.npart):
for jj in range(self.nsub):
self.profs[ii,jj,:] = Num.fromfile(infile, Num.float64, self.proflen)
else:
self.profs = Num.asarray(struct.unpack(swapchar+"d"*self.numprofs*self.proflen, \
infile.read(self.numprofs*self.proflen*8)))
self.profs = Num.reshape(self.profs, (self.npart, self.nsub, self.proflen))
if (self.numchan==1):
try:
idata = infodata.infodata(self.filenm[:self.filenm.rfind('.')]+".inf")
if idata.waveband=="Radio":
self.bestdm = idata.DM
self.numchan = idata.numchan
else: # i.e. for events
self.bestdm = 0.0
self.numchan = 1
except IOError:
print "Warning! Can't open the .inf file for "+filename+"!"
self.binspersec = self.fold_p1*self.proflen
self.chanpersub = self.numchan/self.nsub
self.subdeltafreq = self.chan_wid*self.chanpersub
self.hifreq = self.lofreq + (self.numchan-1)*self.chan_wid
self.losubfreq = self.lofreq + self.subdeltafreq - self.chan_wid
self.subfreqs = Num.arange(self.nsub, dtype='d')*self.subdeltafreq + \
self.losubfreq
self.subdelays_bins = Num.zeros(self.nsub, dtype='d')
# Save current DM
self.currdm = 0
self.killed_subbands = []
self.killed_intervals = []
self.pts_per_fold = []
# Note: a foldstats struct is read in as a group of 7 doubles
# the correspond to, in order:
# numdata, data_avg, data_var, numprof, prof_avg, prof_var, redchi
self.stats = Num.zeros((self.npart, self.nsub, 7), dtype='d')
for ii in range(self.npart):
currentstats = self.stats[ii]
for jj in range(self.nsub):
if (swapchar=='<'): # little endian
currentstats[jj] = Num.fromfile(infile, Num.float64, 7)
else:
currentstats[jj] = Num.asarray(struct.unpack(swapchar+"d"*7, \
infile.read(7*8)))
self.pts_per_fold.append(self.stats[ii][0][0]) # numdata from foldstats
self.start_secs = Num.add.accumulate([0]+self.pts_per_fold[:-1])*self.dt
self.pts_per_fold = Num.asarray(self.pts_per_fold)
self.mid_secs = self.start_secs + 0.5*self.dt*self.pts_per_fold
if (not self.tepoch==0.0):
self.start_topo_MJDs = self.start_secs/86400.0 + self.tepoch
self.mid_topo_MJDs = self.mid_secs/86400.0 + self.tepoch
if (not self.bepoch==0.0):
self.start_bary_MJDs = self.start_secs/86400.0 + self.bepoch
self.mid_bary_MJDs = self.mid_secs/86400.0 + self.bepoch
self.Nfolded = Num.add.reduce(self.pts_per_fold)
self.T = self.Nfolded*self.dt
self.avgprof = (self.profs/self.proflen).sum()
self.varprof = self.calc_varprof()
infile.close()
self.barysubfreqs = None
if self.avgvoverc==0:
if self.candnm.startswith("PSR_"):
# If this doesn't work, we should try to use the barycentering calcs
# in the presto module.
try:
self.polycos = polycos.polycos(self.candnm[4:],
filenm=self.pfd_filename+".polycos")
midMJD = self.tepoch + 0.5*self.T/86400.0
self.avgvoverc = self.polycos.get_voverc(int(midMJD), midMJD-int(midMJD))
#sys.stderr.write("Approximate Doppler velocity (in c) is: %.4g\n"%self.avgvoverc)
# Make the Doppler correction
self.barysubfreqs = self.subfreqs*(1.0+self.avgvoverc)
except IOError:
self.polycos = 0
if self.barysubfreqs is None:
self.barysubfreqs = self.subfreqs
def main():
parser = OptionParser(usage)
parser.add_option(
"-x",
"--xwin",
action="store_true",
dest="xwin",
default=False,
help="Don't make a postscript plot, just use an X-window")
parser.add_option("-p",
"--noplot",
action="store_false",
dest="makeplot",
default=True,
help="Look for pulses but do not generate a plot")
parser.add_option(
"-m",
"--maxwidth",
type="float",
dest="maxwidth",
default=0.0,
help="Set the max downsampling in sec (see below for default)")
parser.add_option("-t",
"--threshold",
type="float",
dest="threshold",
default=5.0,
help="Set a different threshold SNR (default=5.0)")
parser.add_option("-s",
"--start",
type="float",
dest="T_start",
default=0.0,
help="Only plot events occuring after this time (s)")
parser.add_option("-e",
"--end",
type="float",
dest="T_end",
default=1e9,
help="Only plot events occuring before this time (s)")
parser.add_option("-g",
"--glob",
type="string",
dest="globexp",
default=None,
help="Process the files from this glob expression")
parser.add_option("-f",
"--fast",
action="store_true",
dest="fast",
default=False,
help="Use a faster method of de-trending (2x speedup)")
(opts, args) = parser.parse_args()
if len(args) == 0:
if opts.globexp == None:
print full_usage
sys.exit(0)
else:
args = []
for globexp in opts.globexp.split():
args += glob.glob(globexp)
useffts = True
dosearch = True
if opts.xwin:
pgplot_device = "/XWIN"
else:
pgplot_device = ""
fftlen = 8192 # Should be a power-of-two for best speed
chunklen = 8000 # Must be at least max_downfact less than fftlen
detrendlen = 1000 # length of a linear piecewise chunk of data for detrending
blocks_per_chunk = chunklen / detrendlen
overlap = (fftlen - chunklen) / 2
worklen = chunklen + 2 * overlap # currently it is fftlen...
max_downfact = 30
default_downfacts = [2, 3, 4, 6, 9, 14, 20, 30, 45, 70, 100, 150]
if args[0].endswith(".singlepulse"):
filenmbase = args[0][:args[0].rfind(".singlepulse")]
dosearch = False
elif args[0].endswith(".dat"):
filenmbase = args[0][:args[0].rfind(".dat")]
else:
filenmbase = args[0]
# Don't do a search, just read results and plot
if not dosearch:
info, DMs, candlist, num_v_DMstr = \
read_singlepulse_files(args, opts.threshold, opts.T_start, opts.T_end)
orig_N, orig_dt = int(info.N), info.dt
obstime = orig_N * orig_dt
else:
DMs = []
candlist = []
num_v_DMstr = {}
# Loop over the input files
for filenm in args:
if filenm.endswith(".dat"):
filenmbase = filenm[:filenm.rfind(".dat")]
else:
filenmbase = filenm
info = infodata.infodata(filenmbase + ".inf")
DMstr = "%.2f" % info.DM
DMs.append(info.DM)
N, dt = int(info.N), info.dt
obstime = N * dt
# Choose the maximum width to search based on time instead
# of bins. This helps prevent increased S/N when the downsampling
# changes as the DM gets larger.
if opts.maxwidth > 0.0:
downfacts = [
x for x in default_downfacts if x * dt <= opts.maxwidth
]
else:
downfacts = [x for x in default_downfacts if x <= max_downfact]
if len(downfacts) == 0:
downfacts = [default_downfacts[0]]
if (filenm == args[0]):
orig_N = N
orig_dt = dt
if useffts:
fftd_kerns = make_fftd_kerns(downfacts, fftlen)
if info.breaks:
offregions = zip([x[1] for x in info.onoff[:-1]],
[x[0] for x in info.onoff[1:]])
outfile = open(filenmbase + '.singlepulse', mode='w')
# Compute the file length in detrendlens
roundN = N / detrendlen * detrendlen
numchunks = roundN / chunklen
# Read in the file
print 'Reading "%s"...' % filenm
timeseries = Num.fromfile(filenm, dtype=Num.float32, count=roundN)
# Split the timeseries into chunks for detrending
numblocks = roundN / detrendlen
timeseries.shape = (numblocks, detrendlen)
stds = Num.zeros(numblocks, dtype=Num.float64)
# de-trend the data one chunk at a time
print ' De-trending the data and computing statistics...'
for ii, chunk in enumerate(timeseries):
if opts.fast: # use median removal instead of detrending (2x speedup)
tmpchunk = chunk.copy()
tmpchunk.sort()
med = tmpchunk[detrendlen / 2]
chunk -= med
tmpchunk -= med
else:
# The detrend calls are the most expensive in the program
timeseries[ii] = scipy.signal.detrend(chunk, type='linear')
tmpchunk = timeseries[ii].copy()
tmpchunk.sort()
# The following gets rid of (hopefully) most of the
# outlying values (i.e. power dropouts and single pulses)
# If you throw out 5% (2.5% at bottom and 2.5% at top)
# of random gaussian deviates, the measured stdev is ~0.871
# of the true stdev. Thus the 1.0/0.871=1.148 correction below.
# The following is roughly .std() since we already removed the median
stds[ii] = Num.sqrt(
(tmpchunk[detrendlen / 40:-detrendlen / 40]**2.0).sum() /
(0.95 * detrendlen))
stds *= 1.148
# sort the standard deviations and separate those with
# very low or very high values
sort_stds = stds.copy()
sort_stds.sort()
# identify the differences with the larges values (this
# will split off the chunks with very low and very high stds
locut = (sort_stds[1:numblocks / 2 + 1] -
sort_stds[:numblocks / 2]).argmax() + 1
hicut = (sort_stds[numblocks / 2 + 1:] -
sort_stds[numblocks / 2:-1]).argmax() + numblocks / 2 - 2
std_stds = scipy.std(sort_stds[locut:hicut])
median_stds = sort_stds[(locut + hicut) / 2]
lo_std = median_stds - 4.0 * std_stds
hi_std = median_stds + 4.0 * std_stds
# Determine a list of "bad" chunks. We will not search these.
bad_blocks = Num.nonzero((stds < lo_std) | (stds > hi_std))[0]
print " pseudo-median block standard deviation = %.2f" % (
median_stds)
print " identified %d bad blocks out of %d (i.e. %.2f%%)" % \
(len(bad_blocks), len(stds),
100.0*float(len(bad_blocks))/float(len(stds)))
stds[bad_blocks] = median_stds
print " Now searching..."
# Now normalize all of the data and reshape it to 1-D
timeseries /= stds[:, Num.newaxis]
timeseries.shape = (roundN, )
# And set the data in the bad blocks to zeros
# Even though we don't search these parts, it is important
# because of the overlaps for the convolutions
for bad_block in bad_blocks:
loind, hiind = bad_block * detrendlen, (bad_block +
1) * detrendlen
timeseries[loind:hiind] = 0.0
# Convert to a set for faster lookups below
bad_blocks = set(bad_blocks)
# Step through the data
dm_candlist = []
for chunknum in range(numchunks):
loind = chunknum * chunklen - overlap
hiind = (chunknum + 1) * chunklen + overlap
# Take care of beginning and end of file overlap issues
if (chunknum == 0): # Beginning of file
chunk = Num.zeros(worklen, dtype=Num.float32)
chunk[overlap:] = timeseries[loind + overlap:hiind]
elif (chunknum == numchunks - 1): # end of the timeseries
chunk = Num.zeros(worklen, dtype=Num.float32)
chunk[:-overlap] = timeseries[loind:hiind - overlap]
else:
chunk = timeseries[loind:hiind]
# Make a set with the current block numbers
lowblock = blocks_per_chunk * chunknum
currentblocks = set(Num.arange(blocks_per_chunk) + lowblock)
localgoodblocks = Num.asarray(
list(currentblocks - bad_blocks)) - lowblock
# Search this chunk if it is not all bad
if len(localgoodblocks):
# This is the good part of the data (end effects removed)
goodchunk = chunk[overlap:-overlap]
# need to pass blocks/chunklen, localgoodblocks
# dm_candlist, dt, opts.threshold to cython routine
# Search non-downsampled data first
# NOTE: these nonzero() calls are some of the most
# expensive calls in the program. Best bet would
# probably be to simply iterate over the goodchunk
# in C and append to the candlist there.
hibins = Num.flatnonzero(goodchunk > opts.threshold)
hivals = goodchunk[hibins]
hibins += chunknum * chunklen
hiblocks = hibins / detrendlen
# Add the candidates (which are sorted by bin)
for bin, val, block in zip(hibins, hivals, hiblocks):
if block not in bad_blocks:
time = bin * dt
dm_candlist.append(
candidate(info.DM, val, time, bin, 1))
# Prepare our data for the convolution
if useffts: fftd_chunk = rfft(chunk, -1)
# Now do the downsampling...
for ii, downfact in enumerate(downfacts):
if useffts:
# Note: FFT convolution is faster for _all_ downfacts, even 2
goodchunk = fft_convolve(fftd_chunk,
fftd_kerns[ii], overlap,
-overlap)
else:
# The normalization of this kernel keeps the post-smoothing RMS = 1
kernel = Num.ones(downfact, dtype=Num.float32) / \
Num.sqrt(downfact)
smoothed_chunk = scipy.signal.convolve(
chunk, kernel, 1)
goodchunk = smoothed_chunk[overlap:-overlap]
#hibins = Num.nonzero(goodchunk>opts.threshold)[0]
hibins = Num.flatnonzero(goodchunk > opts.threshold)
hivals = goodchunk[hibins]
hibins += chunknum * chunklen
hiblocks = hibins / detrendlen
hibins = hibins.tolist()
hivals = hivals.tolist()
# Now walk through the new candidates and remove those
# that are not the highest but are within downfact/2
# bins of a higher signal pulse
hibins, hivals = prune_related1(
hibins, hivals, downfact)
# Insert the new candidates into the candlist, but
# keep it sorted...
for bin, val, block in zip(hibins, hivals, hiblocks):
if block not in bad_blocks:
time = bin * dt
bisect.insort(
dm_candlist,
candidate(info.DM, val, time, bin,
downfact))
# Now walk through the dm_candlist and remove the ones that
# are within the downsample proximity of a higher
# signal-to-noise pulse
dm_candlist = prune_related2(dm_candlist, downfacts)
print " Found %d pulse candidates" % len(dm_candlist)
# Get rid of those near padding regions
if info.breaks: prune_border_cases(dm_candlist, offregions)
# Write the pulses to an ASCII output file
if len(dm_candlist):
#dm_candlist.sort(cmp_sigma)
outfile.write(
"# DM Sigma Time (s) Sample Downfact\n")
for cand in dm_candlist:
outfile.write(str(cand))
outfile.close()
# Add these candidates to the overall candidate list
for cand in dm_candlist:
candlist.append(cand)
num_v_DMstr[DMstr] = len(dm_candlist)
if (opts.makeplot):
# Step through the candidates to make a SNR list
DMs.sort()
snrs = []
for cand in candlist:
snrs.append(cand.sigma)
if snrs:
maxsnr = max(int(max(snrs)), int(opts.threshold)) + 3
else:
maxsnr = int(opts.threshold) + 3
# Generate the SNR histogram
snrs = Num.asarray(snrs)
(num_v_snr, lo_snr, d_snr, num_out_of_range) = \
scipy.stats.histogram(snrs,
int(maxsnr-opts.threshold+1),
[opts.threshold, maxsnr])
snrs = Num.arange(maxsnr-opts.threshold+1, dtype=Num.float64) * d_snr \
+ lo_snr + 0.5*d_snr
num_v_snr = num_v_snr.astype(Num.float32)
num_v_snr[num_v_snr == 0.0] = 0.001
# Generate the DM histogram
num_v_DM = Num.zeros(len(DMs))
for ii, DM in enumerate(DMs):
num_v_DM[ii] = num_v_DMstr["%.2f" % DM]
DMs = Num.asarray(DMs)
# open the plot device
short_filenmbase = filenmbase[:filenmbase.find("_DM")]
if opts.T_end > obstime:
opts.T_end = obstime
if pgplot_device:
ppgplot.pgopen(pgplot_device)
else:
if (opts.T_start > 0.0 or opts.T_end < obstime):
ppgplot.pgopen(short_filenmbase +
'_%.0f-%.0fs_singlepulse.ps/VPS' %
(opts.T_start, opts.T_end))
else:
ppgplot.pgopen(short_filenmbase + '_singlepulse.ps/VPS')
ppgplot.pgpap(7.5, 1.0) # Width in inches, aspect
# plot the SNR histogram
ppgplot.pgsvp(0.06, 0.31, 0.6, 0.87)
ppgplot.pgswin(opts.threshold, maxsnr, Num.log10(0.5),
Num.log10(2 * max(num_v_snr)))
ppgplot.pgsch(0.8)
ppgplot.pgbox("BCNST", 0, 0, "BCLNST", 0, 0)
ppgplot.pgmtxt('B', 2.5, 0.5, 0.5, "Signal-to-Noise")
ppgplot.pgmtxt('L', 1.8, 0.5, 0.5, "Number of Pulses")
ppgplot.pgsch(1.0)
ppgplot.pgbin(snrs, Num.log10(num_v_snr), 1)
# plot the DM histogram
ppgplot.pgsvp(0.39, 0.64, 0.6, 0.87)
# Add [1] to num_v_DM in YMAX below so that YMIN != YMAX when max(num_v_DM)==0
ppgplot.pgswin(
min(DMs) - 0.5,
max(DMs) + 0.5, 0.0, 1.1 * max(num_v_DM + [1]))
ppgplot.pgsch(0.8)
ppgplot.pgbox("BCNST", 0, 0, "BCNST", 0, 0)
ppgplot.pgmtxt('B', 2.5, 0.5, 0.5, "DM (pc cm\u-3\d)")
ppgplot.pgmtxt('L', 1.8, 0.5, 0.5, "Number of Pulses")
ppgplot.pgsch(1.0)
ppgplot.pgbin(DMs, num_v_DM, 1)
# plot the SNR vs DM plot
ppgplot.pgsvp(0.72, 0.97, 0.6, 0.87)
ppgplot.pgswin(min(DMs) - 0.5, max(DMs) + 0.5, opts.threshold, maxsnr)
ppgplot.pgsch(0.8)
ppgplot.pgbox("BCNST", 0, 0, "BCNST", 0, 0)
ppgplot.pgmtxt('B', 2.5, 0.5, 0.5, "DM (pc cm\u-3\d)")
ppgplot.pgmtxt('L', 1.8, 0.5, 0.5, "Signal-to-Noise")
ppgplot.pgsch(1.0)
cand_ts = Num.zeros(len(candlist), dtype=Num.float32)
cand_SNRs = Num.zeros(len(candlist), dtype=Num.float32)
cand_DMs = Num.zeros(len(candlist), dtype=Num.float32)
for ii, cand in enumerate(candlist):
cand_ts[ii], cand_SNRs[ii], cand_DMs[ii] = \
cand.time, cand.sigma, cand.DM
ppgplot.pgpt(cand_DMs, cand_SNRs, 20)
# plot the DM vs Time plot
ppgplot.pgsvp(0.06, 0.97, 0.08, 0.52)
ppgplot.pgswin(opts.T_start, opts.T_end,
min(DMs) - 0.5,
max(DMs) + 0.5)
ppgplot.pgsch(0.8)
ppgplot.pgbox("BCNST", 0, 0, "BCNST", 0, 0)
ppgplot.pgmtxt('B', 2.5, 0.5, 0.5, "Time (s)")
ppgplot.pgmtxt('L', 1.8, 0.5, 0.5, "DM (pc cm\u-3\d)")
# Circles are symbols 20-26 in increasing order
snr_range = 12.0
cand_symbols = (cand_SNRs - opts.threshold) / snr_range * 6.0 + 20.5
cand_symbols = cand_symbols.astype(Num.int32)
cand_symbols[cand_symbols > 26] = 26
for ii in [26, 25, 24, 23, 22, 21, 20]:
inds = Num.nonzero(cand_symbols == ii)[0]
ppgplot.pgpt(cand_ts[inds], cand_DMs[inds], ii)
# Now fill the infomation area
ppgplot.pgsvp(0.05, 0.95, 0.87, 0.97)
ppgplot.pgsch(1.0)
ppgplot.pgmtxt('T', 0.5, 0.0, 0.0,
"Single pulse results for '%s'" % short_filenmbase)
ppgplot.pgsch(0.8)
# first row
ppgplot.pgmtxt('T', -1.1, 0.02, 0.0, 'Source: %s'%\
info.object)
ppgplot.pgmtxt('T', -1.1, 0.33, 0.0, 'RA (J2000):')
ppgplot.pgmtxt('T', -1.1, 0.5, 0.0, info.RA)
ppgplot.pgmtxt('T', -1.1, 0.73, 0.0, 'N samples: %.0f' % orig_N)
# second row
ppgplot.pgmtxt('T', -2.4, 0.02, 0.0, 'Telescope: %s'%\
info.telescope)
ppgplot.pgmtxt('T', -2.4, 0.33, 0.0, 'DEC (J2000):')
ppgplot.pgmtxt('T', -2.4, 0.5, 0.0, info.DEC)
ppgplot.pgmtxt('T', -2.4, 0.73, 0.0, 'Sampling time: %.2f \gms'%\
(orig_dt*1e6))
# third row
if info.instrument.find("pigot") >= 0:
instrument = "Spigot"
else:
instrument = info.instrument
ppgplot.pgmtxt('T', -3.7, 0.02, 0.0, 'Instrument: %s' % instrument)
if (info.bary):
ppgplot.pgmtxt('T', -3.7, 0.33, 0.0,
'MJD\dbary\u: %.12f' % info.epoch)
else:
ppgplot.pgmtxt('T', -3.7, 0.33, 0.0,
'MJD\dtopo\u: %.12f' % info.epoch)
ppgplot.pgmtxt('T', -3.7, 0.73, 0.0, 'Freq\dctr\u: %.1f MHz'%\
((info.numchan/2-0.5)*info.chan_width+info.lofreq))
ppgplot.pgiden()
ppgplot.pgend()
def main():
parser = optparse.OptionParser(prog="sp_pipeline..py", \
version=" Chitrang Patel (May. 12, 2015)", \
usage="%prog INFILE(PsrFits FILE, SINGLEPULSE FILES)", \
description="Create single pulse plots to show the " \
"frequency sweeps of a single pulse, " \
"DM vs time, and SNR vs DM,"\
"in psrFits data.")
parser.add_option('--infile', dest='infile', type='string', \
help="Give a .inf file to read the appropriate header information.")
parser.add_option('--groupsfile', dest='txtfile', type='string', \
help="Give the groups.txt file to read in the groups information.")
parser.add_option('--mask', dest='maskfile', type='string', \
help="Mask file produced by rfifind. (Default: No Mask).", \
default=None)
options, args = parser.parse_args()
if not hasattr(options, 'infile'):
raise ValueError("A .inf file must be given on the command line! ")
if not hasattr(options, 'txtfile'):
raise ValueError(
"The groups.txt file must be given on the command line! ")
files = get_textfile(options.txtfile)
print_debug("Begining waterfaller... " + strftime("%Y-%m-%d %H:%M:%S"))
Detrendlen = 50
if not args[0].endswith("fits"):
raise ValueError("The first file must be a psrFits file! ")
basename = args[0][:-5]
filetype = "psrfits"
inffile = options.infile
topo, bary = bary_and_topo.bary_to_topo(inffile)
time_shift = bary - topo
inf = infodata.infodata(inffile)
RA = inf.RA
dec = inf.DEC
MJD = inf.epoch
mjd = Popen(["mjd2cal", "%f" % MJD], stdout=PIPE, stderr=PIPE)
date, err = mjd.communicate()
date = date.split()[2:5]
telescope = inf.telescope
N = inf.N
Total_observed_time = inf.dt * N
print_debug('getting file..')
rawdatafile = psrfits.PsrfitsFile(args[0])
print "rawdatafile", memory.resident() / (1024.0**3)
bin_shift = np.round(time_shift / rawdatafile.tsamp).astype('int')
for group in [6, 5, 4, 3, 2]:
rank = group + 1
if files[group] != "Number of rank %i groups: 0 " % rank:
print_debug(files[group])
values = split_parameters(rank, options.txtfile)
lis = np.where(files == '\tRank: %i.000000' % rank)[0]
for ii in range(len(values)):
#### Array for Plotting DM vs SNR
print "DM, S/N", memory.resident() / (1024.0**3)
print_debug("Making arrays for DM vs Signal to Noise...")
temp_list = files[lis[ii] - 6].split()
npulses = int(temp_list[2])
temp_lines = files[(lis[ii] + 3):(lis[ii] + npulses + 1)]
arr = np.split(temp_lines, len(temp_lines))
dm_list = []
time_list = []
for i in range(len(arr)):
dm_val = float(arr[i][0].split()[0])
time_val = float(arr[i][0].split()[2])
dm_list.append(dm_val)
time_list.append(time_val)
arr_2 = np.array([arr[i][0].split() for i in range(len(arr))],
dtype=np.float32)
dm_arr = np.array([arr_2[i][0] for i in range(len(arr))],
dtype=np.float32)
sigma_arr = np.array([arr_2[i][1] for i in range(len(arr))],
dtype=np.float32)
print "After DM, S/N", memory.resident() / (1024.0**3)
#### Array for Plotting DM vs Time is in show_spplots.plot(...)
#### Setting variables up for the waterfall arrays.
j = ii + 1
subdm = dm = sweep_dm = values[ii][0]
integrate_dm = None
sigma = values[ii][1]
sweep_posn = 0.0
topo_start_time = values[ii][2] - topo_timeshift(
values[ii][2], time_shift, topo)[0]
sample_number = values[ii][3]
width_bins = values[ii][4]
binratio = 50
scaleindep = False
zerodm = None
downsamp = np.round((values[ii][2] / sample_number /
6.54761904761905e-05)).astype('int')
duration = binratio * width_bins * rawdatafile.tsamp * downsamp
start = topo_start_time - (0.25 * duration)
if (start < 0.0):
start = 0.0
pulse_width = width_bins * downsamp * 6.54761904761905e-05
if sigma <= 10:
nsub = 32
elif sigma >= 10 and sigma < 15:
nsub = 64
else:
nsub = 96
nbins = np.round(duration / rawdatafile.tsamp).astype('int')
start_bin = np.round(start / rawdatafile.tsamp).astype('int')
dmfac = 4.15e3 * np.abs(1. / rawdatafile.frequencies[0]**2 -
1. / rawdatafile.frequencies[-1]**2)
nbinsextra = np.round(
(duration + dmfac * dm) / rawdatafile.tsamp).astype('int')
if (start_bin + nbinsextra) > N - 1:
nbinsextra = N - 1 - start_bin
data = rawdatafile.get_spectra(start_bin, nbinsextra)
print "After rawdata", memory.resident() / (1024.0**3)
data = maskdata(data, start_bin, nbinsextra, options.maskfile)
#make an array to store header information for the .npz files
temp_filename = basename + "_DM%.1f_%.1fs_rank_%i" % (
subdm, topo_start_time, rank)
# Array for Plotting Dedispersed waterfall plot - zerodm - OFF
print_debug("Running waterfaller with Zero-DM OFF...")
data, Data_dedisp_nozerodm = waterfall_array(
start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm,
integrate_dm, downsamp, scaleindep, width_bins,
rawdatafile, binratio, data)
print "waterfall", memory.resident() / (1024.0**3)
# Add additional information to the header information array
text_array = np.array([
args[0], 'Arecibo', RA, dec, MJD, rank, nsub, nbins, subdm,
sigma, sample_number, duration, width_bins, pulse_width,
rawdatafile.tsamp, Total_observed_time, topo_start_time,
data.starttime, data.dt, data.numspectra,
data.freqs.min(),
data.freqs.max()
])
#### Array for plotting Dedispersed waterfall plot zerodm - ON
print_debug("Running Waterfaller with Zero-DM ON...")
data = rawdatafile.get_spectra(start_bin, nbinsextra)
data = maskdata(data, start_bin, nbinsextra, options.maskfile)
zerodm = True
data, Data_dedisp_zerodm = waterfall_array(
start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm,
integrate_dm, downsamp, scaleindep, width_bins,
rawdatafile, binratio, data)
print "waterfall", memory.resident() / (1024.0**3)
####Sweeped without zerodm
print_debug("Running waterfaller for sweeped arrays.")
start = start + (0.25 * duration)
start_bin = np.round(start / rawdatafile.tsamp).astype('int')
sweep_duration = 4.15e3 * np.abs(
1. / rawdatafile.frequencies[0]**2 -
1. / rawdatafile.frequencies[-1]**2) * sweep_dm
nbins = np.round(sweep_duration /
(rawdatafile.tsamp)).astype('int')
if ((nbins + start_bin) > (N - 1)):
nbins = N - 1 - start_bin
data = rawdatafile.get_spectra(start_bin, nbins)
data = maskdata(data, start_bin, nbins, options.maskfile)
zerodm = None
dm = None
data, Data_nozerodm = waterfall_array(
start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm,
integrate_dm, downsamp, scaleindep, width_bins,
rawdatafile, binratio, data)
print "waterfall", memory.resident() / (1024.0**3)
text_array = np.append(text_array, sweep_duration)
text_array = np.append(text_array, data.starttime)
# Array to Construct the sweep
if sweep_dm is not None:
ddm = sweep_dm - data.dm
delays = psr_utils.delay_from_DM(ddm, data.freqs)
delays -= delays.min()
delays_nozerodm = delays
freqs_nozerodm = data.freqs
# Sweeped with zerodm-on
zerodm = True
downsamp_temp = 1
data, Data_zerodm = waterfall_array(start_bin, dmfac, duration,
nbins, zerodm, nsub, subdm,
dm, integrate_dm,
downsamp_temp, scaleindep,
width_bins, rawdatafile,
binratio, data)
print "waterfall", memory.resident() / (1024.0**3)
# Saving the arrays into the .spd file.
with open(temp_filename + ".spd", 'wb') as f:
np.savez_compressed(
f,
Data_dedisp_nozerodm=Data_dedisp_nozerodm.astype(
np.float16),
Data_dedisp_zerodm=Data_dedisp_zerodm.astype(
np.float16),
Data_nozerodm=Data_nozerodm.astype(np.float16),
delays_nozerodm=delays_nozerodm,
freqs_nozerodm=freqs_nozerodm,
Data_zerodm=Data_zerodm.astype(np.float16),
dm_arr=map(np.float16, dm_arr),
sigma_arr=map(np.float16, sigma_arr),
dm_list=map(np.float16, dm_list),
time_list=map(np.float16, time_list),
text_array=text_array)
print_debug("Now plotting...")
print "Before plot..", memory.resident() / (1024.0**3)
show_spplots.plot(temp_filename + ".spd",
args[1:],
xwin=False,
outfile=basename,
tar=None)
print "After plot..", memory.resident() / (1024.0**3)
print_debug("Finished plot %i " % j +
strftime("%Y-%m-%d %H:%M:%S"))
print_debug("Finished group %i... " % rank +
strftime("%Y-%m-%d %H:%M:%S"))
print_debug("Finished running waterfaller... " +
strftime("%Y-%m-%d %H:%M:%S"))
sys.exit(2)
else:
global inffile
inffile = args[0]
global spfile
spfile = args[1]
except getopt.GetoptError:
print "Wrong option!"
usage (prg)
sys.exit(2)
if __name__=="__main__":
parsecmdline (sys.argv[0].split("/")[-1], sys.argv[1:])
# reading inf-file
id = inf.infodata(inffile)
if is_phase == True:
import polycos as poly
startmjd=id.epoch
tres = id.dt
unc = "%9f" % (tres * 1000000., )
if psrname != "":
source = psrname
else:
source = id.object
cfreq = id.lofreq
totbw = id.BW
chanbw = id.chan_width
freq = cfreq + totbw - chanbw # central freq of the highest channel
def main():
parser = OptionParser(usage)
parser.add_option("-x", "--xwin", action="store_true", dest="xwin",
default=False, help="Don't make a postscript plot, just use an X-window")
parser.add_option("-p", "--noplot", action="store_false", dest="makeplot",
default=True, help="Look for pulses but do not generate a plot")
parser.add_option("-m", "--maxwidth", type="float", dest="maxwidth", default=0.0,
help="Set the max downsampling in sec (see below for default)")
parser.add_option("-t", "--threshold", type="float", dest="threshold", default=5.0,
help="Set a different threshold SNR (default=5.0)")
parser.add_option("-s", "--start", type="float", dest="T_start", default=0.0,
help="Only plot events occuring after this time (s)")
parser.add_option("-e", "--end", type="float", dest="T_end", default=1e9,
help="Only plot events occuring before this time (s)")
parser.add_option("-g", "--glob", type="string", dest="globexp", default=None,
help="Process the files from this glob expression")
parser.add_option("-f", "--fast", action="store_true", dest="fast",
default=False, help="Use a faster method of de-trending (2x speedup)")
parser.add_option("-b", "--nobadblocks", action="store_false", dest="badblocks",
default=True, help="Don't check for bad-blocks (may save strong pulses)")
parser.add_option("-d", "--detrendlen", type="int", dest="detrendfact", default=1,
help="Chunksize for detrending (pow-of-2 in 1000s)")
(opts, args) = parser.parse_args()
if len(args)==0:
if opts.globexp==None:
print full_usage
sys.exit(0)
else:
args = []
for globexp in opts.globexp.split():
args += glob.glob(globexp)
useffts = True
dosearch = True
if opts.xwin:
pgplot_device = "/XWIN"
else:
pgplot_device = ""
fftlen = 8192 # Should be a power-of-two for best speed
chunklen = 8000 # Must be at least max_downfact less than fftlen
assert(opts.detrendfact in [1,2,4,8,16,32])
detrendlen = opts.detrendfact*1000
if (detrendlen > chunklen):
chunklen = detrendlen
fftlen = int(next2_to_n(chunklen))
blocks_per_chunk = chunklen / detrendlen
overlap = (fftlen - chunklen)/2
worklen = chunklen + 2*overlap # currently it is fftlen...
max_downfact = 30
default_downfacts = [2, 3, 4, 6, 9, 14, 20, 30, 45, 70, 100, 150, 220, 300]
if args[0].endswith(".singlepulse"):
filenmbase = args[0][:args[0].rfind(".singlepulse")]
dosearch = False
elif args[0].endswith(".dat"):
filenmbase = args[0][:args[0].rfind(".dat")]
else:
filenmbase = args[0]
# Don't do a search, just read results and plot
if not dosearch:
info, DMs, candlist, num_v_DMstr = \
read_singlepulse_files(args, opts.threshold, opts.T_start, opts.T_end)
orig_N, orig_dt = int(info.N), info.dt
obstime = orig_N * orig_dt
else:
DMs = []
candlist = []
num_v_DMstr = {}
# Loop over the input files
for filenm in args:
if filenm.endswith(".dat"):
filenmbase = filenm[:filenm.rfind(".dat")]
else:
filenmbase = filenm
info = infodata.infodata(filenmbase+".inf")
DMstr = "%.2f"%info.DM
DMs.append(info.DM)
N, dt = int(info.N), info.dt
obstime = N * dt
# Choose the maximum width to search based on time instead
# of bins. This helps prevent increased S/N when the downsampling
# changes as the DM gets larger.
if opts.maxwidth > 0.0:
downfacts = [x for x in default_downfacts if x*dt <= opts.maxwidth]
else:
downfacts = [x for x in default_downfacts if x <= max_downfact]
if len(downfacts) == 0:
downfacts = [default_downfacts[0]]
if (filenm == args[0]):
orig_N = N
orig_dt = dt
if info.breaks:
offregions = zip([x[1] for x in info.onoff[:-1]],
[x[0] for x in info.onoff[1:]])
# If last break spans to end of file, don't read it in (its just padding)
if offregions[-1][1] == N - 1:
N = offregions[-1][0] + 1
outfile = open(filenmbase+'.singlepulse', mode='w')
# Compute the file length in detrendlens
roundN = N/detrendlen * detrendlen
numchunks = roundN / chunklen
# Read in the file
print 'Reading "%s"...'%filenm
timeseries = Num.fromfile(filenm, dtype=Num.float32, count=roundN)
# Split the timeseries into chunks for detrending
numblocks = roundN/detrendlen
timeseries.shape = (numblocks, detrendlen)
stds = Num.zeros(numblocks, dtype=Num.float64)
# de-trend the data one chunk at a time
print ' De-trending the data and computing statistics...'
for ii, chunk in enumerate(timeseries):
if opts.fast: # use median removal instead of detrending (2x speedup)
tmpchunk = chunk.copy()
tmpchunk.sort()
med = tmpchunk[detrendlen/2]
chunk -= med
tmpchunk -= med
else:
# The detrend calls are the most expensive in the program
timeseries[ii] = scipy.signal.detrend(chunk, type='linear')
tmpchunk = timeseries[ii].copy()
tmpchunk.sort()
# The following gets rid of (hopefully) most of the
# outlying values (i.e. power dropouts and single pulses)
# If you throw out 5% (2.5% at bottom and 2.5% at top)
# of random gaussian deviates, the measured stdev is ~0.871
# of the true stdev. Thus the 1.0/0.871=1.148 correction below.
# The following is roughly .std() since we already removed the median
stds[ii] = Num.sqrt((tmpchunk[detrendlen/40:-detrendlen/40]**2.0).sum() /
(0.95*detrendlen))
stds *= 1.148
# sort the standard deviations and separate those with
# very low or very high values
sort_stds = stds.copy()
sort_stds.sort()
# identify the differences with the larges values (this
# will split off the chunks with very low and very high stds
locut = (sort_stds[1:numblocks/2+1] -
sort_stds[:numblocks/2]).argmax() + 1
hicut = (sort_stds[numblocks/2+1:] -
sort_stds[numblocks/2:-1]).argmax() + numblocks/2 - 2
std_stds = scipy.std(sort_stds[locut:hicut])
median_stds = sort_stds[(locut+hicut)/2]
print " pseudo-median block standard deviation = %.2f" % (median_stds)
if (opts.badblocks):
lo_std = median_stds - 4.0 * std_stds
hi_std = median_stds + 4.0 * std_stds
# Determine a list of "bad" chunks. We will not search these.
bad_blocks = Num.nonzero((stds < lo_std) | (stds > hi_std))[0]
print " identified %d bad blocks out of %d (i.e. %.2f%%)" % \
(len(bad_blocks), len(stds),
100.0*float(len(bad_blocks))/float(len(stds)))
stds[bad_blocks] = median_stds
else:
bad_blocks = []
print " Now searching..."
# Now normalize all of the data and reshape it to 1-D
timeseries /= stds[:,Num.newaxis]
timeseries.shape = (roundN,)
# And set the data in the bad blocks to zeros
# Even though we don't search these parts, it is important
# because of the overlaps for the convolutions
for bad_block in bad_blocks:
loind, hiind = bad_block*detrendlen, (bad_block+1)*detrendlen
timeseries[loind:hiind] = 0.0
# Convert to a set for faster lookups below
bad_blocks = set(bad_blocks)
# Step through the data
dm_candlist = []
for chunknum in xrange(numchunks):
loind = chunknum*chunklen-overlap
hiind = (chunknum+1)*chunklen+overlap
# Take care of beginning and end of file overlap issues
if (chunknum==0): # Beginning of file
chunk = Num.zeros(worklen, dtype=Num.float32)
chunk[overlap:] = timeseries[loind+overlap:hiind]
elif (chunknum==numchunks-1): # end of the timeseries
chunk = Num.zeros(worklen, dtype=Num.float32)
chunk[:-overlap] = timeseries[loind:hiind-overlap]
else:
chunk = timeseries[loind:hiind]
# Make a set with the current block numbers
lowblock = blocks_per_chunk * chunknum
currentblocks = set(Num.arange(blocks_per_chunk) + lowblock)
localgoodblocks = Num.asarray(list(currentblocks -
bad_blocks)) - lowblock
# Search this chunk if it is not all bad
if len(localgoodblocks):
# This is the good part of the data (end effects removed)
goodchunk = chunk[overlap:-overlap]
# need to pass blocks/chunklen, localgoodblocks
# dm_candlist, dt, opts.threshold to cython routine
# Search non-downsampled data first
# NOTE: these nonzero() calls are some of the most
# expensive calls in the program. Best bet would
# probably be to simply iterate over the goodchunk
# in C and append to the candlist there.
hibins = Num.flatnonzero(goodchunk>opts.threshold)
hivals = goodchunk[hibins]
hibins += chunknum * chunklen
hiblocks = hibins/detrendlen
# Add the candidates (which are sorted by bin)
for bin, val, block in zip(hibins, hivals, hiblocks):
if block not in bad_blocks:
time = bin * dt
dm_candlist.append(candidate(info.DM, val, time, bin, 1))
# Now do the downsampling...
for downfact in downfacts:
if useffts:
# Note: FFT convolution is faster for _all_ downfacts, even 2
chunk2 = Num.concatenate((Num.zeros(1000), chunk, Num.zeros(1000)))
goodchunk = Num.convolve(chunk2, Num.ones(downfact), mode='same') / Num.sqrt(downfact)
goodchunk = goodchunk[overlap:-overlap]
#O qualcosa di simile, altrimenti non so perche' trova piu' candidati! Controllare!
else:
# The normalization of this kernel keeps the post-smoothing RMS = 1
kernel = Num.ones(downfact, dtype=Num.float32) / \
Num.sqrt(downfact)
smoothed_chunk = scipy.signal.convolve(chunk, kernel, 1)
goodchunk = smoothed_chunk[overlap:-overlap]
#hibins = Num.nonzero(goodchunk>opts.threshold)[0]
hibins = Num.flatnonzero(goodchunk>opts.threshold)
hivals = goodchunk[hibins]
hibins += chunknum * chunklen
hiblocks = hibins/detrendlen
hibins = hibins.tolist()
hivals = hivals.tolist()
# Now walk through the new candidates and remove those
# that are not the highest but are within downfact/2
# bins of a higher signal pulse
hibins, hivals = prune_related1(hibins, hivals, downfact)
# Insert the new candidates into the candlist, but
# keep it sorted...
for bin, val, block in zip(hibins, hivals, hiblocks):
if block not in bad_blocks:
time = bin * dt
bisect.insort(dm_candlist,
candidate(info.DM, val, time, bin, downfact))
# Now walk through the dm_candlist and remove the ones that
# are within the downsample proximity of a higher
# signal-to-noise pulse
dm_candlist = prune_related2(dm_candlist, downfacts)
print " Found %d pulse candidates"%len(dm_candlist)
# Get rid of those near padding regions
if info.breaks: prune_border_cases(dm_candlist, offregions)
# Write the pulses to an ASCII output file
if len(dm_candlist):
#dm_candlist.sort(cmp_sigma)
outfile.write("# DM Sigma Time (s) Sample Downfact\n")
for cand in dm_candlist:
outfile.write(str(cand))
outfile.close()
# Add these candidates to the overall candidate list
for cand in dm_candlist:
candlist.append(cand)
num_v_DMstr[DMstr] = len(dm_candlist)
if (opts.makeplot):
# Step through the candidates to make a SNR list
DMs.sort()
snrs = []
for cand in candlist:
if not Num.isinf(cand.sigma):
snrs.append(cand.sigma)
if snrs:
maxsnr = max(int(max(snrs)), int(opts.threshold)) + 3
else:
maxsnr = int(opts.threshold) + 3
# Generate the SNR histogram
snrs = Num.asarray(snrs)
(num_v_snr, lo_snr, d_snr, num_out_of_range) = \
scipy.stats.histogram(snrs,
int(maxsnr-opts.threshold+1),
[opts.threshold, maxsnr])
snrs = Num.arange(maxsnr-opts.threshold+1, dtype=Num.float64) * d_snr \
+ lo_snr + 0.5*d_snr
num_v_snr = num_v_snr.astype(Num.float32)
num_v_snr[num_v_snr==0.0] = 0.001
# Generate the DM histogram
num_v_DM = Num.zeros(len(DMs))
for ii, DM in enumerate(DMs):
num_v_DM[ii] = num_v_DMstr["%.2f"%DM]
DMs = Num.asarray(DMs)
# open the plot device
short_filenmbase = filenmbase[:filenmbase.find("_DM")]
if opts.T_end > obstime:
opts.T_end = obstime
if pgplot_device:
ppgplot.pgopen(pgplot_device)
else:
if (opts.T_start > 0.0 or opts.T_end < obstime):
ppgplot.pgopen(short_filenmbase+'_%.0f-%.0fs_singlepulse.ps/VPS'%
(opts.T_start, opts.T_end))
else:
ppgplot.pgopen(short_filenmbase+'_singlepulse.ps/VPS')
ppgplot.pgpap(7.5, 1.0) # Width in inches, aspect
# plot the SNR histogram
ppgplot.pgsvp(0.06, 0.31, 0.6, 0.87)
ppgplot.pgswin(opts.threshold, maxsnr,
Num.log10(0.5), Num.log10(2*max(num_v_snr)))
ppgplot.pgsch(0.8)
ppgplot.pgbox("BCNST", 0, 0, "BCLNST", 0, 0)
ppgplot.pgmtxt('B', 2.5, 0.5, 0.5, "Signal-to-Noise")
ppgplot.pgmtxt('L', 1.8, 0.5, 0.5, "Number of Pulses")
ppgplot.pgsch(1.0)
ppgplot.pgbin(snrs, Num.log10(num_v_snr), 1)
# plot the DM histogram
ppgplot.pgsvp(0.39, 0.64, 0.6, 0.87)
# Add [1] to num_v_DM in YMAX below so that YMIN != YMAX when max(num_v_DM)==0
ppgplot.pgswin(min(DMs)-0.5, max(DMs)+0.5, 0.0, 1.1*max(num_v_DM+[1]))
ppgplot.pgsch(0.8)
ppgplot.pgbox("BCNST", 0, 0, "BCNST", 0, 0)
ppgplot.pgmtxt('B', 2.5, 0.5, 0.5, "DM (pc cm\u-3\d)")
ppgplot.pgmtxt('L', 1.8, 0.5, 0.5, "Number of Pulses")
ppgplot.pgsch(1.0)
ppgplot.pgbin(DMs, num_v_DM, 1)
# plot the SNR vs DM plot
ppgplot.pgsvp(0.72, 0.97, 0.6, 0.87)
ppgplot.pgswin(min(DMs)-0.5, max(DMs)+0.5, opts.threshold, maxsnr)
ppgplot.pgsch(0.8)
ppgplot.pgbox("BCNST", 0, 0, "BCNST", 0, 0)
ppgplot.pgmtxt('B', 2.5, 0.5, 0.5, "DM (pc cm\u-3\d)")
ppgplot.pgmtxt('L', 1.8, 0.5, 0.5, "Signal-to-Noise")
ppgplot.pgsch(1.0)
cand_ts = Num.zeros(len(candlist), dtype=Num.float32)
cand_SNRs = Num.zeros(len(candlist), dtype=Num.float32)
cand_DMs = Num.zeros(len(candlist), dtype=Num.float32)
for ii, cand in enumerate(candlist):
cand_ts[ii], cand_SNRs[ii], cand_DMs[ii] = \
cand.time, cand.sigma, cand.DM
ppgplot.pgpt(cand_DMs, cand_SNRs, 20)
# plot the DM vs Time plot
ppgplot.pgsvp(0.06, 0.97, 0.08, 0.52)
ppgplot.pgswin(opts.T_start, opts.T_end, min(DMs)-0.5, max(DMs)+0.5)
ppgplot.pgsch(0.8)
ppgplot.pgbox("BCNST", 0, 0, "BCNST", 0, 0)
ppgplot.pgmtxt('B', 2.5, 0.5, 0.5, "Time (s)")
ppgplot.pgmtxt('L', 1.8, 0.5, 0.5, "DM (pc cm\u-3\d)")
# Circles are symbols 20-26 in increasing order
snr_range = 12.0
cand_symbols = (cand_SNRs-opts.threshold)/snr_range * 6.0 + 20.5
cand_symbols = cand_symbols.astype(Num.int32)
cand_symbols[cand_symbols>26] = 26
for ii in [26, 25, 24, 23, 22, 21, 20]:
inds = Num.nonzero(cand_symbols==ii)[0]
ppgplot.pgpt(cand_ts[inds], cand_DMs[inds], ii)
# Now fill the infomation area
ppgplot.pgsvp(0.05, 0.95, 0.87, 0.97)
ppgplot.pgsch(1.0)
ppgplot.pgmtxt('T', 0.5, 0.0, 0.0,
"Single pulse results for '%s'"%short_filenmbase)
ppgplot.pgsch(0.8)
# first row
ppgplot.pgmtxt('T', -1.1, 0.02, 0.0, 'Source: %s'%\
info.object)
ppgplot.pgmtxt('T', -1.1, 0.33, 0.0, 'RA (J2000):')
ppgplot.pgmtxt('T', -1.1, 0.5, 0.0, info.RA)
ppgplot.pgmtxt('T', -1.1, 0.73, 0.0, 'N samples: %.0f'%orig_N)
# second row
ppgplot.pgmtxt('T', -2.4, 0.02, 0.0, 'Telescope: %s'%\
info.telescope)
ppgplot.pgmtxt('T', -2.4, 0.33, 0.0, 'DEC (J2000):')
ppgplot.pgmtxt('T', -2.4, 0.5, 0.0, info.DEC)
ppgplot.pgmtxt('T', -2.4, 0.73, 0.0, 'Sampling time: %.2f \gms'%\
(orig_dt*1e6))
# third row
if info.instrument.find("pigot") >= 0:
instrument = "Spigot"
else:
instrument = info.instrument
ppgplot.pgmtxt('T', -3.7, 0.02, 0.0, 'Instrument: %s'%instrument)
if (info.bary):
ppgplot.pgmtxt('T', -3.7, 0.33, 0.0, 'MJD\dbary\u: %.12f'%info.epoch)
else:
ppgplot.pgmtxt('T', -3.7, 0.33, 0.0, 'MJD\dtopo\u: %.12f'%info.epoch)
ppgplot.pgmtxt('T', -3.7, 0.73, 0.0, 'Freq\dctr\u: %.1f MHz'%\
((info.numchan/2-0.5)*info.chan_width+info.lofreq))
ppgplot.pgiden()
ppgplot.pgend()
def main():
parser = optparse.OptionParser(prog="rrattrap.py", \
version="Chen Karako, updated by Chitrang Patel(June 23, 2015)",\
usage="%prog --inffile <.inf file> [options] *.singlepulse",\
description="Group single pulse events and rank them based on the sigma behavior. \
Plot DM vs time with different colours for different ranks as follows:\
\t\tRank 1 (Other) : Grey\
\t\tRank 2 (RFI) : Red\
\t\tRank 3 (ok) : Cyan\
\t\tRank 4 (good) : dim blue\
\t\tRank 5 (very good) : dark blue\
\t\tRank 6 (excellent) : Magenta" )
parser.add_option('--CLOSE-DM', dest='close_dm', type='float', \
help="DM to below which the signalis considered RFI(Default: 2", \
default=2.0)
parser.add_option('--use-configfile', dest='use_configfile', action='store_true', \
help="If this flag is set - import the config file for selecting grouping" \
"parameters.(Default: do not use a config file.)", default=False)
parser.add_option('--use-DMplan', dest='use_DMplan', action='store_true', \
help="If this flag is set - Use the ddplan for selecting grouping" \
"parameters. Make sure that you have a corresponding config file containing" \
"the DDplan. (Default: do not use ddplan)", default=False)
parser.add_option('--min-group', dest='min_group', type='int', \
help="minimum number of events in a group to no be considered noise." \
"(Default: 45)", \
default=45)
parser.add_option('--dm-thresh', dest='dm_thresh', type='float', \
help="DM threshold to use for nearest neighbour. Suggest a value greater " \
" than the DM step size(Default: 0.5 pc/cm^3 - will not work if DM " \
"step size is greater than 0.5)", default=0.5)
parser.add_option('--time-thresh', dest='time_thresh', type='float', \
help="Time threshold to use for nearest neighbour. Suggest a value that " \
" is a few times the max pulse width(Default: 0.1 s)", default=0.1)
parser.add_option('--min-sigma', dest='min_sigma', type='float', \
help="minimum signal-to-noise above which the group is highly considered" \
"to be astrophysical. (Default: 8.0)", \
default=8.0)
parser.add_option('--no-plot', dest='plot', action='store_false', \
help="Do not plot the groups in the DM time plot." \
"(Default: Make a plot)", default=True)
parser.add_option('--plottype', dest='plottype', type = 'string',\
help="Make a plot using : 'matplotlib' or 'pgplot'."\
, default='pgplot')
parser.add_option('--min-rank-to-plot', dest='min_ranktoplot', type = 'int',\
help="Only groups with rank upto this will plotted.(default: plot \
all except rank 1)" , default=0)
parser.add_option('--min-rank-to-write', dest='min_ranktowrite', type = 'int',\
help="Only info of the groups with rank upto this will written." \
"(default: write all except rank 1)", default=0)
parser.add_option('--inffile', dest='inffile', type = 'string',\
help="A .inf file. I suggest a .rfifind.inf file."\
, default=None)
parser.add_option('-o', dest='outbasenm', type = 'string',\
help="outfile base name. .groups.txt will be added to the given name."\
, default='')
options, args = parser.parse_args()
if not hasattr(options, 'inffile'):
raise ValueError("You must supply a .inf file. I suggest .rfifind.inf")
if not options.inffile.endswith(".inf"):
raise ValueError("Cannot recognize file type from extension. "
" Only '.inf' types are supported.")
if options.use_DMplan or options.use_configfile:
import singlepulse.rrattrap_config as rrattrap_config
RANKS = np.asarray([2, 0, 3, 4, 5, 6])
if options.use_configfile:
CLOSE_DM = rrattrap_config.CLOSE_DM
MIN_GROUP = rrattrap_config.MIN_GROUP
TIME_THRESH = rrattrap_config.TIME_THRESH
DM_THRESH = rrattrap_config.DM_THRESH
MIN_SIGMA = rrattrap_config.MIN_SIGMA
PLOT = rrattrap_config.PLOT
PLOTTYPE = rrattrap_config.PLOTTYPE
RANKS_TO_WRITE = rrattrap_config.RANKS_TO_WRITE
RANKS_TO_PLOT = rrattrap_config.RANKS_TO_PLOT
else:
CLOSE_DM = options.close_dm
MIN_GROUP = options.min_group
TIME_THRESH = options.time_thresh
DM_THRESH = options.dm_thresh
MIN_SIGMA = options.min_sigma
PLOT = options.plot
PLOTTYPE = options.plottype
RANKS_TO_WRITE = list(RANKS[RANKS > options.min_ranktowrite])
RANKS_TO_PLOT = list(RANKS[RANKS > options.min_ranktoplot])
inffile = options.inffile
inf = infodata.infodata(inffile)
print_debug("Beginning read_sp_files... " + strftime("%Y-%m-%d %H:%M:%S"))
groups = spio.read_sp_files(args[1:])[0]
print_debug("Finished read_sp_files, beginning create_groups... " +
strftime("%Y-%m-%d %H:%M:%S"))
print_debug("Number of single pulse events: %d " % len(groups))
groups = create_groups(
groups,
inffile,
min_nearby=1,
ignore_obs_end=10,
time_thresh=TIME_THRESH,
dm_thresh=DM_THRESH,
use_dmplan=options.use_DMplan
) # ignore the last 10 seconds of the obs, for palfa
print_debug("Number of groups: %d " % len(groups))
print_debug("Finished create_groups, beginning grouping_sp_dmt... " +
strftime("%Y-%m-%d %H:%M:%S"))
grouping_sp_dmt(groups,
use_dmplan=options.use_DMplan,
time_thresh=TIME_THRESH,
dm_thresh=DM_THRESH)
print_debug("Number of groups (after initial grouping): %d " % len(groups))
print_debug("Finished grouping_sp_dmt, beginning flag_noise... " +
strftime("%Y-%m-%d %H:%M:%S"))
flag_noise(
groups, use_dmplan=options.use_DMplan,
min_group=MIN_GROUP) # do an initial coarse noise flagging and removal
pop_by_rank(groups, 1)
print_debug("Number of groups (after removed noise gps w <10 sps): %d " %
len(groups))
print_debug("Beginning grouping_sp_t... " + strftime("%Y-%m-%d %H:%M:%S"))
# Regroup good groups based on proximity in time only (compensate for missing middles):
groups = grouping_sp_t(groups,
use_dmplan=options.use_DMplan,
time_thresh=TIME_THRESH,
dm_thresh=DM_THRESH)
print_debug("Finished grouping_sp_t. " + strftime("%Y-%m-%d %H:%M:%S"))
# Flag RFI groups, noise
flag_rfi(groups, close_dm=CLOSE_DM)
# Rank groups and identify noise (<45/40/35/30 sp events) groups
print_debug("Ranking groups...")
rank_groups(groups,
use_dmplan=options.use_DMplan,
min_group=MIN_GROUP,
min_sigma=MIN_SIGMA)
# Remove noise groups
print_debug("Before removing noise, len(groups): %s" % len(groups))
pop_by_rank(groups, 1)
print_debug("After removing noise, len(groups): %s" % len(groups))
# Group rfi with very close groups
print_debug("len(groups) before grouping_rfi: %s" % len(groups))
print_debug("Beginning grouping_rfi... " + strftime("%Y-%m-%d %H:%M:%S"))
grouping_rfi(groups,
use_dmplan=options.use_DMplan,
time_thresh=TIME_THRESH,
dm_thresh=DM_THRESH)
print_debug("Finished grouping_rfi. " + strftime("%Y-%m-%d %H:%M:%S"))
# Rank groups
print_debug("Finished rank_groups, beginning DM span check... " +
strftime("%Y-%m-%d %H:%M:%S"))
# Remove groups that are likely RFI, based on their large span in DM
print_debug("Beginning DM span check...")
check_dmspan(groups, inf.dt, inf.lofreq, inf.lofreq + inf.BW)
print_debug("Finished DM span check, beginning writing to outfile... " +
strftime("%Y-%m-%d %H:%M:%S"))
outfile = open(options.outbasenm + 'groups.txt', 'w')
summaryfile = open(options.outbasenm + 'spsummary.txt', 'w')
rank_dict = rank_occur(groups)
for rank in sorted(ALL_RANKS_ORDERED):
if rank != 1:
outfile.write("Number of rank %d groups: %d \n" %
(rank, rank_dict.get(rank, 0)))
summaryfile.write("Number of rank %d groups: %d \n" %
(rank, rank_dict.get(rank, 0)))
outfile.write("\n")
summaryfile.close()
# Reverse sort lists so good groups are written at the top of the file
groups.sort(reverse=True)
# write list of events in each group
for grp in groups:
if grp.rank in RANKS_TO_WRITE:
outfile.write(str(grp) + '\n') #print group summary
outfile.write('\n')
outfile.write(
"# DM Sigma Time (s) Sample Downfact \n")
for sp in grp.singlepulses:
outfile.write("%7.2f %7.2f %13.6f %10d %3d \n" % sp)
outfile.write('\n')
outfile.close()
print_debug("Finished writing to outfile, now plotting... " +
strftime("%Y-%m-%d %H:%M:%S"))
if PLOT:
ranks = RANKS_TO_PLOT
# Sort groups so better-ranked groups are plotted on top of worse groups
groups.sort()
# create several DM vs t plots, splitting up DM in overlapping intervals
# DMs 0-30, 20-110, 100-300, 300-1000
if PLOTTYPE.lower() == 'pgplot':
# Use PGPLOT to plot
plot_sp_rated_pgplot(groups, ranks, inffile, 0, 30)
print_debug("Finished PGplotting DMs0-30 " +
strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 20, 110)
print_debug("Finished PGplotting DMs20-110 " +
strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 100, 310)
print_debug("Finished PGplotting DMs100-310 " +
strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 300, 1000)
print_debug("Finished PGplotting DMs100-310 " +
strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 1000, 10000)
print_debug("Finished PGplotting DMs100-310 " +
strftime("%Y-%m-%d %H:%M:%S"))
elif PLOTTYPE.lower() == 'matplotlib':
# Use matplotlib to plot
plot_sp_rated_all(groups, ranks, inffile, 0, 30)
print_debug("Finished plotting DMs0-30 " +
strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 20, 110)
print_debug("Finished plotting DMs20-110 " +
strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 100, 310)
print_debug("Finished plotting DMs100-310 " +
strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 300, 1000)
print_debug("Finished plotting DMs300-1000 " +
strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 1000, 10000)
print_debug("Finished plotting DMs1000-10000 " +
strftime("%Y-%m-%d %H:%M:%S"))
else:
print "Plot type must be one of 'matplotlib' or 'pgplot'. Not plotting."
def sift_directory(dir, outbasenm):
"""Sift candidates in given directory.
Inputs:
dir: The directory containing candidates to sift.
outbasenm: The base name of the output files.
Outputs:
None
"""
import os.path
import glob
import infodata
# TODO: Remove hard-coded values in this function
# replace with command line options.
global sigma_threshold
sigma_threshold = 5.0
print_sift_globals()
# Get list of DMs from *.inf files
inffns = glob.glob(os.path.join(dir, '*.inf'))
dmstrs = ['%.2f'%infodata.infodata(inffn).DM for inffn in inffns]
# Read candidates found in low acceleration searching
lo_accel_fns = glob.glob(os.path.join(dir, '*ACCEL_0'))
lo_accel_cands = read_candidates(lo_accel_fns)
print "Read %d candidates from %d files" % \
(len(lo_accel_cands), len(lo_accel_fns))
print "%d candidates passed default rejection" % len(lo_accel_cands)
if len(lo_accel_cands):
lo_accel_cands.remove_duplicate_candidates()
if len(lo_accel_cands):
lo_accel_cands.remove_DM_problems(2, dmstrs, 2.0)
lo_accel_cands.print_cand_summary()
# Read candidates found in high acceleration searching
hi_accel_fns = glob.glob(os.path.join(dir, '*ACCEL_50'))
hi_accel_cands = read_candidates(hi_accel_fns)
print "Read %d candidates from %d files" % \
(len(hi_accel_cands), len(hi_accel_fns))
print "%d candidates passed default rejection" % len(hi_accel_cands)
if len(hi_accel_cands):
hi_accel_cands.remove_duplicate_candidates()
if len(hi_accel_cands):
hi_accel_cands.remove_DM_problems(2, dmstrs, 2.0)
hi_accel_cands.print_cand_summary()
all_accel_cands = lo_accel_cands + hi_accel_cands
if len(all_accel_cands):
all_accel_cands.remove_harmonics()
# Note: the candidates will be sorted in _sigma_ order, not _SNR_!
all_accel_cands.cands.sort(cmp_sigma)
print "Found %d good candidates" % len(all_accel_cands)
all_accel_cands.to_file(outbasenm+".accelcands")
all_accel_cands.write_cand_report(outbasenm+".accelcands.report")
all_accel_cands.print_cand_summary()
all_accel_cands.plot_goodbad()
plt.savefig(outbasenm+".accelcands.rejects.png")
all_accel_cands.plot_summary()
plt.savefig(outbasenm+".accelcands.summary.png")
def main():
usage = "usage: %prog [options]"
parser = OptionParser(usage)
parser.add_option("-n",
"--number",
type="int",
dest="nM",
default=40,
help="Number of points in each chunk (millions)")
parser.add_option("-o",
"--outdir",
type="string",
dest="outdir",
default=".",
help="Output directory to store results")
parser.add_option("-d",
"--workdir",
type="string",
dest="workdir",
default=".",
help="Working directory for search")
parser.add_option("-l",
"--flo",
type="float",
dest="flo",
default=10.0,
help="Low frequency (Hz) to search")
parser.add_option("-f",
"--frac",
type="float",
dest="frac",
default=0.5,
help="Fraction to overlap")
parser.add_option("-x",
"--fhi",
type="float",
dest="fhi",
default=10000.0,
help="High frequency (Hz) to search")
parser.add_option("-z",
"--zmax",
type="int",
dest="zmax",
default=160,
help="Maximum fourier drift (bins) to search")
parser.add_option("-w",
"--wmax",
type="int",
dest="wmax",
default=400,
help="Maximum fourier drift deriv (bins) to search")
parser.add_option("-a",
"--numharm",
type="int",
dest="numharm",
default=4,
help="Number of harmonics to sum when searching")
parser.add_option("-s",
"--sigma",
type="float",
dest="sigma",
default=2.0,
help="Cutoff sigma to consider a candidate")
(options, args) = parser.parse_args()
if (options.outdir[-1] != "/"):
options.outdir = options.outdir + "/"
if (options.workdir != '.'):
chdir(options.workdir)
if (options.nM >= 1000000):
if (options.nM % 1000000):
print "If you specify --num nM to be > 1000000, it must be divisible by 1000000."
exit(1)
else:
options.nM *= 1000000
short_nM = options.nM / 1000000
# The basename of the data files
if argv[1].endswith(".dat"):
basename = "../" + argv[1][:-4]
else:
basename = "../" + argv[1]
# Get the bird file (the first birdie file in the directory!)
birdname = glob("../*.birds")
if birdname:
birdname = birdname[0]
outnamebase = options.outdir + basename[3:]
inf = read_inffile(basename)
idata = infodata.infodata(basename + ".inf")
N = inf.N
t0i = inf.mjd_i
t0f = inf.mjd_f
num = 0
point = 0
T = options.nM * inf.dt / 86400.0
baryv = get_baryv(idata.RA, idata.DEC, idata.epoch, T, obs='GB')
print "Baryv = ", baryv
inf.N = options.nM
inf.numonoff = 0
nM = options.nM / 1000000
while (point + options.nM < N):
pM = point / 1000000
outname = basename[3:] + '_%03dM' % nM + '_%02d' % num
stdout.write('\n' + outname + '\n\n')
inf.name = outname
tstartf = inf.mjd_f + num * T * options.frac
if (tstartf > 1.0):
tstartf = tstartf - 1.0
inf.mjd_i = inf.mjd_i + 1
inf.mjd_f = tstartf
writeinf(inf)
myexecute('dd if=' + basename + '.dat of=' + outname +
'.dat bs=4000000 skip=' + ` pM ` + ' count=' + ` nM `)
myexecute('realfft ' + outname + '.dat')
myexecute('rm -f ' + outname + '.dat')
myexecute('cp ' + birdname + ' ' + outname + '.birds')
myexecute('makezaplist.py ' + outname + '.birds')
myexecute('rm -f ' + outname + '.birds')
myexecute('zapbirds -zap -zapfile ' + outname +
'.zaplist -baryv %g ' % baryv + outname + '.fft')
myexecute('rm -f ' + outname + '.zaplist')
myexecute(
'accelsearch -sigma %.2f -zmax %d -wmax %d -numharm %d -flo %f -fhi %f '
% (options.sigma, options.zmax, options.wmax, options.numharm,
options.flo, options.fhi) + outname + '.fft')
myexecute('rm ' + outname + '.fft ' + outname +
'_ACCEL_%d.txtcand' % options.zmax)
myexecute('cp ' + outname + '.inf ' + options.outdir)
num = num + 1
point = point + int(options.nM * options.frac)
#!/usr/bin/env python
from __future__ import (print_function, division)
import psr_utils as pu
import sys
from infodata import infodata
if len(sys.argv) != 2:
print("chooseN <file.inf|numpoints>")
print(
" Prints a good value for fast FFTs to be used for -numout in prepdata/prepsubband"
)
sys.exit(1)
if (sys.argv[1].endswith('.inf')):
inf = infodata(sys.argv[1])
n = inf.N
else:
try:
n = int(sys.argv[1])
except:
print("chooseN <file.inf|numpoints>")
print(
" Prints a good value for fast FFTs to be used for -numout in prepdata/prepsubband"
)
sys.exit(2)
print(pu.choose_N(n))
def main():
parser = optparse.OptionParser(prog="rrattrap.py", \
version="Chen Karako, updated by Chitrang Patel(June 23, 2015)",\
usage="%prog --inffile <.inf file> [options] *.singlepulse",\
description="Group single pulse events and rank them based on the sigma behavior. \
Plot DM vs time with different colours for different ranks as follows:\
\t\tRank 1 (Other) : Grey\
\t\tRank 2 (RFI) : Red\
\t\tRank 3 (ok) : Cyan\
\t\tRank 4 (good) : dim blue\
\t\tRank 5 (very good) : dark blue\
\t\tRank 6 (excellent) : Magenta")
parser.add_option('--CLOSE-DM', dest='close_dm', type='float', \
help="DM to below which the signalis considered RFI(Default: 2", \
default=2.0)
parser.add_option('--use-configfile', dest='use_configfile', action='store_true', \
help="If this flag is set - import the config file for selecting grouping" \
"parameters.(Default: do not use a config file.)", default=False)
parser.add_option('--use-DMplan', dest='use_DMplan', action='store_true', \
help="If this flag is set - Use the ddplan for selecting grouping" \
"parameters. Make sure that you have a corresponding config file containing" \
"the DDplan. (Default: do not use ddplan)", default=False)
parser.add_option('--min-group', dest='min_group', type='int', \
help="minimum number of events in a group to no be considered noise." \
"(Default: 45)", \
default=45)
parser.add_option('--dm-thresh', dest='dm_thresh', type='float', \
help="DM threshold to use for nearest neighbour. Suggest a value greater " \
" than the DM step size(Default: 0.5 pc/cm^3 - will not work if DM " \
"step size is greater than 0.5)", default=0.5)
parser.add_option('--time-thresh', dest='time_thresh', type='float', \
help="Time threshold to use for nearest neighbour. Suggest a value that " \
" is a few times the max pulse width(Default: 0.1 s)", default=0.1)
parser.add_option('--min-sigma', dest='min_sigma', type='float', \
help="minimum signal-to-noise above which the group is highly considered" \
"to be astrophysical. (Default: 8.0)", \
default=8.0)
parser.add_option('--no-plot', dest='plot', action='store_false', \
help="Do not plot the groups in the DM time plot." \
"(Default: Make a plot)", default=True)
parser.add_option('--plottype', dest='plottype', type = 'string',\
help="Make a plot using : 'matplotlib' or 'pgplot'."\
, default='pgplot')
parser.add_option('--min-rank-to-plot', dest='min_ranktoplot', type = 'int',\
help="Only groups with rank upto this will plotted.(default: plot \
all except rank 1)", default=0)
parser.add_option('--min-rank-to-write', dest='min_ranktowrite', type = 'int',\
help="Only info of the groups with rank upto this will written." \
"(default: write all except rank 1)", default=0)
parser.add_option('--inffile', dest='inffile', type = 'string',\
help="A .inf file. I suggest a .rfifind.inf file."\
, default=None)
parser.add_option('-o', dest='outbasenm', type = 'string',\
help="outfile base name. .groups.txt will be added to the given name."\
, default='')
options, args = parser.parse_args()
if not hasattr(options, 'inffile'):
raise ValueError("You must supply a .inf file. I suggest .rfifind.inf")
if not options.inffile.endswith(".inf"):
raise ValueError("Cannot recognize file type from extension. "
" Only '.inf' types are supported.")
if options.use_DMplan or options.use_configfile:
import singlepulse.rrattrap_config as rrattrap_config
RANKS = np.asarray([2,0,3,4,5,6])
if options.use_configfile:
CLOSE_DM = rrattrap_config.CLOSE_DM
MIN_GROUP = rrattrap_config.MIN_GROUP
TIME_THRESH = rrattrap_config.TIME_THRESH
DM_THRESH = rrattrap_config.DM_THRESH
MIN_SIGMA = rrattrap_config.MIN_SIGMA
PLOT = rrattrap_config.PLOT
PLOTTYPE = rrattrap_config.PLOTTYPE
RANKS_TO_WRITE = rrattrap_config.RANKS_TO_WRITE
RANKS_TO_PLOT = rrattrap_config.RANKS_TO_PLOT
else:
CLOSE_DM = options.close_dm
MIN_GROUP = options.min_group
TIME_THRESH = options.time_thresh
DM_THRESH = options.dm_thresh
MIN_SIGMA = options.min_sigma
PLOT = options.plot
PLOTTYPE = options.plottype
RANKS_TO_WRITE = list(RANKS[RANKS>options.min_ranktowrite])
RANKS_TO_PLOT = list(RANKS[RANKS>options.min_ranktoplot])
inffile = options.inffile
inf = infodata.infodata(inffile)
print_debug("Beginning read_sp_files... "+strftime("%Y-%m-%d %H:%M:%S"))
groups = spio.read_sp_files(args[1:])[0]
print_debug("Finished read_sp_files, beginning create_groups... " +
strftime("%Y-%m-%d %H:%M:%S"))
print_debug("Number of single pulse events: %d " % len(groups))
groups = create_groups(groups, inffile, min_nearby=1, ignore_obs_end=10, time_thresh=TIME_THRESH, dm_thresh=DM_THRESH, use_dmplan=options.use_DMplan) # ignore the last 10 seconds of the obs, for palfa
print_debug("Number of groups: %d " % len(groups))
print_debug("Finished create_groups, beginning grouping_sp_dmt... " +
strftime("%Y-%m-%d %H:%M:%S"))
grouping_sp_dmt(groups, use_dmplan=options.use_DMplan, time_thresh=TIME_THRESH, dm_thresh=DM_THRESH)
print_debug("Number of groups (after initial grouping): %d " % len(groups))
print_debug("Finished grouping_sp_dmt, beginning flag_noise... " +
strftime("%Y-%m-%d %H:%M:%S"))
flag_noise(groups, use_dmplan=options.use_DMplan, min_group=MIN_GROUP) # do an initial coarse noise flagging and removal
pop_by_rank(groups, 1)
print_debug("Number of groups (after removed noise gps w <10 sps): %d " % len(groups))
print_debug("Beginning grouping_sp_t... " +
strftime("%Y-%m-%d %H:%M:%S"))
# Regroup good groups based on proximity in time only (compensate for missing middles):
groups = grouping_sp_t(groups, use_dmplan=options.use_DMplan, time_thresh=TIME_THRESH, dm_thresh=DM_THRESH)
print_debug("Finished grouping_sp_t. " + strftime("%Y-%m-%d %H:%M:%S"))
# Flag RFI groups, noise
flag_rfi(groups, close_dm=CLOSE_DM)
# Rank groups and identify noise (<45/40/35/30 sp events) groups
print_debug("Ranking groups...")
rank_groups(groups, use_dmplan=options.use_DMplan, min_group=MIN_GROUP, min_sigma=MIN_SIGMA)
# Remove noise groups
print_debug("Before removing noise, len(groups): %s" % len(groups))
pop_by_rank(groups, 1)
print_debug("After removing noise, len(groups): %s" % len(groups))
# Group rfi with very close groups
print_debug("len(groups) before grouping_rfi: %s" % len(groups))
print_debug("Beginning grouping_rfi... " + strftime("%Y-%m-%d %H:%M:%S"))
grouping_rfi(groups, use_dmplan=options.use_DMplan, time_thresh=TIME_THRESH, dm_thresh=DM_THRESH)
print_debug("Finished grouping_rfi. " +
strftime("%Y-%m-%d %H:%M:%S"))
# Rank groups
print_debug("Finished rank_groups, beginning DM span check... " +
strftime("%Y-%m-%d %H:%M:%S"))
# Remove groups that are likely RFI, based on their large span in DM
print_debug("Beginning DM span check...")
check_dmspan(groups, inf.dt, inf.lofreq, inf.lofreq+inf.BW)
print_debug("Finished DM span check, beginning writing to outfile... " +
strftime("%Y-%m-%d %H:%M:%S"))
outfile = open(options.outbasenm+'groups.txt', 'w')
summaryfile = open(options.outbasenm+'spsummary.txt', 'w')
rank_dict = rank_occur(groups)
for rank in sorted(ALL_RANKS_ORDERED):
if rank != 1:
outfile.write("Number of rank %d groups: %d \n" %
(rank, rank_dict.get(rank, 0)))
summaryfile.write("Number of rank %d groups: %d \n" %
(rank, rank_dict.get(rank, 0)))
outfile.write("\n")
summaryfile.close()
# Reverse sort lists so good groups are written at the top of the file
groups.sort(reverse=True)
# write list of events in each group
for grp in groups:
if grp.rank in RANKS_TO_WRITE:
outfile.write(str(grp) + '\n') #print group summary
outfile.write('\n')
outfile.write("# DM Sigma Time (s) Sample Downfact \n")
for sp in grp.singlepulses:
outfile.write("%7.2f %7.2f %13.6f %10d %3d \n" % sp)
outfile.write('\n')
outfile.close()
print_debug("Finished writing to outfile, now plotting... " +
strftime("%Y-%m-%d %H:%M:%S"))
if PLOT:
ranks = RANKS_TO_PLOT
# Sort groups so better-ranked groups are plotted on top of worse groups
groups.sort()
# create several DM vs t plots, splitting up DM in overlapping intervals
# DMs 0-30, 20-110, 100-300, 300-1000
if PLOTTYPE.lower() == 'pgplot':
# Use PGPLOT to plot
plot_sp_rated_pgplot(groups, ranks, inffile, 0, 30)
print_debug("Finished PGplotting DMs0-30 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 20, 110)
print_debug("Finished PGplotting DMs20-110 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 100, 310)
print_debug("Finished PGplotting DMs100-310 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 300, 1000)
print_debug("Finished PGplotting DMs100-310 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 1000, 10000)
print_debug("Finished PGplotting DMs100-310 "+strftime("%Y-%m-%d %H:%M:%S"))
elif PLOTTYPE.lower() == 'matplotlib':
# Use matplotlib to plot
plot_sp_rated_all(groups, ranks, inffile, 0, 30)
print_debug("Finished plotting DMs0-30 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 20, 110)
print_debug("Finished plotting DMs20-110 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 100, 310)
print_debug("Finished plotting DMs100-310 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 300, 1000)
print_debug("Finished plotting DMs300-1000 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 1000, 10000)
print_debug("Finished plotting DMs1000-10000 "+strftime("%Y-%m-%d %H:%M:%S"))
else:
print "Plot type must be one of 'matplotlib' or 'pgplot'. Not plotting."
def main():
parser = optparse.OptionParser(prog="sp_pipeline..py", \
version=" Chitrang Patel (May. 12, 2015)", \
usage="%prog INFILE(PsrFits FILE, SINGLEPULSE FILES)", \
description="Create single pulse plots to show the " \
"frequency sweeps of a single pulse, " \
"DM vs time, and SNR vs DM,"\
"in psrFits data.")
parser.add_option('--infile', dest='infile', type='string', \
help="Give a .inf file to read the appropriate header information.")
parser.add_option('--groupsfile', dest='txtfile', type='string', \
help="Give the groups.txt file to read in the groups information.")
parser.add_option('--mask', dest='maskfile', type='string', \
help="Mask file produced by rfifind. (Default: No Mask).", \
default=None)
parser.add_option('-n', dest='maxnumcands', type='int', \
help="Maximum number of candidates to plot. (Default: 100).", \
default=100)
options, args = parser.parse_args()
if not hasattr(options, 'infile'):
raise ValueError("A .inf file must be given on the command line! ")
if not hasattr(options, 'txtfile'):
raise ValueError(
"The groups.txt file must be given on the command line! ")
files = get_textfile(options.txtfile)
print_debug("Begining waterfaller... " + strftime("%Y-%m-%d %H:%M:%S"))
if not args[0].endswith("fits"):
raise ValueError("The first file must be a psrFits file! ")
print_debug('Maximum number of candidates to plot: %i' %
options.maxnumcands)
basename = args[0][:-5]
filetype = "psrfits"
inffile = options.infile
topo, bary = bary_and_topo.bary_to_topo(inffile)
time_shift = bary - topo
inf = infodata.infodata(inffile)
RA = inf.RA
dec = inf.DEC
MJD = inf.epoch
mjd = Popen(["mjd2cal", "%f" % MJD], stdout=PIPE, stderr=PIPE)
date, err = mjd.communicate()
date = date.split()[2:5]
telescope = inf.telescope
N = inf.N
numcands = 0
Total_observed_time = inf.dt * N
print_debug('getting file..')
values = split_parameters(options.txtfile)
if len(values) > options.maxnumcands:
values = sorted(values, key=itemgetter(
5, 1)) #sorting candidates based on ranks and snr
values = values[-options.maxnumcands:]
print "More than", options.maxnumcands, "candidates, making plots for", options.maxnumcands, "candidates"
values = sorted(values, key=itemgetter(0))
for ii in range(len(values)):
#### Array for Plotting DM vs SNR
print_debug("Making arrays for DM vs Signal to Noise...")
temp_list = files[values[ii][6] - 6].split()
npulses = int(temp_list[2])
temp_lines = files[(values[ii][6] + 3):(values[ii][6] + npulses + 1)]
arr = np.split(temp_lines, len(temp_lines))
dm_list = []
time_list = []
for i in range(len(arr)):
dm_val = float(arr[i][0].split()[0])
time_val = float(arr[i][0].split()[2])
dm_list.append(dm_val)
time_list.append(time_val)
arr_2 = np.array([arr[i][0].split() for i in range(len(arr))],
dtype=np.float32)
dm_arr = np.array([arr_2[i][0] for i in range(len(arr))],
dtype=np.float32)
sigma_arr = np.array([arr_2[i][1] for i in range(len(arr))],
dtype=np.float32)
#### Array for Plotting DM vs Time is in show_spplots.plot(...)
#### Setting variables up for the waterfall arrays.
j = ii + 1
subdm = dm = sweep_dm = values[ii][0]
sample_number = values[ii][3]
rank = values[ii][5]
width_bins = values[ii][4]
#print "dm", dm
#print "width_bins", width_bins
downsamp = np.round(
(values[ii][2] / sample_number / inf.dt)).astype('int')
#print "downsamp", downsamp
pulse_width = width_bins * downsamp * inf.dt
#print "pulse_width", pulse_width
if ii == 0:
mask_subband = rfifind.rfifind("%s_rfifind.mask" % (basename))
mask_subband.set_zap_chans(power=1000, plot=False)
mask_subband.set_weights_and_offsets()
mask_subband.write_weights(filename="%s_weights.txt" % (basename))
cmd = "psrfits_subband -dm %.2f -nsub 128 -o %s_subband_%.2f -weights %s_weights.txt %s" % (
dm, basename, dm, basename, args[0])
call(cmd, shell=True)
#subband args[0] at dm and then generate a file that will be set equal to rawdatafile
subband_file = "%s_subband_%.2f_0001.fits" % (basename, dm)
dm_prev = dm
subband_prev = subband_file
else:
dm_diff = dm - dm_prev
t_smear = 8.3e3 * dm_diff * (350**-3) * (
np.abs(rawdatafile.frequencies[0] -
rawdatafile.frequencies[-1]) / 128.)
if (5 * t_smear) > pulse_width:
cmd = "psrfits_subband -dm %.2f -nsub 128 -o %s_subband_%.2f -weights %s_weights.txt %s" % (
dm, basename, dm, basename, args[0])
call(cmd, shell=True)
#subband args[0] at dm and then generate a file that will be set equal to rawdatafile
subband_file = "%s_subband_%.2f_0001.fits" % (basename, dm)
dm_prev = dm
subband_prev = subband_file
rawdatafile = psrfits.PsrfitsFile(subband_file)
bin_shift = np.round(time_shift / rawdatafile.tsamp).astype('int')
integrate_dm = None
sigma = values[ii][1]
sweep_posn = 0.0
bary_start_time = values[ii][2]
topo_start_time = bary_start_time - topo_timeshift(
bary_start_time, time_shift, topo)[0]
binratio = 50
scaleindep = False
zerodm = None
duration = binratio * width_bins * rawdatafile.tsamp * downsamp
start = topo_start_time - (0.25 * duration)
if (start < 0.0):
start = 0.0
if sigma <= 7:
nsub = 32
elif sigma >= 7 and sigma < 10:
nsub = 64
else:
nsub = 128
nbins = np.round(duration / rawdatafile.tsamp).astype('int')
start_bin = np.round(start / rawdatafile.tsamp).astype('int')
dmfac = 4.15e3 * np.abs(1. / rawdatafile.frequencies[0]**2 -
1. / rawdatafile.frequencies[-1]**2)
nbinsextra = np.round(
(duration + dmfac * dm) / rawdatafile.tsamp).astype('int')
if (start_bin + nbinsextra) > N - 1:
nbinsextra = N - 1 - start_bin
data = rawdatafile.get_spectra(start_bin, nbinsextra)
data = maskdata(data, start_bin, nbinsextra, options.maskfile)
#make an array to store header information for the .npz files
temp_filename = basename + "_DM%.1f_%.1fs_rank_%i" % (
subdm, topo_start_time, rank)
# Array for Plotting Dedispersed waterfall plot - zerodm - OFF
print_debug("Running waterfaller with Zero-DM OFF...")
data, Data_dedisp_nozerodm = waterfall_array(
start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm,
integrate_dm, downsamp, scaleindep, width_bins, rawdatafile,
binratio, data)
#Add additional information to the header information array
text_array = np.array([
subband_file, 'GBT', RA, dec, MJD, rank, nsub, nbins, subdm, sigma,
sample_number, duration, width_bins, pulse_width,
rawdatafile.tsamp, Total_observed_time, topo_start_time,
data.starttime, data.dt, data.numspectra,
data.freqs.min(),
data.freqs.max()
])
#### Array for plotting Dedispersed waterfall plot zerodm - ON
print_debug("Running Waterfaller with Zero-DM ON...")
#print "before get_spectra",memory.resident()/(1024.0**3)
data = rawdatafile.get_spectra(start_bin, nbinsextra)
#print "after get_spectra",memory.resident()/(1024.0**3)
data = maskdata(data, start_bin, nbinsextra, options.maskfile)
zerodm = True
data, Data_dedisp_zerodm = waterfall_array(start_bin, dmfac, duration,
nbins, zerodm, nsub, subdm,
dm, integrate_dm, downsamp,
scaleindep, width_bins,
rawdatafile, binratio, data)
#print "waterfall",memory.resident()/(1024.0**3)
####Sweeped without zerodm
start = start + (0.25 * duration)
start_bin = np.round(start / rawdatafile.tsamp).astype('int')
sweep_duration = 4.15e3 * np.abs(
1. / rawdatafile.frequencies[0]**2 -
1. / rawdatafile.frequencies[-1]**2) * sweep_dm
nbins = np.round(sweep_duration / (rawdatafile.tsamp)).astype('int')
if ((nbins + start_bin) > (N - 1)):
nbins = N - 1 - start_bin
#print "before get_spectra",memory.resident()/(1024.0**3)
data = rawdatafile.get_spectra(start_bin, nbins)
#print "after get_spectra",memory.resident()/(1024.0**3)
data = maskdata(data, start_bin, nbins, options.maskfile)
zerodm = None
dm = None
data, Data_nozerodm = waterfall_array(start_bin, dmfac, duration,
nbins, zerodm, nsub, subdm, dm,
integrate_dm, downsamp,
scaleindep, width_bins,
rawdatafile, binratio, data)
#print "waterfall",memory.resident()/(1024.0**3)
text_array = np.append(text_array, sweep_duration)
text_array = np.append(text_array, data.starttime)
text_array = np.append(text_array, bary_start_time)
# Array to Construct the sweep
if sweep_dm is not None:
ddm = sweep_dm - data.dm
delays = psr_utils.delay_from_DM(ddm, data.freqs)
delays -= delays.min()
delays_nozerodm = delays
freqs_nozerodm = data.freqs
# Sweeped with zerodm-on
zerodm = True
downsamp_temp = 1
data, Data_zerodm = waterfall_array(start_bin, dmfac, duration, nbins,
zerodm, nsub, subdm, dm,
integrate_dm, downsamp_temp,
scaleindep, width_bins,
rawdatafile, binratio, data)
#print "waterfall",memory.resident()/(1024.0**3)
# Saving the arrays into the .spd file.
with open(temp_filename + ".spd", 'wb') as f:
np.savez_compressed(
f,
Data_dedisp_nozerodm=Data_dedisp_nozerodm.astype(np.float16),
Data_dedisp_zerodm=Data_dedisp_zerodm.astype(np.float16),
Data_nozerodm=Data_nozerodm.astype(np.float16),
delays_nozerodm=delays_nozerodm,
freqs_nozerodm=freqs_nozerodm,
Data_zerodm=Data_zerodm.astype(np.float16),
dm_arr=map(np.float16, dm_arr),
sigma_arr=map(np.float16, sigma_arr),
dm_list=map(np.float16, dm_list),
time_list=map(np.float16, time_list),
text_array=text_array)
print_debug("Now plotting...")
#print "Before plot..",memory.resident()/(1024.0**3)
show_spplots.plot(temp_filename + ".spd",
args[1:],
xwin=False,
outfile=basename,
tar=None)
print_debug("Finished plot %i " % j + strftime("%Y-%m-%d %H:%M:%S"))
#print "After plot..",memory.resident()/(1024.0**3)
numcands += 1
print_debug('Finished sp_candidate : %i' % numcands)
print_debug("Finished running waterfaller... " +
strftime("%Y-%m-%d %H:%M:%S"))
def main():
freqs = []
freqerrs = []
filenums = []
intervals = []
filenum = 0
inffiles = glob.glob("*.inf")
for inffile in inffiles:
accelfile = inffile[:-4] + "_ACCEL_0.cand"
if not os.path.exists(accelfile):
continue
filenum += 1
rzws = presto.read_rzwcands(accelfile)
inf = infodata.infodata(inffile)
T = inf.dt * inf.N
for rzw in rzws:
freq = rzw.r / T
freqerr = rzw.rerr / T
freqs.append(freq)
freqerrs.append(freqerr)
filenums.append(filenum)
fint = FreqInterval(freq, freqerr)
# Traverse list of intervals backwards
for ii in range(len(intervals))[::-1]:
if fint in intervals[ii]:
fint = fint + intervals[ii]
matchfound = True
del intervals[ii]
intervals.append(fint)
freqs = np.array(freqs)
freqerrs = np.array(freqerrs)
filenums = np.array(filenums)
plt.figure(figsize=(11, 8.5))
ebax = plt.axes((0.1, 0.1, 0.7, 0.7))
plt.errorbar(freqs,
filenums,
xerr=freqerrs,
fmt=None,
zorder=1,
ecolor='k')
# Plot intervals worth masking
for i in intervals:
if i.numelements > 7:
r = matplotlib.patches.Rectangle((i.fcent-i.width/2.0,0), i.width, max(filenums), \
fill=True, fc='r', ec='none', \
alpha=0.25, zorder=-1)
plt.gca().add_patch(r)
print i.zaplist_string()
plt.xlabel("Spin Frequency (Hz)")
plt.ylabel("File number (index)")
hax = plt.axes((0.8, 0.1, 0.15, 0.7), sharey=ebax)
plt.hist(filenums,
bins=max(filenums),
range=(0, max(filenums)),
orientation='horizontal',
fc='none')
plt.savefig("accelcands.ps", orientation="landscape", papertype="letter")
plt.show()
def sift_directory(dir, outbasenm):
"""Sift candidates in given directory.
Inputs:
dir: The directory containing candidates to sift.
outbasenm: The base name of the output files.
Outputs:
None
"""
import os.path
import glob
import infodata
# TODO: Remove hard-coded values in this function
# replace with command line options.
global sigma_threshold
sigma_threshold = 5.0
print_sift_globals()
# Get list of DMs from *.inf files
inffns = glob.glob(os.path.join(dir, '*.inf'))
dmstrs = ['%.2f' % infodata.infodata(inffn).DM for inffn in inffns]
# Read candidates found in low acceleration searching
lo_accel_fns = glob.glob(os.path.join(dir, '*ACCEL_0'))
lo_accel_cands = read_candidates(lo_accel_fns)
print "Read %d candidates from %d files" % \
(len(lo_accel_cands), len(lo_accel_fns))
print "%d candidates passed default rejection" % len(lo_accel_cands)
if len(lo_accel_cands):
lo_accel_cands.remove_duplicate_candidates()
if len(lo_accel_cands):
lo_accel_cands.remove_DM_problems(2, dmstrs, 2.0)
lo_accel_cands.print_cand_summary()
# Read candidates found in high acceleration searching
hi_accel_fns = glob.glob(os.path.join(dir, '*ACCEL_50'))
hi_accel_cands = read_candidates(hi_accel_fns)
print "Read %d candidates from %d files" % \
(len(hi_accel_cands), len(hi_accel_fns))
print "%d candidates passed default rejection" % len(hi_accel_cands)
if len(hi_accel_cands):
hi_accel_cands.remove_duplicate_candidates()
if len(hi_accel_cands):
hi_accel_cands.remove_DM_problems(2, dmstrs, 2.0)
hi_accel_cands.print_cand_summary()
all_accel_cands = lo_accel_cands + hi_accel_cands
if len(all_accel_cands):
all_accel_cands.remove_harmonics()
# Note: the candidates will be sorted in _sigma_ order, not _SNR_!
all_accel_cands.cands.sort(cmp_sigma)
print "Found %d good candidates" % len(all_accel_cands)
all_accel_cands.to_file(outbasenm + ".accelcands")
all_accel_cands.write_cand_report(outbasenm + ".accelcands.report")
all_accel_cands.print_cand_summary()
all_accel_cands.plot_goodbad()
plt.savefig(outbasenm + ".accelcands.rejects.png")
all_accel_cands.plot_summary()
plt.savefig(outbasenm + ".accelcands.summary.png")
def main():
parser = optparse.OptionParser(prog="Group_sp_events.py", \
version="Chen Karako, updated by Chitrang Patel(June 23, 2015)",\
usage="%prog args inf files(produced by prepsubband) singlepulse files",\
description="Group single pulse events and rank them based \
on the sigma behavior. Plot DM vs time with \
different colours for different ranks.")
parser.add_option('--rank', dest='min_ranktoplot', type = 'int',\
help="Only groups with rank upto this will plotted.(default: plot \
all except rank 1)", default=0)
parser.add_option('-o', dest='outbasenm', type = 'string',\
help="outfile base name. .groups.txt will be added to the given name."\
, default='')
parser.add_option('--DM', dest='MAX_DMRANGE', type = 'float',\
help="DM range above which a group is considered RFI.(Default = 300.0)\
", default=300.0)
options, args = parser.parse_args()
#RANKS_TO_PLOT = [2,0,3,4,7,5,6]
RANKS_TO_PLOT = [2,0,3,4,5,6]
ranks = ranks_to_plot(RANKS_TO_PLOT, options.min_ranktoplot)
inffile = glob.glob('*.inf')[0] # Take the 1st .inf file in the current directory
if len(inffile) == 0: # no inf files exist in this directory
print "No inf files available in the current directory!"
inf = infodata.infodata(inffile)
print ranks
print_debug("Beginning read_sp_files... "+strftime("%Y-%m-%d %H:%M:%S"))
#singlepulses = read_sp_files(args[1:])[0]
groups = read_sp_files(args[1:])[0]
print_debug("Finished read_sp_files, beginning create_groups... " +
strftime("%Y-%m-%d %H:%M:%S"))
print_debug("Number of single pulse events: %d " % len(groups))
groups = create_groups(groups, inffile, min_nearby=1, ignore_obs_end=10) # ignore the last 10 seconds of the obs, for palfa
print_debug("Number of groups: %d " % len(groups))
print_debug("Finished create_groups, beginning grouping_sp_dmt... " +
strftime("%Y-%m-%d %H:%M:%S"))
grouping_sp_dmt(groups)
print_debug("Number of groups (after initial grouping): %d " % len(groups))
print_debug("Finished grouping_sp_dmt, beginning flag_noise... " +
strftime("%Y-%m-%d %H:%M:%S"))
flag_noise(groups) # do an initial coarse noise flagging and removal
pop_by_rank(groups, 1)
print_debug("Number of groups (after removed noise gps w <10 sps): %d " % len(groups))
print_debug("Beginning grouping_sp_t... " +
strftime("%Y-%m-%d %H:%M:%S"))
# Regroup good groups based on proximity in time only (compensate for missing middles):
groups = grouping_sp_t(groups)
print_debug("Finished grouping_sp_t. " + strftime("%Y-%m-%d %H:%M:%S"))
# Flag RFI groups, noise
flag_rfi(groups)
# Rank groups and identify noise (<45/40/35/30 sp events) groups
print_debug("Ranking groups...")
rank_groups(groups)
# Remove noise groups
print_debug("Before removing noise, len(groups): %s" % len(groups))
pop_by_rank(groups, 1)
print_debug("After removing noise, len(groups): %s" % len(groups))
# Group rfi with very close groups
print_debug("len(groups) before grouping_rfi: %s" % len(groups))
print_debug("Beginning grouping_rfi... " + strftime("%Y-%m-%d %H:%M:%S"))
grouping_rfi(groups)
print_debug("Finished grouping_rfi. " +
strftime("%Y-%m-%d %H:%M:%S"))
# Rank groups
#rank_groups(groups) # don't need this again
#print_debug("group summary after rank_groups: " + str(rank_occur(groups)))
print_debug("Finished rank_groups, beginning DM span check... " +
strftime("%Y-%m-%d %H:%M:%S"))
# Remove groups that are likely RFI, based on their large span in DM
if CHECKDMSPAN:
print_debug("Beginning DM span check...")
check_dmspan(groups, options.MAX_DMRANGE, inf.dt)
else:
print_debug("Skipping DM span check.")
print_debug("Finished DM span check, beginning writing to outfile... " +
strftime("%Y-%m-%d %H:%M:%S"))
outfile = open(options.outbasenm+'groups.txt', 'w')
summaryfile = open(options.outbasenm+'spsummary.txt', 'w')
rank_dict = rank_occur(groups)
for rank in sorted(ALL_RANKS_ORDERED):
if rank != 1:
outfile.write("Number of rank %d groups: %d \n" %
(rank, rank_dict.get(rank, 0)))
summaryfile.write("Number of rank %d groups: %d \n" %
(rank, rank_dict.get(rank, 0)))
outfile.write("\n")
summaryfile.close()
# Reverse sort lists so good groups are written at the top of the file
groups.sort(reverse=True)
# write list of events in each group
for grp in groups:
if grp.rank in RANKS_TO_WRITE:
outfile.write(str(grp) + '\n') #print group summary
outfile.write('\n')
outfile.write("# DM Sigma Time (s) Sample Downfact \n")
for sp in grp.singlepulses:
outfile.write("%7.2f %7.2f %13.6f %10d %3d \n" % sp)
outfile.write('\n')
outfile.close()
print_debug("Finished writing to outfile, now plotting... " +
strftime("%Y-%m-%d %H:%M:%S"))
if PLOT:
# Sort groups so better-ranked groups are plotted on top of worse groups
groups.sort()
# create several DM vs t plots, splitting up DM in overlapping intervals
# DMs 0-30, 20-110, 100-300, 300-1000
if PLOTTYPE.lower() == 'pgplot':
# Use PGPLOT to plot
plot_sp_rated_pgplot(groups, ranks, inffile, 0, 30)
print_debug("Finished PGplotting DMs0-30 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 20, 110)
print_debug("Finished PGplotting DMs20-110 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 100, 310)
print_debug("Finished PGplotting DMs100-310 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 300, 1000)
print_debug("Finished PGplotting DMs100-310 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_pgplot(groups, ranks, inffile, 1000, 10000)
print_debug("Finished PGplotting DMs100-310 "+strftime("%Y-%m-%d %H:%M:%S"))
elif PLOTTYPE.lower() == 'matplotlib':
# Use matplotlib to plot
plot_sp_rated_all(groups, ranks, inffile, 0, 30)
print_debug("Finished plotting DMs0-30 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 20, 110)
print_debug("Finished plotting DMs20-110 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 100, 310)
print_debug("Finished plotting DMs100-310 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 300, 1000)
print_debug("Finished plotting DMs300-1000 "+strftime("%Y-%m-%d %H:%M:%S"))
plot_sp_rated_all(groups, ranks, inffile, 1000, 10000)
print_debug("Finished plotting DMs1000-10000 "+strftime("%Y-%m-%d %H:%M:%S"))
else:
print "Plot type must be one of 'matplotlib' or 'pgplot'. Not plotting."
#for i in range(50,51,50):
# for j in [0.0,0.10,0.20,0.30,0.40,0.50,0.60,0.70,0.80,0.90]:
#for i in [10,20,30,50,100,200]:
for i in [40.0, 80.0, 120.0, 200.0, 400.0, 800.0, 1600.0, 3200.0]:
for j in [0.40]:
searchpulse1 = scatter(t1, i, beta=j)
searchpulse1 = searchpulse1 / np.amax(searchpulse1)
searchpulse.append(searchpulse1)
taud.append([i, j])
areaundercurve = sum(searchpulse1)
area.append(areaundercurve)
filenm = sys.argv[1]
filenmbase = filenm[:filenm.rfind(".dat")]
signal = np.fromfile(filenm, dtype=np.float32, count=numpoints)
info = infodata.infodata(filenmbase + ".inf")
DMs = []
DMstr = "%.4f" % info.DM
DMs.append(info.DM)
DM = info.DM
N, dt = int(info.N), info.dt
obstime = N * dt
outfile = open(filenmbase + '.scatteredsearch', mode='w')
outfile.write("# DM SNR Time (s) Sample t_d beta\n")
outfile2 = open(filenmbase + '.statistics', mode='w')
outfile.write(
"# DM SNR SNR2 Red_Chi2 Time (s) Sample t_d beta\n"
)
data = np.fromfile(filenm, dtype=np.float32, count=N)
offsetatbeginning = 500
startN = offsetatbeginning
def __init__(self, filename):
self.pfd_filename = filename
infile = open(filename, "rb")
# See if the .bestprof file is around
try:
self.bestprof = bestprof(filename+".bestprof")
except IOError:
self.bestprof = 0
swapchar = '<' # this is little-endian
data = infile.read(5*4)
testswap = struct.unpack(swapchar+"i"*5, data)
# This is a hack to try and test the endianness of the data.
# None of the 5 values should be a large positive number.
if (Num.fabs(Num.asarray(testswap))).max() > 100000:
swapchar = '>' # this is big-endian
(self.numdms, self.numperiods, self.numpdots, self.nsub, self.npart) = \
struct.unpack(swapchar+"i"*5, data)
(self.proflen, self.numchan, self.pstep, self.pdstep, self.dmstep, \
self.ndmfact, self.npfact) = struct.unpack(swapchar+"i"*7, infile.read(7*4))
self.filenm = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
self.candnm = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
self.telescope = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
self.pgdev = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
test = infile.read(16)
if not test[:8]=="Unknown":
self.rastr = test[:test.find('\0')]
test = infile.read(16)
self.decstr = test[:test.find('\0')]
else:
self.rastr = "Unknown"
self.decstr = "Unknown"
(self.dt, self.startT) = struct.unpack(swapchar+"dd", infile.read(2*8))
(self.endT, self.tepoch, self.bepoch, self.avgvoverc, self.lofreq, \
self.chan_wid, self.bestdm) = struct.unpack(swapchar+"d"*7, infile.read(7*8))
# The following "fixes" (we think) the observing frequency of the Spigot
# based on tests done by Ingrid on 0737 (comparing it to GASP)
# The same sorts of corrections should be made to WAPP data as well...
# The tepoch corrections are empirically determined timing corrections
# Note that epoch is only double precision and so the floating
# point accuracy is ~1 us!
if self.telescope=='GBT':
if (Num.fabs(Num.fmod(self.dt, 8.192e-05) < 1e-12) and \
("spigot" in filename.lower() or "guppi" not in filename.lower()) and \
(self.tepoch < 54832.0)):
sys.stderr.write("Assuming SPIGOT data...\n")
if self.chan_wid==800.0/1024: # Spigot 800 MHz mode 2
self.lofreq -= 0.5 * self.chan_wid
# original values
#if self.tepoch > 0.0: self.tepoch += 0.039334/86400.0
#if self.bestprof: self.bestprof.epochf += 0.039334/86400.0
# values measured with 1713+0747 wrt BCPM2 on 13 Sept 2007
if self.tepoch > 0.0: self.tepoch += 0.039365/86400.0
if self.bestprof: self.bestprof.epochf += 0.039365/86400.0
elif self.chan_wid==800.0/2048:
self.lofreq -= 0.5 * self.chan_wid
if self.tepoch < 53700.0: # Spigot 800 MHz mode 16 (downsampled)
if self.tepoch > 0.0: self.tepoch += 0.039352/86400.0
if self.bestprof: self.bestprof.epochf += 0.039352/86400.0
else: # Spigot 800 MHz mode 14
# values measured with 1713+0747 wrt BCPM2 on 13 Sept 2007
if self.tepoch > 0.0: self.tepoch += 0.039365/86400.0
if self.bestprof: self.bestprof.epochf += 0.039365/86400.0
elif self.chan_wid==50.0/1024 or self.chan_wid==50.0/2048: # Spigot 50 MHz modes
self.lofreq += 0.5 * self.chan_wid
# Note: the offset has _not_ been measured for the 2048-lag mode
if self.tepoch > 0.0: self.tepoch += 0.039450/86400.0
if self.bestprof: self.bestprof.epochf += 0.039450/86400.0
(self.topo_pow, tmp) = struct.unpack(swapchar+"f"*2, infile.read(2*4))
(self.topo_p1, self.topo_p2, self.topo_p3) = struct.unpack(swapchar+"d"*3, \
infile.read(3*8))
(self.bary_pow, tmp) = struct.unpack(swapchar+"f"*2, infile.read(2*4))
(self.bary_p1, self.bary_p2, self.bary_p3) = struct.unpack(swapchar+"d"*3, \
infile.read(3*8))
(self.fold_pow, tmp) = struct.unpack(swapchar+"f"*2, infile.read(2*4))
(self.fold_p1, self.fold_p2, self.fold_p3) = struct.unpack(swapchar+"d"*3, \
infile.read(3*8))
# Save current p, pd, pdd
# NOTE: Fold values are actually frequencies!
self.curr_p1, self.curr_p2, self.curr_p3 = \
psr_utils.p_to_f(self.fold_p1, self.fold_p2, self.fold_p3)
self.pdelays_bins = Num.zeros(self.npart, dtype='d')
(self.orb_p, self.orb_e, self.orb_x, self.orb_w, self.orb_t, self.orb_pd, \
self.orb_wd) = struct.unpack(swapchar+"d"*7, infile.read(7*8))
self.dms = Num.asarray(struct.unpack(swapchar+"d"*self.numdms, \
infile.read(self.numdms*8)))
if self.numdms==1:
self.dms = self.dms[0]
self.periods = Num.asarray(struct.unpack(swapchar+"d"*self.numperiods, \
infile.read(self.numperiods*8)))
self.pdots = Num.asarray(struct.unpack(swapchar+"d"*self.numpdots, \
infile.read(self.numpdots*8)))
self.numprofs = self.nsub*self.npart
if (swapchar=='<'): # little endian
self.profs = Num.zeros((self.npart, self.nsub, self.proflen), dtype='d')
for ii in range(self.npart):
for jj in range(self.nsub):
self.profs[ii,jj,:] = Num.fromfile(infile, Num.float64, self.proflen)
else:
self.profs = Num.asarray(struct.unpack(swapchar+"d"*self.numprofs*self.proflen, \
infile.read(self.numprofs*self.proflen*8)))
self.profs = Num.reshape(self.profs, (self.npart, self.nsub, self.proflen))
if (self.numchan==1):
try:
idata = infodata.infodata(self.filenm[:self.filenm.rfind('.')]+".inf")
try:
if idata.waveband=="Radio":
self.bestdm = idata.DM
self.numchan = idata.numchan
except:
self.bestdm = 0.0
self.numchan = 1
except IOError:
print "Warning! Can't open the .inf file for "+filename+"!"
self.binspersec = self.fold_p1*self.proflen
self.chanpersub = self.numchan/self.nsub
self.subdeltafreq = self.chan_wid*self.chanpersub
self.hifreq = self.lofreq + (self.numchan-1)*self.chan_wid
self.losubfreq = self.lofreq + self.subdeltafreq - self.chan_wid
self.subfreqs = Num.arange(self.nsub, dtype='d')*self.subdeltafreq + \
self.losubfreq
self.subdelays_bins = Num.zeros(self.nsub, dtype='d')
# Save current DM
self.currdm = 0
self.killed_subbands = []
self.killed_intervals = []
self.pts_per_fold = []
# Note: a foldstats struct is read in as a group of 7 doubles
# the correspond to, in order:
# numdata, data_avg, data_var, numprof, prof_avg, prof_var, redchi
self.stats = Num.zeros((self.npart, self.nsub, 7), dtype='d')
for ii in range(self.npart):
currentstats = self.stats[ii]
for jj in range(self.nsub):
if (swapchar=='<'): # little endian
currentstats[jj] = Num.fromfile(infile, Num.float64, 7)
else:
currentstats[jj] = Num.asarray(struct.unpack(swapchar+"d"*7, \
infile.read(7*8)))
self.pts_per_fold.append(self.stats[ii][0][0]) # numdata from foldstats
self.start_secs = Num.add.accumulate([0]+self.pts_per_fold[:-1])*self.dt
self.pts_per_fold = Num.asarray(self.pts_per_fold)
self.mid_secs = self.start_secs + 0.5*self.dt*self.pts_per_fold
if (not self.tepoch==0.0):
self.start_topo_MJDs = self.start_secs/86400.0 + self.tepoch
self.mid_topo_MJDs = self.mid_secs/86400.0 + self.tepoch
if (not self.bepoch==0.0):
self.start_bary_MJDs = self.start_secs/86400.0 + self.bepoch
self.mid_bary_MJDs = self.mid_secs/86400.0 + self.bepoch
self.Nfolded = Num.add.reduce(self.pts_per_fold)
self.T = self.Nfolded*self.dt
self.avgprof = (self.profs/self.proflen).sum()
self.varprof = self.calc_varprof()
# nominal number of degrees of freedom for reduced chi^2 calculation
self.DOFnom = float(self.proflen) - 1.0
# corrected number of degrees of freedom due to inter-bin correlations
self.dt_per_bin = self.curr_p1 / self.proflen / self.dt
self.DOFcor = self.DOFnom * self.DOF_corr()
infile.close()
self.barysubfreqs = None
if self.avgvoverc==0:
if self.candnm.startswith("PSR_"):
# If this doesn't work, we should try to use the barycentering calcs
# in the presto module.
try:
self.polycos = polycos.polycos(self.candnm[4:],
filenm=self.pfd_filename+".polycos")
midMJD = self.tepoch + 0.5*self.T/86400.0
self.avgvoverc = self.polycos.get_voverc(int(midMJD), midMJD-int(midMJD))
#sys.stderr.write("Approximate Doppler velocity (in c) is: %.4g\n"%self.avgvoverc)
# Make the Doppler correction
self.barysubfreqs = self.subfreqs*(1.0+self.avgvoverc)
except IOError:
self.polycos = 0
if self.barysubfreqs is None:
self.barysubfreqs = self.subfreqs
#!/usr/bin/env python
from __future__ import (print_function,division)
import psr_utils as pu
import sys
from infodata import infodata
if len(sys.argv) != 2:
print("chooseN <file.inf|numpoints>")
print(" Prints a good value for fast FFTs to be used for -numout in prepdata/prepsubband")
sys.exit(1)
if (sys.argv[1].endswith('.inf')):
inf = infodata(sys.argv[1])
n = inf.N
else:
try:
n = int(sys.argv[1])
except:
print("chooseN <file.inf|numpoints>")
print(" Prints a good value for fast FFTs to be used for -numout in prepdata/prepsubband")
sys.exit(2)
print(pu.choose_N(n))
def main():
parser = optparse.OptionParser(prog="sp_pipeline..py", \
version=" Chitrang Patel (May. 12, 2015)", \
usage="%prog INFILE(PsrFits FILE, SINGLEPULSE FILES)", \
description="Create single pulse plots to show the " \
"frequency sweeps of a single pulse, " \
"DM vs time, and SNR vs DM,"\
"in psrFits data.")
parser.add_option('--infile', dest='infile', type='string', \
help="Give a .inf file to read the appropriate header information.")
parser.add_option('--groupsfile', dest='txtfile', type='string', \
help="Give the groups.txt file to read in the groups information.")
parser.add_option('--mask', dest='maskfile', type='string', \
help="Mask file produced by rfifind. (Default: No Mask).", \
default=None)
parser.add_option('-n', dest='maxnumcands', type='int', \
help="Maximum number of candidates to plot. (Default: 100).", \
default=100)
options, args = parser.parse_args()
if not hasattr(options, 'infile'):
raise ValueError("A .inf file must be given on the command line! ")
if not hasattr(options, 'txtfile'):
raise ValueError("The groups.txt file must be given on the command line! ")
files = sp_utils.spio.get_textfile(options.txtfile)
print_debug("Begining waterfaller... "+strftime("%Y-%m-%d %H:%M:%S"))
if not args[0].endswith("fits"):
raise ValueError("The first file must be a psrFits file! ")
print_debug('Maximum number of candidates to plot: %i'%options.maxnumcands)
basename = args[0][:-5]
filetype = "psrfits"
inffile = options.infile
topo, bary = bary_and_topo.bary_to_topo(inffile)
time_shift = bary-topo
inf = infodata.infodata(inffile)
RA = inf.RA
dec = inf.DEC
MJD = inf.epoch
mjd = Popen(["mjd2cal", "%f"%MJD], stdout=PIPE, stderr=PIPE)
date, err = mjd.communicate()
date = date.split()[2:5]
telescope = inf.telescope
N = inf.N
Total_observed_time = inf.dt *N
print_debug('getting file..')
rawdatafile = psrfits.PsrfitsFile(args[0])
bin_shift = np.round(time_shift/rawdatafile.tsamp).astype('int')
numcands = 0 # candidate counter. Use this to decide the maximum bumber of candidates to plot.
loop_must_break = False # dont break the loop unless num of cands >100.
for group in [6, 5, 4, 3, 2]:
rank = group+1
if files[group] != "Number of rank %i groups: 0 "%rank:
print_debug(files[group])
values = sp_utils.spio.split_parameters(rank, options.txtfile)
lis = np.where(files == '\tRank: %i.000000'%rank)[0]
for ii in range(len(values)):
#### Array for Plotting DM vs SNR
print_debug("Making arrays for DM vs Signal to Noise...")
temp_list = files[lis[ii]-6].split()
npulses = int(temp_list[2])
temp_lines = files[(lis[ii]+3):(lis[ii]+npulses+1)]
arr = np.split(temp_lines, len(temp_lines))
dm_list = []
time_list = []
for i in range(len(arr)):
dm_val= float(arr[i][0].split()[0])
time_val = float(arr[i][0].split()[2])
dm_list.append(dm_val)
time_list.append(time_val)
arr_2 = np.array([arr[i][0].split() for i in range(len(arr))], dtype = np.float32)
dm_arr = np.array([arr_2[i][0] for i in range(len(arr))], dtype = np.float32)
sigma_arr = np.array([arr_2[i][1] for i in range(len(arr))], dtype = np.float32)
#### Array for Plotting DM vs Time is in show_spplots.plot(...)
#### Setting variables up for the waterfall arrays.
j = ii+1
subdm = dm = sweep_dm= values[ii][0]
integrate_dm = None
sigma = values[ii][1]
sweep_posn = 0.0
bary_start_time = values[ii][2]
topo_start_time = bary_start_time - topo_timeshift(bary_start_time, time_shift, topo)[0]
sample_number = values[ii][3]
width_bins = values[ii][4]
binratio = 50
scaleindep = False
zerodm = None
downsamp = np.round((values[ii][2]/sample_number/rawdatafile.tsamp)).astype('int')
duration = binratio * width_bins * rawdatafile.tsamp * downsamp
start = topo_start_time - (0.25 * duration)
if (start<0.0):
start = 0.0
pulse_width = width_bins*downsamp*rawdatafile.tsamp
if sigma < 10:
nsub = 32
elif sigma >= 10 and sigma < 15:
nsub = 64
else:
nsub = 96
if nsub > inf.numchan:
nsub = inf.numchan
nbins = np.round(duration/rawdatafile.tsamp).astype('int')
start_bin = np.round(start/rawdatafile.tsamp).astype('int')
dmfac = 4.15e3 * np.abs(1./rawdatafile.frequencies[0]**2 - 1./rawdatafile.frequencies[-1]**2)
nbinsextra = np.round((duration + dmfac * dm)/rawdatafile.tsamp).astype('int')
if (start_bin+nbinsextra) > N-1:
nbinsextra = N-1-start_bin
data = rawdatafile.get_spectra(start_bin, nbinsextra)
data = maskdata(data, start_bin, nbinsextra, options.maskfile)
#make an array to store header information for the .npz files
temp_filename = basename+"_DM%.1f_%.1fs_rank_%i"%(subdm, topo_start_time, rank)
# Array for Plotting Dedispersed waterfall plot - zerodm - OFF
print_debug("Running waterfaller with Zero-DM OFF...")
data, Data_dedisp_nozerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data)
# Add additional information to the header information array
text_array = np.array([args[0], 'Arecibo', RA, dec, MJD, rank, nsub, nbins, subdm, sigma, sample_number, duration, width_bins, pulse_width, rawdatafile.tsamp, Total_observed_time, topo_start_time, data.starttime, data.dt, data.numspectra, data.freqs.min(), data.freqs.max()])
#### Array for plotting Dedispersed waterfall plot zerodm - ON
print_debug("Running Waterfaller with Zero-DM ON...")
data = rawdatafile.get_spectra(start_bin, nbinsextra)
data = maskdata(data, start_bin, nbinsextra, options.maskfile)
zerodm = True
data, Data_dedisp_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data)
####Sweeped without zerodm
start = start + (0.25*duration)
start_bin = np.round(start/rawdatafile.tsamp).astype('int')
sweep_duration = 4.15e3 * np.abs(1./rawdatafile.frequencies[0]**2-1./rawdatafile.frequencies[-1]**2)*sweep_dm
nbins = np.round(sweep_duration/(rawdatafile.tsamp)).astype('int')
if ((nbins+start_bin)> (N-1)):
nbins = N-1-start_bin
data = rawdatafile.get_spectra(start_bin, nbins)
data = maskdata(data, start_bin, nbins, options.maskfile)
zerodm = None
dm = None
data, Data_nozerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data)
text_array = np.append(text_array, sweep_duration)
text_array = np.append(text_array, data.starttime)
text_array = np.append(text_array, bary_start_time)
# Array to Construct the sweep
if sweep_dm is not None:
ddm = sweep_dm-data.dm
delays = psr_utils.delay_from_DM(ddm, data.freqs)
delays -= delays.min()
delays_nozerodm = delays
freqs_nozerodm = data.freqs
# Sweeped with zerodm-on
zerodm = True
downsamp_temp = 1
data, Data_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp_temp, scaleindep, width_bins, rawdatafile, binratio, data)
# Saving the arrays into the .spd file.
with open(temp_filename+".spd", 'wb') as f:
np.savez_compressed(f, Data_dedisp_nozerodm = Data_dedisp_nozerodm.astype(np.float16), Data_dedisp_zerodm = Data_dedisp_zerodm.astype(np.float16), Data_nozerodm = Data_nozerodm.astype(np.float16), delays_nozerodm = delays_nozerodm, freqs_nozerodm = freqs_nozerodm, Data_zerodm = Data_zerodm.astype(np.float16), dm_arr= map(np.float16, dm_arr), sigma_arr = map(np.float16, sigma_arr), dm_list= map(np.float16, dm_list), time_list = map(np.float16, time_list), text_array = text_array)
print_debug("Now plotting...")
show_spplots.plot(temp_filename+".spd", args[1:], xwin=False, outfile = basename, tar = None)
print_debug("Finished plot %i " %j+strftime("%Y-%m-%d %H:%M:%S"))
numcands+= 1
print_debug('Finished sp_candidate : %i'%numcands)
if numcands >= options.maxnumcands: # Max number of candidates to plot 100.
loop_must_break = True
break
if loop_must_break:
break
print_debug("Finished group %i... "%rank+strftime("%Y-%m-%d %H:%M:%S"))
print_debug("Finished running waterfaller... "+strftime("%Y-%m-%d %H:%M:%S"))
def main():
parser = optparse.OptionParser(prog="sp_pipeline..py", \
version=" Chitrang Patel (May. 12, 2015)", \
usage="%prog INFILE(PsrFits FILE, SINGLEPULSE FILES)", \
description="Create single pulse plots to show the " \
"frequency sweeps of a single pulse, " \
"DM vs time, and SNR vs DM,"\
"in psrFits data.")
parser.add_option('--infile', dest='infile', type='string', \
help="Give a .inf file to read the appropriate header information.")
parser.add_option('--groupsfile', dest='txtfile', type='string', \
help="Give the groups.txt file to read in the groups information.")
parser.add_option('--mask', dest='maskfile', type='string', \
help="Mask file produced by rfifind. (Default: No Mask).", \
default=None)
parser.add_option('-n', dest='maxnumcands', type='int', \
help="Maximum number of candidates to plot. (Default: 100).", \
default=100)
options, args = parser.parse_args()
if not hasattr(options, 'infile'):
raise ValueError("A .inf file must be given on the command line! ")
if not hasattr(options, 'txtfile'):
raise ValueError("The groups.txt file must be given on the command line! ")
files = get_textfile(options.txtfile)
print_debug("Begining waterfaller... "+strftime("%Y-%m-%d %H:%M:%S"))
if not args[0].endswith("fits"):
raise ValueError("The first file must be a psrFits file! ")
print_debug('Maximum number of candidates to plot: %i'%options.maxnumcands)
basename = args[0][:-5]
filetype = "psrfits"
inffile = options.infile
topo, bary = bary_and_topo.bary_to_topo(inffile)
time_shift = bary-topo
inf = infodata.infodata(inffile)
RA = inf.RA
dec = inf.DEC
MJD = inf.epoch
mjd = Popen(["mjd2cal", "%f"%MJD], stdout=PIPE, stderr=PIPE)
date, err = mjd.communicate()
date = date.split()[2:5]
telescope = inf.telescope
N = inf.N
numcands=0
Total_observed_time = inf.dt *N
print_debug('getting file..')
values = split_parameters(options.txtfile)
if len(values)> options.maxnumcands:
values=sorted(values, key=itemgetter(5,1)) #sorting candidates based on ranks and snr
values=values[-options.maxnumcands:]
print "More than", options.maxnumcands, "candidates, making plots for", options.maxnumcands, "candidates"
values = sorted(values, key=itemgetter(0))
for ii in range(len(values)):
#### Array for Plotting DM vs SNR
print_debug("Making arrays for DM vs Signal to Noise...")
temp_list = files[values[ii][6]-6].split()
npulses = int(temp_list[2])
temp_lines = files[(values[ii][6]+3):(values[ii][6]+npulses+1)]
arr = np.split(temp_lines, len(temp_lines))
dm_list = []
time_list = []
for i in range(len(arr)):
dm_val= float(arr[i][0].split()[0])
time_val = float(arr[i][0].split()[2])
dm_list.append(dm_val)
time_list.append(time_val)
arr_2 = np.array([arr[i][0].split() for i in range(len(arr))], dtype = np.float32)
dm_arr = np.array([arr_2[i][0] for i in range(len(arr))], dtype = np.float32)
sigma_arr = np.array([arr_2[i][1] for i in range(len(arr))], dtype = np.float32)
#### Array for Plotting DM vs Time is in show_spplots.plot(...)
#### Setting variables up for the waterfall arrays.
j = ii+1
subdm = dm = sweep_dm= values[ii][0]
sample_number = values[ii][3]
rank=values[ii][5]
width_bins = values[ii][4]
#print "dm", dm
#print "width_bins", width_bins
downsamp = np.round((values[ii][2]/sample_number/inf.dt)).astype('int')
#print "downsamp", downsamp
pulse_width = width_bins*downsamp*inf.dt
#print "pulse_width", pulse_width
if ii == 0:
mask_subband=rfifind.rfifind("%s_rfifind.mask"%(basename))
mask_subband.set_zap_chans(power=1000,plot=False)
mask_subband.set_weights_and_offsets()
mask_subband.write_weights(filename="%s_weights.txt"%(basename))
cmd="psrfits_subband -dm %.2f -nsub 128 -o %s_subband_%.2f -weights %s_weights.txt %s"%(dm,basename,dm,basename,args[0])
call(cmd, shell=True)
#subband args[0] at dm and then generate a file that will be set equal to rawdatafile
subband_file="%s_subband_%.2f_0001.fits" %(basename,dm)
dm_prev=dm
subband_prev= subband_file
else:
dm_diff=dm-dm_prev
t_smear=8.3e3*dm_diff*(350**-3)*(np.abs(rawdatafile.frequencies[0]-rawdatafile.frequencies[-1])/128.)
if (5*t_smear) > pulse_width:
cmd="psrfits_subband -dm %.2f -nsub 128 -o %s_subband_%.2f -weights %s_weights.txt %s"%(dm,basename,dm,basename,args[0])
call(cmd, shell=True)
#subband args[0] at dm and then generate a file that will be set equal to rawdatafile
subband_file="%s_subband_%.2f_0001.fits" %(basename,dm)
dm_prev=dm
subband_prev=subband_file
rawdatafile = psrfits.PsrfitsFile(subband_file)
bin_shift = np.round(time_shift/rawdatafile.tsamp).astype('int')
integrate_dm = None
sigma = values[ii][1]
sweep_posn = 0.0
bary_start_time = values[ii][2]
topo_start_time = bary_start_time - topo_timeshift(bary_start_time, time_shift, topo)[0]
binratio = 50
scaleindep = False
zerodm = None
duration = binratio * width_bins * rawdatafile.tsamp * downsamp
start = topo_start_time - (0.25 * duration)
if (start<0.0):
start = 0.0
if sigma <= 7:
nsub = 32
elif sigma >= 7 and sigma < 10:
nsub = 64
else:
nsub = 128
nbins = np.round(duration/rawdatafile.tsamp).astype('int')
start_bin = np.round(start/rawdatafile.tsamp).astype('int')
dmfac = 4.15e3 * np.abs(1./rawdatafile.frequencies[0]**2 - 1./rawdatafile.frequencies[-1]**2)
nbinsextra = np.round((duration + dmfac * dm)/rawdatafile.tsamp).astype('int')
if (start_bin+nbinsextra) > N-1:
nbinsextra = N-1-start_bin
data = rawdatafile.get_spectra(start_bin, nbinsextra)
data = maskdata(data, start_bin, nbinsextra, options.maskfile)
#make an array to store header information for the .npz files
temp_filename = basename+"_DM%.1f_%.1fs_rank_%i"%(subdm, topo_start_time, rank)
# Array for Plotting Dedispersed waterfall plot - zerodm - OFF
print_debug("Running waterfaller with Zero-DM OFF...")
data, Data_dedisp_nozerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data)
#Add additional information to the header information array
text_array = np.array([subband_file, 'GBT', RA, dec, MJD, rank, nsub, nbins, subdm, sigma, sample_number, duration, width_bins, pulse_width, rawdatafile.tsamp, Total_observed_time, topo_start_time, data.starttime, data.dt, data.numspectra, data.freqs.min(), data.freqs.max()])
#### Array for plotting Dedispersed waterfall plot zerodm - ON
print_debug("Running Waterfaller with Zero-DM ON...")
#print "before get_spectra",memory.resident()/(1024.0**3)
data = rawdatafile.get_spectra(start_bin, nbinsextra)
#print "after get_spectra",memory.resident()/(1024.0**3)
data = maskdata(data, start_bin, nbinsextra, options.maskfile)
zerodm = True
data, Data_dedisp_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data)
#print "waterfall",memory.resident()/(1024.0**3)
####Sweeped without zerodm
start = start + (0.25*duration)
start_bin = np.round(start/rawdatafile.tsamp).astype('int')
sweep_duration = 4.15e3 * np.abs(1./rawdatafile.frequencies[0]**2-1./rawdatafile.frequencies[-1]**2)*sweep_dm
nbins = np.round(sweep_duration/(rawdatafile.tsamp)).astype('int')
if ((nbins+start_bin)> (N-1)):
nbins = N-1-start_bin
#print "before get_spectra",memory.resident()/(1024.0**3)
data = rawdatafile.get_spectra(start_bin, nbins)
#print "after get_spectra",memory.resident()/(1024.0**3)
data = maskdata(data, start_bin, nbins, options.maskfile)
zerodm = None
dm = None
data, Data_nozerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data)
#print "waterfall",memory.resident()/(1024.0**3)
text_array = np.append(text_array, sweep_duration)
text_array = np.append(text_array, data.starttime)
text_array = np.append(text_array, bary_start_time)
# Array to Construct the sweep
if sweep_dm is not None:
ddm = sweep_dm-data.dm
delays = psr_utils.delay_from_DM(ddm, data.freqs)
delays -= delays.min()
delays_nozerodm = delays
freqs_nozerodm = data.freqs
# Sweeped with zerodm-on
zerodm = True
downsamp_temp = 1
data, Data_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp_temp, scaleindep, width_bins, rawdatafile, binratio, data)
#print "waterfall",memory.resident()/(1024.0**3)
# Saving the arrays into the .spd file.
with open(temp_filename+".spd", 'wb') as f:
np.savez_compressed(f, Data_dedisp_nozerodm = Data_dedisp_nozerodm.astype(np.float16), Data_dedisp_zerodm = Data_dedisp_zerodm.astype(np.float16), Data_nozerodm = Data_nozerodm.astype(np.float16), delays_nozerodm = delays_nozerodm, freqs_nozerodm = freqs_nozerodm, Data_zerodm = Data_zerodm.astype(np.float16), dm_arr= map(np.float16, dm_arr), sigma_arr = map(np.float16, sigma_arr), dm_list= map(np.float16, dm_list), time_list = map(np.float16, time_list), text_array = text_array)
print_debug("Now plotting...")
#print "Before plot..",memory.resident()/(1024.0**3)
show_spplots.plot(temp_filename+".spd", args[1:], xwin=False, outfile = basename, tar = None)
print_debug("Finished plot %i " %j+strftime("%Y-%m-%d %H:%M:%S"))
#print "After plot..",memory.resident()/(1024.0**3)
numcands+=1
print_debug('Finished sp_candidate : %i'%numcands)
print_debug("Finished running waterfaller... "+strftime("%Y-%m-%d %H:%M:%S"))