def preprocess(cls, fns):
"""Given a list of filenames apply any preprocessing steps
required. Return a list of file names of the output
files.
Input:
fns: List of names of files to preprocess.
Output:
outfns: List of names of preprocessed files.
"""
infiles = " ".join(fns)
fnmatchdict = cls.fnmatch(fns[0]).groupdict()
outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
fnmatchdict
outfile = outbasenm + "_0001.fits" # '0001' added is the filenumber
# Clobber output file
#if os.path.exists(outfile):
# os.remove(outfile)
# Merge mock subbands
mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
pipeline_utils.execute(mergecmd, stdout=outbasenm+"_merge.out")
# Remove first 7 rows from file
rowdelcmd = "fitsdelrow %s[SUBINT] 1 7" % outfile
pipeline_utils.execute(rowdelcmd)
# Rename file to remove the '_0001' that was added
os.rename(outfile, outbasenm+'.fits')
return [outbasenm+'.fits']
def preprocess(cls, fns):
"""Given a list of filenames apply any preprocessing steps
required. Return a list of file names of the output
files.
Input:
fns: List of names of files to preprocess.
Output:
outfns: List of names of preprocessed files.
"""
infiles = " ".join(fns)
fnmatchdict = cls.fnmatch(fns[0]).groupdict()
outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
fnmatchdict
outfile = outbasenm + "_0001.fits" # '0001' added is the filenumber
# Clobber output file
#if os.path.exists(outfile):
# os.remove(outfile)
# Merge mock subbands
mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
pipeline_utils.execute(mergecmd, stdout=outbasenm + "_merge.out")
# Remove first 7 rows from file
rowdelcmd = "fitsdelrow %s[SUBINT] 1 7" % outfile
pipeline_utils.execute(rowdelcmd)
# Rename file to remove the '_0001' that was added
os.rename(outfile, outbasenm + '.fits')
return [outbasenm + '.fits']
def get_subints_with_cal(self, nsigma=20, margin_of_error=1):
"""Return a list of subint numbers with the cal turned on.
Input:
nsigma: The number of sigma above the median a
subint needs to be in order to be flagged as
having the cal on. (Default: 50)
NOTE: The median absolute deviation is used in
place of the standard deviation here.
margin_of_error: For each subint with the cal on
also flag 'margin_of_error' subints on either
side. (Default: 1)
Output:
subints_with_cal: A sorted list of subints with the
cal on.
"""
fn = self.fns[0]
if not fn.endswith(".fits"):
raise ValueError("Filename doesn't end with '.fits'!")
basenm = fn[:-5] # Chop off '.fits'
# Make dat file
prepdatacmd = "prepdata -noclip -nobary -dm 0 -o %s_pre_DM0.00 %s" % (
basenm, fn)
pipeline_utils.execute(prepdatacmd,
stdout=basenm + "_pre_prepdata.out")
datfn = basenm + "_pre_DM0.00.dat"
samp_per_rec = self.num_samples_per_record
dat = np.memmap(datfn, mode='r', dtype='float32')
dat = dat[:dat.size / samp_per_rec * samp_per_rec]
dat.shape = (dat.size / samp_per_rec, samp_per_rec)
meds = np.median(dat, axis=1)
med_of_meds = np.median(meds)
mad_of_meds = np.median(np.abs(meds - med_of_meds))
#print "Median of medians:", med_of_meds
#print "MAD of medians:", mad_of_meds
#for ii, (med, nsig) in enumerate(zip(meds, (meds-med_of_meds)/mad_of_meds)):
# print "%d: %g (%g)" % (ii, med, nsig)
has_cal = (meds - med_of_meds) / mad_of_meds > nsigma
subints_with_cal = set()
for isub in np.flatnonzero(has_cal):
subints_with_cal.add(isub)
for x in range(1, margin_of_error + 1):
subints_with_cal.add(isub - x)
subints_with_cal.add(isub + x)
#print "Subints with cal: %s" % sorted(list(subints_with_cal))
#print "Conservative list of subints to remove: %s" % sorted(list(subints_with_cal))
# Remove dat file created
#os.remove(datfn)
return sorted(list(subints_with_cal))
def get_subints_with_cal(self, nsigma=15, margin_of_error=1):
"""Return a list of subint numbers with the cal turned on.
Input:
nsigma: The number of sigma above the median a
subint needs to be in order to be flagged as
having the cal on. (Default: 15)
NOTE: The median absolute deviation is used in
place of the standard deviation here.
margin_of_error: For each subint with the cal on
also flag 'margin_of_error' subints on either
side. (Default: 1)
Output:
subints_with_cal: A sorted list of subints with the
cal on.
"""
fn = self.fns[0]
if not fn.endswith(".fits"):
raise ValueError("Filename doesn't end with '.fits'!")
basenm = fn[:-5] # Chop off '.fits'
# Make dat file
prepdatacmd = "prepdata -noclip -nobary -dm 0 -o %s_pre_DM0.00 %s" % (basenm, fn)
pipeline_utils.execute(prepdatacmd, stdout=basenm+"_pre_prepdata.out")
datfn = basenm+"_pre_DM0.00.dat"
samp_per_rec = self.num_samples_per_record
dat = np.memmap(datfn, mode='r', dtype='float32')
dat = dat[:dat.size/samp_per_rec*samp_per_rec]
dat.shape = (dat.size/samp_per_rec, samp_per_rec)
meds = np.median(dat, axis=1)
med_of_meds = np.median(meds)
mad_of_meds = np.median(np.abs(meds-med_of_meds))
print "Median of medians:", med_of_meds
print "MAD of medians:", mad_of_meds
for ii, (med, nsig) in enumerate(zip(meds, (meds-med_of_meds)/mad_of_meds)):
print "%d: %g (%g)" % (ii, med, nsig)
has_cal = (meds-med_of_meds)/mad_of_meds > nsigma
subints_with_cal = set()
print "Subints with cal: %s" % sorted(list(subints_with_cal))
for isub in np.flatnonzero(has_cal):
subints_with_cal.add(isub)
for x in range(1,margin_of_error+1):
subints_with_cal.add(isub-x)
subints_with_cal.add(isub+x)
print "Conservative list of subints to remove: %s" % sorted(list(subints_with_cal))
# Remove dat file created
os.remove(datfn)
return sorted(list(subints_with_cal))
def version_num():
"""Compute version number and return it as a string.
"""
prestohash = pipeline_utils.execute("git rev-parse HEAD", \
dir=os.getenv('PRESTO'))[0].strip()
pipelinehash = pipeline_utils.execute("git rev-parse HEAD", \
dir=config.basic.pipelinedir)[0].strip()
psrfits_utilshash = pipeline_utils.execute("git rev-parse HEAD", \
dir=config.basic.psrfits_utilsdir)[0].strip()
vernum = 'PRESTO:%s;PIPELINE:%s;PSRFITS_UTILS:%s' % \
(prestohash, pipelinehash, psrfits_utilshash)
return vernum
def version_num():
"""Compute version number and return it as a string.
"""
prestohash = pipeline_utils.execute("git rev-parse HEAD", \
dir=os.getenv('PRESTO'))[0].strip()
pipelinehash = pipeline_utils.execute("git rev-parse HEAD", \
dir=config.basic.pipelinedir)[0].strip()
psrfits_utilshash = pipeline_utils.execute("git rev-parse HEAD", \
dir=config.basic.psrfits_utilsdir)[0].strip()
vernum = 'PRESTO:%s;PIPELINE:%s;PSRFITS_UTILS:%s' % \
(prestohash, pipelinehash, psrfits_utilshash)
return vernum
def preprocess(cls, fns):
"""Given a list of filenames apply any preprocessing steps
required. Return a list of file names of the output
files.
Input:
fns: List of names of files to preprocess.
Output:
outfns: List of names of preprocessed files.
"""
infiles = " ".join(fns)
fnmatchdict = cls.fnmatch(fns[0]).groupdict()
outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
fnmatchdict
outfile = outbasenm + "_0001.fits" # '0001' added is the filenumber
# Clobber output file
#if os.path.exists(outfile):
# os.remove(outfile)
# Merge mock subbands
mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
pipeline_utils.execute(mergecmd, stdout=outbasenm+"_merge.out")
# Anti-center commensal data taken after 2011 Nov. 4 has the
# cal turned on at the end of the observation for 6-7 seconds.
# This needs to be removed instead of the first 7 seconds.
# Changes made by Ryan Lynch 2012 June 17.
if self.galactic_longitude > 170.0 and self.galactic_longitude < 210.0 \
and self.timestamp_mjd > 55868:
if self.specinfo.num_subint >= 270:
rowdelcmd = "fitsdelrow %[SUBINT] 270 %i" % \
(outfile,self.specinfo.num_subint)
elif self.specinfo.num_subint >= 180 and self.specinfo.num_subint < 190:
rowdelcmd = "fitsdelrow %[SUBINT] 180 %i" % \
(outfile,self.specinfo.num_subint)
else:
# Otherwisse, remove first 7 rows from file as usual
rowdelcmd = "fitsdelrow %s[SUBINT] 1 7" % outfile
pipeline_utils.execute(rowdelcmd)
# Rename file to remove the '_0001' that was added
os.rename(outfile, outbasenm+'.fits')
return [outbasenm+'.fits']
def preprocess(cls, fns):
"""Given a list of filenames apply any preprocessing steps
required. Return a list of file names of the output
files.
Input:
fns: List of names of files to preprocess.
Output:
outfns: List of names of preprocessed files.
"""
infiles = " ".join(fns)
fnmatchdict = cls.fnmatch(fns[0]).groupdict()
outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
fnmatchdict
outfile = outbasenm + "_0001.fits" # '0001' added is the filenumber
# Clobber output file
#if os.path.exists(outfile):
# os.remove(outfile)
# Merge mock subbands
mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
pipeline_utils.execute(mergecmd, stdout=outbasenm + "_merge.out")
# Anti-center commensal data taken after 2011 Nov. 4 has the
# cal turned on at the end of the observation for 6-7 seconds.
# This needs to be removed instead of the first 7 seconds.
# Changes made by Ryan Lynch 2012 June 17.
if self.galactic_longitude > 170.0 and self.galactic_longitude < 210.0 \
and self.timestamp_mjd > 55868:
if self.specinfo.num_subint >= 270:
rowdelcmd = "fitsdelrow %[SUBINT] 270 %i" % \
(outfile,self.specinfo.num_subint)
elif self.specinfo.num_subint >= 180 and self.specinfo.num_subint < 190:
rowdelcmd = "fitsdelrow %[SUBINT] 180 %i" % \
(outfile,self.specinfo.num_subint)
else:
# Otherwisse, remove first 7 rows from file as usual
rowdelcmd = "fitsdelrow %s[SUBINT] 1 7" % outfile
pipeline_utils.execute(rowdelcmd)
# Rename file to remove the '_0001' that was added
os.rename(outfile, outbasenm + '.fits')
return [outbasenm + '.fits']
def preprocess(cls, fns):
"""Given a list of filenames apply any preprocessing steps
required. Return a list of file names of the output
files.
Input:
fns: List of names of files to preprocess.
Output:
outfns: List of names of preprocessed files.
"""
infiles = " ".join(fns)
fnmatchdict = cls.fnmatch(fns[0]).groupdict()
obsdata = cls(fns)
outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
fnmatchdict
outfile = outbasenm + "_0001.fits" # '0001' added is the filenumber
# Clobber output file
#if os.path.exists(outfile):
# os.remove(outfile)
# Merge mock subbands
mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
pipeline_utils.execute(mergecmd, stdout=outbasenm+"_merge.out")
# Rename file to remove the '_0001' that was added
mergedfn = outbasenm+'.fits'
os.rename(outfile, mergedfn)
merged = autogen_dataobj([mergedfn])
if not isinstance(merged, MergedMockPsrfitsData):
raise ValueError("Preprocessing of Mock data has not produced " \
"a recognized merged file!")
subints_with_cal = merged.get_subints_with_cal()
num_subints = merged.num_samples/merged.num_samples_per_record
subints_with_cal = [isub for isub in subints_with_cal \
if isub >=0 and isub < num_subints]
if len(subints_with_cal):
rowdelcmds = []
startrow = subints_with_cal[0]
numrows = 1
numdelrows = 0 # Number of rows deleted so far
# Add infinity to the list of subints with cal to
# make sure we remove the last cal-block.
for isub in subints_with_cal[1:]+[np.inf]:
isub -= numdelrows # Adjust based on the number of missing rows
if isub == startrow+numrows:
numrows+=1
else:
if startrow+1 < 0.1*num_subints:
print "Cal-affected region is within 10%% of start of obs " \
"remove all rows before cal. (cal start: %d; " \
"total num rows: %d)" % (startrow, num_subints)
numrows += startrow
startrow = 0
elif startrow+numrows > 0.9*num_subints:
print "Cal-affected region is within 10%% of end of obs " \
"remove all rows after cal. (cal start: %d; " \
"total num rows: %d)" % (startrow, num_subints)
numrows = num_subints - startrow
numdelrows += numrows # Keep track of number of rows deleted
print "Will delete %d rows starting at %d" % (numrows, startrow+1)
rowdelcmds.append("fitsdelrow %s[SUBINT] %d %d" % \
(mergedfn, startrow+1, numrows))
startrow = isub-numrows # NOTE: only delete numrows because
# numdelrows has already been deleted
# from isub
num_subints -= numrows # Adjust number of subints in the file
print "resetting startrow to", startrow
numrows = 1
for rowdelcmd in rowdelcmds:
pipeline_utils.execute(rowdelcmd)
# Make dat file
prepdatacmd = "prepdata -noclip -nobary -dm 0 -o %s_post_DM0.00 %s" % (outbasenm, mergedfn)
pipeline_utils.execute(prepdatacmd, stdout=outbasenm+"_post_prepdata.out")
return [mergedfn]
def preprocess(cls, fns):
"""Given a list of filenames apply any preprocessing steps
required. Return a list of file names of the output
files.
Input:
fns: List of names of files to preprocess.
Output:
outfns: List of names of preprocessed files.
"""
infiles = " ".join(fns)
fnmatchdict = cls.fnmatch(fns[0]).groupdict()
obsdata = cls(fns)
outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
fnmatchdict
outfile = outbasenm + "_0001.fits" # '0001' added is the filenumber
# Clobber output file
#if os.path.exists(outfile):
# os.remove(outfile)
# Merge mock subbands
mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
pipeline_utils.execute(mergecmd, stdout=outbasenm + "_merge.out")
# Rename file to remove the '_0001' that was added
mergedfn = outbasenm + '.fits'
os.rename(outfile, mergedfn)
merged = autogen_dataobj([mergedfn])
if not isinstance(merged, MergedMockPsrfitsData):
raise ValueError("Preprocessing of Mock data has not produced " \
"a recognized merged file!")
subints_with_cal = merged.get_subints_with_cal()
num_subints = merged.num_samples / merged.num_samples_per_record
subints_with_cal = [isub for isub in subints_with_cal \
if isub >=0 and isub < num_subints]
total_removed = 0
calrowsfile = open(outbasenm + "_calrows.txt", 'w')
if len(subints_with_cal):
calrowsfile.write("Subints with cal: %s\n" % \
",".join(["%d" % ii for ii in sorted(subints_with_cal)]))
rowdelcmds = []
# Enlarge list of rows to remove if some are near
# the beginning or end of the obs
to_remove = []
nearstart = [
ii for ii in subints_with_cal if ii < 0.1 * num_subints
]
nearend = [ii for ii in subints_with_cal if ii > 0.9 * num_subints]
others = [
ii for ii in subints_with_cal if ii not in nearstart + nearend
]
if len(nearend):
cal_start = min(nearend)
msg = "Cal-affected region is within 10%% of end of obs.\n" \
"Remove all rows within cal up-to end of obs.\n" \
"(cal start: %d; total num rows: %d)" % \
(cal_start, num_subints)
print msg
calrowsfile.write(msg + '\n')
numrows = num_subints - cal_start
rowdelcmds.append("fitsdelrow %s[SUBINT] %d %d" % \
(mergedfn, cal_start+1, numrows))
total_removed += numrows
if len(nearstart):
cal_end = max(nearstart)
msg = "Cal-affected region is within 10%% of start of obs.\n" \
"remove all rows up-to and including cal.\n" \
"(cal end: %d; total num rows: %d)" % \
(cal_end, num_subints)
print msg
calrowsfile.write(msg + '\n')
numrows = cal_end + 1 # Subints are counted starting with 0
rowdelcmds.append("fitsdelrow %s[SUBINT] %d %d" % \
(mergedfn, 1, numrows))
total_removed += numrows
if len(others):
msg = "Not removing rows in middle of obs " \
"(%s)." % ",".join(["%d" % ii for ii in sorted(others)])
print msg
calrowsfile.write(msg + '\n')
if rowdelcmds:
calrowsfile.write("Row-deletion commands:\n")
else:
calrowsfile.write("No rows to delete.\n")
for rowdelcmd in rowdelcmds:
calrowsfile.write(rowdelcmd + '\n')
pipeline_utils.execute(rowdelcmd)
msg = "Total number of rows removed: %d" % total_removed
print msg
calrowsfile.write(msg + '\n')
calrowsfile.close()
# Make dat file
prepdatacmd = "prepdata -noclip -nobary -dm 0 -o %s_post_DM0.00 %s" % (
outbasenm, mergedfn)
pipeline_utils.execute(prepdatacmd,
stdout=outbasenm + "_post_prepdata.out")
return [mergedfn]