def __plot_nofreq(self, tmpfn, params):
info = self.__get_info(params)
cmd = [
"psrplot",
"-O",
"-j",
"D",
"-c",
"above:c=,x:range=0:2",
self.fn,
"-D",
"%s/PNG" % tmpfn,
"-p",
"flux",
"-c",
":0:x:view=0.075:0.95,"
"y:view=0.5:0.7,"
"subint=I,"
"chan=I,"
"pol=I,"
"x:opt=BCTS,"
"x:lab=,"
"below:l=,"
"%s" % info,
"-p",
"time",
"-c",
":1:x:view=0.075:0.95," "y:view=0.15:0.5," "chan=I," "pol=I," "cmap:map=plasma",
]
utils.execute(cmd)
def show_templates(templates):
if len(templates):
print "--"*25
for tdict in templates:
print colour.cstring("Template ID:", underline=True, bold=True) + \
colour.cstring(" %d" % tdict['template_id'], bold=True)
fn = os.path.join(tdict['filepath'], tdict['filename'])
print "\nTemplate: %s" % fn
print "Pulsar name: %s" % tdict['pulsar_name']
print "Master template? %s" % \
(((tdict['mtempid'] is not None) and "Yes") or "No")
print "Number of phase bins: %d" % tdict['nbin']
print "Uploaded by: %s (%s)" % (tdict['real_name'],
tdict['email_address'])
print "Uploader's comments: %s" % tdict['comments']
print "Date and time template was added: %s" % \
tdict['add_time'].isoformat(' ')
# Show extra information if verbosity is >= 1
lines = ["Observing System ID: %d" % tdict['obssystem_id'],
"Observing System Name: %s" % tdict['obssys_name'],
"Telescope: %s" % tdict['telescope_name'],
"Frontend: %s" % tdict['frontend'],
"Backend: %s" % tdict['backend'],
"Clock: %s" % tdict['clock']]
notify.print_info("\n".join(lines), 1)
try:
# Show the template if verbosity is >= 2
cmd = ["psrtxt", fn]
psrtxtout, stderr = utils.execute(cmd)
gnuplotcode = """set term dumb
set format y ""
set nokey
set border 1
set tics out
set xtics nomirror
set ytics 0,1,0
set xlabel "Phase Bin"
set xrange [0:%d]
plot "-" using 3:4 w l
%s
end
""" % (tdict.nbin-1, psrtxtout)
plot, stderr = utils.execute(["gnuplot"],
stderr=open(os.devnull),
stdinstr=gnuplotcode)
notify.print_info(plot, 2)
except errors.SystemCallError:
# gnuplot is probably not installed
pass
print "--"*25
else:
raise errors.ToasterError("No templates match parameters provided!")
def parse_psrfits_header(fn, hdritems):
"""Get a set of header params from the given file.
Returns a dictionary.
Inputs:
fn: The name of the file to get params for.
hdritems: List of parameter names to fetch.
Output:
params: A dictionary. The keys are values requested from 'psredit'
the values are the values reported by 'psredit'.
"""
hdrstr = ",".join(hdritems)
if '=' in hdrstr:
raise ValueError("'hdritems' passed to 'parse_psrfits_header' "
"should not perform and assignments!")
cmd = ["psredit", "-q", "-Q", "-c", hdrstr, fn]
outstr, errstr = utils.execute(cmd)
outvals = outstr.split()
if errstr:
raise errors.SystemCallError("The command: %s\nprinted to stderr:\n%s" %
(cmd, errstr))
elif len(outvals) != len(hdritems):
raise errors.SystemCallError("The command: %s\nreturn the wrong "
"number of values. (Was expecting %d, got %d.)" %
(cmd, len(hdritems), len(outvals)))
params = {}
for key, val in zip(hdritems, outstr.split()):
params[key] = val
return params
def _compute(self):
notify.print_info("Calling psrstat to get weights for %s" % self.fn, 3)
cmd = ["psrstat", self.fn, "-c", "int:wt", "-Qq"]
outstr, errstr = utils.execute(cmd)
wtstrs = outstr.strip().split(',')
weights = np.array([float(wt) for wt in wtstrs])
maskpcnt = 100.0*np.sum(weights > 0)/weights.size
return maskpcnt
def __plot_profonly(self, tmpfn, params):
info = self.__get_info(params)
cmd = [
"psrplot",
"-O",
"-j",
"D",
"-c",
"above:c=,x:range=0:2",
self.fn,
"-D",
"%s/PNG" % tmpfn,
"-p",
"flux",
"-c",
":0:x:view=0.075:0.95," "y:view=0.15:0.7," "subint=I," "chan=I," "pol=I," "below:l=," "%s" % info,
]
utils.execute(cmd)
def get_githash(repodir):
"""Get the pipeline's git hash.
Inputs:
repodir: The location of the git repository.
Output:
githash: The githash
"""
if is_gitrepo_dirty(repodir):
warnings.warn("Git repository has uncommitted changes!", \
errors.ToasterWarning)
cmd = ["git", "rev-parse", "HEAD"]
stdout, stderr = utils.execute(cmd, execdir=repodir)
githash = stdout.strip()
return githash
def get_header_vals(fn, hdritems):
"""Get a set of header params from the given file.
Returns a dictionary.
Inputs:
fn: The name of the file to get params for.
hdritems: List of parameters (recognized by vap) to fetch.
Output:
params: A dictionary. The keys are values requested from 'vap'
the values are the values reported by 'vap'.
"""
if not len(hdritems):
raise ValueError("No 'hdritems' requested to get from file header!")
hdrstr = ",".join(hdritems)
if '=' in hdrstr:
raise ValueError("'hdritems' passed to 'get_header_vals' "
"should not perform and assignments!")
cmd = ["vap", "-n", "-c", hdrstr, fn]
outstr, errstr = utils.execute(cmd)
outvals = outstr.split()[(0 - len(hdritems)):] # First value is filename (we don't need it)
if errstr:
raise errors.SystemCallError("The command: %s\nprinted to stderr:\n%s" %
(cmd, errstr))
elif len(outvals) != len(hdritems):
raise errors.SystemCallError("The command: %s\nreturned the wrong "
"number of values. (Was expecting %d, got %d.)" %
(cmd, len(hdritems), len(outvals)))
params = HeaderParams(fn)
for key, val in zip(hdritems, outvals):
if val == "INVALID":
raise errors.SystemCallError("The vap header key '%s' "
"is invalid!" % key)
elif val == "*" or val == "UNDEF":
warnings.warn("The vap header key '%s' is not "
"defined in this file (%s)" % (key, fn),
errors.ToasterWarning)
params[key] = None
else:
# Get param's type to cast value
caster = header_param_types.get(key, str)
params[key] = caster(val)
return params
def is_gitrepo_dirty(repodir):
"""Return True if the git repository has local changes.
Inputs:
repodir: The location of the git repository.
Output:
is_dirty: True if git repository has local changes. False otherwise.
"""
notify.print_info("Checking if Git repo at '%s' is dirty..." % repodir, 2)
try:
cmd = ["git", "diff", "--quiet"]
stdout, stderr = utils.execute(cmd, execdir=repodir)
except errors.SystemCallError:
# Exit code is non-zero
return True
else:
# Success error code (i.e. no differences)
return False
def is_gitrepo(repodir):
"""Return True if the given dir is a git repository.
Input:
repodir: The location of the git repository.
Output:
is_git: True if directory is part of a git repository. False otherwise.
"""
notify.print_info("Checking if directory '%s' contains a Git repo..." % repodir, 2)
try:
cmd = ["git", "rev-parse"]
stdout, stderr = utils.execute(cmd, execdir=repodir, \
stderr=open(os.devnull))
except errors.SystemCallError:
# Exit code is non-zero
return False
else:
# Success error code (i.e. dir is in a git repo)
return True
def pipeline_core(manip, rawfile_id, parfile_id, template_id,
existdb=None):
"""Run a prepared manipulator function on the raw file with
ID 'rawfile_id'. Then generate TOAs and load them into the DB.
Inputs:
manip: A manipulator instance.
rawfile_id: The ID number of the raw data file to generate TOAs from.
parfile_id: The ID number of the parfile to install into the
raw file. If this is None, then no new parfile will be installed.
template_id: The ID number of the template to use.
existdb: An existing database connection object.
(Default: establish a new DB connection)
Outputs:
None
"""
# Initialise these so the 'finally' clause doesn't throw an exception of
# it's own if an error is caught before these filenames are determined
manipfn = ''
adjustfn = ''
#Start pipeline
print "###################################################"
print "Starting to toast data"
print "Start time: %s" % utils.give_utc_now()
print "###################################################"
db = existdb or database.Database()
db.connect()
try:
trans = db.begin() # Open a transaction
# Get version ID
version_id = version.get_version_id(db)
# Get raw data from rawfile_id and verify MD5SUM
rawfile = rawfiles_general.get_rawfile_from_id(rawfile_id,
db, verify_md5=True)
# Manipulate the raw file
notify.print_info("Manipulating file", 1)
# Create a temporary file for the adjusted results
tmpfile, adjustfn = tempfile.mkstemp(prefix='toaster_tmp',
suffix='_newephem.ar',
dir=config.cfg.base_tmp_dir)
os.close(tmpfile)
shutil.copy(rawfile, adjustfn)
if parfile_id is not None:
# Re-install ephemeris
# Get ephemeris from parfile_id and verify MD5SUM
parfile = parfiles_general.get_parfile_from_id(parfile_id,
db, verify_md5=True)
cmd = ["pam", "-m", "-E", parfile, "--update_dm", adjustfn]
utils.execute(cmd)
# Create a temporary file for the manipulated results
tmpfile, manipfn = tempfile.mkstemp(prefix='toaster_tmp',
suffix='_manip.ar',
dir=config.cfg.base_tmp_dir)
os.close(tmpfile)
# Run the manipulator
manip.run([adjustfn], manipfn, tmpdir=config.cfg.base_tmp_dir)
# Get template from template_id and verify MD5SUM
template = templates_general.get_template_from_id(template_id,
db, verify_md5=True)
# Create a temporary file for the toa diagnostic plots
tmpfile, toadiagfn = tempfile.mkstemp(prefix='toaster_tmp',
suffix='_TOAdiag.png',
dir=config.cfg.base_tmp_dir)
os.close(tmpfile)
# Generate TOAs with pat
notify.print_info("Computing TOAs", 0)
cmd = ["pat", "-f", "tempo2", "-A", config.cfg.toa_fitting_method,
"-s", template, "-C", "gof length bw nbin nchan nsubint",
"-t", "-K", "%s/PNG" % toadiagfn, manipfn]
patout, paterr = utils.execute(cmd)
# Check version ID is still the same. Just in case.
new_version_id = version.get_version_id(db)
if version_id != new_version_id:
raise errors.ToasterError("Weird... Version ID at the start "
"of processing (%s) is different "
"from at the end (%d)!" %
(version_id, new_version_id))
# Read some header values from the manipulated archive
hdr = datafile.get_header_vals(manipfn, ['nchan', 'nsub', 'name',
'intmjd', 'fracmjd'])
hdr['secs'] = int(hdr['fracmjd']*24*3600+0.5) # Add 0.5 so result is
# rounded to nearest int
# Fill pipeline table
cmdline = " ".join(sys.argv)
process_id = fill_process_table(version_id, rawfile_id, parfile_id,
template_id, manip, hdr['nchan'],
hdr['nsub'], db)
# Parse pat output
toainfo = toas_general.parse_pat_output(patout)
rawfile_info = rawfiles_general.get_rawfile_info(rawfile_id)
# Insert TOAs into DB
for ti in toainfo:
ti['process_id'] = process_id
ti['template_id'] = template_id
ti['rawfile_id'] = rawfile_id
ti['pulsar_id'] = rawfile_info['pulsar_id']
ti['obssystem_id'] = rawfile_info['obssystem_id']
toa_ids = load_toa.load_toas(toainfo, db)
# Create processing diagnostics
notify.print_info("Generating processing diagnostics", 1)
diagdir = make_proc_diagnostics_dir(manipfn, process_id)
suffix = "_procid%d.%s" % (process_id, manip.name)
diags = []
for diagname in config.cfg.default_rawfile_diagnostics:
diagcls = diagnostics.get_diagnostic_class(diagname)
try:
diags.append(diagcls(manipfn))
except errors.DiagnosticNotApplicable, e:
notify.print_info("Diagnostic isn't applicable: %s. "
"Skipping..." % str(e), 1)
if diags:
# Load processing diagnostics
diagnose_processing.insert_processing_diagnostics(process_id,
diags, diagdir,
suffix, existdb=db)
# Copy TOA diagnostic plots and register them into DB
basefn = "%(name)s_%(intmjd)05d_%(secs)05d" % hdr
values = []
for ii, toa_id in enumerate(toa_ids):
outfn = basefn+"_procid%d.TOA%d.png" % (process_id, ii+1)
if ii == 0:
fn = toadiagfn
else:
fn = "%s_%d" % (toadiagfn, ii+1)
shutil.move(fn, os.path.join(diagdir, outfn))
ins = db.toa_diagnostic_plots.insert()
values.append({'toa_id': toa_id,
'filename': outfn,
'filepath': diagdir,
'plot_type': 'Prof-Temp Resids'})
result = db.execute(ins, values)
result.close()
notify.print_info("Inserted %d TOA diagnostic plots." % len(toa_ids), 2)
def _compute(self):
notify.print_info("Calling psrstat to get SNR for %s" % self.fn, 3)
cmd = ["psrstat", "-Qq", "-j", "DTFp", "-c", "snr", self.fn]
outstr, errstr = utils.execute(cmd)
snr = float(outstr)
return snr
def get_version_id(existdb=None):
"""Get the pipeline version number.
If the version number isn't in the database, add it.
Input:
existdb: A (optional) existing database connection object.
(Default: Establish a db connection)
Output:
version_id: The version ID for the current pipeline/psrchive
combination.
"""
# Check to make sure the repositories are clean
check_repos()
# Get git hashes
pipeline_githash = get_githash(os.path.dirname(__file__))
if is_gitrepo(config.cfg.psrchive_dir):
psrchive_githash = get_githash(config.cfg.psrchive_dir)
else:
warnings.warn("PSRCHIVE directory (%s) is not a git repository! " \
"Falling back to 'psrchive --version' for version " \
"information." % config.cfg.psrchive_dir, \
errors.ToasterWarning)
cmd = ["psrchive", "--version"]
stdout, stderr = utils.execute(cmd)
psrchive_githash = stdout.strip()
# Use the exisitng DB connection, or open a new one if None was provided
db = existdb or database.Database()
db.connect()
db.begin() # open a transaction
# Check to see if this combination of versions is in the database
select = db.select([db.versions.c.version_id]).\
where((db.versions.c.pipeline_githash==pipeline_githash) & \
(db.versions.c.psrchive_githash==psrchive_githash) & \
(db.versions.c.tempo2_cvsrevno=='Not available'))
result = db.execute(select)
rows = result.fetchall()
result.close()
if len(rows) > 1:
db.rollback()
if not existdb:
# Close the DB connection we opened
db.close()
raise errors.DatabaseError("There are too many (%d) matching " \
"version IDs" % len(rows))
elif len(rows) == 1:
version_id = rows[0].version_id
else:
# Insert the current versions
ins = db.versions.insert()
values = {'pipeline_githash':pipeline_githash, \
'psrchive_githash':psrchive_githash, \
'tempo2_cvsrevno':'Not available'}
result = db.execute(ins, values)
# Get the newly add version ID
version_id = result.inserted_primary_key[0]
result.close()
db.commit()
if not existdb:
# Close the DB connection we opened
db.close()
return version_id
