def __call__(self, parser, namespace, values, option_string):
colour.cprint("Available Diagnostics:", \
bold=True, underline=True)
descwrapper = textwrap.TextWrapper(initial_indent=" ",
subsequent_indent=" ")
for key in sorted(diagnostics.registered_diagnostics):
diagcls = diagnostics.get_diagnostic_class(key)
wrapper = textwrap.TextWrapper(subsequent_indent=" "*(len(key)+4))
print "%s -- %s" % (colour.cstring(key, bold=True),
wrapper.fill(diagcls.name))
if diagcls.description is not None:
print descwrapper.fill(diagcls.description)
sys.exit(1)
def diagnose_rawfile(rawfile_id, diagnostic, value=None, existdb=None):
"""Diagnose a rawfile (specified by its ID number).
Inputs:
rawfile_id: The ID number of the rawfile to diagnose.
diagnostic: The name of the diagnostic.
value: The value of the diagnostic.
- If the value provided is None, try to compute
a pre-defined diagnostic.
- If the value provided is a string, assume it
is a plot-diagnostic.
- If the value provided is a floating-point
number, assume it is a numeric-diagnostic.
existdb: An (optional) existing database connection object.
(Default: Establish a db connection)
Outputs:
diag: The Diagnostic object.
"""
db = existdb or database.Database()
db.connect()
fn = utils.get_rawfile_from_id(rawfile_id, existdb=db)
if value is None:
# Pre-defined diagnostic
diagcls = diagnostics.get_diagnostic_class(diagnostic)
# Pre-check if a diagnostic with this name already exists for
# the rawfile provided
check_rawfile_diagnostic_existence(rawfile_id, diagcls.name, \
existdb=db)
diag = diagcls(fn)
elif type(value) == types.FloatType:
# Numeric diagnostic
notify.print_info("Custom floating-point rawfile diagnostic provided", 2)
diag = diagnostics.get_custom_float_diagnostic(fn, diagnostic, \
value)
else:
# Plot diagnostic
notify.print_info("Custom rawfile diagnostic plot provided", 2)
diag = diagnostics.get_custom_diagnostic_plot(fn, diagnostic, \
value)
if existdb is None:
db.close()
return diag
def pipeline_core(manip, rawfile_id, parfile_id, template_id,
existdb=None):
"""Run a prepared manipulator function on the raw file with
ID 'rawfile_id'. Then generate TOAs and load them into the DB.
Inputs:
manip: A manipulator instance.
rawfile_id: The ID number of the raw data file to generate TOAs from.
parfile_id: The ID number of the parfile to install into the
raw file. If this is None, then no new parfile will be installed.
template_id: The ID number of the template to use.
existdb: An existing database connection object.
(Default: establish a new DB connection)
Outputs:
None
"""
# Initialise these so the 'finally' clause doesn't throw an exception of
# it's own if an error is caught before these filenames are determined
manipfn = ''
adjustfn = ''
#Start pipeline
print "###################################################"
print "Starting to toast data"
print "Start time: %s" % utils.give_utc_now()
print "###################################################"
db = existdb or database.Database()
db.connect()
try:
trans = db.begin() # Open a transaction
# Get version ID
version_id = version.get_version_id(db)
# Get raw data from rawfile_id and verify MD5SUM
rawfile = rawfiles_general.get_rawfile_from_id(rawfile_id,
db, verify_md5=True)
# Manipulate the raw file
notify.print_info("Manipulating file", 1)
# Create a temporary file for the adjusted results
tmpfile, adjustfn = tempfile.mkstemp(prefix='toaster_tmp',
suffix='_newephem.ar',
dir=config.cfg.base_tmp_dir)
os.close(tmpfile)
shutil.copy(rawfile, adjustfn)
if parfile_id is not None:
# Re-install ephemeris
# Get ephemeris from parfile_id and verify MD5SUM
parfile = parfiles_general.get_parfile_from_id(parfile_id,
db, verify_md5=True)
cmd = ["pam", "-m", "-E", parfile, "--update_dm", adjustfn]
utils.execute(cmd)
# Create a temporary file for the manipulated results
tmpfile, manipfn = tempfile.mkstemp(prefix='toaster_tmp',
suffix='_manip.ar',
dir=config.cfg.base_tmp_dir)
os.close(tmpfile)
# Run the manipulator
manip.run([adjustfn], manipfn, tmpdir=config.cfg.base_tmp_dir)
# Get template from template_id and verify MD5SUM
template = templates_general.get_template_from_id(template_id,
db, verify_md5=True)
# Create a temporary file for the toa diagnostic plots
tmpfile, toadiagfn = tempfile.mkstemp(prefix='toaster_tmp',
suffix='_TOAdiag.png',
dir=config.cfg.base_tmp_dir)
os.close(tmpfile)
# Generate TOAs with pat
notify.print_info("Computing TOAs", 0)
cmd = ["pat", "-f", "tempo2", "-A", config.cfg.toa_fitting_method,
"-s", template, "-C", "gof length bw nbin nchan nsubint",
"-t", "-K", "%s/PNG" % toadiagfn, manipfn]
patout, paterr = utils.execute(cmd)
# Check version ID is still the same. Just in case.
new_version_id = version.get_version_id(db)
if version_id != new_version_id:
raise errors.ToasterError("Weird... Version ID at the start "
"of processing (%s) is different "
"from at the end (%d)!" %
(version_id, new_version_id))
# Read some header values from the manipulated archive
hdr = datafile.get_header_vals(manipfn, ['nchan', 'nsub', 'name',
'intmjd', 'fracmjd'])
hdr['secs'] = int(hdr['fracmjd']*24*3600+0.5) # Add 0.5 so result is
# rounded to nearest int
# Fill pipeline table
cmdline = " ".join(sys.argv)
process_id = fill_process_table(version_id, rawfile_id, parfile_id,
template_id, manip, hdr['nchan'],
hdr['nsub'], db)
# Parse pat output
toainfo = toas_general.parse_pat_output(patout)
rawfile_info = rawfiles_general.get_rawfile_info(rawfile_id)
# Insert TOAs into DB
for ti in toainfo:
ti['process_id'] = process_id
ti['template_id'] = template_id
ti['rawfile_id'] = rawfile_id
ti['pulsar_id'] = rawfile_info['pulsar_id']
ti['obssystem_id'] = rawfile_info['obssystem_id']
toa_ids = load_toa.load_toas(toainfo, db)
# Create processing diagnostics
notify.print_info("Generating processing diagnostics", 1)
diagdir = make_proc_diagnostics_dir(manipfn, process_id)
suffix = "_procid%d.%s" % (process_id, manip.name)
diags = []
for diagname in config.cfg.default_rawfile_diagnostics:
diagcls = diagnostics.get_diagnostic_class(diagname)
try:
diags.append(diagcls(manipfn))
except errors.DiagnosticNotApplicable, e:
notify.print_info("Diagnostic isn't applicable: %s. "
"Skipping..." % str(e), 1)
if diags:
# Load processing diagnostics
diagnose_processing.insert_processing_diagnostics(process_id,
diags, diagdir,
suffix, existdb=db)
# Copy TOA diagnostic plots and register them into DB
basefn = "%(name)s_%(intmjd)05d_%(secs)05d" % hdr
values = []
for ii, toa_id in enumerate(toa_ids):
outfn = basefn+"_procid%d.TOA%d.png" % (process_id, ii+1)
if ii == 0:
fn = toadiagfn
else:
fn = "%s_%d" % (toadiagfn, ii+1)
shutil.move(fn, os.path.join(diagdir, outfn))
ins = db.toa_diagnostic_plots.insert()
values.append({'toa_id': toa_id,
'filename': outfn,
'filepath': diagdir,
'plot_type': 'Prof-Temp Resids'})
result = db.execute(ins, values)
result.close()
notify.print_info("Inserted %d TOA diagnostic plots." % len(toa_ids), 2)
def populate_rawfiles_table(db, archivefn, params):
# md5sum helper function in utils
md5 = datafile.get_md5sum(archivefn)
path, fn = os.path.split(os.path.abspath(archivefn))
size = os.path.getsize(archivefn) # File size in bytes
trans = db.begin()
# Does this file exist already?
select = db.select([db.rawfiles.c.rawfile_id,
db.rawfiles.c.pulsar_id]).\
where(db.rawfiles.c.md5sum == md5)
result = db.execute(select)
rows = result.fetchall()
result.close()
if len(rows) > 1:
trans.rollback()
raise errors.InconsistentDatabaseError("There are %d rawfiles "
"with MD5 (%s) in the "
"database already" %
(len(rows), md5))
elif len(rows) == 1:
rawfile_id = rows[0]['rawfile_id']
psr_id = rows[0]['pulsar_id']
if psr_id == params['pulsar_id']:
warnings.warn("A rawfile with this MD5 (%s) already exists "
"in the DB for this pulsar (ID: %d). "
"The file will not be re-registed into the DB. "
"Doing nothing..." % (md5, psr_id),
errors.ToasterWarning)
trans.commit()
return rawfile_id
else:
trans.rollback()
raise errors.InconsistentDatabaseError("A rawfile with this "
"MD5 (%s) already exists "
"in the DB, but for "
"a different pulsar "
"(ID: %d)!" %
(md5, psr_id))
else:
notify.print_info("Inserting rawfile (%s) into DB." % fn, 3)
# Based on its MD5, this rawfile doesn't already
# exist in the DB. Insert it.
# Insert the file
ins = db.rawfiles.insert()
values = {'md5sum': md5,
'filename': fn,
'filepath': path,
'filesize': size,
'coord': '%s,%s' % (params['ra'], params['dec'])}
values.update(params)
result = db.execute(ins, values)
rawfile_id = result.inserted_primary_key[0]
result.close()
# Create rawfile diagnostics
diags = []
for diagname in config.cfg.default_rawfile_diagnostics:
diagcls = diagnostics.get_diagnostic_class(diagname)
try:
diags.append(diagcls(archivefn))
except errors.DiagnosticNotApplicable, e:
notify.print_info("Diagnostic isn't applicable: %s. "
"Skipping..." % str(e), 1)
if diags:
# Load processing diagnostics
diagnose_rawfile.insert_rawfile_diagnostics(rawfile_id, diags,
existdb=db)