def main(args):
# Establish a DB connection
db = database.Database()
db.connect()
trans = db.begin()
try:
template = general.get_template_from_id(args.template_id, existdb=db)
# First remove the template entry from the DB
remove_template_entry(args.template_id)
# Now deal with the template itself
if args.action == "leave":
# Do nothing
pass
elif args.action == "move":
if not args.dest:
raise errors.BadInputError("Destination must be provided " "when moving template.")
notify.print_info("Moving template %s to %s" % (template, args.dest))
shutil.move(template, args.dest)
elif args.action == "delete":
notify.print_info("Deleting template %s" % template)
os.remove(template)
else:
raise errors.UnrecognizedValueError("The action provided (%s) " "is not recognized." % args.action)
except:
trans.rollback()
raise
else:
trans.commit()
finally:
db.close()
def pipeline_core(manip, rawfile_id, parfile_id, template_id,
existdb=None):
"""Run a prepared manipulator function on the raw file with
ID 'rawfile_id'. Then generate TOAs and load them into the DB.
Inputs:
manip: A manipulator instance.
rawfile_id: The ID number of the raw data file to generate TOAs from.
parfile_id: The ID number of the parfile to install into the
raw file. If this is None, then no new parfile will be installed.
template_id: The ID number of the template to use.
existdb: An existing database connection object.
(Default: establish a new DB connection)
Outputs:
None
"""
# Initialise these so the 'finally' clause doesn't throw an exception of
# it's own if an error is caught before these filenames are determined
manipfn = ''
adjustfn = ''
#Start pipeline
print "###################################################"
print "Starting to toast data"
print "Start time: %s" % utils.give_utc_now()
print "###################################################"
db = existdb or database.Database()
db.connect()
try:
trans = db.begin() # Open a transaction
# Get version ID
version_id = version.get_version_id(db)
# Get raw data from rawfile_id and verify MD5SUM
rawfile = rawfiles_general.get_rawfile_from_id(rawfile_id,
db, verify_md5=True)
# Manipulate the raw file
notify.print_info("Manipulating file", 1)
# Create a temporary file for the adjusted results
tmpfile, adjustfn = tempfile.mkstemp(prefix='toaster_tmp',
suffix='_newephem.ar',
dir=config.cfg.base_tmp_dir)
os.close(tmpfile)
shutil.copy(rawfile, adjustfn)
if parfile_id is not None:
# Re-install ephemeris
# Get ephemeris from parfile_id and verify MD5SUM
parfile = parfiles_general.get_parfile_from_id(parfile_id,
db, verify_md5=True)
cmd = ["pam", "-m", "-E", parfile, "--update_dm", adjustfn]
utils.execute(cmd)
# Create a temporary file for the manipulated results
tmpfile, manipfn = tempfile.mkstemp(prefix='toaster_tmp',
suffix='_manip.ar',
dir=config.cfg.base_tmp_dir)
os.close(tmpfile)
# Run the manipulator
manip.run([adjustfn], manipfn, tmpdir=config.cfg.base_tmp_dir)
# Get template from template_id and verify MD5SUM
template = templates_general.get_template_from_id(template_id,
db, verify_md5=True)
# Create a temporary file for the toa diagnostic plots
tmpfile, toadiagfn = tempfile.mkstemp(prefix='toaster_tmp',
suffix='_TOAdiag.png',
dir=config.cfg.base_tmp_dir)
os.close(tmpfile)
# Generate TOAs with pat
notify.print_info("Computing TOAs", 0)
cmd = ["pat", "-f", "tempo2", "-A", config.cfg.toa_fitting_method,
"-s", template, "-C", "gof length bw nbin nchan nsubint",
"-t", "-K", "%s/PNG" % toadiagfn, manipfn]
patout, paterr = utils.execute(cmd)
# Check version ID is still the same. Just in case.
new_version_id = version.get_version_id(db)
if version_id != new_version_id:
raise errors.ToasterError("Weird... Version ID at the start "
"of processing (%s) is different "
"from at the end (%d)!" %
(version_id, new_version_id))
# Read some header values from the manipulated archive
hdr = datafile.get_header_vals(manipfn, ['nchan', 'nsub', 'name',
'intmjd', 'fracmjd'])
hdr['secs'] = int(hdr['fracmjd']*24*3600+0.5) # Add 0.5 so result is
# rounded to nearest int
# Fill pipeline table
cmdline = " ".join(sys.argv)
process_id = fill_process_table(version_id, rawfile_id, parfile_id,
template_id, manip, hdr['nchan'],
hdr['nsub'], db)
# Parse pat output
toainfo = toas_general.parse_pat_output(patout)
rawfile_info = rawfiles_general.get_rawfile_info(rawfile_id)
# Insert TOAs into DB
for ti in toainfo:
ti['process_id'] = process_id
ti['template_id'] = template_id
ti['rawfile_id'] = rawfile_id
ti['pulsar_id'] = rawfile_info['pulsar_id']
ti['obssystem_id'] = rawfile_info['obssystem_id']
toa_ids = load_toa.load_toas(toainfo, db)
# Create processing diagnostics
notify.print_info("Generating processing diagnostics", 1)
diagdir = make_proc_diagnostics_dir(manipfn, process_id)
suffix = "_procid%d.%s" % (process_id, manip.name)
diags = []
for diagname in config.cfg.default_rawfile_diagnostics:
diagcls = diagnostics.get_diagnostic_class(diagname)
try:
diags.append(diagcls(manipfn))
except errors.DiagnosticNotApplicable, e:
notify.print_info("Diagnostic isn't applicable: %s. "
"Skipping..." % str(e), 1)
if diags:
# Load processing diagnostics
diagnose_processing.insert_processing_diagnostics(process_id,
diags, diagdir,
suffix, existdb=db)
# Copy TOA diagnostic plots and register them into DB
basefn = "%(name)s_%(intmjd)05d_%(secs)05d" % hdr
values = []
for ii, toa_id in enumerate(toa_ids):
outfn = basefn+"_procid%d.TOA%d.png" % (process_id, ii+1)
if ii == 0:
fn = toadiagfn
else:
fn = "%s_%d" % (toadiagfn, ii+1)
shutil.move(fn, os.path.join(diagdir, outfn))
ins = db.toa_diagnostic_plots.insert()
values.append({'toa_id': toa_id,
'filename': outfn,
'filepath': diagdir,
'plot_type': 'Prof-Temp Resids'})
result = db.execute(ins, values)
result.close()
notify.print_info("Inserted %d TOA diagnostic plots." % len(toa_ids), 2)