def load_rawfile(fn, existdb=None):
# Connect to the database
db = existdb or database.Database()
db.connect()
try:
# Enter information in rawfiles table
notify.print_info("Working on %s (%s)" % (fn, utils.give_utc_now()), 1)
# Check the file and parse the header
params = datafile.prep_file(fn)
# Move the File
destdir = datafile.get_archive_dir(fn, params=params)
newfn = datafile.archive_file(fn, destdir)
notify.print_info("%s moved to %s (%s)" % (fn, newfn,
utils.give_utc_now()), 1)
# Register the file into the database
rawfile_id = populate_rawfiles_table(db, newfn, params)
notify.print_info("Successfully loaded %s - rawfile_id=%d (%s)" %
(fn, rawfile_id, utils.give_utc_now()), 1)
finally:
if not existdb:
# Close DB connection
db.close()
return rawfile_id
def __insert_rawfile_diagnostic_plot(rawfile_id, diag, existdb=None):
"""Insert rawfile plot diagnostic.
Inputs:
rawfile_id: The ID number of the raw file for which to
insert the diagnostic.
diag: A FloatDiagnostic object.
existdb: An (optional) existing database connection object.
(Default: Establish a db connection)
Outputs:
None
"""
# Connect to the database
db = existdb or database.Database()
db.connect()
diagpath = None # Initialise diagplot in case an exception is raised
try:
# Put diagnostic plot next to the data file
diagplot = os.path.abspath(diag.diagnostic)
archivedir = os.path.split(os.path.abspath(diag.fn))[0]
diagpath = datafile.archive_file(diagplot, archivedir)
diagdir, diagfn = os.path.split(os.path.abspath(diagpath))
ins = db.raw_diagnostic_plots.insert()
values = {'rawfile_id':rawfile_id, \
'filename':diagfn, \
'filepath':diagdir, \
'plot_type':diag.name}
result = db.execute(ins, values)
diag_id = result.inserted_primary_key[0]
result.close()
notify.print_info("Inserted rawfile diagnostic plot (type: %s)." % \
diag.name, 2)
except:
# Move the diagnostic plot back if it has already been archived.
if diagpath and os.path.isfile(diagpath):
shutil.move(diagpath, diagplot)
raise
finally:
if not existdb:
# Close DB connection
db.close()
return diag_id
def load_template(fn, comments, is_master=False, existdb=None):
# Connect to the database
db = existdb or database.Database()
db.connect()
try:
# Now load the template file into database
notify.print_info("Working on %s (%s)" % (fn, utils.give_utc_now()), 1)
# Check the template and parse the header
params = datafile.prep_file(fn)
# Move the file
destdir = datafile.get_archive_dir(fn, params=params)
newfn = datafile.archive_file(fn, destdir)
# Register the template into the database
template_id = populate_templates_table(db, newfn, params,
comments=comments)
mastertemp_id, tempfn = general.get_master_template(params['pulsar_id'],
params['obssystem_id'])
if mastertemp_id is None:
# If this is the only template for this pulsar
# make sure it will be set as the master
is_master = True
if is_master:
notify.print_info("Setting %s as master template (%s)" %
(newfn, utils.give_utc_now()), 1)
general.set_as_master_template(template_id, db)
notify.print_info("Finished with %s - template_id=%d (%s)" %
(fn, template_id, utils.give_utc_now()), 1)
finally:
if not existdb:
# Close DB connection
db.close()
return template_id
def load_parfile(fn, is_master=False, existdb=None):
# Connect to the database
db = existdb or database.Database()
db.connect()
try:
# Now load the parfile file into database
notify.print_info("Working on %s (%s)" % (fn, utils.give_utc_now()), 1)
# Check the parfile and parse it
params = general.prep_parfile(fn)
# Archive the parfile
destdir = os.path.join(config.cfg.data_archive_location,
'parfiles', params['name'])
newfn = datafile.archive_file(fn, destdir)
# Register the parfile into the database
parfile_id = populate_parfiles_table(db, newfn, params)
masterpar_id, parfn = general.get_master_parfile(params['pulsar_id'])
if masterpar_id is None:
# If this is the only parfile for this pulsar
# make sure it will be set as the master
is_master = True
if is_master:
notify.print_info("Setting %s as master parfile (%s)" %
(newfn, utils.give_utc_now()), 1)
general.set_as_master_parfile(parfile_id, db)
notify.print_info("Finished with %s - parfile_id=%d (%s)" %
(fn, parfile_id, utils.give_utc_now()), 1)
finally:
if not existdb:
# Close DB connection
db.close()
return parfile_id
def __insert_processing_diagnostic_plot(proc_id, diag, archivedir=None,
suffix="", existdb=None):
"""Insert processing plot diagnostic.
Inputs:
proc_id: The ID number of the processing job for which
the diagnostic describes.
diag: A PlotDiagnostic object.
archivedir: The location where diagnostic plots should be
archived. (Default: put diagnostic plots in same directory
as the input file.)
suffix: Add a suffix just before the extension of diagnostic
plots' filenames. (Default: Do not insert a suffix)
existdb: An (optional) existing database connection object.
(Default: Establish a db connection)
Outputs:
None
"""
# Connect to the database
db = existdb or database.Database()
db.connect()
diagpath = None # Initialise diagpath in case an exception is raised
try:
# Put diagnostic plot next to the data file
origfn = os.path.abspath(diag.diagnostic)
if archivedir is None:
archivedir = os.path.split(os.path.abspath(diag.fn))[0]
if suffix:
# Rename file
stem, ext = os.path.splitext(origfn)
newfn = stem+suffix+ext
if os.path.dirname(newfn) != os.path.dirname(origfn):
raise errors.FileError("Adding processing diagnostic "
"plot suffix will cause plot to "
"be moved to annother directory "
"(new file name: %s)!" % newfn)
shutil.move(origfn, newfn)
diagplot = newfn
else:
diagplot = origfn
diagpath = datafile.archive_file(diagplot, archivedir)
diagdir, diagfn = os.path.split(os.path.abspath(diagpath))
ins = db.proc_diagnostic_plots.insert()
values = {'process_id': proc_id,
'filename': diagfn,
'filepath': diagdir,
'plot_type': diag.name}
result = db.execute(ins, values)
diag_id = result.inserted_primary_key[0]
result.close()
notify.print_info("Inserted process diagnostic plot (type: %s)." %
diag.name, 2)
except:
# Move the diagnostic plot back if it has already been archived.
if diagpath and os.path.isfile(diagpath):
shutil.move(diagpath, origfn)
raise
finally:
if not existdb:
# Close DB connection
db.close()
return diag_id
