def main():
# Collect input files
infiles = set(args.infiles)
for glob_expr in args.glob_exprs:
infiles.update(glob.glob(glob_expr))
infiles = list(infiles)
if not infiles:
sys.stderr.write("You didn't provide any files to load. " \
"You should consider including some next time...\n")
sys.exit(1)
# Enter information in rawfiles table
# create diagnostic plots and metrics.
# Also fill-in raw_diagnostics and raw_diagnostic_plots tables
for fn in infiles:
try:
if config.cfg.verbosity:
print "Checking %s (%s)" % (fn, utils.give_utc_now())
# Check the file and parse the header
params = utils.prep_file(fn)
# Find where the file will be moved to.
destdir = utils.get_archive_dir(fn, params=params)
utils.print_info("%s will get archived to %s (%s)" % \
(fn, destdir, utils.give_utc_now()), 1)
utils.print_info("Finished with %s - pre-check successful (%s)" % \
(fn, utils.give_utc_now()), 1)
except errors.ToasterError, msg:
sys.stderr.write("Pre-check of %s failed!\n%s\nSkipping...\n" % \
(fn, msg))
def _compute(self):
utils.print_info("Creating profile plot for %s" % self.fn, 3)
params = utils.prep_file(self.fn)
handle, tmpfn = tempfile.mkstemp(suffix=".png")
os.close(handle)
cmd = ["psrplot", "-p", "flux", "-j", "TDFp", "-c", \
"above:c=%s" % os.path.split(self.fn)[-1], \
"-D", "%s/PNG" % tmpfn, self.fn]
utils.execute(cmd)
tmpdir = os.path.split(tmpfn)[0]
pngfn = os.path.join(tmpdir, self.fn+".flux.png")
shutil.move(tmpfn, pngfn)
return pngfn
def _compute(self):
utils.print_info("Creating freq vs. phase plot for %s" % self.fn, 3)
params = utils.prep_file(self.fn)
if not (params['nchan'] > 1):
raise errors.DiagnosticNotApplicable("Archive (%s) only has " \
"a single channel. Freq vs. phase diagnostic " \
"doesn't apply to this data file." % self.fn)
handle, tmpfn = tempfile.mkstemp(suffix=".png")
os.close(handle)
cmd = ["psrplot", "-p", "freq", "-j", "DTp", "-c", \
"above:c=%s" % os.path.split(self.fn)[-1], \
"-D", "%s/PNG" % tmpfn, "%s" % self.fn]
utils.execute(cmd)
tmpdir = os.path.split(tmpfn)[0]
pngfn = os.path.join(tmpdir, self.fn+".freq.png")
shutil.move(tmpfn, pngfn)
return pngfn
def find_overlaps(rawfile, existdb=None):
"""Given the name of a raw data file find rawfile entries
in the database of the same pulsar/observing system
that overlap in time.
Inputs:
rawfile: The name of the raw file to compare database
entries with.
existdb: An (optional) existing database connection object.
(Default: Establish a db connection)
Output:
matches: A list of rawfile_id numbers of overlapping files.
"""
# Connect to the database
db = existdb or database.Database()
db.connect()
# Get info about the input raw file
params = utils.prep_file(rawfile)
mjdstart = params['mjd']
mjdend = mjdstart + params['length']/86400.0
select = db.select([db.rawfiles.c.rawfile_id, \
db.rawfiles.c.filepath, \
db.rawfiles.c.filename]).\
where((db.rawfiles.c.pulsar_id==params['pulsar_id']) & \
(db.rawfiles.c.obssystem_id==params['obssystem_id']) & \
((mjdstart < (db.rawfiles.c.mjd+(db.rawfiles.c.length/86400.0))) & \
(mjdend > db.rawfiles.c.mjd)))
result = db.execute(select)
rows = result.fetchall()
result.close()
if not existdb:
db.close()
return rows
def verify_replacement(obsolete_id, replacement, db):
"""Verify that the replacement file is a suitable replacement
for the obsolete file. The following conditions must be
satisfied:
- Both observations refer to the same pulsar
(i.e. same pulsar_id)
- Both observations come from the same observing system
(i.e. same obssystem_id)
- The observations overlap
(this is checked by comparing the start/end MJDs)
Inputs:
obsolete_id: The rawfile_id value of the file being replaced.
replacement: The name of the replacement file.
db: An connected database object.
Outputs:
None
"""
# Get info about the replacement file from its header
replaceparams = utils.prep_file(replacement)
# Get info about the obsolete file from database
select = db.select([db.rawfiles.c.pulsar_id, \
db.rawfiles.c.obssystem_id, \
db.rawfiles.c.mjd, \
db.rawfiles.c.length]).\
where(db.rawfiles.c.rawfile_id==obsolete_id)
result = db.execute(select)
rows = result.fetchall()
result.close()
if len(rows) > 1:
raise errors.InconsistentDatabaseError("Multiple matches (%s) " \
"for rawfile_id (%d)! This should not happen..." % \
(len(rows), obsolete_id))
elif len(rows) < 1:
raise errors.BadInputError("The rawfile_id provided (%d) does not " \
"exist!" % obsolete_id)
else:
obsoleteparams = rows[0]
# Now check that the obsolete file and its replacement are compatible
# Check the replacement is from the same obssystem
if obsoleteparams['obssystem_id'] != replaceparams['obssystem_id']:
raise errors.FileError("The observing system of the replacement " \
"(ID: %d) doesn't match the observing system of " \
"the file it's replacing (ID: %d)." % \
(obsoleteparams['obssystem_id'], \
replaceparams['obssystem_id']))
# Check the replacement is data on the same pulsar
if obsoleteparams['pulsar_id'] != replaceparams['pulsar_id']:
raise errors.FileError("The pulsar name in the replacement file " \
"(%s) doesn't match the pulsar name in the file it's "
"replacing (%s)." % \
(utils.get_pulsarname(obsoleteparams['pulsar_id']), \
utils.get_pulsarname(replaceparams['pulsar_id'])))
# Check the replacement overlaps with the obsolete file
omjdstart = obsoleteparams['mjd']
omjdend = omjdstart + obsoleteparams['length']/86400.0
rmjdstart = replaceparams['mjd']
rmjdend = rmjdstart + replaceparams['length']/86400.0
if (omjdstart >= rmjdend or omjdend <= rmjdstart):
raise errors.FileError("The replacement file (MJD: %f - %f) " \
"doesn't overlap with the file it is replacing " \
"(MJD: %f - %f)." % \
(rmjdstart, rmjdend, omjdstart, omjdend))