def print_debug(msg, category, stepsback=1):
"""Print a debugging message if the given debugging category
is turned on.
The message will be colourized as 'debug'.
Inputs:
msg: The message to print.
category: The debugging category of the message.
stepsback: The number of steps back into the call stack
to get function calling information from.
(Default: 1).
Outputs:
None
"""
if debug.is_on(category):
if config.cfg.helpful_debugging:
# Get caller info
fn, lineno, funcnm = inspect.stack()[stepsback][1:4]
to_print = colour.cstring("DEBUG %s [%s:%d - %s(...)]:\n" %
(category.upper(),
os.path.split(fn)[-1], lineno,
funcnm), 'debughdr')
msg = msg.replace('\n', '\n ')
to_print += colour.cstring(" %s" % msg, 'debug')
else:
to_print = colour.cstring(msg, 'debug')
sys.stderr.write(to_print + '\n')
sys.stderr.flush()
def run_manipulator(prepped_manipfunc, infns, outname=None,
tmpdir=None):
"""Set up a temporary directory to run the manipulator
method in, load the archives, run the manipulator,
get the result of the manipulator, break down the
temporary directory and return the result of the
manipulation.
Inputs:
prepped_manipfunc: A prepared manipulator function to use.
infns: Names of the input files to be passed to the
manipulator
outname: File name of the manipulated archive.
tmpdir: Location of the temporary directory.
(Default: let python's 'tempfile' module
put the temp dir in a standard location.)
Outputs:
None
"""
workdir = tempfile.mkdtemp(dir=tmpdir, prefix='toaster_tmp',
suffix='_workdir')
newfns = []
for fn in infns:
newfn = os.path.join(workdir, os.path.split(fn)[-1])
shutil.copy(fn, newfn)
newfns.append(newfn)
try:
prepped_manipfunc(newfns, outname)
finally:
if debug.is_on('MANIPULATOR'):
notify.print_debug("Manipulator worked in %s. Not removing it.",
'manipulator')
else:
shutil.rmtree(workdir)
def get_toaster_engine(url=None):
"""Given a DB URL string return the corresponding DB engine.
Create the Engine object if necessary. If the engine
already exists return it rather than creating a new one.
Input:
url: A DB URL string.
Output:
engine: The corresponding DB engine.
"""
global engines
if url is None:
url = config.cfg.dburl
if url not in engines:
# Create the database engine
engine = sa.create_engine(config.cfg.dburl)
sa.event.listen(engine, "before_cursor_execute",
before_cursor_execute)
if debug.is_on('database'):
sa.event.listen(engine, "commit", commit_event)
sa.event.listen(engine, "rollback", rollback_event)
sa.event.listen(engine, "begin", begin_event)
engines[url] = engine
return engines[url]
def main(args):
# Check to make sure user provided a comment
if not args.dry_run and args.comments is None:
raise errors.BadInputError("A comment describing the timfile is "
"required!")
if args.from_file is not None:
# Re-create parser, so we can read arguments from file
parser = utils.DefaultArguments()
add_arguments(parser)
if args.from_file == '-':
argfile = sys.stdin
else:
if not os.path.exists(args.from_file):
raise errors.FileError("The list of cmd line args (%s) "
"does not exist." % args.from_file)
argfile = open(args.from_file, 'r')
for line in argfile:
# Strip comments
line = line.partition('#')[0].strip()
if not line:
# Skip empty line
continue
arglist = shlex.split(line.strip())
args = parser.parse_args(arglist, namespace=args)
# Establish a database connection
db = database.Database()
db.connect()
trans = db.begin()
try:
cmdline = " ".join(sys.argv)
toas = get_toas(args, db)
if debug.is_on('TIMFILE'):
# Check for / handle conflicts
conflict_handler = CONFLICT_HANDLERS[args.on_conflict]
toas = conflict_handler(toas)
wt.write_timfile(toas, {'comments': args.comments,
'user_id': cache.get_userid(),
'add_time': "Not in DB!",
'timfile_id': -1})
elif args.dry_run:
print_summary(toas, args.comments)
else:
conflict_handler = CONFLICT_HANDLERS[args.on_conflict]
timfile_id = add_timfile_entry(toas, cmdline, args.comments,
conflict_handler)
notify.print_info("Created new timfile entry - timfile_id=%d (%s)" %
(timfile_id, utils.give_utc_now()), 1)
except:
db.rollback()
db.close()
raise
else:
db.commit()
db.close()
def check_repos():
"""Check git repositories for the pipeline code, and for PSRCHIVE.
If the repos are dirty raise and error.
Inputs:
None
Outputs:
None
"""
if is_gitrepo_dirty(os.path.abspath(os.path.dirname(__file__))):
if debug.is_on('GITTEST'):
warnings.warn("Git repository is dirty! Will tolerate because "
"pipeline debugging is on.",
errors.ToasterWarning)
else:
raise errors.ToasterError("Pipeline's git repository is dirty. "
"Aborting!")
if not is_gitrepo(config.cfg.psrchive_dir):
warnings.warn("PSRCHIVE directory (%s) is not a git repository!" %
config.cfg.psrchive_dir, errors.ToasterWarning)
elif is_gitrepo_dirty(config.cfg.psrchive_dir):
raise errors.ToasterError("PSRCHIVE's git repository is dirty. "
"Clean up your act!")
def parse_timfile(timfn, reader=readers.tempo2_reader,
determine_obssystem=True,
get_telescope_id=True, **obssys_discovery_kwargs):
"""Read the input timfile and parse the TOAs contained.
Inputs:
timfn: The timfile to parse.
reader: The reader function. Reader functions take
a single line as input and return a dictionary
of TOA info. (Default: a Tempo2 TOA format reader)
determine_obssystem: Try to automatically discover
the observing system. (Default: True)
get_telescope_id: Query the database to get the telescope
ID number. This argument is passed on to the
TOA-line reader. (Default: True)
** Additional keyword arguments are directly passed on
to __determine_obssystem(...) for observing system
discovery
Output:
toas: A list of TOA info dictionaries.
"""
if not os.path.exists(timfn):
raise errors.FileError("The input timfile (%s) does not "
"appear to exist." % timfn)
if isinstance(reader, str):
# Assume reader is actually the name of the reader.
if reader not in READERS:
raise errors.UnrecognizedValueError("Requested reader (%s) is "
"not recognized!" % reader)
else:
reader = READERS[reader]
notify.print_info("Starting to parse timfile (%s)" % timfn, 2)
toas = []
timfile = open(timfn, 'r')
for ii, line in enumerate(timfile):
line = line.rstrip()
if line.startswith("INCLUDE"):
includefn = os.path.abspath(os.path.join(os.path.dirname(timfn),
line.split()[1]))
# Recursively parse included files
parsed = parse_timfile(includefn, reader=reader,
determine_obssystem=determine_obssystem,
get_telescope_id=get_telescope_id,
**obssys_discovery_kwargs)
utils.notify.print_info("Parsed %d TOAs from included file (%s)" %
(len(parsed), includefn), 1)
toas.extend(parsed)
try:
toainfo = reader(line, get_telescope_id=get_telescope_id)
if toainfo is not None:
if determine_obssystem:
toainfo['obssystem_id'] = __determine_obssystem(toainfo,
**obssys_discovery_kwargs)
toas.append(toainfo)
except Exception, e:
if debug.is_on('TOAPARSE'):
raise
raise errors.BadTOAFormat("Error occurred while parsing "
"TOA line (%s:%d):\n %s\n\n"
"Original exception message:\n %s" %
(timfn, ii+1, line, str(e)))
