def begin_event(conn):
"""An event to be executed when a transaction is opened.
See SQLAlchemy for details about event triggers.
"""
notify.print_debug("Opening database transaction.", 'database',
stepsback=7)
def run_manipulator(prepped_manipfunc, infns, outname=None,
tmpdir=None):
"""Set up a temporary directory to run the manipulator
method in, load the archives, run the manipulator,
get the result of the manipulator, break down the
temporary directory and return the result of the
manipulation.
Inputs:
prepped_manipfunc: A prepared manipulator function to use.
infns: Names of the input files to be passed to the
manipulator
outname: File name of the manipulated archive.
tmpdir: Location of the temporary directory.
(Default: let python's 'tempfile' module
put the temp dir in a standard location.)
Outputs:
None
"""
workdir = tempfile.mkdtemp(dir=tmpdir, prefix='toaster_tmp',
suffix='_workdir')
newfns = []
for fn in infns:
newfn = os.path.join(workdir, os.path.split(fn)[-1])
shutil.copy(fn, newfn)
newfns.append(newfn)
try:
prepped_manipfunc(newfns, outname)
finally:
if debug.is_on('MANIPULATOR'):
notify.print_debug("Manipulator worked in %s. Not removing it.",
'manipulator')
else:
shutil.rmtree(workdir)
def rollback_event(conn):
"""An event to be executed when a transaction is rolled back.
See SQLAlchemy for details about event triggers.
"""
notify.print_debug("Rolling back database transaction.", 'database',
stepsback=7)
def rollback(self):
"""Roll back the most recently opened transaction.
Inputs:
None
Outputs:
None
"""
notify.print_debug("Attempting to roll back a transaction via "
"database object", 'database', stepsback=2)
trans = self.open_transactions.pop()
trans.rollback()
def before_cursor_execute(conn, cursor, statement, parameters,
context, executemany):
"""An event to be executed before execution of SQL queries.
See SQLAlchemy for details about event triggers.
"""
# Step back 7 levels through the call stack to find
# the function that called 'execute'
msg = str(statement)
if executemany and len(parameters) > 1:
msg += "\n Executing %d statements" % len(parameters)
elif parameters:
msg += "\n Params: %s" % str(parameters)
notify.print_debug(msg, "queries", stepsback=7)
def parkes_reader(line, get_telescope_id=True):
"""Parse line, assuming it is a TOA in parkes format.
Return a dictionary of information.
Input:
line: A single TOA line in parkes format.
get_telescope_id: Query the database to get the telescope
ID number. (Default: True)
Output:
toainfo: A dictionary of TOA information.
"""
parkes_toa_re = re.compile(r'^ *?(?P<bad>(#|(C ))(?P<comment1>.*?))?'
r' (?P<info>.{24})(?P<freq>.{9})(?P<imjd>.{7})(?P<fmjd>\..{12}) '
r'(?P<phaseoffset>.{8}) (?P<err>.{7})(?P<info2>.{7}) '
r'(?P<site>.)(?P<dmcorr>[^#]*)')
comment_re = re.compile(r'#(?P<comment>.*)$')
match = parkes_toa_re.search(line.rstrip())
if match is None:
toainfo = None
notify.print_debug("Line is not a Parkes-format TOA:\n %s" % line, 'toaparse')
else:
grp = match.groupdict()
toainfo = {}
toainfo['is_bad'] = (grp['bad'] is not None)
toainfo['freq'] = float(grp['freq'])
toainfo['imjd'] = int(grp['imjd'])
toainfo['fmjd'] = float(grp['fmjd'])
toainfo['toa_unc_us'] = float(grp['err'])
toainfo['telescope'] = grp['site']
if get_telescope_id:
toainfo['telescope_id'] = cache.get_telescope_info(grp['site'])['telescope_id']
toainfo['extras'] = {'phaseoffset': float(grp['phaseoffset']),
'infostr': grp['info'].strip() + ' -- ' + grp['info2'].strip()}
if grp['dmcorr']:
toainfo['extras']['dmcorr'] = float(grp['dmcorr'])
comments = []
if grp['comment1']:
comments.append(grp['comment1'].strip())
match2 = comment_re.search(line[match.end():])
if match2:
grp2 = match2.groupdict()
if grp2['comment']:
comments.append(grp2['comment'].strip())
toainfo['comment'] = " -- ".join(comments)
return toainfo
def close(self):
"""Close the established connection.
Also, close any result set that may be present.
Inputs:
None
Outputs:
None
"""
if self.is_connected():
notify.print_debug("Database connection closed.", 'database',
stepsback=2)
self.conn.close()
if self.result is not None:
self.result.close()
def commit(self):
"""Commit the most recently opened transaction.
Inputs:
None
Outputs:
None
"""
notify.print_debug("Attempting to commit a transaction via "
"database object", 'database', stepsback=2)
if self.open_transactions:
trans = self.open_transactions.pop()
else:
raise errors.DatabaseError("Cannot commit. No open database transactions.")
trans.commit()
def begin(self):
"""Begin a transaction.
Inputs:
None
Outputs:
None
"""
notify.print_debug("Attempting to begin a transaction via "
"database object", 'database', stepsback=2)
if not self.is_connected():
raise errors.DatabaseError("Connection to database not "
"established. Be sure "
"self.connect(...) is called "
"before attempting to execute "
"queries.")
if self.open_transactions:
warnings.warn("A transaction already appears to be in progress.",
errors.ToasterWarning)
trans = self.conn.begin()
self.open_transactions.append(trans)
return trans
def connect(self):
"""Connect to the database, setting self.conn.
Inputs:
None
Output:
conn: The established SQLAlchemy Connection object,
which is also available as self.conn.
"""
# Only open a connection if not already connected
if not self.is_connected():
# Establish a connection
self.conn = self.engine.connect()
self.conn.execution_options(autocommit=self.autocommit)
self.open_transactions = []
self.result = None
if self.engine.dialect.name == 'sqlite':
result = self.execute("PRAGMA foreign_keys=ON")
result.close()
notify.print_debug("Database connection established.",
'database',
stepsback=2)
return self.conn
def tempo2_reader(line, get_telescope_id=True):
"""Parse line, assuming it is a TOA in tempo2 format.
Return a dictionary of information.
Input:
line: A single TOA line in Tempo2 format.
get_telescope_id: Query the database to get the telescope
ID number. (Default: True)
Output:
toainfo: A dictionary of TOA information.
"""
tempo2_toa_re = re.compile(r'^ *(?P<bad>(#|(C )|(c ))(?P<comment1>.*?))? *'
r'(?P<file>[^ ]+) +'
r'(?P<freq>\d+(\.\d+)?) +(?P<imjd>\d+)(?P<fmjd>\.\d+) +'
r'(?P<err>\d+(\.\d+)?) +(?P<site>[^ ]+)')
comment_re = re.compile(r'#(?P<comment>.*)$')
tempo2_flag_re = re.compile(r'-(?P<flagkey>[^ ]+) +(?P<flagval>[^ ]+)')
match = tempo2_toa_re.search(line)
if match is None:
toainfo = None
notify.print_debug("Line is not a Tempo2 TOA:\n %s" % line, 'toaparse')
else:
grp = match.groupdict()
toainfo = {}
toainfo['grp'] = grp
toainfo['is_bad'] = (line.strip().startswith('#') or line.strip().lower().startswith('c ')) #(grp['bad'] is not None)
toainfo['file'] = grp['file']
toainfo['freq'] = float(grp['freq'])
toainfo['imjd'] = int(grp['imjd'])
toainfo['fmjd'] = float(grp['fmjd'])
toainfo['toa_unc_us'] = float(grp['err'])
toainfo['telescope'] = grp['site']
toainfo['line'] = line
if get_telescope_id:
toainfo['telescope_id'] = cache.get_telescope_info(grp['site'])['telescope_id']
comments = []
if grp['comment1']:
comments.append(grp['comment1'].strip())
match2 = comment_re.search(line[match.end():])
if match2:
grp2 = match2.groupdict()
if grp2['comment']:
comments.append(grp2['comment'].strip())
toainfo['comment'] = " -- ".join(comments)
toainfo['extras'] = {}
for key, val in tempo2_flag_re.findall(line[match.end():]):
key = key.lower()
key = KNOWN_FLAG_ALIASES.get(key, key)
caster = KNOWN_FLAG_TYPES.get(key, str)
try:
toainfo['extras'][key] = caster(val.strip())
except:
notify.print_info("Couldn't cast %s:%s" % (key, val), 2)
notify.print_debug("TOA line: %s\nParsed info: %s" % (line, toainfo),
'toaparse')
return toainfo
def execute(cmd, stdout=subprocess.PIPE, stderr=sys.stderr,
execdir=None, stdinstr=None):
"""Execute the command 'cmd' after logging the command
to STDOUT. Execute the command in the directory 'execdir',
which defaults to the current directory is not provided.
Output standard output to 'stdout' and standard
error to 'stderr'. Both are strings containing filenames.
If values are None, the out/err streams are not recorded.
By default stdout is subprocess.PIPE and stderr is sent
to sys.stderr.
If stdinstr is not None, send the string to the command as
data in the stdin stream.
Returns (stdoutdata, stderrdata). These will both be None,
unless subprocess.PIPE is provided.
"""
# Log command to stdout
if execdir is not None:
msg = "(In %s)\n%s" % (execdir, str(cmd))
else:
msg = str(cmd)
notify.print_debug(msg, "syscalls", stepsback=2)
stdoutfile = False
stderrfile = False
if isinstance(stdout, str):
stdout = open(stdout, 'w')
stdoutfile = True
if isinstance(stderr, str):
stderr = open(stderr, 'w')
stderrfile = True
if stdinstr is not None:
notify.print_debug("Sending the following to cmd's stdin: %s" % stdinstr, \
"syscalls")
# Run (and time) the command. Check for errors.
pipe = subprocess.Popen(cmd, shell=False, cwd=execdir,
stdin=subprocess.PIPE,
stdout=stdout, stderr=stderr)
(stdoutdata, stderrdata) = pipe.communicate(stdinstr)
else:
# Run (and time) the command. Check for errors.
pipe = subprocess.Popen(cmd, shell=False, cwd=execdir,
stdout=stdout)#, stderr=stderr)
(stdoutdata, stderrdata) = pipe.communicate()
retcode = pipe.returncode
if retcode < 0:
raise errors.SystemCallError("Execution of command (%s) terminated by signal (%s)!" % \
(cmd, -retcode))
elif retcode > 0:
raise errors.SystemCallError("Execution of command (%s) failed with status (%s)!" % \
(cmd, retcode))
else:
# Exit code is 0, which is "Success". Do nothing.
pass
# Close file objects, if any
if stdoutfile:
stdout.close()
if stderrfile:
stderr.close()
return stdoutdata, stderrdata