def etransfer_send_output(job_id, dry_run=False, force=False):
"""High level e-transfer function for use from scripts.
This function makes some basic checks and then launches
the private function _etransfer_send under the control
of the ErrorDecorator so that any subsequent errors
are captured.
"""
logger.debug('Preparing to e-transfer output for job {0}'.format(job_id))
# When not in dry run mode, check that etransfer is being
# run on the correct machine by the correct user and with
# sufficient available disk space.
if not dry_run:
etransfer_check_config()
_etransfer_check_space()
logger.debug('Connecting to JSA processing database')
db = get_database()
if not force:
job = db.get_job(id_=job_id)
if job.state != JSAProcState.PROCESSED:
message = 'Job {0} cannot be e-transferred as it is in ' \
'state {1}'.format(job_id,
JSAProcState.get_name(job.state))
logger.error(message)
raise CommandError(message)
_etransfer_send(job_id, dry_run=dry_run, db=db, force=force)
logger.debug('Done adding output for job {0} to e-transfer'.format(job_id))
def etransfer_send_output(job_id, dry_run=False, force=False):
"""High level e-transfer function for use from scripts.
This function makes some basic checks and then launches
the private function _etransfer_send under the control
of the ErrorDecorator so that any subsequent errors
are captured.
"""
logger.debug('Preparing to e-transfer output for job {0}'.format(job_id))
# When not in dry run mode, check that etransfer is being
# run on the correct machine by the correct user and with
# sufficient available disk space.
if not dry_run:
etransfer_check_config()
_etransfer_check_space()
logger.debug('Connecting to JSA processing database')
db = get_database()
if not force:
job = db.get_job(id_=job_id)
if job.state != JSAProcState.PROCESSED:
message = 'Job {0} cannot be e-transferred as it is in ' \
'state {1}'.format(job_id,
JSAProcState.get_name(job.state))
logger.error(message)
raise CommandError(message)
_etransfer_send(job_id, dry_run=dry_run, db=db, force=force)
logger.debug('Done adding output for job {0} to e-transfer'.format(job_id))
def fetch_output(job_id=None,
location=None,
task=None,
dry_run=False,
force=False):
"""Output fetch function for use from scripts."""
logger.debug('Connecting to JSA processing database')
db = get_database()
# Find next job if not specified.
if job_id is None:
force = False
logger.debug('Looking for a job to fetch output data for')
jobs = db.find_jobs(state=JSAProcState.INGEST_QUEUE,
location=location,
task=task,
prioritize=True,
number=1,
sort=True)
if jobs:
job_id = jobs[0].id
else:
logger.warning('Did not find a job to fetch output data for!')
return
_fetch_job_output(job_id, db=db, force=force, dry_run=dry_run)
def run_job(job_id=None, db=None, force=False, task=None):
"""
Run the JSA processing of the next job. This will select the highest
priority job in state 'WAITING' with location 'JAC'.
Optionally an integer job_id can be given isntead to specify a specific job
By default it will look in the database determined by the JSA_proc config.
Optionally a database object can be given for testing purposes.
Any errors raised will be logged in the 'log' table for the job_id.
If insufficient disk space is available (as configured by the disk_limit
section of the configuration file) then this function returns without
doing anything.
"""
# Check we have sufficient disk space for running to occur.
config = get_config()
output_limit = float(config.get('disk_limit', 'run_min_output_space'))
scratch_limit = float(config.get('disk_limit', 'run_min_scratch_space'))
output_space = get_output_dir_space()
scratch_space = get_scratch_dir_space()
if output_space < output_limit:
logger.warning('Insufficient output disk space: %f / %f GiB required',
output_space, output_limit)
return
if scratch_space < scratch_limit:
logger.warning('Insufficient scratch disk space: %f / %f GiB required',
scratch_space, scratch_limit)
return
# Get a link to the database.
if not db:
db = get_database()
# Get next job if a job id is not specified
if not job_id:
force = False
logger.debug('Looking for a job to run')
jobs = db.find_jobs(state=JSAProcState.WAITING,
location='JAC',
prioritize=True,
number=1,
sort=True,
task=task)
if jobs:
job_id = jobs[0].id
else:
logger.warning('Did not find a job to run!')
return
run_a_job(job_id, db=db, force=force)
def ingest_output(
job_id, location=None, task=None, dry_run=False, force=False):
"""High-level output ingestion function for use from scripts."""
logger.debug('Connecting to JSA processing database')
db = get_database()
if job_id is not None:
jobs = [db.get_job(id_=job_id)]
else:
jobs = db.find_jobs(
state=JSAProcState.INGESTION,
location=location,
task=task, prioritize=True)
# Get full list of tasks.
task_info = db.get_task_info()
for job in jobs:
job_task_info = task_info.get(job.task)
command_ingest = None
description = 'into CAOM-2'
if ((job_task_info is not None)
and (job_task_info.command_ingest is not None)):
command_ingest = job_task_info.command_ingest
description = 'via custom process'
if not dry_run:
try:
# Change the state from INGESTION to INGESTING, raising an
# error if the job was not already in that state.
db.change_state(
job.id, JSAProcState.INGESTING,
'Job output is being ingested {}'.format(description),
state_prev=(None if force else JSAProcState.INGESTION))
except NoRowsError:
# This would normally be a "logger.error", but we routinely
# run multiple copies of this ingestion routine simultaneously
# and it is therefore expected that a lot of jobs will have
# already been moved out of the INGESTION state by other
# processes. Therefore a warning or error should not be
# logged as these lead to unnecessary warnings in the cron
# job monitor.
logger.debug('Job %i can not be ingested as it is not ready',
job.id)
continue
_perform_ingestion(job_id=job.id, db=db, command_ingest=command_ingest)
else:
logger.info(
'Skipping ingestion %s of job %i (DRY RUN)',
description, job.id)
def reset_jobs(task,
date_start,
date_end,
instrument=None,
state=None,
force=False,
dry_run=False):
"""Change the state of the specified jobs back to "Unknown".
If a state is specified, select only that state.
Active jobs are skipped unless the force argument is set.
"""
db = get_database()
obsquery = {}
if date_start is not None and date_end is not None:
obsquery['utdate'] = Range(date_start, date_end)
elif date_start is None and date_end is None:
pass
else:
raise CommandError('only one of start and end date specified')
if instrument is not None:
obsquery['instrument'] = instrument
if state is not None:
state = JSAProcState.lookup_name(state)
n_active = 0
for job in db.find_jobs(location='JAC',
task=task,
obsquery=obsquery,
state=state):
state_info = JSAProcState.get_info(job.state)
# Check if the job is in an "active" state.
if state_info.active and not force:
logger.warning('Skipping active job %i (%s)', job.id,
state_info.name)
n_active += 1
continue
logger.info('Resetting status of job %i (was %s)', job.id,
state_info.name)
if not dry_run:
db.change_state(job.id,
JSAProcState.UNKNOWN,
'Resetting job',
state_prev=job.state)
if n_active:
raise CommandError('Could not reset {0} active jobs'.format(n_active))
def run_job(job_id=None, db=None, force=False, task=None):
"""
Run the JSA processing of the next job. This will select the highest
priority job in state 'WAITING' with location 'JAC'.
Optionally an integer job_id can be given isntead to specify a specific job
By default it will look in the database determined by the JSA_proc config.
Optionally a database object can be given for testing purposes.
Any errors raised will be logged in the 'log' table for the job_id.
If insufficient disk space is available (as configured by the disk_limit
section of the configuration file) then this function returns without
doing anything.
"""
# Check we have sufficient disk space for running to occur.
config = get_config()
output_limit = float(config.get('disk_limit', 'run_min_output_space'))
scratch_limit = float(config.get('disk_limit', 'run_min_scratch_space'))
output_space = get_output_dir_space()
scratch_space = get_scratch_dir_space()
if output_space < output_limit:
logger.warning('Insufficient output disk space: %f / %f GiB required',
output_space, output_limit)
return
if scratch_space < scratch_limit:
logger.warning('Insufficient scratch disk space: %f / %f GiB required',
scratch_space, scratch_limit)
return
# Get a link to the database.
if not db:
db = get_database()
# Get next job if a job id is not specified
if not job_id:
force = False
logger.debug('Looking for a job to run')
jobs = db.find_jobs(state=JSAProcState.WAITING, location='JAC',
prioritize=True, number=1, sort=True, task=task)
if jobs:
job_id = jobs[0].id
else:
logger.warning('Did not find a job to run!')
return
run_a_job(job_id, db=db, force=force)
def fetch(job_id=None, db=None, force=False, replaceparent=False, task=None):
"""
Assemble the files required to process a job.
If it is not given a job_id, it will take the next JAC job
with the highest priority and a state of MISSING.
Optionally allows a database object to be given for testing purposes.
Otherwise uses usual database from config file.
This will raise an error if job is not in MISSING state to start with.
This will advance the state of the job to WAITING on completion.
Any error's raised in the process will be logged to the job log.
"""
# Check we have sufficient disk space for fetching to occur.
input_space = get_input_dir_space()
required_space = float(get_config().get('disk_limit', 'fetch_min_space'))
if input_space < required_space and not force:
logger.warning('Insufficient disk space: %f / %f GiB required',
input_space, required_space)
return
# Get the database.
if not db:
db = get_database()
# Get next job if a job_id is not specified.
if not job_id:
force = False
logger.debug('Looking for a job for which to fetch data')
jobs = db.find_jobs(state=JSAProcState.MISSING,
location='JAC',
prioritize=True,
number=1,
sort=True,
task=task)
if jobs:
job_id = jobs[0].id
else:
logger.warning('Did not find a job to fetch!')
return
fetch_a_job(job_id, db=db, force=force, replaceparent=replaceparent)
def _clean_job_directories(dir_function,
state,
task=None,
count=None,
clean_function=None,
dry_run=False,
clean_function_kwargs={}):
"""Generic directory deletion function.
If a clean_function is given, it should return True when it is
able to clean a directory and False otherwise. It will
be passed the extra clean_function_kwargs keyword arguments.
"""
db = get_database()
jobs = db.find_jobs(location='JAC', state=state, task=task)
n = 0
for job in jobs:
directory = dir_function(job.id)
if not os.path.exists(directory):
logger.debug('Directory for job %i does not exist', job.id)
continue
try:
if clean_function is None:
logger.info('Removing directory for job %i: %s', job.id,
directory)
if not dry_run:
shutil.rmtree(directory)
n += 1
else:
if clean_function(directory,
job_id=job.id,
db=db,
dry_run=dry_run,
**clean_function_kwargs):
n += 1
if (count is not None) and not (n < count):
break
except:
logger.exception('Error removing directory for job %i: %s', job.id,
directory)
def move_from_cadc(job_ids, dry_run=False):
"""Move a list of jobs from CADC to JAC.
This sets up a database connection and then passes it to _move_job_from
to move each job individually.
"""
db = get_database()
logger.info('Starting move of jobs from CADC to JAC')
for job_id in job_ids:
_move_job_from(job_id, db=db, dry_run=dry_run)
logger.info('Done moving jobs')
def reset_jobs(task, date_start, date_end, instrument=None,
state=None, force=False, dry_run=False):
"""Change the state of the specified jobs back to "Unknown".
If a state is specified, select only that state.
Active jobs are skipped unless the force argument is set.
"""
db = get_database()
obsquery = {}
if date_start is not None and date_end is not None:
obsquery['utdate'] = Range(date_start, date_end)
elif date_start is None and date_end is None:
pass
else:
raise CommandError('only one of start and end date specified')
if instrument is not None:
obsquery['instrument'] = instrument
if state is not None:
state = JSAProcState.lookup_name(state)
n_active = 0
for job in db.find_jobs(location='JAC', task=task, obsquery=obsquery,
state=state):
state_info = JSAProcState.get_info(job.state)
# Check if the job is in an "active" state.
if state_info.active and not force:
logger.warning('Skipping active job %i (%s)',
job.id, state_info.name)
n_active += 1
continue
logger.info('Resetting status of job %i (was %s)',
job.id, state_info.name)
if not dry_run:
db.change_state(job.id, JSAProcState.UNKNOWN,
'Resetting job', state_prev=job.state)
if n_active:
raise CommandError(
'Could not reset {0} active jobs'.format(n_active))
def submit_one_coadd_job(tile, parenttask, mode, parameters, location,
exclude_pointing_jobs=False,
science_obs_only=False,
never_update=False,
dryrun=True, priority=0,
output_task=None, pointings_only=False):
"""
Submit a single coadd job.
"""
# Generate tag, task name, and filter.
if not output_task:
coadd_task = generate_hpx_coadd_task(parenttask)
else:
coadd_task = output_task
tag = generate_hpx_coadd_tag(tile, coadd_task)
filt = create_hpx_filter(tile, parenttask)
db = get_database()
# Check what current parent values should be.
try:
parent_jobs = get_parents(tile, parenttask,
exclude_pointing_jobs=exclude_pointing_jobs,
pointings_only=pointings_only,
science_obs_only=science_obs_only)
except NoRowsError:
parent_jobs = []
# Perform upsert operation and return job ID (or None).
return add_upd_del_job(
db=db,
tag=tag,
location=location,
mode=mode,
parameters=parameters,
task=coadd_task,
priority=priority,
parent_jobs=parent_jobs,
filters=([filt] * len(parent_jobs)),
tilelist=[tile],
allow_upd=(not never_update),
allow_del=(not never_update),
description='coadd for tile {}'.format(tile),
dry_run=dryrun)
def etransfer_query_output(job_id):
"""Investigate the e-transfer status of the output of a job."""
db = get_database()
config = get_config()
transdir = config.get('etransfer', 'transdir')
files = db.get_output_files(job_id)
problem_files = []
print('{0:110} {1:5} {2:12} {3:5}'.format('File', 'ET', 'Directory', 'AD'))
for file in zip(files,
etransfer_file_status(files),
check_cadc_files(files)):
(filename, etransfer_status, ad_status) = file
if etransfer_status is None:
(ok, dir) = (None, '')
else:
(ok, dir) = etransfer_status
print('{0:110} {1:5} {2:12} {3:5}'.format(
filename, repr(ok), dir, repr(ad_status)))
if ok is False:
problem_files.append(
os.path.join(transdir, 'reject', dir, filename))
if problem_files:
if yes_or_no_question(
'Delete rejected files from e-transfer directories?'):
for file in problem_files:
logger.debug('Deleting file %s', file)
os.unlink(file)
if yes_or_no_question('Re-try e-transfer?'):
# Clear cache before attempting to e-transfer since we just
# removed the files from the e-transfer directories.
_etransfer_clear_cache()
etransfer_send_output(job_id, dry_run=False, force=True)
def etransfer_query_output(job_id):
"""Investigate the e-transfer status of the output of a job."""
db = get_database()
config = get_config()
transdir = config.get('etransfer', 'transdir')
files = db.get_output_files(job_id)
problem_files = []
print('{0:110} {1:5} {2:12} {3:5}'.format('File', 'ET', 'Directory', 'AD'))
for file in zip(files, etransfer_file_status(files),
check_cadc_files(files)):
(filename, etransfer_status, ad_status) = file
if etransfer_status is None:
(ok, dir) = (None, '')
else:
(ok, dir) = etransfer_status
print('{0:110} {1:5} {2:12} {3:5}'.format(filename, repr(ok), dir,
repr(ad_status)))
if ok is False:
problem_files.append(
os.path.join(transdir, 'reject', dir, filename))
if problem_files:
if yes_or_no_question(
'Delete rejected files from e-transfer directories?'):
for file in problem_files:
logger.debug('Deleting file %s', file)
os.unlink(file)
if yes_or_no_question('Re-try e-transfer?'):
# Clear cache before attempting to e-transfer since we just
# removed the files from the e-transfer directories.
_etransfer_clear_cache()
etransfer_send_output(job_id, dry_run=False, force=True)
def _clean_job_directories(dir_function, state, task=None, count=None,
clean_function=None, dry_run=False,
clean_function_kwargs={}):
"""Generic directory deletion function.
If a clean_function is given, it should return True when it is
able to clean a directory and False otherwise. It will
be passed the extra clean_function_kwargs keyword arguments.
"""
db = get_database()
jobs = db.find_jobs(location='JAC', state=state, task=task)
n = 0
for job in jobs:
directory = dir_function(job.id)
if not os.path.exists(directory):
logger.debug('Directory for job %i does not exist', job.id)
continue
try:
if clean_function is None:
logger.info('Removing directory for job %i: %s',
job.id, directory)
if not dry_run:
shutil.rmtree(directory)
n += 1
else:
if clean_function(directory, job_id=job.id, db=db,
dry_run=dry_run,
**clean_function_kwargs):
n += 1
if (count is not None) and not (n < count):
break
except:
logger.exception('Error removing directory for job %i: %s',
job.id, directory)
def __call__(self, job_id, *args, **kwargs):
try:
return self.function(job_id, *args, **kwargs)
except Exception as theexception:
logger.exception('Error caught running function %s',
self.function.__name__)
if kwargs.get('dry_run', False):
logger.info('Skipping inserting error into database (DRY RUN)')
else:
if 'db' in kwargs and kwargs['db'] is not None:
db = kwargs['db']
else:
db = get_database()
db.change_state(job_id, JSAProcState.ERROR,
' '.join([str(i) for i in theexception.args]))
raise
def namecheck_output(task, outfile):
db = get_database()
# Look for post-run or error states (as the error may be from namecheck!).
states = JSAProcState.STATE_POST_RUN.copy()
states.add(JSAProcState.ERROR)
with open(outfile, 'w') as f:
for job in db.find_jobs(task=task, location='JAC', state=states):
job_id = job.id
logger.info('Considering job %i', job_id)
try:
for file in db.get_output_files(job_id):
if check_file_name(file):
logger.debug('Job %i file %s OK', job_id, file)
else:
logger.warning('Job %i file %s FAILURE', job_id, file)
print(file, file=f)
except NoRowsError:
# Ignore jobs for which we have no output files.
pass
def fetch_a_job(job_id, db=None, force=False, replaceparent=False):
"""
Assemble the files required to process a job.
Requires an integer job_id.
Optionally allows a db to be given, for testing purposes. Otherwise
uses usual database from config file.
Option 'replace' will force it to overwrite parent data already in the
input directory.
This will raise an error if job is not in MISSING state to start with.
This will advance the state of the job to WAITING on completion.
"""
if not db:
# Get link to database
db = get_database()
logger.info('About to fetch data for job %i', job_id)
try:
# Change status of job to 'Fetching', raise error if not in MISSING
db.change_state(job_id,
JSAProcState.FETCHING,
'Data is being assembled',
state_prev=(None if force else JSAProcState.MISSING))
except NoRowsError:
# If the job was not in the MISSING state, it is likely that another
# process is also trying to fetch it. Trap the error so that the
# ErrorDecorator does not put the job into the ERROR state as that
# will cause the other process to fail to set the job to WAITING.
logger.error('Job %i cannot be fetched because it is not missing',
job_id)
return
# Assemble any files listed in the input files tree
try:
input_files = db.get_input_files(job_id)
input_files_with_paths = assemble_input_data_for_job(
job_id, input_files)
except NoRowsError:
input_files_with_paths = []
# Assemble any files from the parent jobs
try:
parents = db.get_parents(job_id, with_state=True)
parent_files_with_paths = []
for p, f, parent_state in parents:
if parent_state not in JSAProcState.STATE_POST_RUN:
logger.error(
'Job %i cannot be fetch because its parent %i is not ready',
job_id, p)
raise ParentNotReadyError(
'Parent job {} is not ready'.format(p))
outputs = db.get_output_files(p)
parent_files = filter_file_list(outputs, f)
parent_files_with_paths += assemble_parent_data_for_job(
job_id, p, parent_files, force_new=replaceparent)
except NoRowsError:
parent_files_with_paths = []
# Write out list of all input files with full path list
files_list = input_files_with_paths + parent_files_with_paths
list_name_path = write_input_list(job_id, files_list)
# Advance the state of the job to 'Waiting'.
db.change_state(
job_id,
JSAProcState.WAITING,
'Data has been assembled for job and job can now be executed',
state_prev=JSAProcState.FETCHING)
logger.info('Done fetching data for job %i', job_id)
return job_id
def get_parents(tile, parenttask, exclude_pointing_jobs=False,
science_obs_only=False, pointings_only=False):
"""
get parent jobs for the requested tile and coaddtask,
using the parettask to look for jobs.
required parameters:
Raises a JSAProcError if there are no parent jobs that fit.
tile (int)
Tile number to perform coadd on.
parenttask (string)
input task name to look for jobs for.
"""
# Find all jobs from the parent task which include the requested tile and
# 1) Have a JSAQA State that is not BAD or INVALID
# 2) Have not been marked as deleted.
logger.debug(
'Finding all jobs in task %s that fall on tile %i',
parenttask, tile)
db = get_database()
qa_state = [JSAQAState.GOOD,
JSAQAState.QUESTIONABLE,
JSAQAState.UNKNOWN]
obsquery = {'omp_status': Not(list(OMPState.STATE_NO_COADD))}
if science_obs_only:
obsquery['obstype'] = {'science'}
if pointings_only:
obsquery['obstype'] = {'pointing'}
# Get the parent jobs.
parentjobs = db.find_jobs(tiles=[tile],
task=parenttask,
qa_state=qa_state,
state=Not([JSAProcState.DELETED]),
obsquery=obsquery)
parentjobs = [p.id for p in parentjobs]
# Do some other queries to give the user info about what is not being
# included.
excludedjobs_ompstatus = db.find_jobs(
tiles=[tile],
task=parenttask,
qa_state=qa_state,
state=Not([JSAProcState.DELETED]),
obsquery={'omp_status': OMPState.STATE_NO_COADD}
)
if science_obs_only or exclude_pointing_jobs:
obsquery = {
'obstype': 'pointing',
'omp_status': Not(list(OMPState.STATE_NO_COADD)),
}
state = Not([JSAProcState.DELETED])
excludedjobs_pointings = db.find_jobs(tiles=[tile],
task=parenttask,
qa_state=qa_state,
state=state,
obsquery=obsquery)
# If it was requested to exclude entirely any job containing a
# pointing:
if exclude_pointing_jobs and len(excludedjobs_pointings) > 0:
logger.debug('Tile %i contains pointing obs.', tile)
raise JSAProcError('Pointings fall on this tile.')
# Log information about which tasks where excluded.
# TODO: check what logger level is being used before going through for
# loops.
logger.debug(
'%i jobs in task %s fall on tile %i with appropriate QA States'
', OMP States and obstype states', len(parentjobs), parenttask, tile)
if len(excludedjobs_ompstatus) > 0:
logger.debug(
'%i jobs were excluded due to wrong OMP status',
len(excludedjobs_ompstatus))
for i in excludedjobs_ompstatus:
omp_status = db.get_obs_info(i.id)[0].omp_status
logger.debug(
'Job %i NOT INCLUDED (omp status of %s)',
i.id, OMPState.get_name(omp_status))
if science_obs_only:
if len(excludedjobs_pointings) > 0:
logger.debug(
'%i additional jobs were excluded as pointings',
len(excludedjobs_pointings))
for i in excludedjobs_pointings:
logger.debug('Job %i NOT INCLUDED (pointing)', i.id)
if len(parentjobs) == 0:
logger.debug('Tile %i has no acceptable parent jobs', tile)
raise JSAProcError('No acceptable observations.')
# Return the parent jobs
return parentjobs
def etransfer_poll_output(dry_run):
"""High level polling function to use from scripts."""
logger.debug('Preparing to poll the e-transfer system for job output')
# When not in dry run mode, check that etransfer is being
# run on the correct machine by the correct user.
if not dry_run:
etransfer_check_config(any_user=True)
logger.debug('Connecting to JSA processing database')
db = get_database()
logger.debug('Retrieving task information from database')
task_info = db.get_task_info()
n_err = 0
for job in db.find_jobs(location='JAC', state=JSAProcState.TRANSFERRING):
# Retrieve this job's task information.
job_task_info = task_info.get(job.task)
if ((job_task_info is None)
or (job_task_info.command_xfer is not None)
or (not job_task_info.etransfer)):
# Job not from an e-transfer task: skip it.
continue
job_id = job.id
logger.debug('Checking state of job %i', job_id)
logger.debug('Retrieving list of output files')
try:
file_info = db.get_output_files(job_id, with_info=True)
files = [x.filename for x in file_info]
except NoRowsError:
logger.error('Did not find output files for job %i', job_id)
n_err += 1
continue
try:
logger.debug('Checking if files are in the e-transfer directories')
etransfer_status = etransfer_file_status(files)
if any(etransfer_status):
rejection = []
for (file, status) in zip(files, etransfer_status):
if status is not None:
(ok, dir) = status
if not ok:
logger.error('File {0} was rejected, reason: {1}'.
format(file, dir))
rejection.append('{0} ({1})'.format(file, dir))
if rejection:
raise ETransferError('files rejected: {0}'.format(
', '.join(rejection)))
# Otherwise we found files in the "in progress" directories
# so proceed to the next job.
continue
logger.debug('Checking if all files are at CADC')
lost = []
for info in file_info:
cadc_file_info = fetch_cadc_file_info(info.filename)
if cadc_file_info is None:
logger.error('Job %i file %s gone from e-transfer '
'but not at CADC', job_id, info.filename)
lost.append(info.filename)
if cadc_file_info['content-md5'] != info.md5:
logger.error('Job %i file %s gone from e-transfer '
'but MD5 sum does not match',
job_id, info.filename)
lost.append(info.filename)
if lost:
raise ETransferError('files lost or corrupt: {0}'.format(
', '.join(lost)))
else:
# All files present and with correct MD5 sums.
logger.info('Job %i appears to have all files at CADC',
job_id)
if not dry_run:
db.change_state(job_id, JSAProcState.INGESTION,
'Output files finished e-transfer',
state_prev=JSAProcState.TRANSFERRING)
except ETransferError as e:
logger.error('Job %i failed e-transfer: %s', job_id, e.message)
if not dry_run:
db.change_state(
job_id, JSAProcState.ERROR,
'Job failed e-transfer: {0}'.format(e.message),
state_prev=JSAProcState.TRANSFERRING)
logger.debug('Done polling the e-transfer system')
if n_err:
raise CommandError('Errors were encountered polling e-transfer')
import logging
logger = logging.getLogger(__name__)
import numpy as np
from jsa_proc.config import get_database
from jsa_proc.state import JSAProcState
from jsa_proc.qastate import JSAQAState
from astropy.table import Table
db = get_database()
# Find all completed jobs
allcompleted = db.find_jobs(task=['hpx-s2-850-r2', 'hpx-s2-850-r1'],
state=JSAProcState.COMPLETE)
# Get observation info about them all
obsinfo = []
for i in allcompleted:
obs = db.get_obs_info(i.id)
if len(obs) > 1:
logger.warning('job {}:Multiple observations found!'.format(i.id))
obsinfo.append(obs)
else:
obsinfo.append(obs[0])
tiles = []
for i in allcompleted:
tile = db.get_tilelist(i.id)
tiles.append(list(tile))
tiles = np.array(tiles)
def search_log_files(
pattern, filename_pattern, task,
project=None, state=None, after_context=None):
db = get_database()
re_pattern = re.compile(pattern)
re_filename = re.compile(filename_pattern)
if state is None:
state = JSAProcState.COMPLETE
else:
state = JSAProcState.lookup_name(state)
if after_context is None:
after_context = 0
search_kwargs = {
'task': task,
'state': state,
}
if project is not None:
search_kwargs['obsquery'] = {'project': project}
jobs = [x.id for x in db.find_jobs(**search_kwargs)]
for job_id in jobs:
logger.debug('Checking log files for job %i', job_id)
log_dir = get_log_dir(job_id)
# Find the latest matching log by iterating through them in reverse
# order and "breaking" after the first match.
for filename in sorted(os.listdir(log_dir), reverse=True):
if not re_filename.search(filename):
continue
logger.debug('Found log file for job %i: %s', job_id, filename)
matched = 0
matched_lines = []
pathname = os.path.join(log_dir, filename)
with open(pathname, 'r') as f:
for line in f:
if matched or re_pattern.search(line):
matched += 1
matched_lines.append(line.rstrip())
if matched > after_context:
break
if matched:
logger.info(
'Found match for job %i: %s', job_id, matched_lines[0])
for matched_line in matched_lines[1:]:
logger.info(
'... continuation %i: %s', job_id, matched_line)
break
def create_web_app():
"""Function to prepare the Flask web application."""
home = get_home()
db = get_database()
database_name = get_config().get('database', 'database')
app = Flask(
'jsa_proc',
static_folder=os.path.join(home, 'web', 'static'),
template_folder=os.path.join(home, 'web', 'templates'),
)
app.secret_key = get_config().get('web', 'key')
# Web authorization -- mostly take from flask docs snippets 8
# http://flask.pocoo.org/snippets/8
def check_auth(password):
"""
Check that the staff pasword has been used.
(Note that we don't care what the username is).
"""
return check_staff_password(password)
def authenticate():
"""
Send a 401 response so that we can log in.
"""
return Response(
render_template('logout.html', redirect=request.referrer), 401,
{'WWW-Authenticate': loginstring})
def requires_auth(f):
"""
A decorator to wrap functions that require authorization.
"""
@wraps(f)
def decorated(*args, **kwargs):
auth = request.authorization
if not auth or not check_auth(auth.password):
return authenticate()
return f(*args, **kwargs)
return decorated
# Route Handlers.
@app.route('/')
def home_page():
raise HTTPRedirect(url_for('task_summary'))
@app.route('/job/')
@templated('job_list.html')
def job_list():
# Prepare query arguments list: special parameters first.
kwargs = {
'state': request.args.getlist('state'),
'mode': request.args.get('mode', 'JSAProc'),
'tiles': request.args.get('tiles')
}
# Now add regular string parameters, including those from
# jcmtobsinfo.
params = [
'location',
'task',
'date_min',
'date_max',
'qa_state',
'sourcename',
'obsnum',
'project',
'number',
'tau_min',
'tau_max',
]
params.extend(ObsQueryDict.keys())
for key in params:
kwargs[key] = request.args.get(key, None)
# Process empty strings used as null form parameters.
for argname in kwargs:
if kwargs[argname] == '':
kwargs[argname] = None
# Store the query in the session.
session['job_query'] = kwargs
# Finally prepare the template context.
return prepare_job_list(db,
page=request.args.get('page', None),
**kwargs)
@app.route('/image/<task>/piechart')
def summary_piechart(task='None'):
if task == 'None':
task = None
obsquerydict = {}
for key in ObsQueryDict.keys():
obsquerydict[key] = request.args.get(key, None)
date_min = request.args.get('date_min', None)
date_max = request.args.get('date_max', None)
return prepare_summary_piechart(db,
task=task,
obsquerydict=obsquerydict,
date_min=date_min,
date_max=date_max)
@app.route('/summary/')
@templated('task_summary.html')
def task_summary():
return prepare_task_summary(db)
@app.route('/qa')
@templated('task_qa_summary.html')
def task_qa_summary():
return prepare_task_qa_summary(db)
@app.route('/job_summary/')
@templated('job_summary.html')
def job_summary():
task = request.args.get('task', None)
date_min = request.args.get('date_min', None)
date_max = request.args.get('date_max', None)
return prepare_job_summary(db,
task=task,
date_min=date_min,
date_max=date_max)
@app.route('/error_summary/')
@templated('error_summary.html')
def error_summary():
return prepare_error_summary(
db,
redirect_url=request.full_path,
filtering=request.args.get('filtering', None),
chosentask=request.args.get('chosentask', None),
extrafilter=request.args.get('extrafilter', None),
state_prev=request.args.get('state_prev', None),
error_state=request.args.get('error_state', None),
filter_done=('submit_filter' in request.args),
)
@app.route('/job/<int:job_id>', methods=['GET'])
@templated('job_info.html')
def job_info(job_id):
return prepare_job_info(db, job_id, session.get('job_query'))
@app.route('/job/<int:job_id>/qa', methods=['GET'])
@templated('job_qa.html')
def job_qa(job_id):
return prepare_job_qa_info(db, job_id, session.get('job_query'))
@app.route('/job/<int:job_id>/add_note', methods=['POST'])
@requires_auth
def job_add_note(job_id):
message = request.form['message']
username = request.authorization['username']
try:
# Add the note.
prepare_add_note(db, job_id, message, username)
# Redirect back to the job info page.
flash('The note has been saved.')
raise HTTPRedirect(url_for('job_info', job_id=job_id))
except ErrorPage as err:
return error_page_response(err)
@app.route('/job_change_state', methods=['POST'])
@requires_auth
def job_change_state():
# Get the variables from POST
newstate = request.form['newstate']
state_prev = request.form['state_prev']
message = request.form['message']
job_ids = request.form.getlist('job_id')
url = request.form['url']
username = request.authorization['username']
try:
# Change the state.
prepare_change_state(db, job_ids, newstate, state_prev, message,
username)
# Redirect the page to correct info.
flash('The status has been changed to %s.' %
JSAProcState.get_name(newstate))
raise HTTPRedirect(url)
except ErrorPage as err:
return error_page_response(err)
@app.route('/job_change_qa', methods=['POST'])
@requires_auth
def job_change_qa():
# Get the variables from POST
if "action_next" in request.form:
url = request.form['url-next']
else:
url = request.form['url']
qa_state = request.form['qa_state']
message = request.form['message']
job_ids = request.form.getlist('job_id')
username = request.authorization['username']
try:
# Change the state.
prepare_change_qa(
db,
job_ids,
qa_state,
message,
username,
)
# Redirect the page to correct info.
flash('The QA status of job %s has been changed to %s.' %
(str(' '.join(job_ids)), JSAQAState.get_name(qa_state)))
raise HTTPRedirect(url)
except ErrorPage as err:
return error_page_response(err)
# QA Nightly Summary pages
@app.route('/qa-nightly')
@templated('task_qa_summary_nightly.html')
def qa_night_page():
"""
By default, show the previous week.
Note that prepare_task_qa_summary interprets dates as
inclusive, so use 6 days for the time delta to get a week
"""
date_min = request.args.get('date_min', None)
if date_min is None or date_min == '':
date_min = (datetime.date.today() -
datetime.timedelta(days=6)).strftime('%Y-%m-%d')
date_max = request.args.get('date_max', None)
if date_max is None or date_max == '':
date_max = datetime.date.today().strftime('%Y-%m-%d')
return prepare_task_qa_summary(db,
date_min=date_min,
date_max=date_max,
task='jcmt-nightly',
byDate=True)
@app.route('/login')
@requires_auth
def login():
raise HTTPRedirect(request.referrer)
@app.route('/logout')
def logout():
return authenticate()
# Image handling.
@app.route('/job/<int:job_id>/preview/<preview>')
def job_preview(job_id, preview):
path = prepare_job_preview(job_id, preview)
return send_file(path, mimetype='image/png')
@app.route('/job/<int:job_id>/pdf/<preview>')
def job_preview_pdf(job_id, preview):
path = prepare_job_preview(job_id, preview, 'pdf')
return send_file(path, mimetype='application/pdf')
@app.route('/job/<int:job_id>/text/<text_file>')
def job_text_file(job_id, text_file):
path = prepare_job_preview(job_id, text_file, 'txt')
return send_file(path, mimetype='text/plain')
@app.route('/job/<int:job_id>/log/<log>')
def job_log_html(job_id, log):
path = prepare_job_log(job_id, log)
return send_file(path, mimetype='text/html')
@app.route('/job/<int:job_id>/log_text/<log>')
def job_log_text(job_id, log):
path = prepare_job_log(job_id, log)
return send_file(path, mimetype='text/plain')
@app.route('/fop_summary', methods=['GET'])
@templated('fop_summary.html')
def fop_summary():
userid = request.args.get('userid', None)
semester = request.args.get('semester', None)
projdict = {}
if userid and semester:
ompdb = get_omp_database(write_access=None)
projects = ompdb.get_support_projects(str(userid), str(semester))
for p in projects:
jobs = db.find_jobs(obsquery={'project': str(p)},
task='jcmt-nightly')
projdict[str(p)] = [
len(jobs),
sum(1 for j in jobs if j.state == JSAProcState.ERROR),
sum(1 for j in jobs if j.qa_state == JSAQAState.BAD),
sum(1 for j in jobs
if j.qa_state == JSAQAState.QUESTIONABLE),
sum(1 for j in jobs if j.qa_state == JSAQAState.UNKNOWN),
sum(1 for j in jobs if j.qa_state == JSAQAState.GOOD)
]
else:
projects = None
return {'userid': userid, 'semester': semester, 'projects': projdict}
# Filters and Tests.
@app.route('/fop_summary_getres', methods=['POST'])
def fop_summary_getres():
userid = request.form['userid']
semester = request.form['semester']
raise HTTPRedirect(
url_for('fop_summary', userid=userid, semester=semester))
@app.template_filter('state_name')
def state_name_filter(state):
return JSAProcState.get_name(state)
@app.template_test('state_active')
def state_active_test(state):
return JSAProcState.get_info(state).active
@app.template_filter('state_phase')
def state_phase_filter(state):
phase = JSAProcState.get_info(state).phase
if phase == JSAProcState.PHASE_QUEUE:
return 'queue'
elif phase == JSAProcState.PHASE_FETCH:
return 'fetch'
elif phase == JSAProcState.PHASE_RUN:
return 'run'
elif phase == JSAProcState.PHASE_COMPLETE:
return 'complete'
elif phase == JSAProcState.PHASE_ERROR:
return 'error'
raise HTTPError('Unknown phase {0}'.format(phase))
@app.template_filter('qa_state_name')
def qa_state_name(qa_state):
if qa_state.lower() != 'total':
name = JSAQAState.get_name(qa_state)
else:
name = 'Total'
return name
@app.template_filter('omp_state_name')
def omp_state_name_filter(ompstate):
return OMPState.get_name(ompstate)
@app.template_filter('uniq')
def uniq_filter(xs):
return set(xs)
@app.template_filter('datetimeformat')
def datetimeformat(value, format='%Y-%m-%d<br>%H:%M'):
return value.strftime(format)
@app.template_filter('replace0')
def replace_zero(value):
if value == 0:
return '-'
else:
return value
@app.context_processor
def add_to_context():
return {
'url_for_omp': url_for_omp,
'url_for_omp_comment': url_for_omp_comment,
'database_name': database_name,
}
# Return the Application.
return app
def run_a_job(job_id, db=None, force=False):
"""
Run the JSA processing of the given job_id (integer).
By default it will look in the database determined by the JSA_proc
config. Optionally a database object can be given for testing
purposes.
"""
if not db:
# Get link to database
db = get_database()
logger.info('About to run job %i', job_id)
try:
# Change status of job to Running, raise an error if not currently in
# WAITING state.
db.change_state(job_id,
JSAProcState.RUNNING,
'Job is about to be run on host {0}'.format(
gethostname().partition('.')[0]),
state_prev=(None if force else JSAProcState.WAITING))
except NoRowsError:
# If the job was not in the WAITING state, it is likely that another
# process is also trying to run it. Trap the error so that the
# ErrorDecorator does not put the job into the ERROR state as that
# will cause the other process to fail to set the job to PROCESSED.
logger.error('Job %i cannot be run because it is not waiting', job_id)
return
# Input file_list -- this should be better? or in jsawrapdr?
input_dir = get_input_dir(job_id)
input_file_list_path = os.path.join(input_dir, input_list_name)
if not os.path.exists(input_file_list_path):
raise JSAProcError('Input file list %s not found for job_id %i' %
(input_file_list_path, job_id))
# Check every file on input_file list exists.
inputfl = open(input_file_list_path, 'r')
for input_file in inputfl:
input_file = input_file.strip()
if not os.path.isfile(input_file):
# If a file is missing, get log.
logstring = 'Input file %s for job %i has gone missing' % (
input_file, job_id)
logger.error(logstring)
logs = db.get_logs(job_id)
states = [i.state_new for i in logs]
# If it has only been in the state MISSING twice before, then try
# again.
if states.count(JSAProcState.MISSING) <= 2:
logstring += ': moving to missing.'
logger.warning(
'Moving job %i to state MISSING due to '
'missing file(s) %s', job_id, input_file)
db.change_state(job_id,
JSAProcState.MISSING,
logstring,
state_prev=JSAProcState.RUNNING)
return job_id
else:
# If it has been in the missing STATE more than two times,
# give up and move it into ERROR state to be fixed manually.
logstring += ': moving to error.'
logger.info(
'Moving job %s to state ERROR due to missing'
' file(s).', job_id)
inputfl.close()
raise JSAProcError(
'Input file %s for job %i has gone missing.' %
(input_file, job_id))
inputfl.close()
logger.debug('All input files found for job %s.', job_id)
# Get the mode and drparameters of the job.
job = db.get_job(id_=job_id)
mode = job.mode
drparameters = job.parameters
# Get the starlink to be used from the task table.
starpath = None
version = None
command_run = None
raw_output = None
log_ingest_command = None
try:
task_info = db.get_task_info(job.task)
starpath = task_info.starlink_dir
version = task_info.version
command_run = task_info.command_run
raw_output = task_info.raw_output
log_ingest_command = task_info.log_ingest
except NoRowsError:
# If the task doesn't have task info, leave "starpath" as None
# so that jsawrapdr_run uses the default value from the configuration
# file.
pass
# Run the processing job.
logger.debug('Launching jsawrapdr: mode=%s, parameters=%s', mode,
drparameters)
# First of all remove the output files and log_files from the database.
db.set_log_files(job_id, [])
db.set_output_files(job_id, [])
log = jsawrapdr_run(job_id,
input_file_list_path,
mode,
drparameters,
cleanup='cadc',
location='JAC',
starlink_dir=starpath,
persist=True,
version=version,
command_run=command_run,
raw_output=raw_output)
# Create list of output files.
logger.debug('Preparing list of output files')
output_files = get_output_files(job_id)
# write output files to table
logger.debug('Storing list of output files')
db.set_output_files(job_id, output_files)
# Create list of output log files.
logger.debug('Preparing list of output log files (log.*)')
log_files = get_output_log_files(job_id)
# Write output log files to table.
logger.debug('Storing list of output log files')
db.set_log_files(job_id, log_files)
# If a log ingest command is set, run it here.
if log_ingest_command:
logger.debug('Will try and ingest log files')
try:
with open_log_file(job.id, 'ingest_log') as logingest_log:
subprocess.check_call(
[log_ingest_command, str(job_id)],
shell=False,
cwd='/tmp',
stdout=logingest_log,
stderr=subprocess.STDOUT,
preexec_fn=restore_signals)
except subprocess.CalledProcessError as e:
logger.exception('Custom log ingest failed ' 'for job %i', job.id)
db.change_state(job.id,
JSAProcState.ERROR,
'Custom log ingestion failed',
state_prev=JSAProcState.RUNNING)
# If task begins with hpx, get tiles from list of output_files
# and write to tile table in db.
if hpx_task.search(job.task):
logger.debug('Storing list of output tiles for HPX job ' + str(job_id))
tiles = hpx_tiles_from_filenames([x.filename for x in output_files])
db.set_tilelist(job_id, tiles)
logger.debug('Job ' + str(job_id) + ' produced output on tiles ' +
', '.join(str(i) for i in tiles))
# Change state of job.
db.change_state(job_id,
JSAProcState.PROCESSED,
'Job has been successfully processed',
state_prev=JSAProcState.RUNNING)
logger.info('Done running job %i', job_id)
return job_id
def search_log_files(pattern,
filename_pattern,
task,
project=None,
state=None,
after_context=None):
db = get_database()
re_pattern = re.compile(pattern)
re_filename = re.compile(filename_pattern)
if state is None:
state = JSAProcState.COMPLETE
else:
state = JSAProcState.lookup_name(state)
if after_context is None:
after_context = 0
search_kwargs = {
'task': task,
'state': state,
}
if project is not None:
search_kwargs['obsquery'] = {'project': project}
jobs = [x.id for x in db.find_jobs(**search_kwargs)]
for job_id in jobs:
logger.debug('Checking log files for job %i', job_id)
log_dir = get_log_dir(job_id)
# Find the latest matching log by iterating through them in reverse
# order and "breaking" after the first match.
for filename in sorted(os.listdir(log_dir), reverse=True):
if not re_filename.search(filename):
continue
logger.debug('Found log file for job %i: %s', job_id, filename)
matched = 0
matched_lines = []
pathname = os.path.join(log_dir, filename)
with open(pathname, 'r') as f:
for line in f:
if matched or re_pattern.search(line):
matched += 1
matched_lines.append(line.rstrip())
if matched > after_context:
break
if matched:
logger.info('Found match for job %i: %s', job_id,
matched_lines[0])
for matched_line in matched_lines[1:]:
logger.info('... continuation %i: %s', job_id,
matched_line)
break
def investigate_unauthorized_errors(location, check_at_cadc=True):
logger.debug('Starting to investigate unauthorized errors')
logger.debug('Connecting to JSA processing database')
db = get_database()
logger.debug('Connecting to OMP/JCMT database')
ompdb = get_omp_database()
logger.debug('Preparing CADC TAP object')
caom2 = CADCTap()
logger.debug('Fetching list of jobs in the error state')
job_logs = db.find_errors_logs(location=location)
logger.debug('Filtering for jobs with unauthorized errors')
filter = JSAProcErrorFilter('unauthorized')
filter(job_logs)
now = datetime.now(UTC)
category = {'unknown': []}
job_info = {}
for job_id in job_logs.keys():
logger.debug('Checking job %i', job_id)
# Python doesn't let us break inner loops, so use exceptions to
# signal when the cause of the problem is identified.
try:
# Find the observation IDs and use it to determine whether the
# job uses any observations which are not yet public.
logger.debug('Fetching observation info')
obs_info = db.get_obs_info(job_id)
job_info[job_id] = {'obs': obs_info}
if not obs_info:
logger.warning('No observation info available for this job')
continue
obsids = set(latin_1_encode(x.obsid)[0] for x in obs_info)
for obsid in obsids:
logger.debug('Fetching COMMON info for %s', obsid)
common = ompdb.get_obsid_common(obsid)
if common is None:
raise IdentifiedProblem('omp', 'obsid not in common table')
release_date = common.release_date
# Keep the last release date inspected in the info dictionary
# so that if it's the one that causes a problem, we see
# it in the output.
job_info[job_id]['release'] = release_date
if release_date > now:
raise IdentifiedProblem(
'release',
'future release date ' +
release_date.strftime('%Y-%m-%d'))
logger.debug('Fetching OMP obslog status for %s', obsid)
status = ompdb.get_obsid_status(obsid)
if status is not None:
logger.debug('Got obslog status: %i', status)
if status == OMPState.JUNK:
raise IdentifiedProblem('junk', 'observation is junk')
# Check whether all of the files are at CADC.
if check_at_cadc:
logger.debug('Retrieving input file list')
files = db.get_input_files(job_id)
logger.debug('Checking for files at CADC')
found = check_cadc_files(files)
if not all(found):
raise IdentifiedProblem(
'missing',
'file {0} missing at CADC'.format(
files[found.index(False)]))
# Check whether all the observations are in CAOM-2.
if True:
logger.debug('Checking for observatons in CAOM-2')
obsid_list = list(obsids)
found = caom2.check_obsids(obsid_list)
if not all(found):
raise IdentifiedProblem(
'caom2',
'observation {0} missing from CAOM-2'.format(
obsid_list[found.index(False)]))
except IdentifiedProblem as problem:
logger.info('Job {0}: {1}'.format(job_id, problem.message))
if problem.category in category:
category[problem.category].append(job_id)
else:
category[problem.category] = [job_id]
else:
logger.info('Job {0}: problem unknown'.format(job_id))
category['unknown'].append(job_id)
# Now go through the categories and output information about them.
for (cat, jobs) in category.items():
if jobs:
print('Category {0}: {1} job(s)'.format(cat, len(jobs)))
if yes_or_no_question('Show detail?', False):
for job in jobs:
info = job_info[job]
print(job,
info['obs'][0].instrument,
info['obs'][0].utdate,
info['obs'][0].obsnum,
info['obs'][0].project,
info['obs'][0].obstype,
info['obs'][0].scanmode,
info['release'])
if yes_or_no_question('Resubmit jobs?', False):
for job in jobs:
db.change_state(
job, JSAProcState.QUEUED,
'Resubmitting job after unauthorized error',
state_prev=JSAProcState.ERROR)
def etransfer_poll_output(dry_run):
"""High level polling function to use from scripts."""
logger.debug('Preparing to poll the e-transfer system for job output')
# When not in dry run mode, check that etransfer is being
# run on the correct machine by the correct user.
if not dry_run:
etransfer_check_config(any_user=True)
logger.debug('Connecting to JSA processing database')
db = get_database()
logger.debug('Retrieving task information from database')
task_info = db.get_task_info()
n_err = 0
for job in db.find_jobs(location='JAC', state=JSAProcState.TRANSFERRING):
# Retrieve this job's task information.
job_task_info = task_info.get(job.task)
if ((job_task_info is None) or (job_task_info.command_xfer is not None)
or (not job_task_info.etransfer)):
# Job not from an e-transfer task: skip it.
continue
job_id = job.id
logger.debug('Checking state of job %i', job_id)
logger.debug('Retrieving list of output files')
try:
file_info = db.get_output_files(job_id, with_info=True)
files = [x.filename for x in file_info]
except NoRowsError:
logger.error('Did not find output files for job %i', job_id)
n_err += 1
continue
try:
logger.debug('Checking if files are in the e-transfer directories')
etransfer_status = etransfer_file_status(files)
if any(etransfer_status):
rejection = []
for (file, status) in zip(files, etransfer_status):
if status is not None:
(ok, dir) = status
if not ok:
logger.error(
'File {0} was rejected, reason: {1}'.format(
file, dir))
rejection.append('{0} ({1})'.format(file, dir))
if rejection:
raise ETransferError('files rejected: {0}'.format(
', '.join(rejection)))
# Otherwise we found files in the "in progress" directories
# so proceed to the next job.
continue
logger.debug('Checking if all files are at CADC')
lost = []
for info in file_info:
cadc_file_info = fetch_cadc_file_info(info.filename)
if cadc_file_info is None:
logger.error(
'Job %i file %s gone from e-transfer '
'but not at CADC', job_id, info.filename)
lost.append(info.filename)
if cadc_file_info['content-md5'] != info.md5:
logger.error(
'Job %i file %s gone from e-transfer '
'but MD5 sum does not match', job_id, info.filename)
lost.append(info.filename)
if lost:
raise ETransferError('files lost or corrupt: {0}'.format(
', '.join(lost)))
else:
# All files present and with correct MD5 sums.
logger.info('Job %i appears to have all files at CADC', job_id)
if not dry_run:
db.change_state(job_id,
JSAProcState.INGESTION,
'Output files finished e-transfer',
state_prev=JSAProcState.TRANSFERRING)
except ETransferError as e:
logger.error('Job %i failed e-transfer: %s', job_id, e.message)
if not dry_run:
db.change_state(job_id,
JSAProcState.ERROR,
'Job failed e-transfer: {0}'.format(e.message),
state_prev=JSAProcState.TRANSFERRING)
logger.debug('Done polling the e-transfer system')
if n_err:
raise CommandError('Errors were encountered polling e-transfer')
def create_web_app():
"""Function to prepare the Flask web application."""
home = get_home()
db = get_database()
database_name = get_config().get('database', 'database')
app = Flask(
'jsa_proc',
static_folder=os.path.join(home, 'web', 'static'),
template_folder=os.path.join(home, 'web', 'templates'),
)
app.secret_key = get_config().get('web', 'key')
# Web authorization -- mostly take from flask docs snippets 8
# http://flask.pocoo.org/snippets/8
def check_auth(password):
"""
Check that the staff pasword has been used.
(Note that we don't care what the username is).
"""
return check_staff_password(password)
def authenticate():
"""
Send a 401 response so that we can log in.
"""
return Response(render_template('logout.html',
redirect=request.referrer),
401, {'WWW-Authenticate': loginstring})
def requires_auth(f):
"""
A decorator to wrap functions that require authorization.
"""
@wraps(f)
def decorated(*args, **kwargs):
auth = request.authorization
if not auth or not check_auth(auth.password):
return authenticate()
return f(*args, **kwargs)
return decorated
# Route Handlers.
@app.route('/')
def home_page():
raise HTTPRedirect(url_for('task_summary'))
@app.route('/job/')
@templated('job_list.html')
def job_list():
# Prepare query arguments list: special parameters first.
kwargs = {
'state': request.args.getlist('state'),
'mode': request.args.get('mode', 'JSAProc'),
'tiles': request.args.get('tiles')
}
# Now add regular string parameters, including those from
# jcmtobsinfo.
params = [
'location',
'task',
'date_min',
'date_max',
'qa_state',
'sourcename',
'obsnum',
'project',
'number',
'tau_min',
'tau_max',
]
params.extend(ObsQueryDict.keys())
for key in params:
kwargs[key] = request.args.get(key, None)
# Process empty strings used as null form parameters.
for argname in kwargs:
if kwargs[argname] == '':
kwargs[argname] = None
# Store the query in the session.
session['job_query'] = kwargs
# Finally prepare the template context.
return prepare_job_list(
db,
page=request.args.get('page', None),
**kwargs)
@app.route('/image/<task>/piechart')
def summary_piechart(task='None'):
if task == 'None':
task = None
obsquerydict = {}
for key in ObsQueryDict.keys():
obsquerydict[key] = request.args.get(key, None)
date_min = request.args.get('date_min', None)
date_max = request.args.get('date_max', None)
return prepare_summary_piechart(db, task=task,
obsquerydict=obsquerydict,
date_min=date_min, date_max=date_max)
@app.route('/summary/')
@templated('task_summary.html')
def task_summary():
return prepare_task_summary(db)
@app.route('/qa')
@templated('task_qa_summary.html')
def task_qa_summary():
return prepare_task_qa_summary(db)
@app.route('/job_summary/')
@templated('job_summary.html')
def job_summary():
task = request.args.get('task', None)
date_min = request.args.get('date_min', None)
date_max = request.args.get('date_max', None)
return prepare_job_summary(db, task=task, date_min=date_min,
date_max=date_max)
@app.route('/error_summary/')
@templated('error_summary.html')
def error_summary():
return prepare_error_summary(
db,
redirect_url=request.full_path,
filtering=request.args.get('filtering', None),
chosentask=request.args.get('chosentask', None),
extrafilter=request.args.get('extrafilter', None),
state_prev=request.args.get('state_prev', None),
error_state=request.args.get('error_state', None),
filter_done=('submit_filter' in request.args),
)
@app.route('/job/<int:job_id>', methods=['GET'])
@templated('job_info.html')
def job_info(job_id):
return prepare_job_info(db, job_id, session.get('job_query'))
@app.route('/job/<int:job_id>/qa', methods=['GET'])
@templated('job_qa.html')
def job_qa(job_id):
return prepare_job_qa_info(db, job_id, session.get('job_query'))
@app.route('/job/<int:job_id>/add_note', methods=['POST'])
@requires_auth
def job_add_note(job_id):
message = request.form['message']
username = request.authorization['username']
try:
# Add the note.
prepare_add_note(db, job_id, message, username)
# Redirect back to the job info page.
flash('The note has been saved.')
raise HTTPRedirect(url_for('job_info', job_id=job_id))
except ErrorPage as err:
return error_page_response(err)
@app.route('/job_change_state', methods=['POST'])
@requires_auth
def job_change_state():
# Get the variables from POST
newstate = request.form['newstate']
state_prev = request.form['state_prev']
message = request.form['message']
job_ids = request.form.getlist('job_id')
url = request.form['url']
username = request.authorization['username']
try:
# Change the state.
prepare_change_state(db, job_ids,
newstate,
state_prev,
message,
username)
# Redirect the page to correct info.
flash('The status has been changed to %s.' % JSAProcState.get_name(
newstate))
raise HTTPRedirect(url)
except ErrorPage as err:
return error_page_response(err)
@app.route('/job_change_qa', methods=['POST'])
@requires_auth
def job_change_qa():
# Get the variables from POST
if "action_next" in request.form:
url = request.form['url-next']
else:
url = request.form['url']
qa_state = request.form['qa_state']
message = request.form['message']
job_ids = request.form.getlist('job_id')
username = request.authorization['username']
try:
# Change the state.
prepare_change_qa(db, job_ids,
qa_state,
message,
username,
)
# Redirect the page to correct info.
flash(
'The QA status of job %s has been changed to %s.' %
(str(' '.join(job_ids)), JSAQAState.get_name(qa_state))
)
raise HTTPRedirect(url)
except ErrorPage as err:
return error_page_response(err)
# QA Nightly Summary pages
@app.route('/qa-nightly')
@templated('task_qa_summary_nightly.html')
def qa_night_page():
"""
By default, show the previous week.
Note that prepare_task_qa_summary interprets dates as
inclusive, so use 6 days for the time delta to get a week
"""
date_min = request.args.get('date_min', None)
if date_min is None or date_min == '':
date_min = (
datetime.date.today() - datetime.timedelta(days=6)
).strftime('%Y-%m-%d')
date_max = request.args.get('date_max', None)
if date_max is None or date_max == '':
date_max = datetime.date.today().strftime('%Y-%m-%d')
return prepare_task_qa_summary(db, date_min=date_min,
date_max=date_max, task='jcmt-nightly',
byDate=True)
@app.route('/login')
@requires_auth
def login():
raise HTTPRedirect(request.referrer)
@app.route('/logout')
def logout():
return authenticate()
# Image handling.
@app.route('/job/<int:job_id>/preview/<preview>')
def job_preview(job_id, preview):
path = prepare_job_preview(job_id, preview)
return send_file(path, mimetype='image/png')
@app.route('/job/<int:job_id>/pdf/<preview>')
def job_preview_pdf(job_id, preview):
path = prepare_job_preview(job_id, preview, 'pdf')
return send_file(path, mimetype='application/pdf')
@app.route('/job/<int:job_id>/text/<text_file>')
def job_text_file(job_id, text_file):
path = prepare_job_preview(job_id, text_file, 'txt')
return send_file(path, mimetype='text/plain')
@app.route('/job/<int:job_id>/log/<log>')
def job_log_html(job_id, log):
path = prepare_job_log(job_id, log)
return send_file(path, mimetype='text/html')
@app.route('/job/<int:job_id>/log_text/<log>')
def job_log_text(job_id, log):
path = prepare_job_log(job_id, log)
return send_file(path, mimetype='text/plain')
@app.route('/fop_summary', methods=['GET'])
@templated('fop_summary.html')
def fop_summary():
userid = request.args.get('userid', None)
semester = request.args.get('semester', None)
projdict = {}
if userid and semester:
ompdb = get_omp_database(write_access=None)
projects = ompdb.get_support_projects(str(userid), str(semester))
for p in projects:
jobs = db.find_jobs(
obsquery={'project': p}, task='jcmt-nightly')
projdict[p] = [
len(jobs),
sum(1 for j in jobs if j.state == JSAProcState.ERROR),
sum(1 for j in jobs if j.qa_state == JSAQAState.BAD),
sum(1 for j in jobs if j.qa_state == JSAQAState.QUESTIONABLE),
sum(1 for j in jobs if j.qa_state == JSAQAState.UNKNOWN),
sum(1 for j in jobs if j.qa_state == JSAQAState.GOOD)]
else:
projects = None
return {'userid': userid, 'semester': semester, 'projects': projdict}
# Filters and Tests.
@app.route('/fop_summary_getres', methods=['POST'])
def fop_summary_getres():
userid = request.form['userid']
semester = request.form['semester']
raise HTTPRedirect(url_for(
'fop_summary', userid=userid, semester=semester))
@app.template_filter('state_name')
def state_name_filter(state):
return JSAProcState.get_name(state)
@app.template_test('state_active')
def state_active_test(state):
return JSAProcState.get_info(state).active
@app.template_filter('state_phase')
def state_phase_filter(state):
phase = JSAProcState.get_info(state).phase
if phase == JSAProcState.PHASE_QUEUE:
return 'queue'
elif phase == JSAProcState.PHASE_FETCH:
return 'fetch'
elif phase == JSAProcState.PHASE_RUN:
return 'run'
elif phase == JSAProcState.PHASE_COMPLETE:
return 'complete'
elif phase == JSAProcState.PHASE_ERROR:
return 'error'
raise HTTPError('Unknown phase {0}'.format(phase))
@app.template_filter('qa_state_name')
def qa_state_name(qa_state):
if qa_state.lower() != 'total':
name = JSAQAState.get_name(qa_state)
else:
name = 'Total'
return name
@app.template_filter('omp_state_name')
def omp_state_name_filter(ompstate):
return OMPState.get_name(ompstate)
@app.template_filter('uniq')
def uniq_filter(xs):
return set(xs)
@app.template_filter('datetimeformat')
def datetimeformat(value, format='%Y-%m-%d<br>%H:%M'):
return value.strftime(format)
@app.template_filter('replace0')
def replace_zero(value):
if value == 0:
return '-'
else:
return value
@app.context_processor
def add_to_context():
return {
'url_for_omp': url_for_omp,
'url_for_omp_comment': url_for_omp_comment,
'database_name': database_name,
}
# Return the Application.
return app
def run_a_job(job_id, db=None, force=False):
"""
Run the JSA processing of the given job_id (integer).
By default it will look in the database determined by the JSA_proc
config. Optionally a database object can be given for testing
purposes.
"""
if not db:
# Get link to database
db = get_database()
logger.info('About to run job %i', job_id)
try:
# Change status of job to Running, raise an error if not currently in
# WAITING state.
db.change_state(job_id, JSAProcState.RUNNING,
'Job is about to be run on host {0}'.format(
gethostname().partition('.')[0]),
state_prev=(None if force else JSAProcState.WAITING))
except NoRowsError:
# If the job was not in the WAITING state, it is likely that another
# process is also trying to run it. Trap the error so that the
# ErrorDecorator does not put the job into the ERROR state as that
# will cause the other process to fail to set the job to PROCESSED.
logger.error('Job %i cannot be run because it is not waiting',
job_id)
return
# Input file_list -- this should be better? or in jsawrapdr?
input_dir = get_input_dir(job_id)
input_file_list_path = os.path.join(input_dir, input_list_name)
if not os.path.exists(input_file_list_path):
raise JSAProcError('Input file list %s not found for job_id %i'
% (input_file_list_path, job_id))
# Check every file on input_file list exists.
inputfl = open(input_file_list_path, 'r')
for input_file in inputfl:
input_file = input_file.strip()
if not os.path.isfile(input_file):
# If a file is missing, get log.
logstring = 'Input file %s for job %i has gone missing' % (
input_file, job_id)
logger.error(logstring)
logs = db.get_logs(job_id)
states = [i.state_new for i in logs]
# If it has only been in the state MISSING twice before, then try
# again.
if states.count(JSAProcState.MISSING) <= 2:
logstring += ': moving to missing.'
logger.warning('Moving job %i to state MISSING due to '
'missing file(s) %s',
job_id, input_file)
db.change_state(job_id, JSAProcState.MISSING,
logstring, state_prev=JSAProcState.RUNNING)
return job_id
else:
# If it has been in the missing STATE more than two times,
# give up and move it into ERROR state to be fixed manually.
logstring += ': moving to error.'
logger.info('Moving job %s to state ERROR due to missing'
' file(s).', job_id)
inputfl.close()
raise JSAProcError('Input file %s for job %i has gone missing.'
% (input_file, job_id))
inputfl.close()
logger.debug('All input files found for job %s.', job_id)
# Get the mode and drparameters of the job.
job = db.get_job(id_=job_id)
mode = job.mode
drparameters = job.parameters
# Get the starlink to be used from the task table.
starpath = None
version = None
command_run = None
raw_output = None
try:
task_info = db.get_task_info(job.task)
starpath = task_info.starlink_dir
version = task_info.version
command_run = task_info.command_run
raw_output = task_info.raw_output
except NoRowsError:
# If the task doesn't have task info, leave "starpath" as None
# so that jsawrapdr_run uses the default value from the configuration
# file.
pass
# Run the processing job.
logger.debug('Launching jsawrapdr: mode=%s, parameters=%s',
mode, drparameters)
log = jsawrapdr_run(
job_id, input_file_list_path, mode, drparameters,
cleanup='cadc', location='JAC', starlink_dir=starpath,
persist=True, version=version, command_run=command_run,
raw_output=raw_output)
# Create list of output files.
logger.debug('Preparing list of output files')
output_files = get_output_files(job_id)
# write output files to table
logger.debug('Storing list of output files')
db.set_output_files(job_id, output_files)
# If task begins with hpx, get tiles from list of output_files
# and write to tile table in db.
if hpx_task.search(job.task):
logger.debug('Storing list of output tiles for HPX job ' + str(job_id))
tiles = hpx_tiles_from_filenames([x.filename for x in output_files])
db.set_tilelist(job_id, tiles)
logger.debug('Job ' + str(job_id) + ' produced output on tiles ' +
', '.join(str(i) for i in tiles))
# Change state of job.
db.change_state(
job_id, JSAProcState.PROCESSED,
'Job has been successfully processed',
state_prev=JSAProcState.RUNNING)
logger.info('Done running job %i', job_id)
return job_id
