def test_state_name(self):
"""Test lookup of state names."""
states = {
JSAProcState.UNKNOWN: 'Unknown',
JSAProcState.QUEUED: 'Queued',
JSAProcState.MISSING: 'Missing',
JSAProcState.FETCHING: 'Fetching',
JSAProcState.WAITING: 'Waiting',
JSAProcState.RUNNING: 'Running',
JSAProcState.PROCESSED: 'Processed',
JSAProcState.TRANSFERRING: 'Transferring',
JSAProcState.INGEST_QUEUE: 'Queued to reingest',
JSAProcState.INGEST_FETCH: 'Fetching to reingest',
JSAProcState.INGESTION: 'Waiting to ingest',
JSAProcState.INGESTING: 'Ingesting',
JSAProcState.COMPLETE: 'Complete',
JSAProcState.ERROR: 'Error',
JSAProcState.DELETED: 'Deleted',
JSAProcState.WONTWORK: 'Won\'t work',
}
for (state, name) in states.items():
self.assertEqual(JSAProcState.get_name(state), name)
with self.assertRaises(JSAProcError):
JSAProcState.get_name('!')
def test_state_info(self):
"""Test retrieval of state information."""
# We should get an error if the state does not exist.
with self.assertRaises(JSAProcError):
JSAProcState.get_info('!')
# Try a small sample of info values
self.assertEqual(
JSAProcState.get_info(JSAProcState.WAITING).active, False)
self.assertEqual(
JSAProcState.get_info(JSAProcState.FETCHING).active, True)
self.assertEqual(
JSAProcState.get_info(JSAProcState.INGESTION).phase,
JSAProcState.PHASE_RUN)
self.assertEqual(
JSAProcState.get_info(JSAProcState.UNKNOWN).phase,
JSAProcState.PHASE_QUEUE)
self.assertEqual(
JSAProcState.get_info(JSAProcState.RUNNING).name, 'Running')
self.assertEqual(
JSAProcState.get_info(JSAProcState.COMPLETE).name, 'Complete')
self.assertEqual(
JSAProcState.get_info(JSAProcState.INGESTING).final, False)
self.assertEqual(
JSAProcState.get_info(JSAProcState.DELETED).final, True)
def test_state_name(self):
"""Test lookup of state names."""
states = {
JSAProcState.UNKNOWN: 'Unknown',
JSAProcState.QUEUED: 'Queued',
JSAProcState.MISSING: 'Missing',
JSAProcState.FETCHING: 'Fetching',
JSAProcState.WAITING: 'Waiting',
JSAProcState.RUNNING: 'Running',
JSAProcState.PROCESSED: 'Processed',
JSAProcState.TRANSFERRING: 'Transferring',
JSAProcState.INGEST_QUEUE: 'Queued to reingest',
JSAProcState.INGEST_FETCH: 'Fetching to reingest',
JSAProcState.INGESTION: 'Waiting to ingest',
JSAProcState.INGESTING: 'Ingesting',
JSAProcState.COMPLETE: 'Complete',
JSAProcState.ERROR: 'Error',
JSAProcState.DELETED: 'Deleted',
JSAProcState.WONTWORK: 'Won\'t work',
}
for (state, name) in states.items():
self.assertEqual(JSAProcState.get_name(state), name)
with self.assertRaises(JSAProcError):
JSAProcState.get_name('!')
def reset_jobs(task,
date_start,
date_end,
instrument=None,
state=None,
force=False,
dry_run=False):
"""Change the state of the specified jobs back to "Unknown".
If a state is specified, select only that state.
Active jobs are skipped unless the force argument is set.
"""
db = get_database()
obsquery = {}
if date_start is not None and date_end is not None:
obsquery['utdate'] = Range(date_start, date_end)
elif date_start is None and date_end is None:
pass
else:
raise CommandError('only one of start and end date specified')
if instrument is not None:
obsquery['instrument'] = instrument
if state is not None:
state = JSAProcState.lookup_name(state)
n_active = 0
for job in db.find_jobs(location='JAC',
task=task,
obsquery=obsquery,
state=state):
state_info = JSAProcState.get_info(job.state)
# Check if the job is in an "active" state.
if state_info.active and not force:
logger.warning('Skipping active job %i (%s)', job.id,
state_info.name)
n_active += 1
continue
logger.info('Resetting status of job %i (was %s)', job.id,
state_info.name)
if not dry_run:
db.change_state(job.id,
JSAProcState.UNKNOWN,
'Resetting job',
state_prev=job.state)
if n_active:
raise CommandError('Could not reset {0} active jobs'.format(n_active))
def prepare_change_state(db, job_ids, newstate, state_prev, message, username):
if not JSAProcState.is_valid(newstate):
raise ErrorPage('Unknown state %s' % (newstate))
if not JSAProcState.is_valid(state_prev):
raise ErrorPage('Unknown (previous) state %s' % (newstate))
if message == '':
raise ErrorPage('You must provide a message to change state!')
for job_id in job_ids:
db.change_state(job_id, newstate, message, state_prev=state_prev,
username=username)
def reset_jobs(task, date_start, date_end, instrument=None,
state=None, force=False, dry_run=False):
"""Change the state of the specified jobs back to "Unknown".
If a state is specified, select only that state.
Active jobs are skipped unless the force argument is set.
"""
db = get_database()
obsquery = {}
if date_start is not None and date_end is not None:
obsquery['utdate'] = Range(date_start, date_end)
elif date_start is None and date_end is None:
pass
else:
raise CommandError('only one of start and end date specified')
if instrument is not None:
obsquery['instrument'] = instrument
if state is not None:
state = JSAProcState.lookup_name(state)
n_active = 0
for job in db.find_jobs(location='JAC', task=task, obsquery=obsquery,
state=state):
state_info = JSAProcState.get_info(job.state)
# Check if the job is in an "active" state.
if state_info.active and not force:
logger.warning('Skipping active job %i (%s)',
job.id, state_info.name)
n_active += 1
continue
logger.info('Resetting status of job %i (was %s)',
job.id, state_info.name)
if not dry_run:
db.change_state(job.id, JSAProcState.UNKNOWN,
'Resetting job', state_prev=job.state)
if n_active:
raise CommandError(
'Could not reset {0} active jobs'.format(n_active))
def prepare_change_state(db, job_ids, newstate, state_prev, message, username):
if not JSAProcState.is_valid(newstate):
raise ErrorPage('Unknown state %s' % (newstate))
if not JSAProcState.is_valid(state_prev):
raise ErrorPage('Unknown (previous) state %s' % (newstate))
if message == '':
raise ErrorPage('You must provide a message to change state!')
for job_id in job_ids:
db.change_state(job_id,
newstate,
message,
state_prev=state_prev,
username=username)
def etransfer_send_output(job_id, dry_run=False, force=False):
"""High level e-transfer function for use from scripts.
This function makes some basic checks and then launches
the private function _etransfer_send under the control
of the ErrorDecorator so that any subsequent errors
are captured.
"""
logger.debug('Preparing to e-transfer output for job {0}'.format(job_id))
# When not in dry run mode, check that etransfer is being
# run on the correct machine by the correct user and with
# sufficient available disk space.
if not dry_run:
etransfer_check_config()
_etransfer_check_space()
logger.debug('Connecting to JSA processing database')
db = get_database()
if not force:
job = db.get_job(id_=job_id)
if job.state != JSAProcState.PROCESSED:
message = 'Job {0} cannot be e-transferred as it is in ' \
'state {1}'.format(job_id,
JSAProcState.get_name(job.state))
logger.error(message)
raise CommandError(message)
_etransfer_send(job_id, dry_run=dry_run, db=db, force=force)
logger.debug('Done adding output for job {0} to e-transfer'.format(job_id))
def etransfer_send_output(job_id, dry_run=False, force=False):
"""High level e-transfer function for use from scripts.
This function makes some basic checks and then launches
the private function _etransfer_send under the control
of the ErrorDecorator so that any subsequent errors
are captured.
"""
logger.debug('Preparing to e-transfer output for job {0}'.format(job_id))
# When not in dry run mode, check that etransfer is being
# run on the correct machine by the correct user and with
# sufficient available disk space.
if not dry_run:
etransfer_check_config()
_etransfer_check_space()
logger.debug('Connecting to JSA processing database')
db = get_database()
if not force:
job = db.get_job(id_=job_id)
if job.state != JSAProcState.PROCESSED:
message = 'Job {0} cannot be e-transferred as it is in ' \
'state {1}'.format(job_id,
JSAProcState.get_name(job.state))
logger.error(message)
raise CommandError(message)
_etransfer_send(job_id, dry_run=dry_run, db=db, force=force)
logger.debug('Done adding output for job {0} to e-transfer'.format(job_id))
def job_change_state():
# Get the variables from POST
newstate = request.form['newstate']
state_prev = request.form['state_prev']
message = request.form['message']
job_ids = request.form.getlist('job_id')
url = request.form['url']
username = request.authorization['username']
try:
# Change the state.
prepare_change_state(db, job_ids,
newstate,
state_prev,
message,
username)
# Redirect the page to correct info.
flash('The status has been changed to %s.' % JSAProcState.get_name(
newstate))
raise HTTPRedirect(url)
except ErrorPage as err:
return error_page_response(err)
def state_phase_filter(state):
phase = JSAProcState.get_info(state).phase
if phase == JSAProcState.PHASE_QUEUE:
return 'queue'
elif phase == JSAProcState.PHASE_FETCH:
return 'fetch'
elif phase == JSAProcState.PHASE_RUN:
return 'run'
elif phase == JSAProcState.PHASE_COMPLETE:
return 'complete'
elif phase == JSAProcState.PHASE_ERROR:
return 'error'
raise HTTPError('Unknown phase {0}'.format(phase))
def state_phase_filter(state):
phase = JSAProcState.get_info(state).phase
if phase == JSAProcState.PHASE_QUEUE:
return 'queue'
elif phase == JSAProcState.PHASE_FETCH:
return 'fetch'
elif phase == JSAProcState.PHASE_RUN:
return 'run'
elif phase == JSAProcState.PHASE_COMPLETE:
return 'complete'
elif phase == JSAProcState.PHASE_ERROR:
return 'error'
raise HTTPError('Unknown phase {0}'.format(phase))
def test_state_info(self):
"""Test retrieval of state information."""
# We should get an error if the state does not exist.
with self.assertRaises(JSAProcError):
JSAProcState.get_info('!')
# Try a small sample of info values
self.assertEqual(JSAProcState.get_info(JSAProcState.WAITING).active,
False)
self.assertEqual(JSAProcState.get_info(JSAProcState.FETCHING).active,
True)
self.assertEqual(JSAProcState.get_info(JSAProcState.INGESTION).phase,
JSAProcState.PHASE_RUN)
self.assertEqual(JSAProcState.get_info(JSAProcState.UNKNOWN).phase,
JSAProcState.PHASE_QUEUE)
self.assertEqual(JSAProcState.get_info(JSAProcState.RUNNING).name,
'Running')
self.assertEqual(JSAProcState.get_info(JSAProcState.COMPLETE).name,
'Complete')
self.assertEqual(JSAProcState.get_info(JSAProcState.INGESTING).final,
False)
self.assertEqual(JSAProcState.get_info(JSAProcState.DELETED).final,
True)
def job_change_state():
# Get the variables from POST
newstate = request.form['newstate']
state_prev = request.form['state_prev']
message = request.form['message']
job_ids = request.form.getlist('job_id')
url = request.form['url']
username = request.authorization['username']
try:
# Change the state.
prepare_change_state(db, job_ids, newstate, state_prev, message,
username)
# Redirect the page to correct info.
flash('The status has been changed to %s.' %
JSAProcState.get_name(newstate))
raise HTTPRedirect(url)
except ErrorPage as err:
return error_page_response(err)
def prepare_summary_piechart(db,
task=None,
obsquerydict=None,
date_min=None,
date_max=None):
"""
Create a piechart of number of jobs in each state for a given task
and obsquery.
*task*: name of task in database
*obsquerydict*: dictionary of values that match the
jcmtobsinfo.ObsQueryDict.
Returns a sendfile object of mime-type image/png.
"""
# Dictionaries for the result
job_summary_dict = OrderedDict()
# Fix up the obsquery to the right format for find jobs
obsquery = {}
for key, value in obsquerydict.items():
if value:
obsquery.update(ObsQueryDict[key][value].where)
# Sort out dates
if date_min is not None or date_max is not None:
obsquery['utdate'] = Range(date_min, date_max)
# Perform the find_jobs task for the given constraints in each
# JSAProcState.
for s in JSAProcState.STATE_ALL:
# Don't include deleted jobs in pie chart
if JSAProcState.get_name(s) != 'Deleted':
job_summary_dict[s] = db.find_jobs(state=s,
task=task,
obsquery=obsquery,
count=True)
# Get numbers, names and colors for the pie chart.
values = job_summary_dict.values()
names = [JSAProcState.get_name(i) for i in JSAProcState.STATE_ALL[:-1]]
# This should probably be done better...
phase_colors = {}
phase_colors[JSAProcState.PHASE_QUEUE] = 'red'
phase_colors[JSAProcState.PHASE_FETCH] = 'yellow'
phase_colors[JSAProcState.PHASE_RUN] = 'green'
phase_colors[JSAProcState.PHASE_COMPLETE] = 'blue'
phase_colors[JSAProcState.PHASE_ERROR] = 'black'
phases = [
phase_colors[JSAProcState.get_info(s).phase]
for s in JSAProcState.STATE_ALL[:-1]
]
# Remove any states that don't have any jobs in them
i = 0
while i < len(values):
if values[i] == 0:
values.pop(i)
names.pop(i)
phases.pop(i)
else:
i += 1
# Create pie chart
fig = Figure(figsize=(6, 5))
ax = fig.add_subplot(111)
ax.set_aspect(1)
p, t, a = ax.pie(values, labels=names, colors=phases, autopct='%.1F')
for i in range(len(a)):
if p[i].get_facecolor() == (1.0, 1.0, 0.0, 1.0):
a[i].set_color('black')
else:
a[i].set_color('white')
p[i].set_edgecolor('none')
ax.patch.set_visible(False)
fig.patch.set_visible(False)
# Put figure into a send_file object
canvas = FigureCanvas(fig)
img = StringIO()
canvas.print_png(img)
img.seek(0)
return send_file(img, mimetype='image/png')
def search_log_files(
pattern, filename_pattern, task,
project=None, state=None, after_context=None):
db = get_database()
re_pattern = re.compile(pattern)
re_filename = re.compile(filename_pattern)
if state is None:
state = JSAProcState.COMPLETE
else:
state = JSAProcState.lookup_name(state)
if after_context is None:
after_context = 0
search_kwargs = {
'task': task,
'state': state,
}
if project is not None:
search_kwargs['obsquery'] = {'project': project}
jobs = [x.id for x in db.find_jobs(**search_kwargs)]
for job_id in jobs:
logger.debug('Checking log files for job %i', job_id)
log_dir = get_log_dir(job_id)
# Find the latest matching log by iterating through them in reverse
# order and "breaking" after the first match.
for filename in sorted(os.listdir(log_dir), reverse=True):
if not re_filename.search(filename):
continue
logger.debug('Found log file for job %i: %s', job_id, filename)
matched = 0
matched_lines = []
pathname = os.path.join(log_dir, filename)
with open(pathname, 'r') as f:
for line in f:
if matched or re_pattern.search(line):
matched += 1
matched_lines.append(line.rstrip())
if matched > after_context:
break
if matched:
logger.info(
'Found match for job %i: %s', job_id, matched_lines[0])
for matched_line in matched_lines[1:]:
logger.info(
'... continuation %i: %s', job_id, matched_line)
break
def state_active_test(state):
return JSAProcState.get_info(state).active
def state_name_filter(state):
return JSAProcState.get_name(state)
def state_active_test(state):
return JSAProcState.get_info(state).active
def state_name_filter(state):
return JSAProcState.get_name(state)
def prepare_summary_piechart(db, task=None, obsquerydict=None, date_min=None,
date_max=None):
"""
Create a piechart of number of jobs in each state for a given task
and obsquery.
*task*: name of task in database
*obsquerydict*: dictionary of values that match the
jcmtobsinfo.ObsQueryDict.
Returns a sendfile object of mime-type image/png.
"""
# Dictionaries for the result
job_summary_dict = OrderedDict()
# Fix up the obsquery to the right format for find jobs
obsquery = {}
for key, value in obsquerydict.items():
if value:
obsquery.update(ObsQueryDict[key][value].where)
# Sort out dates
if date_min is not None or date_max is not None:
obsquery['utdate'] = Range(date_min, date_max)
# Perform the find_jobs task for the given constraints in each
# JSAProcState.
for s in JSAProcState.STATE_ALL:
# Don't include deleted jobs in pie chart
if JSAProcState.get_name(s) != 'Deleted':
job_summary_dict[s] = db.find_jobs(state=s, task=task,
obsquery=obsquery,
count=True)
# Get numbers, names and colors for the pie chart.
values = job_summary_dict.values()
names = [JSAProcState.get_name(i) for i in JSAProcState.STATE_ALL[:-1]]
# This should probably be done better...
phase_colors = {}
phase_colors[JSAProcState.PHASE_QUEUE] = 'red'
phase_colors[JSAProcState.PHASE_FETCH] = 'yellow'
phase_colors[JSAProcState.PHASE_RUN] = 'green'
phase_colors[JSAProcState.PHASE_COMPLETE] = 'blue'
phase_colors[JSAProcState.PHASE_ERROR] = 'black'
phases = [phase_colors[JSAProcState.get_info(s).phase]
for s in JSAProcState.STATE_ALL[:-1]]
# Remove any states that don't have any jobs in them
i = 0
while i < len(values):
if values[i] == 0:
values.pop(i)
names.pop(i)
phases.pop(i)
else:
i += 1
# Create pie chart
fig = Figure(figsize=(6, 5))
ax = fig.add_subplot(111)
ax.set_aspect(1)
p, t, a = ax.pie(values, labels=names, colors=phases, autopct='%.1F')
for i in range(len(a)):
if p[i].get_facecolor() == (1.0, 1.0, 0.0, 1.0):
a[i].set_color('black')
else:
a[i].set_color('white')
p[i].set_edgecolor('none')
ax.patch.set_visible(False)
fig.patch.set_visible(False)
# Put figure into a send_file object
canvas = FigureCanvas(fig)
img = StringIO.StringIO()
canvas.print_png(img)
img.seek(0)
return send_file(img, mimetype='image/png')
def search_log_files(pattern,
filename_pattern,
task,
project=None,
state=None,
after_context=None):
db = get_database()
re_pattern = re.compile(pattern)
re_filename = re.compile(filename_pattern)
if state is None:
state = JSAProcState.COMPLETE
else:
state = JSAProcState.lookup_name(state)
if after_context is None:
after_context = 0
search_kwargs = {
'task': task,
'state': state,
}
if project is not None:
search_kwargs['obsquery'] = {'project': project}
jobs = [x.id for x in db.find_jobs(**search_kwargs)]
for job_id in jobs:
logger.debug('Checking log files for job %i', job_id)
log_dir = get_log_dir(job_id)
# Find the latest matching log by iterating through them in reverse
# order and "breaking" after the first match.
for filename in sorted(os.listdir(log_dir), reverse=True):
if not re_filename.search(filename):
continue
logger.debug('Found log file for job %i: %s', job_id, filename)
matched = 0
matched_lines = []
pathname = os.path.join(log_dir, filename)
with open(pathname, 'r') as f:
for line in f:
if matched or re_pattern.search(line):
matched += 1
matched_lines.append(line.rstrip())
if matched > after_context:
break
if matched:
logger.info('Found match for job %i: %s', job_id,
matched_lines[0])
for matched_line in matched_lines[1:]:
logger.info('... continuation %i: %s', job_id,
matched_line)
break