def test_state_name(self): """Test lookup of state names.""" states = { JSAProcState.UNKNOWN: 'Unknown', JSAProcState.QUEUED: 'Queued', JSAProcState.MISSING: 'Missing', JSAProcState.FETCHING: 'Fetching', JSAProcState.WAITING: 'Waiting', JSAProcState.RUNNING: 'Running', JSAProcState.PROCESSED: 'Processed', JSAProcState.TRANSFERRING: 'Transferring', JSAProcState.INGEST_QUEUE: 'Queued to reingest', JSAProcState.INGEST_FETCH: 'Fetching to reingest', JSAProcState.INGESTION: 'Waiting to ingest', JSAProcState.INGESTING: 'Ingesting', JSAProcState.COMPLETE: 'Complete', JSAProcState.ERROR: 'Error', JSAProcState.DELETED: 'Deleted', JSAProcState.WONTWORK: 'Won\'t work', } for (state, name) in states.items(): self.assertEqual(JSAProcState.get_name(state), name) with self.assertRaises(JSAProcError): JSAProcState.get_name('!')
def test_state_info(self): """Test retrieval of state information.""" # We should get an error if the state does not exist. with self.assertRaises(JSAProcError): JSAProcState.get_info('!') # Try a small sample of info values self.assertEqual( JSAProcState.get_info(JSAProcState.WAITING).active, False) self.assertEqual( JSAProcState.get_info(JSAProcState.FETCHING).active, True) self.assertEqual( JSAProcState.get_info(JSAProcState.INGESTION).phase, JSAProcState.PHASE_RUN) self.assertEqual( JSAProcState.get_info(JSAProcState.UNKNOWN).phase, JSAProcState.PHASE_QUEUE) self.assertEqual( JSAProcState.get_info(JSAProcState.RUNNING).name, 'Running') self.assertEqual( JSAProcState.get_info(JSAProcState.COMPLETE).name, 'Complete') self.assertEqual( JSAProcState.get_info(JSAProcState.INGESTING).final, False) self.assertEqual( JSAProcState.get_info(JSAProcState.DELETED).final, True)
def reset_jobs(task, date_start, date_end, instrument=None, state=None, force=False, dry_run=False): """Change the state of the specified jobs back to "Unknown". If a state is specified, select only that state. Active jobs are skipped unless the force argument is set. """ db = get_database() obsquery = {} if date_start is not None and date_end is not None: obsquery['utdate'] = Range(date_start, date_end) elif date_start is None and date_end is None: pass else: raise CommandError('only one of start and end date specified') if instrument is not None: obsquery['instrument'] = instrument if state is not None: state = JSAProcState.lookup_name(state) n_active = 0 for job in db.find_jobs(location='JAC', task=task, obsquery=obsquery, state=state): state_info = JSAProcState.get_info(job.state) # Check if the job is in an "active" state. if state_info.active and not force: logger.warning('Skipping active job %i (%s)', job.id, state_info.name) n_active += 1 continue logger.info('Resetting status of job %i (was %s)', job.id, state_info.name) if not dry_run: db.change_state(job.id, JSAProcState.UNKNOWN, 'Resetting job', state_prev=job.state) if n_active: raise CommandError('Could not reset {0} active jobs'.format(n_active))
def prepare_change_state(db, job_ids, newstate, state_prev, message, username): if not JSAProcState.is_valid(newstate): raise ErrorPage('Unknown state %s' % (newstate)) if not JSAProcState.is_valid(state_prev): raise ErrorPage('Unknown (previous) state %s' % (newstate)) if message == '': raise ErrorPage('You must provide a message to change state!') for job_id in job_ids: db.change_state(job_id, newstate, message, state_prev=state_prev, username=username)
def reset_jobs(task, date_start, date_end, instrument=None, state=None, force=False, dry_run=False): """Change the state of the specified jobs back to "Unknown". If a state is specified, select only that state. Active jobs are skipped unless the force argument is set. """ db = get_database() obsquery = {} if date_start is not None and date_end is not None: obsquery['utdate'] = Range(date_start, date_end) elif date_start is None and date_end is None: pass else: raise CommandError('only one of start and end date specified') if instrument is not None: obsquery['instrument'] = instrument if state is not None: state = JSAProcState.lookup_name(state) n_active = 0 for job in db.find_jobs(location='JAC', task=task, obsquery=obsquery, state=state): state_info = JSAProcState.get_info(job.state) # Check if the job is in an "active" state. if state_info.active and not force: logger.warning('Skipping active job %i (%s)', job.id, state_info.name) n_active += 1 continue logger.info('Resetting status of job %i (was %s)', job.id, state_info.name) if not dry_run: db.change_state(job.id, JSAProcState.UNKNOWN, 'Resetting job', state_prev=job.state) if n_active: raise CommandError( 'Could not reset {0} active jobs'.format(n_active))
def etransfer_send_output(job_id, dry_run=False, force=False): """High level e-transfer function for use from scripts. This function makes some basic checks and then launches the private function _etransfer_send under the control of the ErrorDecorator so that any subsequent errors are captured. """ logger.debug('Preparing to e-transfer output for job {0}'.format(job_id)) # When not in dry run mode, check that etransfer is being # run on the correct machine by the correct user and with # sufficient available disk space. if not dry_run: etransfer_check_config() _etransfer_check_space() logger.debug('Connecting to JSA processing database') db = get_database() if not force: job = db.get_job(id_=job_id) if job.state != JSAProcState.PROCESSED: message = 'Job {0} cannot be e-transferred as it is in ' \ 'state {1}'.format(job_id, JSAProcState.get_name(job.state)) logger.error(message) raise CommandError(message) _etransfer_send(job_id, dry_run=dry_run, db=db, force=force) logger.debug('Done adding output for job {0} to e-transfer'.format(job_id))
def job_change_state(): # Get the variables from POST newstate = request.form['newstate'] state_prev = request.form['state_prev'] message = request.form['message'] job_ids = request.form.getlist('job_id') url = request.form['url'] username = request.authorization['username'] try: # Change the state. prepare_change_state(db, job_ids, newstate, state_prev, message, username) # Redirect the page to correct info. flash('The status has been changed to %s.' % JSAProcState.get_name( newstate)) raise HTTPRedirect(url) except ErrorPage as err: return error_page_response(err)
def state_phase_filter(state): phase = JSAProcState.get_info(state).phase if phase == JSAProcState.PHASE_QUEUE: return 'queue' elif phase == JSAProcState.PHASE_FETCH: return 'fetch' elif phase == JSAProcState.PHASE_RUN: return 'run' elif phase == JSAProcState.PHASE_COMPLETE: return 'complete' elif phase == JSAProcState.PHASE_ERROR: return 'error' raise HTTPError('Unknown phase {0}'.format(phase))
def test_state_info(self): """Test retrieval of state information.""" # We should get an error if the state does not exist. with self.assertRaises(JSAProcError): JSAProcState.get_info('!') # Try a small sample of info values self.assertEqual(JSAProcState.get_info(JSAProcState.WAITING).active, False) self.assertEqual(JSAProcState.get_info(JSAProcState.FETCHING).active, True) self.assertEqual(JSAProcState.get_info(JSAProcState.INGESTION).phase, JSAProcState.PHASE_RUN) self.assertEqual(JSAProcState.get_info(JSAProcState.UNKNOWN).phase, JSAProcState.PHASE_QUEUE) self.assertEqual(JSAProcState.get_info(JSAProcState.RUNNING).name, 'Running') self.assertEqual(JSAProcState.get_info(JSAProcState.COMPLETE).name, 'Complete') self.assertEqual(JSAProcState.get_info(JSAProcState.INGESTING).final, False) self.assertEqual(JSAProcState.get_info(JSAProcState.DELETED).final, True)
def job_change_state(): # Get the variables from POST newstate = request.form['newstate'] state_prev = request.form['state_prev'] message = request.form['message'] job_ids = request.form.getlist('job_id') url = request.form['url'] username = request.authorization['username'] try: # Change the state. prepare_change_state(db, job_ids, newstate, state_prev, message, username) # Redirect the page to correct info. flash('The status has been changed to %s.' % JSAProcState.get_name(newstate)) raise HTTPRedirect(url) except ErrorPage as err: return error_page_response(err)
def prepare_summary_piechart(db, task=None, obsquerydict=None, date_min=None, date_max=None): """ Create a piechart of number of jobs in each state for a given task and obsquery. *task*: name of task in database *obsquerydict*: dictionary of values that match the jcmtobsinfo.ObsQueryDict. Returns a sendfile object of mime-type image/png. """ # Dictionaries for the result job_summary_dict = OrderedDict() # Fix up the obsquery to the right format for find jobs obsquery = {} for key, value in obsquerydict.items(): if value: obsquery.update(ObsQueryDict[key][value].where) # Sort out dates if date_min is not None or date_max is not None: obsquery['utdate'] = Range(date_min, date_max) # Perform the find_jobs task for the given constraints in each # JSAProcState. for s in JSAProcState.STATE_ALL: # Don't include deleted jobs in pie chart if JSAProcState.get_name(s) != 'Deleted': job_summary_dict[s] = db.find_jobs(state=s, task=task, obsquery=obsquery, count=True) # Get numbers, names and colors for the pie chart. values = job_summary_dict.values() names = [JSAProcState.get_name(i) for i in JSAProcState.STATE_ALL[:-1]] # This should probably be done better... phase_colors = {} phase_colors[JSAProcState.PHASE_QUEUE] = 'red' phase_colors[JSAProcState.PHASE_FETCH] = 'yellow' phase_colors[JSAProcState.PHASE_RUN] = 'green' phase_colors[JSAProcState.PHASE_COMPLETE] = 'blue' phase_colors[JSAProcState.PHASE_ERROR] = 'black' phases = [ phase_colors[JSAProcState.get_info(s).phase] for s in JSAProcState.STATE_ALL[:-1] ] # Remove any states that don't have any jobs in them i = 0 while i < len(values): if values[i] == 0: values.pop(i) names.pop(i) phases.pop(i) else: i += 1 # Create pie chart fig = Figure(figsize=(6, 5)) ax = fig.add_subplot(111) ax.set_aspect(1) p, t, a = ax.pie(values, labels=names, colors=phases, autopct='%.1F') for i in range(len(a)): if p[i].get_facecolor() == (1.0, 1.0, 0.0, 1.0): a[i].set_color('black') else: a[i].set_color('white') p[i].set_edgecolor('none') ax.patch.set_visible(False) fig.patch.set_visible(False) # Put figure into a send_file object canvas = FigureCanvas(fig) img = StringIO() canvas.print_png(img) img.seek(0) return send_file(img, mimetype='image/png')
def search_log_files( pattern, filename_pattern, task, project=None, state=None, after_context=None): db = get_database() re_pattern = re.compile(pattern) re_filename = re.compile(filename_pattern) if state is None: state = JSAProcState.COMPLETE else: state = JSAProcState.lookup_name(state) if after_context is None: after_context = 0 search_kwargs = { 'task': task, 'state': state, } if project is not None: search_kwargs['obsquery'] = {'project': project} jobs = [x.id for x in db.find_jobs(**search_kwargs)] for job_id in jobs: logger.debug('Checking log files for job %i', job_id) log_dir = get_log_dir(job_id) # Find the latest matching log by iterating through them in reverse # order and "breaking" after the first match. for filename in sorted(os.listdir(log_dir), reverse=True): if not re_filename.search(filename): continue logger.debug('Found log file for job %i: %s', job_id, filename) matched = 0 matched_lines = [] pathname = os.path.join(log_dir, filename) with open(pathname, 'r') as f: for line in f: if matched or re_pattern.search(line): matched += 1 matched_lines.append(line.rstrip()) if matched > after_context: break if matched: logger.info( 'Found match for job %i: %s', job_id, matched_lines[0]) for matched_line in matched_lines[1:]: logger.info( '... continuation %i: %s', job_id, matched_line) break
def state_active_test(state): return JSAProcState.get_info(state).active
def state_name_filter(state): return JSAProcState.get_name(state)
def prepare_summary_piechart(db, task=None, obsquerydict=None, date_min=None, date_max=None): """ Create a piechart of number of jobs in each state for a given task and obsquery. *task*: name of task in database *obsquerydict*: dictionary of values that match the jcmtobsinfo.ObsQueryDict. Returns a sendfile object of mime-type image/png. """ # Dictionaries for the result job_summary_dict = OrderedDict() # Fix up the obsquery to the right format for find jobs obsquery = {} for key, value in obsquerydict.items(): if value: obsquery.update(ObsQueryDict[key][value].where) # Sort out dates if date_min is not None or date_max is not None: obsquery['utdate'] = Range(date_min, date_max) # Perform the find_jobs task for the given constraints in each # JSAProcState. for s in JSAProcState.STATE_ALL: # Don't include deleted jobs in pie chart if JSAProcState.get_name(s) != 'Deleted': job_summary_dict[s] = db.find_jobs(state=s, task=task, obsquery=obsquery, count=True) # Get numbers, names and colors for the pie chart. values = job_summary_dict.values() names = [JSAProcState.get_name(i) for i in JSAProcState.STATE_ALL[:-1]] # This should probably be done better... phase_colors = {} phase_colors[JSAProcState.PHASE_QUEUE] = 'red' phase_colors[JSAProcState.PHASE_FETCH] = 'yellow' phase_colors[JSAProcState.PHASE_RUN] = 'green' phase_colors[JSAProcState.PHASE_COMPLETE] = 'blue' phase_colors[JSAProcState.PHASE_ERROR] = 'black' phases = [phase_colors[JSAProcState.get_info(s).phase] for s in JSAProcState.STATE_ALL[:-1]] # Remove any states that don't have any jobs in them i = 0 while i < len(values): if values[i] == 0: values.pop(i) names.pop(i) phases.pop(i) else: i += 1 # Create pie chart fig = Figure(figsize=(6, 5)) ax = fig.add_subplot(111) ax.set_aspect(1) p, t, a = ax.pie(values, labels=names, colors=phases, autopct='%.1F') for i in range(len(a)): if p[i].get_facecolor() == (1.0, 1.0, 0.0, 1.0): a[i].set_color('black') else: a[i].set_color('white') p[i].set_edgecolor('none') ax.patch.set_visible(False) fig.patch.set_visible(False) # Put figure into a send_file object canvas = FigureCanvas(fig) img = StringIO.StringIO() canvas.print_png(img) img.seek(0) return send_file(img, mimetype='image/png')
def search_log_files(pattern, filename_pattern, task, project=None, state=None, after_context=None): db = get_database() re_pattern = re.compile(pattern) re_filename = re.compile(filename_pattern) if state is None: state = JSAProcState.COMPLETE else: state = JSAProcState.lookup_name(state) if after_context is None: after_context = 0 search_kwargs = { 'task': task, 'state': state, } if project is not None: search_kwargs['obsquery'] = {'project': project} jobs = [x.id for x in db.find_jobs(**search_kwargs)] for job_id in jobs: logger.debug('Checking log files for job %i', job_id) log_dir = get_log_dir(job_id) # Find the latest matching log by iterating through them in reverse # order and "breaking" after the first match. for filename in sorted(os.listdir(log_dir), reverse=True): if not re_filename.search(filename): continue logger.debug('Found log file for job %i: %s', job_id, filename) matched = 0 matched_lines = [] pathname = os.path.join(log_dir, filename) with open(pathname, 'r') as f: for line in f: if matched or re_pattern.search(line): matched += 1 matched_lines.append(line.rstrip()) if matched > after_context: break if matched: logger.info('Found match for job %i: %s', job_id, matched_lines[0]) for matched_line in matched_lines[1:]: logger.info('... continuation %i: %s', job_id, matched_line) break