def get_log_files(job_id): """Get a dictionary of recognised logs for a given job. Scans the log directory of the given job looking for log files which match one of the patterns in the log_types dictionary. Returns a dictionary where the keys are the log types for which logs were found, and the values are lists of logs of that type. """ log_files = {} log_dir = get_log_dir(job_id) if os.path.isdir(log_dir): for file in sorted(os.listdir(get_log_dir(job_id)), reverse=True): for (type_, pattern) in log_types.items(): if pattern.match(file): if file.endswith('.html'): url = url_for('job_log_html', job_id=job_id, log=file) else: url = url_for('job_log_text', job_id=job_id, log=file) if type_ in log_files: log_files[type_].append(LogInfo(file, url, None)) else: log_files[type_] = [LogInfo(file, url, None)] return log_files
def get_log_files(job_id): """Get a dictionary of recognised logs for a given job. Scans the log directory of the given job looking for log files which match one of the patterns in the log_types dictionary. Returns a dictionary where the keys are the log types for which logs were found, and the values are lists of logs of that type. """ log_files = {} log_dir = get_log_dir(job_id) if os.path.isdir(log_dir): for file in sorted(os.listdir(get_log_dir(job_id)), reverse=True): for (type_, pattern) in log_types.items(): if pattern.match(file): if file.endswith('.html'): url = url_for('job_log_html', job_id=job_id, log=file) else: url = url_for('job_log_text', job_id=job_id, log=file) if type_ in log_files: log_files[type_].append(LogInfo(file, url)) else: log_files[type_] = [LogInfo(file, url)] return log_files
def prepare_job_list(db, page, **kwargs): # Generate query objects based on the parameters. (query, job_query) = job_search(**kwargs) # Identify number of jobs. count = db.find_jobs(count=True, **job_query) (number, page, pagination) = calculate_pagination(count, 24, page, 'job_list', query) # If no number in kwargs, add in default. if 'number' not in job_query: job_query['number'] = number jobs = [] for job in db.find_jobs(outputs='%_64.png', offset=(number * page), **job_query): if job.outputs: preview = url_for('job_preview', job_id=job.id, preview=job.outputs[0]) else: preview = None jobs.append({ 'url': url_for('job_info', job_id=job.id), 'qaurl': url_for('job_qa', job_id=job.id), 'id': job.id, 'state': job.state, 'tag': job.tag, 'location': job.location, 'preview': preview, 'qa_state': job.qa_state }) return { 'title': 'Job List', 'jobs': jobs, 'locations': ('JAC', 'CADC'), 'states': JSAProcState.STATE_ALL, 'qa_states': JSAQAState.STATE_ALL, 'tasks': db.get_tasks(), 'number': number, 'pagination': pagination, 'obsqueries': ObsQueryDict, 'query': query, 'mode': query['mode'], 'count': count, }
def job_add_note(job_id): message = request.form['message'] username = request.authorization['username'] try: # Add the note. prepare_add_note(db, job_id, message, username) # Redirect back to the job info page. flash('The note has been saved.') raise HTTPRedirect(url_for('job_info', job_id=job_id)) except ErrorPage as err: return error_page_response(err)
def job_add_note(job_id): message = request.form['message'] username = request.authorization['username'] try: # Add the note. prepare_add_note(db, job_id, message, username) # Redirect back to the job info page. flash('The note has been saved.') raise HTTPRedirect(url_for('job_info', job_id=job_id)) except ErrorPage as err: return error_page_response(err)
def get_orac_log_files(job_id): """ Get a dictionary of ORAC-DR (log.*) files for a job. Scans the log directory to get all log.* files. Skips them if they have a date stamp older than the last run of the system. """ pattern = re.compile('log.*') log_dir = get_log_dir(job_id) log_files = [] if os.path.isdir(log_dir): files = os.listdir(log_dir) for f in sorted(files): if pattern.match(f): mtime = time.ctime(os.path.getmtime(os.path.join(log_dir, f))) url = url_for('job_log_text', job_id=job_id, log=f) log_files.append(LogInfo(f, url, mtime)) return log_files
def fop_summary_getres(): userid = request.form['userid'] semester = request.form['semester'] raise HTTPRedirect( url_for('fop_summary', userid=userid, semester=semester))
def home_page(): raise HTTPRedirect(url_for('task_summary'))
def fop_summary_getres(): userid = request.form['userid'] semester = request.form['semester'] raise HTTPRedirect(url_for( 'fop_summary', userid=userid, semester=semester))
def home_page(): raise HTTPRedirect(url_for('task_summary'))
def make_output_file_list(db, job_id, preview_filter=None): """Prepare output file lists for job information pages. """ output_files = [] previews1024 = [] previews256 = [] try: for i in sorted(db.get_output_files(job_id)): url = None mtype = None if i.endswith('.png'): url = url_for('job_preview', job_id=job_id, preview=i) if preview_filter is None or any((f in i for f in preview_filter)): caption = i caption = re.sub('^jcmt_', '', caption) caption = re.sub('_(preview_)?\d+\.png', '', caption) if '_256.png' in i: previews256.append(PreviewInfo(url, caption)) if '_1024.png' in i: previews1024.append(PreviewInfo(url, caption)) elif i.endswith('.pdf'): url = url_for('job_preview_pdf', job_id=job_id, preview=i) elif i.endswith('.txt'): url = url_for('job_text_file', job_id=job_id, text_file=i) elif i.endswith('.fits'): url = 'file://{0}/{1}'.format(get_output_dir(job_id), i) if re.search('-cat[0-9]{6}', i): mtype = 'table.load.fits' elif re.search('-moc[0-9]{6}', i): # This should be "coverage.load.moc.fits" but neither GAIA # nor Aladin appear to subscribe to that mtype yet. # mtype = 'coverage.load.moc.fits' mtype = 'image.load.fits' elif '_rsp_' in i: # Prevent a broadcast button being shown for spectra # for now. mtype = None else: mtype = 'image.load.fits' # Remove URL for types we can't broadcast. if mtype is None: url = None output_files.append(FileInfo(i, url, mtype)) except NoRowsError: pass return (output_files, previews1024, previews256)
def prepare_job_info(db, job_id, query): # Fetch job information from the database. try: job = db.get_job(job_id) except NoRowsError: raise HTTPNotFound() # Convert the information to a dictionary so that we can augment it. info = job._asdict() if info['foreign_id'] is not None: if info['location'] == 'CADC': info['foreign_url'] = \ 'http://beta.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/' \ 'dp/recipe/{0}'.format(info['foreign_id']) else: info['foreign_url'] = None # Try to get tiles. try: tiles = db.get_tilelist(job_id) except NoRowsError: tiles = None if tiles == []: tiles = None # Try to get input files (if any) try: input_files = db.get_input_files(job_id) except NoRowsError: input_files = None # Try to get parent jobs (if any). # Dictionary with parent as key and filter as item. try: parents = db.get_parents(job_id) parents = dict(parents) parent_obs = OrderedDict() pjobs = list(parents.keys()) pjobs.sort() for i in pjobs: parent_obs[i] = [o._asdict() for o in db.get_obs_info(i)] except NoRowsError: parents = None parent_obs = None # See if there are any child jobs. try: children = db.get_children(job_id) except NoRowsError: children = None (output_files, previews1024, previews256) = \ make_output_file_list(db, job.id) obs_info = db.get_obs_info(job.id) if obs_info: obs_info = [o._asdict() for o in obs_info] else: obs_info = None # Logged entries in the database (newest first). log = db.get_logs(job_id) log.reverse() # Get the log files on disk (if any) log_files = get_log_files(job_id) # Get the ORAC-DR log.* files from the database try: orac_log_files = db.get_log_files(job_id) orac_log_files = [(i, url_for('job_log_text', job_id=job_id, log=i)) for i in orac_log_files] except NoRowsError: orac_log_files = None # Get notes. notes = db.get_notes(job_id) # If we know what the user's job query was (from the session information) # then set up pagination based on the previous and next job identifiers. if query is not None: (url_query, job_query) = job_search(**query) # Need to remove 'number' option from job_query to. pnquery = job_query.copy() if 'number' in pnquery: del(pnquery['number']) (prev, next) = db.job_prev_next(job_id, **pnquery) count = db.find_jobs(count=True, **job_query) pagination = Pagination( None, None if prev is None else url_for('job_info', job_id=prev), None if next is None else url_for('job_info', job_id=next), None, url_for('job_list', **url_query), count,) else: pagination = None return { 'title': 'Job {}'.format(job_id), 'info': info, 'tiles': tiles, 'log': log, 'notes': notes, 'input_files': input_files, 'parents': parents, 'children': children, 'output_files': output_files, 'log_files': log_files, 'orac_log_files': orac_log_files, 'previews': list(zip(previews256, previews1024)), 'states': JSAProcState.STATE_ALL, 'obsinfo': obs_info, 'parent_obs': parent_obs, 'pagination': pagination, }
def prepare_job_qa_info(db, job_id, query): # Fetch job and qa information from the database. try: job = db.get_job(job_id) except NoRowsError: raise HTTPNotFound() # Convert the information to a dictionary so that we can augment it. info = job._asdict() if info['foreign_id'] is not None: if info['location'] == 'CADC': info['foreign_url'] = \ 'http://beta.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/' \ 'dp/recipe/{0}'.format(info['foreign_id']) else: info['foreign_url'] = None try: input_files = db.get_input_files(job_id) except NoRowsError: input_files = None # Try to get parent jobs (if any). # Dictionary with parent as key and filter as item. try: parents = db.get_parents(job_id) parents = dict(parents) parent_obs = OrderedDict() pjobs = list(parents.keys()) pjobs.sort() for i in pjobs: obsinfo = db.get_obs_info(i) if obsinfo != []: obsinfo = [o._asdict() for o in db.get_obs_info(i)] qa_state = db.get_job(i).qa_state for o in obsinfo: o['qa_state'] = qa_state parent_obs[i] = obsinfo if parent_obs.keys() == []: parent_obs = None except NoRowsError: parents = None parent_obs = None # See if there are any child jobs. try: children = db.get_children(job_id) except NoRowsError: children = None (output_files, previews1024, _) = \ make_output_file_list(db, job.id) obs_info = db.get_obs_info(job.id) if obs_info: obs_info = [o._asdict() for o in obs_info] else: obs_info = None # Get the log files on disk (if any) log_files = get_log_files(job_id) # QA log (f any) qalog = db.get_qas(job_id) qalog.reverse() # If we know what the user's job query was (from the session information) # then set up pagination based on the previous and next job identifiers. if query is not None: (url_query, job_query) = job_search(**query) # prev_next query should not contain kwarg 'number'. pnquery = job_query.copy() if 'number' in pnquery: pnquery.pop('number') (prev, next) = db.job_prev_next(job_id, **pnquery) count = db.find_jobs(count=True, **job_query) pagination = Pagination( None, None if prev is None else url_for('job_qa', job_id=prev), None if next is None else url_for('job_qa', job_id=next), None, url_for('job_list', **url_query), count, ) else: pagination = None # In the case of task='*-cat' and there are no output preview # images, show the preview image from the 1st parent job. if '-cat' in info['task'] and previews1024 == []: (_, previews1024, _) = make_output_file_list(db, parents.keys()[0]) nopreview = True else: nopreview = False # Get parent output .fits files. parent_fits = [] if parents: for i in parents.keys(): (parent_outputs, _, _) = make_output_file_list(db, i) # remove everything that isn't a .fits file from output list. [ parent_fits.append(i) for i in parent_outputs if '.fits' in i.name ] return { 'title': 'Job {}'.format(job_id), 'info': info, 'qalog': qalog, 'output_files': output_files, 'parents': parents, 'children': children, 'log_files': log_files, 'previews': list(zip(previews1024, previews1024)), 'states': JSAProcState.STATE_ALL, 'obsinfo': obs_info, 'parent_obs': parent_obs, 'qa_states': JSAQAState.STATE_ALL, 'pagination': pagination, 'nopreview': nopreview, 'parent_fits': parent_fits, }
def make_output_file_list(db, job_id, preview_filter=None): """Prepare output file lists for job information pages. """ output_files = [] previews1024 = [] previews256 = [] try: for i in sorted(db.get_output_files(job_id)): url = None mtype = None if i.endswith('.png'): url = url_for('job_preview', job_id=job_id, preview=i) if preview_filter is None or any( (f in i for f in preview_filter)): caption = i caption = re.sub('^jcmt_', '', caption) caption = re.sub('_(preview_)?\d+\.png', '', caption) if '_256.png' in i: previews256.append(PreviewInfo(url, caption)) if '_1024.png' in i: previews1024.append(PreviewInfo(url, caption)) elif i.endswith('.pdf'): url = url_for('job_preview_pdf', job_id=job_id, preview=i) elif i.endswith('.txt'): url = url_for('job_text_file', job_id=job_id, text_file=i) elif i.endswith('.fits'): url = 'file://{0}/{1}'.format(get_output_dir(job_id), i) if re.search('-cat[0-9]{6}', i): mtype = 'table.load.fits' elif re.search('-moc[0-9]{6}', i): # This should be "coverage.load.moc.fits" but neither GAIA # nor Aladin appear to subscribe to that mtype yet. # mtype = 'coverage.load.moc.fits' mtype = 'image.load.fits' elif '_rsp_' in i: # Prevent a broadcast button being shown for spectra # for now. mtype = None else: mtype = 'image.load.fits' # Remove URL for types we can't broadcast. if mtype is None: url = None output_files.append(FileInfo(i, url, mtype)) except NoRowsError: pass return (output_files, previews1024, previews256)
def prepare_job_info(db, job_id, query): # Fetch job information from the database. try: job = db.get_job(job_id) except NoRowsError: raise HTTPNotFound() # Convert the information to a dictionary so that we can augment it. info = job._asdict() if info['foreign_id'] is not None: if info['location'] == 'CADC': info['foreign_url'] = \ 'http://beta.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/' \ 'dp/recipe/{0}'.format(info['foreign_id']) else: info['foreign_url'] = None # Try to get tiles. try: tiles = db.get_tilelist(job_id) except NoRowsError: tiles = None if tiles == []: tiles = None # Try to get input files (if any) try: input_files = db.get_input_files(job_id) except NoRowsError: input_files = None # Try to get parent jobs (if any). # Dictionary with parent as key and filter as item. try: parents = db.get_parents(job_id) parents = dict(parents) parent_obs = OrderedDict() pjobs = parents.keys() pjobs.sort() for i in pjobs: parent_obs[i] = [o._asdict() for o in db.get_obs_info(i)] except NoRowsError: parents = None parent_obs = None # See if there are any child jobs. try: children = db.get_children(job_id) except NoRowsError: children = None (output_files, previews1024, previews256) = \ make_output_file_list(db, job.id) obs_info = db.get_obs_info(job.id) if obs_info: obs_info = [o._asdict() for o in obs_info] else: obs_info = None # Logged entries in the database (newest first). log = db.get_logs(job_id) log.reverse() # Get the log files on disk (if any) log_files = get_log_files(job_id) # Get notes. notes = db.get_notes(job_id) # If we know what the user's job query was (from the session information) # then set up pagination based on the previous and next job identifiers. if query is not None: (url_query, job_query) = job_search(**query) # Need to remove 'number' option from job_query to. pnquery = job_query.copy() if 'number' in pnquery: del(pnquery['number']) (prev, next) = db.job_prev_next(job_id, **pnquery) count = db.find_jobs(count=True, **job_query) pagination = Pagination( None, None if prev is None else url_for('job_info', job_id=prev), None if next is None else url_for('job_info', job_id=next), None, url_for('job_list', **url_query), count,) else: pagination = None return { 'title': 'Job {}'.format(job_id), 'info': info, 'tiles': tiles, 'log': log, 'notes': notes, 'input_files': input_files, 'parents': parents, 'children': children, 'output_files': output_files, 'log_files': log_files, 'previews': zip(previews256, previews1024), 'states': JSAProcState.STATE_ALL, 'obsinfo': obs_info, 'parent_obs': parent_obs, 'pagination': pagination, }