def get_log_files(job_id): """Get a dictionary of recognised logs for a given job. Scans the log directory of the given job looking for log files which match one of the patterns in the log_types dictionary. Returns a dictionary where the keys are the log types for which logs were found, and the values are lists of logs of that type. """ log_files = {} log_dir = get_log_dir(job_id) if os.path.isdir(log_dir): for file in sorted(os.listdir(get_log_dir(job_id)), reverse=True): for (type_, pattern) in log_types.items(): if pattern.match(file): if file.endswith('.html'): url = url_for('job_log_html', job_id=job_id, log=file) else: url = url_for('job_log_text', job_id=job_id, log=file) if type_ in log_files: log_files[type_].append(LogInfo(file, url, None)) else: log_files[type_] = [LogInfo(file, url, None)] return log_files
def get_log_files(job_id): """Get a dictionary of recognised logs for a given job. Scans the log directory of the given job looking for log files which match one of the patterns in the log_types dictionary. Returns a dictionary where the keys are the log types for which logs were found, and the values are lists of logs of that type. """ log_files = {} log_dir = get_log_dir(job_id) if os.path.isdir(log_dir): for file in sorted(os.listdir(get_log_dir(job_id)), reverse=True): for (type_, pattern) in log_types.items(): if pattern.match(file): if file.endswith('.html'): url = url_for('job_log_html', job_id=job_id, log=file) else: url = url_for('job_log_text', job_id=job_id, log=file) if type_ in log_files: log_files[type_].append(LogInfo(file, url)) else: log_files[type_] = [LogInfo(file, url)] return log_files
def prepare_job_log(job_id, log): """ Prepare a log to show it. Return the path to the log file. """ log_path = os.path.join(get_log_dir(job_id), log) return log_path
def prepare_job_log(job_id, log): """ Prepare a log to show it. Return the path to the log file. """ log_path = os.path.join(get_log_dir(job_id), log) return log_path
def test_directories(self): # Test all the directory functions. self.assertEqual( get_input_dir(18), '/net/kamaka/export/data/jsa_proc/input/000/000000/000000018') self.assertEqual( get_output_dir(46), '/net/kamaka/export/data/jsa_proc/output/000/000000/000000046') self.assertEqual( get_scratch_dir(92), '/export/data/jsa_proc/scratch/000/000000/000000092') self.assertEqual( get_log_dir(844), '/net/kamaka/export/data/jsa_proc/log/000/000000/000000844') # Test longer job ID numbers (we know all the functions use the same # private function to prepare the decimal number internally). self.assertEqual( get_log_dir(123456789), '/net/kamaka/export/data/jsa_proc/log/123/123456/123456789') self.assertEqual( get_log_dir(22333), '/net/kamaka/export/data/jsa_proc/log/000/000022/000022333') self.assertEqual( get_log_dir(22333999), '/net/kamaka/export/data/jsa_proc/log/022/022333/022333999') # Test what happens with a billion or more job IDs. self.assertEqual( get_log_dir(1999000999), '/net/kamaka/export/data/jsa_proc/log/1999/1999000/1999000999') with self.assertRaises(JSAProcError): get_input_dir('not an integer')
def test_directories(self): # Test all the directory functions. self.assertEqual( get_input_dir(18), '/net/kamaka/export/data/jsa_proc/input/000/000000/000000018') self.assertEqual( get_output_dir(46), '/net/kamaka/export/data/jsa_proc/output/000/000000/000000046') self.assertEqual(get_scratch_dir(92), '/export/data/jsa_proc/scratch/000/000000/000000092') self.assertEqual( get_log_dir(844), '/net/kamaka/export/data/jsa_proc/log/000/000000/000000844') # Test longer job ID numbers (we know all the functions use the same # private function to prepare the decimal number internally). self.assertEqual( get_log_dir(123456789), '/net/kamaka/export/data/jsa_proc/log/123/123456/123456789') self.assertEqual( get_log_dir(22333), '/net/kamaka/export/data/jsa_proc/log/000/000022/000022333') self.assertEqual( get_log_dir(22333999), '/net/kamaka/export/data/jsa_proc/log/022/022333/022333999') # Test what happens with a billion or more job IDs. self.assertEqual( get_log_dir(1999000999), '/net/kamaka/export/data/jsa_proc/log/1999/1999000/1999000999') with self.assertRaises(JSAProcError): get_input_dir('not an integer')
def find_offsets(jobs): offset_dict = {} for j in jobs: logdir = get_log_dir(j.id) oraclogs = glob.glob(os.path.join(logdir, 'oracdr_*.html')) oraclogs.sort(key=os.path.getmtime) oraclog = oraclogs[-1] try: pointingoffsets = get_pointing_offsets(oraclog) for p in pointingoffsets: offset_dict[j.id] = p except: sourcename = db.get_obs_info(j.id)[0].sourcename print('No pointing for %s, job=%i' % (sourcename, j.id)) return offset_dict
def get_output_log_files(job_id): """ Get the current list of output log.* files from the log directory. This command trusts whatever is in the log directory and starts with log.* is the correct list of output log files. Returns: list of bare file names """ log_dir = get_log_dir(job_id) if not os.path.exists(log_dir) or not os.path.isdir: raise JSAProcError( 'The log directory %s for job %i does not exist.' % (log_dir, job_id)) pattern = re.compile('log.*') logs = [i for i in os.listdir(log_dir) if pattern.match(i)] return logs
def get_orac_log_files(job_id): """ Get a dictionary of ORAC-DR (log.*) files for a job. Scans the log directory to get all log.* files. Skips them if they have a date stamp older than the last run of the system. """ pattern = re.compile('log.*') log_dir = get_log_dir(job_id) log_files = [] if os.path.isdir(log_dir): files = os.listdir(log_dir) for f in sorted(files): if pattern.match(f): mtime = time.ctime(os.path.getmtime(os.path.join(log_dir, f))) url = url_for('job_log_text', job_id=job_id, log=f) log_files.append(LogInfo(f, url, mtime)) return log_files
def search_log_files( pattern, filename_pattern, task, project=None, state=None, after_context=None): db = get_database() re_pattern = re.compile(pattern) re_filename = re.compile(filename_pattern) if state is None: state = JSAProcState.COMPLETE else: state = JSAProcState.lookup_name(state) if after_context is None: after_context = 0 search_kwargs = { 'task': task, 'state': state, } if project is not None: search_kwargs['obsquery'] = {'project': project} jobs = [x.id for x in db.find_jobs(**search_kwargs)] for job_id in jobs: logger.debug('Checking log files for job %i', job_id) log_dir = get_log_dir(job_id) # Find the latest matching log by iterating through them in reverse # order and "breaking" after the first match. for filename in sorted(os.listdir(log_dir), reverse=True): if not re_filename.search(filename): continue logger.debug('Found log file for job %i: %s', job_id, filename) matched = 0 matched_lines = [] pathname = os.path.join(log_dir, filename) with open(pathname, 'r') as f: for line in f: if matched or re_pattern.search(line): matched += 1 matched_lines.append(line.rstrip()) if matched > after_context: break if matched: logger.info( 'Found match for job %i: %s', job_id, matched_lines[0]) for matched_line in matched_lines[1:]: logger.info( '... continuation %i: %s', job_id, matched_line) break
def jsawrapdr_run(job_id, input_file_list, mode, drparameters, cleanup='cadc', location='JAC', persist=False, jsawrapdr=None, starlink_dir=None, version=None, command_run=None, raw_output=None): """ Execute jsawrapdr script from python. This function calls jsawrapdr with following options: jsawrapdr --outdir=configbase/scratch/$job_id --inputs=input_file_list --id = jac-$job_id --mode=$mode --drparameters=$drparameters --cleanup=$cleanup (cadc by default) --location=$location (JAC by default) --fileversion=$version (if not None) --drcommand=$command_run (if not None) if persist is True, then it adds the flag: -persist if raw_output is True, it adds the option: --rawoutput Args: job_id (int): Job identifier from jsaproc database. input_file_list (str): List of files (with extensions and full path). mode (str): Can be 'night', 'obs', 'public' or 'project'. drparameters (str): cleanup (str, optional): Type of cleanup. Can be one of 'cadc'|'none'|'all', defaults to 'cadc'. persist (bool, optional): Defaults to False If persist is turned on, then dpCapture will copy acceptable products to the default output directory. Otherwise it won't (used for debugging purposes). The output directory is determined by jsa_proc.admin.directories 'get_output_dir' for the given job_id. location (str, optional): One of |'cadc'|'JAC'| (NOT CURRENTLY IMPLEMENTED, default is 'JAC') jsawrapdr (str, optional): The path to jsawrapdr. If not given, the one in configured starlink will be used. starlink_dir (str, optional): The path of a starlink install to use. If not given, the one found in the configuration file will be used. version: CADC file name "version" or None to use default. command_run: custom "run" command to be passed to jsawrapdr. Returns: str: The filename (including path) of the logfile. """ # Get log directory. Note that opening a log file in this # directory using open_log_file will ensure that it exists. log_dir = get_log_dir(job_id) # Prepare scratch directory. scratch_dir = make_temp_scratch_dir(job_id) # Get output directory name. out_dir = get_output_dir(job_id) # If output dir currently exists, delete the directory. if os.path.exists(out_dir): shutil.rmtree(out_dir) # Make the "transfer" directory in advance. (This saves dpCapture # or another copying routine from having to do so.) os.makedirs(out_dir) # Find paths to starlink, jsawrapdr and orac_dr. config = get_config() if starlink_dir is None: starpath = config.get('job_run', 'starpath') else: starpath = starlink_dir if not jsawrapdr: jsawrapdr = os.path.join(starpath, 'Perl', 'bin', 'jsawrapdr') orac_dir = os.path.join(starpath, 'bin', 'oracdr', 'src') # Set thejac recipe id. jacid = 'jac-'+str(job_id) # Collect the jsawrapdr arguments. jsawrapdrcom = [jsawrapdr, '--debugxfer', '--outdir='+scratch_dir, '--inputs='+input_file_list, '--id='+jacid, '--mode='+mode, '--cleanup='+cleanup, '--drparameters='+drparameters] if persist: jsawrapdrcom.append('-persist') jsawrapdrcom.append('--transdir='+out_dir) if raw_output: jsawrapdrcom.append('--rawoutput') if version is not None: jsawrapdrcom.append('--fileversion={0}'.format(version)) if command_run is not None: jsawrapdrcom.append('--drcommand={0}'.format(command_run)) # Set up the environment for running jsawrapdr. jsa_env = os.environ.copy() jsa_env = setup_starlink(starpath, jsa_env) # Add in the LOGDIR jsa_env['ORAC_LOGDIR'] = log_dir # Ensure that we delete the results of previous log.* files the ORAC_LOGDIR if they exist. if os.path.exists(log_dir): calculation_logs = glob.glob(os.path.join(log_dir, 'log.*')) for cl in calculation_logs: os.remove(cl) # Open a log file and run jsawrapdr while saving output to log. with open_log_file(job_id, 'jsawrapdr') as log: # Save the log file name. log_name = log.name # Run jsawrapdr. retcode = subprocess.call(jsawrapdrcom, env=jsa_env, bufsize=1, stdout=log, stderr=subprocess.STDOUT, preexec_fn=restore_signals) # Handle jsawrapdr errors. if retcode != 0: errormessage = 'jsawrapdr exited with Retcode %i ' % (retcode) # Find the first ORAC error message in the jsawrapdr log. jsalogfile = open(log_name, 'r') lines = jsalogfile.read() jsalogfile.close() result = re.search(r'.*(STDERR:\s*.*)$', lines, re.DOTALL) if result: firsterror = result.group(1).split('\n')[1] # Insert the ORAC error at the start of the error message. if firsterror: errormessage = 'ORAC ERROR: ' + firsterror + '.\n' + \ errormessage # Raise the error. raise JSAProcError(errormessage) return log_name
def search_log_files(pattern, filename_pattern, task, project=None, state=None, after_context=None): db = get_database() re_pattern = re.compile(pattern) re_filename = re.compile(filename_pattern) if state is None: state = JSAProcState.COMPLETE else: state = JSAProcState.lookup_name(state) if after_context is None: after_context = 0 search_kwargs = { 'task': task, 'state': state, } if project is not None: search_kwargs['obsquery'] = {'project': project} jobs = [x.id for x in db.find_jobs(**search_kwargs)] for job_id in jobs: logger.debug('Checking log files for job %i', job_id) log_dir = get_log_dir(job_id) # Find the latest matching log by iterating through them in reverse # order and "breaking" after the first match. for filename in sorted(os.listdir(log_dir), reverse=True): if not re_filename.search(filename): continue logger.debug('Found log file for job %i: %s', job_id, filename) matched = 0 matched_lines = [] pathname = os.path.join(log_dir, filename) with open(pathname, 'r') as f: for line in f: if matched or re_pattern.search(line): matched += 1 matched_lines.append(line.rstrip()) if matched > after_context: break if matched: logger.info('Found match for job %i: %s', job_id, matched_lines[0]) for matched_line in matched_lines[1:]: logger.info('... continuation %i: %s', job_id, matched_line) break