def get_log_files(job_id):
"""Get a dictionary of recognised logs for a given job.
Scans the log directory of the given job looking for log
files which match one of the patterns in the log_types
dictionary.
Returns a dictionary where the keys are the log types
for which logs were found, and the values are lists of
logs of that type.
"""
log_files = {}
log_dir = get_log_dir(job_id)
if os.path.isdir(log_dir):
for file in sorted(os.listdir(get_log_dir(job_id)), reverse=True):
for (type_, pattern) in log_types.items():
if pattern.match(file):
if file.endswith('.html'):
url = url_for('job_log_html', job_id=job_id, log=file)
else:
url = url_for('job_log_text', job_id=job_id, log=file)
if type_ in log_files:
log_files[type_].append(LogInfo(file, url, None))
else:
log_files[type_] = [LogInfo(file, url, None)]
return log_files
def get_log_files(job_id):
"""Get a dictionary of recognised logs for a given job.
Scans the log directory of the given job looking for log
files which match one of the patterns in the log_types
dictionary.
Returns a dictionary where the keys are the log types
for which logs were found, and the values are lists of
logs of that type.
"""
log_files = {}
log_dir = get_log_dir(job_id)
if os.path.isdir(log_dir):
for file in sorted(os.listdir(get_log_dir(job_id)), reverse=True):
for (type_, pattern) in log_types.items():
if pattern.match(file):
if file.endswith('.html'):
url = url_for('job_log_html', job_id=job_id, log=file)
else:
url = url_for('job_log_text', job_id=job_id, log=file)
if type_ in log_files:
log_files[type_].append(LogInfo(file, url))
else:
log_files[type_] = [LogInfo(file, url)]
return log_files
def prepare_job_log(job_id, log):
"""
Prepare a log to show it.
Return the path to the log file.
"""
log_path = os.path.join(get_log_dir(job_id), log)
return log_path
def prepare_job_log(job_id, log):
"""
Prepare a log to show it.
Return the path to the log file.
"""
log_path = os.path.join(get_log_dir(job_id), log)
return log_path
def test_directories(self):
# Test all the directory functions.
self.assertEqual(
get_input_dir(18),
'/net/kamaka/export/data/jsa_proc/input/000/000000/000000018')
self.assertEqual(
get_output_dir(46),
'/net/kamaka/export/data/jsa_proc/output/000/000000/000000046')
self.assertEqual(
get_scratch_dir(92),
'/export/data/jsa_proc/scratch/000/000000/000000092')
self.assertEqual(
get_log_dir(844),
'/net/kamaka/export/data/jsa_proc/log/000/000000/000000844')
# Test longer job ID numbers (we know all the functions use the same
# private function to prepare the decimal number internally).
self.assertEqual(
get_log_dir(123456789),
'/net/kamaka/export/data/jsa_proc/log/123/123456/123456789')
self.assertEqual(
get_log_dir(22333),
'/net/kamaka/export/data/jsa_proc/log/000/000022/000022333')
self.assertEqual(
get_log_dir(22333999),
'/net/kamaka/export/data/jsa_proc/log/022/022333/022333999')
# Test what happens with a billion or more job IDs.
self.assertEqual(
get_log_dir(1999000999),
'/net/kamaka/export/data/jsa_proc/log/1999/1999000/1999000999')
with self.assertRaises(JSAProcError):
get_input_dir('not an integer')
def test_directories(self):
# Test all the directory functions.
self.assertEqual(
get_input_dir(18),
'/net/kamaka/export/data/jsa_proc/input/000/000000/000000018')
self.assertEqual(
get_output_dir(46),
'/net/kamaka/export/data/jsa_proc/output/000/000000/000000046')
self.assertEqual(get_scratch_dir(92),
'/export/data/jsa_proc/scratch/000/000000/000000092')
self.assertEqual(
get_log_dir(844),
'/net/kamaka/export/data/jsa_proc/log/000/000000/000000844')
# Test longer job ID numbers (we know all the functions use the same
# private function to prepare the decimal number internally).
self.assertEqual(
get_log_dir(123456789),
'/net/kamaka/export/data/jsa_proc/log/123/123456/123456789')
self.assertEqual(
get_log_dir(22333),
'/net/kamaka/export/data/jsa_proc/log/000/000022/000022333')
self.assertEqual(
get_log_dir(22333999),
'/net/kamaka/export/data/jsa_proc/log/022/022333/022333999')
# Test what happens with a billion or more job IDs.
self.assertEqual(
get_log_dir(1999000999),
'/net/kamaka/export/data/jsa_proc/log/1999/1999000/1999000999')
with self.assertRaises(JSAProcError):
get_input_dir('not an integer')
def find_offsets(jobs):
offset_dict = {}
for j in jobs:
logdir = get_log_dir(j.id)
oraclogs = glob.glob(os.path.join(logdir, 'oracdr_*.html'))
oraclogs.sort(key=os.path.getmtime)
oraclog = oraclogs[-1]
try:
pointingoffsets = get_pointing_offsets(oraclog)
for p in pointingoffsets:
offset_dict[j.id] = p
except:
sourcename = db.get_obs_info(j.id)[0].sourcename
print('No pointing for %s, job=%i' % (sourcename, j.id))
return offset_dict
def get_output_log_files(job_id):
"""
Get the current list of output log.* files from the log directory.
This command trusts whatever is in the log directory and starts
with log.* is the correct list of output log files.
Returns: list of bare file names
"""
log_dir = get_log_dir(job_id)
if not os.path.exists(log_dir) or not os.path.isdir:
raise JSAProcError(
'The log directory %s for job %i does not exist.' % (log_dir, job_id))
pattern = re.compile('log.*')
logs = [i for i in os.listdir(log_dir) if pattern.match(i)]
return logs
def get_orac_log_files(job_id):
"""
Get a dictionary of ORAC-DR (log.*) files for a job.
Scans the log directory to get all log.* files.
Skips them if they have a date stamp older than the last run of the system.
"""
pattern = re.compile('log.*')
log_dir = get_log_dir(job_id)
log_files = []
if os.path.isdir(log_dir):
files = os.listdir(log_dir)
for f in sorted(files):
if pattern.match(f):
mtime = time.ctime(os.path.getmtime(os.path.join(log_dir, f)))
url = url_for('job_log_text', job_id=job_id, log=f)
log_files.append(LogInfo(f, url, mtime))
return log_files
def search_log_files(
pattern, filename_pattern, task,
project=None, state=None, after_context=None):
db = get_database()
re_pattern = re.compile(pattern)
re_filename = re.compile(filename_pattern)
if state is None:
state = JSAProcState.COMPLETE
else:
state = JSAProcState.lookup_name(state)
if after_context is None:
after_context = 0
search_kwargs = {
'task': task,
'state': state,
}
if project is not None:
search_kwargs['obsquery'] = {'project': project}
jobs = [x.id for x in db.find_jobs(**search_kwargs)]
for job_id in jobs:
logger.debug('Checking log files for job %i', job_id)
log_dir = get_log_dir(job_id)
# Find the latest matching log by iterating through them in reverse
# order and "breaking" after the first match.
for filename in sorted(os.listdir(log_dir), reverse=True):
if not re_filename.search(filename):
continue
logger.debug('Found log file for job %i: %s', job_id, filename)
matched = 0
matched_lines = []
pathname = os.path.join(log_dir, filename)
with open(pathname, 'r') as f:
for line in f:
if matched or re_pattern.search(line):
matched += 1
matched_lines.append(line.rstrip())
if matched > after_context:
break
if matched:
logger.info(
'Found match for job %i: %s', job_id, matched_lines[0])
for matched_line in matched_lines[1:]:
logger.info(
'... continuation %i: %s', job_id, matched_line)
break
def jsawrapdr_run(job_id, input_file_list, mode, drparameters,
cleanup='cadc', location='JAC', persist=False,
jsawrapdr=None, starlink_dir=None,
version=None, command_run=None,
raw_output=None):
"""
Execute jsawrapdr script from python.
This function calls jsawrapdr with following options:
jsawrapdr --outdir=configbase/scratch/$job_id
--inputs=input_file_list
--id = jac-$job_id
--mode=$mode
--drparameters=$drparameters
--cleanup=$cleanup (cadc by default)
--location=$location (JAC by default)
--fileversion=$version (if not None)
--drcommand=$command_run (if not None)
if persist is True, then it adds the flag:
-persist
if raw_output is True, it adds the option:
--rawoutput
Args:
job_id (int): Job identifier from jsaproc database.
input_file_list (str): List of files (with extensions and full
path).
mode (str): Can be 'night', 'obs', 'public' or 'project'.
drparameters (str):
cleanup (str, optional): Type of cleanup. Can be one of
'cadc'|'none'|'all', defaults to 'cadc'.
persist (bool, optional): Defaults to False If persist is turned
on, then dpCapture will copy acceptable products to the
default output directory. Otherwise it won't (used for
debugging purposes). The output directory is determined by
jsa_proc.admin.directories 'get_output_dir' for the given
job_id.
location (str, optional): One of |'cadc'|'JAC'| (NOT CURRENTLY
IMPLEMENTED, default is 'JAC')
jsawrapdr (str, optional): The path to jsawrapdr. If not given,
the one in configured starlink will be used.
starlink_dir (str, optional): The path of a starlink install to
use. If not given, the one found in the configuration file will be
used.
version: CADC file name "version" or None to use default.
command_run: custom "run" command to be passed to jsawrapdr.
Returns:
str: The filename (including path) of the logfile.
"""
# Get log directory. Note that opening a log file in this
# directory using open_log_file will ensure that it exists.
log_dir = get_log_dir(job_id)
# Prepare scratch directory.
scratch_dir = make_temp_scratch_dir(job_id)
# Get output directory name.
out_dir = get_output_dir(job_id)
# If output dir currently exists, delete the directory.
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
# Make the "transfer" directory in advance. (This saves dpCapture
# or another copying routine from having to do so.)
os.makedirs(out_dir)
# Find paths to starlink, jsawrapdr and orac_dr.
config = get_config()
if starlink_dir is None:
starpath = config.get('job_run', 'starpath')
else:
starpath = starlink_dir
if not jsawrapdr:
jsawrapdr = os.path.join(starpath, 'Perl', 'bin', 'jsawrapdr')
orac_dir = os.path.join(starpath, 'bin', 'oracdr', 'src')
# Set thejac recipe id.
jacid = 'jac-'+str(job_id)
# Collect the jsawrapdr arguments.
jsawrapdrcom = [jsawrapdr,
'--debugxfer',
'--outdir='+scratch_dir,
'--inputs='+input_file_list,
'--id='+jacid,
'--mode='+mode,
'--cleanup='+cleanup,
'--drparameters='+drparameters]
if persist:
jsawrapdrcom.append('-persist')
jsawrapdrcom.append('--transdir='+out_dir)
if raw_output:
jsawrapdrcom.append('--rawoutput')
if version is not None:
jsawrapdrcom.append('--fileversion={0}'.format(version))
if command_run is not None:
jsawrapdrcom.append('--drcommand={0}'.format(command_run))
# Set up the environment for running jsawrapdr.
jsa_env = os.environ.copy()
jsa_env = setup_starlink(starpath, jsa_env)
# Add in the LOGDIR
jsa_env['ORAC_LOGDIR'] = log_dir
# Ensure that we delete the results of previous log.* files the ORAC_LOGDIR if they exist.
if os.path.exists(log_dir):
calculation_logs = glob.glob(os.path.join(log_dir, 'log.*'))
for cl in calculation_logs:
os.remove(cl)
# Open a log file and run jsawrapdr while saving output to log.
with open_log_file(job_id, 'jsawrapdr') as log:
# Save the log file name.
log_name = log.name
# Run jsawrapdr.
retcode = subprocess.call(jsawrapdrcom, env=jsa_env, bufsize=1,
stdout=log, stderr=subprocess.STDOUT,
preexec_fn=restore_signals)
# Handle jsawrapdr errors.
if retcode != 0:
errormessage = 'jsawrapdr exited with Retcode %i ' % (retcode)
# Find the first ORAC error message in the jsawrapdr log.
jsalogfile = open(log_name, 'r')
lines = jsalogfile.read()
jsalogfile.close()
result = re.search(r'.*(STDERR:\s*.*)$', lines, re.DOTALL)
if result:
firsterror = result.group(1).split('\n')[1]
# Insert the ORAC error at the start of the error message.
if firsterror:
errormessage = 'ORAC ERROR: ' + firsterror + '.\n' + \
errormessage
# Raise the error.
raise JSAProcError(errormessage)
return log_name
def search_log_files(pattern,
filename_pattern,
task,
project=None,
state=None,
after_context=None):
db = get_database()
re_pattern = re.compile(pattern)
re_filename = re.compile(filename_pattern)
if state is None:
state = JSAProcState.COMPLETE
else:
state = JSAProcState.lookup_name(state)
if after_context is None:
after_context = 0
search_kwargs = {
'task': task,
'state': state,
}
if project is not None:
search_kwargs['obsquery'] = {'project': project}
jobs = [x.id for x in db.find_jobs(**search_kwargs)]
for job_id in jobs:
logger.debug('Checking log files for job %i', job_id)
log_dir = get_log_dir(job_id)
# Find the latest matching log by iterating through them in reverse
# order and "breaking" after the first match.
for filename in sorted(os.listdir(log_dir), reverse=True):
if not re_filename.search(filename):
continue
logger.debug('Found log file for job %i: %s', job_id, filename)
matched = 0
matched_lines = []
pathname = os.path.join(log_dir, filename)
with open(pathname, 'r') as f:
for line in f:
if matched or re_pattern.search(line):
matched += 1
matched_lines.append(line.rstrip())
if matched > after_context:
break
if matched:
logger.info('Found match for job %i: %s', job_id,
matched_lines[0])
for matched_line in matched_lines[1:]:
logger.info('... continuation %i: %s', job_id,
matched_line)
break