def call_reformat(args, log_path, exe, out_file, dependency=None):
"""Reformat outputs using the script provided."""
import pyorac.local_defaults as defaults
from pyorac.colour_print import colour_print
from pyorac.definitions import OracError, COLOURING
from subprocess import check_call, check_output, CalledProcessError
# Optionally print command and driver file contents to StdOut
if args.verbose or args.script_verbose or args.dry_run:
colour_print('{} {} 1 <<<'.format(exe, out_file), COLOURING['header'])
if args.dry_run:
return -1
job_name = args.File.job_name(args.revision, 'format')
if not args.batch:
try:
check_call([exe, out_file, "1"])
except CalledProcessError as err:
raise OracError('{:s} failed with error code {:d}. {}'.format(
' '.join(err.cmd), err.returncode, err.output))
else:
try:
# Collect batch settings from defaults, command line, and script
batch_params = defaults.batch_values.copy()
batch_params['job_name'] = job_name
batch_params['log_file'] = os.path.join(log_path,
job_name + '.log')
batch_params['err_file'] = os.path.join(log_path,
job_name + '.err')
batch_params['duration'] = '01:00'
batch_params['ram'] = 5000
batch_params['procs'] = 1
if dependency is not None:
batch_params['depend'] = dependency
batch_params.update({key: val for key, val in args.batch_settings})
# Form batch queue command and call batch queuing system
cmd = defaults.batch.list_batch(batch_params,
exe=[exe, out_file, "1"])
if args.verbose or args.script_verbose:
colour_print(' '.join(cmd), COLOURING['header'])
out = check_output(cmd, universal_newlines=True)
# Parse job ID # and return it to the caller
jid = defaults.batch.parse_out(out, 'ID')
return jid
except CalledProcessError as err:
raise OracError('Failed to queue job ' + exe)
except SyntaxError as err:
raise OracError(str(err))
if orig_args.out_dir:
base_out_dir = orig_args.out_dir
else:
base_out_dir = defaults.data_dir +'/testoutput'
# Increment version number (as this is usually run on uncommited code)
if orig_args.revision is None:
orig_args.revision = get_repository_revision()
if not orig_args.benchmark:
orig_args.revision += 1
try:
for test in orig_args.tests:
colour_print(test, COLOURING['header'])
args = deepcopy(orig_args)
# Set filename to be processed and output folder
args.out_dir = os.path.join(base_out_dir, test)
try:
args.target, args.limit, args.preset_settings = REGRESSION_TESTS[test]
except KeyError:
raise OracError("Invalid regression test for given phases.")
args.preset_settings += "_" + args.test_type
jid, out_file = process_all(args)
log_path = os.path.join(args.out_dir, log_dir)
# Check for regressions
from pyorac.colour_print import colour_print
from pyorac.definitions import COLOURING, OracError, OracWarning
from pyorac.run import process_all
from pyorac.util import warning_format
import warnings
warnings.formatwarning = warning_format
warnings.filterwarnings('always', category=OracWarning)
# Define parser
pars = ArgumentParser(
description='Run the full ORAC suite on a given Level 1B file.')
pars.add_argument('target', type=str, help='Level 1B file to be processed')
args_common(pars)
args_cc4cl(pars)
args_preproc(pars)
args_main(pars)
args_postproc(pars)
args = pars.parse_args()
try:
jid, _ = process_all(args)
if args.script_verbose and args.batch:
print("Job queued with ID {}".format(jid))
except OracError as err:
colour_print('ERROR) ' + str(err), COLOURING['error'])
except KeyboardInterrupt:
colour_print('Execution halted by user.', COLOURING['error'])
def call_exe(args, exe, driver, values=dict()):
"""Call an ORAC executable, managing the necessary driver file.
Args:
:list args: Arguments of the script.
:str exe: Name of the executable.
:str driver: Contents of the driver file to pass.
:dict values: Arguments for the batch queueing system."""
import pyorac.local_defaults as defaults
from pyorac.colour_print import colour_print
from pyorac.definitions import OracError, COLOURING
from subprocess import check_call, check_output, CalledProcessError
from tempfile import mkstemp
from time import time
# Optionally print command and driver file contents to StdOut
if args.verbose or args.script_verbose or args.dry_run:
colour_print(exe + ' <<<', COLOURING['header'])
colour_print(driver, COLOURING['text'])
if args.dry_run:
return
# Write driver file
(fd, driver_file) = mkstemp('.driver',
os.path.basename(exe) + '.', args.out_dir,
True)
f = os.fdopen(fd, "w")
f.write(driver)
f.close()
if not args.batch:
# Form processing environment
os.environ["LD_LIBRARY_PATH"] = build_orac_library_path()
# Define a directory for EMOS to put it's gridding
try:
os.environ["PPDIR"] = args.emos_dir
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["OMP_NUM_THREADS"] = str(args.procs)
except AttributeError:
pass
# Call program
try:
st = time()
check_call([exe, driver_file])
if args.timing:
colour_print(exe + ' took {:f}s'.format(time() - st),
COLOURING['timing'])
return True
except CalledProcessError as err:
raise OracError('{:s} failed with error code {:d}. {}'.format(
' '.join(err.cmd), err.returncode, err.output))
finally:
if not args.keep_driver:
os.remove(driver_file)
elif args.verbose or args.script_verbose:
print("Driver file stored at " + driver_file)
else:
# Write temporary script to call executable
(gd, script_file) = mkstemp('.sh',
os.path.basename(exe) + '.', args.out_dir,
True)
g = os.fdopen(gd, "w")
g.write(defaults.batch_script)
# Define processing environment
libs = read_orac_libraries(args.orac_lib)
g.write("export LD_LIBRARY_PATH=" + build_orac_library_path(libs) +
"\n")
g.write("export OPENBLAS_NUM_THREADS=1\n")
try:
g.write("export PPDIR=" + args.emos_dir + "\n")
except AttributeError:
pass
defaults.batch.add_openmp_to_script(g)
# Call executable and give the script permission to execute
g.write(exe + ' ' + driver_file + "\n")
if not args.keep_driver:
g.write("rm -f " + driver_file + "\n")
g.write("rm -f " + script_file + "\n")
g.close()
os.chmod(script_file, 0o700)
try:
# Collect batch settings from defaults, command line, and script
batch_params = defaults.batch_values.copy()
batch_params.update(values)
batch_params.update({key: val for key, val in args.batch_settings})
batch_params['procs'] = args.procs
# Form batch queue command and call batch queuing system
cmd = defaults.batch.ListBatch(batch_params, exe=script_file)
if args.verbose or args.script_verbose:
colour_print(' '.join(cmd), COLOURING['header'])
out = check_output(cmd, universal_newlines=True)
# Parse job ID # and return it to the caller
jid = defaults.batch.ParseOut(out, 'ID')
return jid
except CalledProcessError as err:
raise OracError('Failed to queue job ' + exe)
except SyntaxError as err:
raise OracError(str(err))
