args.out_dir = os.path.join(base_out_dir, test)
try:
args.target, args.limit, args.preset_settings = REGRESSION_TESTS[test]
except KeyError:
raise OracError("Invalid regression test for given phases.")
args.preset_settings += "_" + args.test_type
jid, out_file = process_all(args)
log_path = os.path.join(args.out_dir, log_dir)
# Check for regressions
if not args.benchmark and not args.dry_run:
inst = FileName(args.out_dir, out_file)
if not args.batch:
args = check_args_common(args)
args = check_args_preproc(args)
try:
run_regression(inst)
except Regression as err:
colour_print('REGRESSION) ' + str(err), COLOURING['error'])
else:
if os.path.isdir(os.path.join(defaults.orac_dir, "tools")):
path = [os.path.join(defaults.orac_dir, "tools"), ]
else:
path = [defaults.orac_dir, ]
path.extend(filter(None, sys.path))
job_name = inst.job_name(tag='regression')
def process_all(orig_args):
"""Run the ORAC pre, main, and post processors on a file."""
from argparse import ArgumentParser
from copy import deepcopy
from pyorac.arguments import args_common, args_main
from pyorac.local_defaults import log_dir, pre_dir
# Generate main-processor-only parser
pars = ArgumentParser()
args_common(pars)
args_main(pars)
# We need one argument from args_cc4cl()
pars.add_argument("--sub_dir", default="")
compare = pars.parse_args("")
# Copy input arguments as we'll need to fiddle with them
orig_args = check_args_common(orig_args)
orig_args = check_args_cc4cl(orig_args)
log_path = os.path.join(orig_args.out_dir, log_dir)
args = deepcopy(orig_args)
written_dirs = set() # The folders we actually wrote to
# Work out output filename
args.out_dir = os.path.join(orig_args.out_dir, pre_dir)
jid_pre, _ = process_pre(args, log_path, tag="pre{}".format(args.label))
if jid_pre is not None:
written_dirs.add(args.out_dir)
# Run main processor -------------------------------------------------------
root_name = args.File.root_name(args.revision, args.processor,
args.project, args.product_name)
args.target = root_name + ".config.nc"
out_files = [] # All files that would be made (facilitates --dry_run)
jid_main = [] # ID no. for each queued job
args.in_dir = [args.out_dir]
for sett in args.settings:
phs_args = deepcopy(args)
parsed_settings_arguments = pars.parse_args(sett.split())
for key, val in compare.__dict__.items():
if val == parsed_settings_arguments.__dict__[key]:
parsed_settings_arguments.__dict__.pop(key)
phs_args.__dict__.update(parsed_settings_arguments.__dict__)
phs_args.out_dir = os.path.join(orig_args.out_dir, phs_args.sub_dir)
jid, out = process_main(phs_args,
log_path,
dependency=jid_pre,
tag=phs_args.sub_dir + phs_args.label)
out_files.append(out)
if jid is not None:
jid_main.append(jid)
written_dirs.add(args.out_dir)
# Run postprocessor if necessary
if len(args.settings) > 1:
args.target = root_name + "NULL.primary.nc"
args.in_dir = written_dirs
args.out_dir = orig_args.out_dir
jid, out_file = process_post(args,
log_path,
out_files,
dependency=jid_main,
tag="post{}".format(args.label))
if jid is not None:
written_dirs.add(args.out_dir)
else:
out_file = out_files[0]
# Run CCI formatting
if args.reformat != "":
call_reformat(args, log_path, args.reformat, out_file, dependency=jid)
# Output root filename and output folders for regression tests
return jid, out_file
args.out_dir = os.path.join(base_out_dir, test)
try:
args.target, args.limit, args.preset_settings = REGRESSION_TESTS[test]
except KeyError:
raise OracError("Invalid regression test for given phases.")
args.preset_settings += "_" + args.test_type
jid, out_file = process_all(args)
log_path = os.path.join(args.out_dir, log_dir)
# Check for regressions
if not args.benchmark and not args.dry_run:
inst = FileName(args.out_dir, out_file)
if not args.batch:
check_args_common(args)
check_args_preproc(args)
try:
run_regression(inst)
except Regression as err:
colour_print('REGRESSION) ' + str(err), COLOURING['error'])
else:
if os.path.isdir(os.path.join(defaults.orac_dir, "tools")):
path = [os.path.join(defaults.orac_dir, "tools"), ]
else:
path = [defaults.orac_dir, ]
path.extend(filter(None, sys.path))
job_name = inst.job_name(tag='regression')