args.out_dir = os.path.join(base_out_dir, test) try: args.target, args.limit, args.preset_settings = REGRESSION_TESTS[test] except KeyError: raise OracError("Invalid regression test for given phases.") args.preset_settings += "_" + args.test_type jid, out_file = process_all(args) log_path = os.path.join(args.out_dir, log_dir) # Check for regressions if not args.benchmark and not args.dry_run: inst = FileName(args.out_dir, out_file) if not args.batch: args = check_args_common(args) args = check_args_preproc(args) try: run_regression(inst) except Regression as err: colour_print('REGRESSION) ' + str(err), COLOURING['error']) else: if os.path.isdir(os.path.join(defaults.orac_dir, "tools")): path = [os.path.join(defaults.orac_dir, "tools"), ] else: path = [defaults.orac_dir, ] path.extend(filter(None, sys.path)) job_name = inst.job_name(tag='regression')
def process_all(orig_args): """Run the ORAC pre, main, and post processors on a file.""" from argparse import ArgumentParser from copy import deepcopy from pyorac.arguments import args_common, args_main from pyorac.local_defaults import log_dir, pre_dir # Generate main-processor-only parser pars = ArgumentParser() args_common(pars) args_main(pars) # We need one argument from args_cc4cl() pars.add_argument("--sub_dir", default="") compare = pars.parse_args("") # Copy input arguments as we'll need to fiddle with them orig_args = check_args_common(orig_args) orig_args = check_args_cc4cl(orig_args) log_path = os.path.join(orig_args.out_dir, log_dir) args = deepcopy(orig_args) written_dirs = set() # The folders we actually wrote to # Work out output filename args.out_dir = os.path.join(orig_args.out_dir, pre_dir) jid_pre, _ = process_pre(args, log_path, tag="pre{}".format(args.label)) if jid_pre is not None: written_dirs.add(args.out_dir) # Run main processor ------------------------------------------------------- root_name = args.File.root_name(args.revision, args.processor, args.project, args.product_name) args.target = root_name + ".config.nc" out_files = [] # All files that would be made (facilitates --dry_run) jid_main = [] # ID no. for each queued job args.in_dir = [args.out_dir] for sett in args.settings: phs_args = deepcopy(args) parsed_settings_arguments = pars.parse_args(sett.split()) for key, val in compare.__dict__.items(): if val == parsed_settings_arguments.__dict__[key]: parsed_settings_arguments.__dict__.pop(key) phs_args.__dict__.update(parsed_settings_arguments.__dict__) phs_args.out_dir = os.path.join(orig_args.out_dir, phs_args.sub_dir) jid, out = process_main(phs_args, log_path, dependency=jid_pre, tag=phs_args.sub_dir + phs_args.label) out_files.append(out) if jid is not None: jid_main.append(jid) written_dirs.add(args.out_dir) # Run postprocessor if necessary if len(args.settings) > 1: args.target = root_name + "NULL.primary.nc" args.in_dir = written_dirs args.out_dir = orig_args.out_dir jid, out_file = process_post(args, log_path, out_files, dependency=jid_main, tag="post{}".format(args.label)) if jid is not None: written_dirs.add(args.out_dir) else: out_file = out_files[0] # Run CCI formatting if args.reformat != "": call_reformat(args, log_path, args.reformat, out_file, dependency=jid) # Output root filename and output folders for regression tests return jid, out_file
args.out_dir = os.path.join(base_out_dir, test) try: args.target, args.limit, args.preset_settings = REGRESSION_TESTS[test] except KeyError: raise OracError("Invalid regression test for given phases.") args.preset_settings += "_" + args.test_type jid, out_file = process_all(args) log_path = os.path.join(args.out_dir, log_dir) # Check for regressions if not args.benchmark and not args.dry_run: inst = FileName(args.out_dir, out_file) if not args.batch: check_args_common(args) check_args_preproc(args) try: run_regression(inst) except Regression as err: colour_print('REGRESSION) ' + str(err), COLOURING['error']) else: if os.path.isdir(os.path.join(defaults.orac_dir, "tools")): path = [os.path.join(defaults.orac_dir, "tools"), ] else: path = [defaults.orac_dir, ] path.extend(filter(None, sys.path)) job_name = inst.job_name(tag='regression')