def call_reformat(args, log_path, exe, out_file, dependency=None): """Reformat outputs using the script provided.""" import pyorac.local_defaults as defaults from pyorac.colour_print import colour_print from pyorac.definitions import OracError, COLOURING from subprocess import check_call, check_output, CalledProcessError # Optionally print command and driver file contents to StdOut if args.verbose or args.script_verbose or args.dry_run: colour_print('{} {} 1 <<<'.format(exe, out_file), COLOURING['header']) if args.dry_run: return -1 job_name = args.File.job_name(args.revision, 'format') if not args.batch: try: check_call([exe, out_file, "1"]) except CalledProcessError as err: raise OracError('{:s} failed with error code {:d}. {}'.format( ' '.join(err.cmd), err.returncode, err.output)) else: try: # Collect batch settings from defaults, command line, and script batch_params = defaults.batch_values.copy() batch_params['job_name'] = job_name batch_params['log_file'] = os.path.join(log_path, job_name + '.log') batch_params['err_file'] = os.path.join(log_path, job_name + '.err') batch_params['duration'] = '01:00' batch_params['ram'] = 5000 batch_params['procs'] = 1 if dependency is not None: batch_params['depend'] = dependency batch_params.update({key: val for key, val in args.batch_settings}) # Form batch queue command and call batch queuing system cmd = defaults.batch.list_batch(batch_params, exe=[exe, out_file, "1"]) if args.verbose or args.script_verbose: colour_print(' '.join(cmd), COLOURING['header']) out = check_output(cmd, universal_newlines=True) # Parse job ID # and return it to the caller jid = defaults.batch.parse_out(out, 'ID') return jid except CalledProcessError as err: raise OracError('Failed to queue job ' + exe) except SyntaxError as err: raise OracError(str(err))
if orig_args.out_dir: base_out_dir = orig_args.out_dir else: base_out_dir = defaults.data_dir +'/testoutput' # Increment version number (as this is usually run on uncommited code) if orig_args.revision is None: orig_args.revision = get_repository_revision() if not orig_args.benchmark: orig_args.revision += 1 try: for test in orig_args.tests: colour_print(test, COLOURING['header']) args = deepcopy(orig_args) # Set filename to be processed and output folder args.out_dir = os.path.join(base_out_dir, test) try: args.target, args.limit, args.preset_settings = REGRESSION_TESTS[test] except KeyError: raise OracError("Invalid regression test for given phases.") args.preset_settings += "_" + args.test_type jid, out_file = process_all(args) log_path = os.path.join(args.out_dir, log_dir) # Check for regressions
from pyorac.colour_print import colour_print from pyorac.definitions import COLOURING, OracError, OracWarning from pyorac.run import process_all from pyorac.util import warning_format import warnings warnings.formatwarning = warning_format warnings.filterwarnings('always', category=OracWarning) # Define parser pars = ArgumentParser( description='Run the full ORAC suite on a given Level 1B file.') pars.add_argument('target', type=str, help='Level 1B file to be processed') args_common(pars) args_cc4cl(pars) args_preproc(pars) args_main(pars) args_postproc(pars) args = pars.parse_args() try: jid, _ = process_all(args) if args.script_verbose and args.batch: print("Job queued with ID {}".format(jid)) except OracError as err: colour_print('ERROR) ' + str(err), COLOURING['error']) except KeyboardInterrupt: colour_print('Execution halted by user.', COLOURING['error'])
def call_exe(args, exe, driver, values=dict()): """Call an ORAC executable, managing the necessary driver file. Args: :list args: Arguments of the script. :str exe: Name of the executable. :str driver: Contents of the driver file to pass. :dict values: Arguments for the batch queueing system.""" import pyorac.local_defaults as defaults from pyorac.colour_print import colour_print from pyorac.definitions import OracError, COLOURING from subprocess import check_call, check_output, CalledProcessError from tempfile import mkstemp from time import time # Optionally print command and driver file contents to StdOut if args.verbose or args.script_verbose or args.dry_run: colour_print(exe + ' <<<', COLOURING['header']) colour_print(driver, COLOURING['text']) if args.dry_run: return # Write driver file (fd, driver_file) = mkstemp('.driver', os.path.basename(exe) + '.', args.out_dir, True) f = os.fdopen(fd, "w") f.write(driver) f.close() if not args.batch: # Form processing environment os.environ["LD_LIBRARY_PATH"] = build_orac_library_path() # Define a directory for EMOS to put it's gridding try: os.environ["PPDIR"] = args.emos_dir os.environ["OPENBLAS_NUM_THREADS"] = "1" os.environ["OMP_NUM_THREADS"] = str(args.procs) except AttributeError: pass # Call program try: st = time() check_call([exe, driver_file]) if args.timing: colour_print(exe + ' took {:f}s'.format(time() - st), COLOURING['timing']) return True except CalledProcessError as err: raise OracError('{:s} failed with error code {:d}. {}'.format( ' '.join(err.cmd), err.returncode, err.output)) finally: if not args.keep_driver: os.remove(driver_file) elif args.verbose or args.script_verbose: print("Driver file stored at " + driver_file) else: # Write temporary script to call executable (gd, script_file) = mkstemp('.sh', os.path.basename(exe) + '.', args.out_dir, True) g = os.fdopen(gd, "w") g.write(defaults.batch_script) # Define processing environment libs = read_orac_libraries(args.orac_lib) g.write("export LD_LIBRARY_PATH=" + build_orac_library_path(libs) + "\n") g.write("export OPENBLAS_NUM_THREADS=1\n") try: g.write("export PPDIR=" + args.emos_dir + "\n") except AttributeError: pass defaults.batch.add_openmp_to_script(g) # Call executable and give the script permission to execute g.write(exe + ' ' + driver_file + "\n") if not args.keep_driver: g.write("rm -f " + driver_file + "\n") g.write("rm -f " + script_file + "\n") g.close() os.chmod(script_file, 0o700) try: # Collect batch settings from defaults, command line, and script batch_params = defaults.batch_values.copy() batch_params.update(values) batch_params.update({key: val for key, val in args.batch_settings}) batch_params['procs'] = args.procs # Form batch queue command and call batch queuing system cmd = defaults.batch.ListBatch(batch_params, exe=script_file) if args.verbose or args.script_verbose: colour_print(' '.join(cmd), COLOURING['header']) out = check_output(cmd, universal_newlines=True) # Parse job ID # and return it to the caller jid = defaults.batch.ParseOut(out, 'ID') return jid except CalledProcessError as err: raise OracError('Failed to queue job ' + exe) except SyntaxError as err: raise OracError(str(err))