def process_post(args, log_path, files=None, dependency=None, tag='post'):
"""Call sequence for post processor"""
from glob import glob
from pyorac.drivers import build_postproc_driver
from pyorac.definitions import FileMissing, SETTINGS
from pyorac.local_defaults import DIR_PERMISSIONS
args = check_args_postproc(args)
job_name = args.File.job_name(args.revision, tag)
root_name = args.File.root_name(args.revision)
if not os.path.isdir(args.out_dir):
os.makedirs(args.out_dir, DIR_PERMISSIONS)
if files is None:
# Find all primary files of requested phases in given input folders.
files = []
for phs in set(args.phases):
for fdr in args.in_dir:
files.extend(
glob(
os.path.join(
fdr,
root_name + SETTINGS[phs].name + '.primary.nc')))
if len(files) < 2:
raise FileMissing('sufficient processed files', args.target)
out_file = os.path.join(
args.out_dir,
'.'.join(filter(None, (root_name, args.suffix, 'primary', 'nc'))))
if args.clobber >= CLOBBER['post'] or not os.path.isfile(out_file):
# Settings for batch processing
values = {
'job_name': job_name,
'log_file': os.path.join(log_path, job_name + '.log'),
'err_file': os.path.join(log_path, job_name + '.err'),
'duration': args.dur[2],
'ram': args.ram[2]
}
if dependency is not None:
values['depend'] = dependency
args.target = out_file
driver = build_postproc_driver(args, files)
exe = os.path.join(args.orac_dir, 'post_processing', 'orac_postproc')
if not os.path.isfile(exe):
exe = os.path.join(args.orac_dir, 'orac_postproc')
jid = call_exe(args, exe, driver, values)
else:
jid = None
return jid, out_file
def process_main(args, log_path, tag='', dependency=None):
"""Call sequence for main processor"""
from pyorac.definitions import SETTINGS
from pyorac.drivers import build_main_driver
from pyorac.local_defaults import DIR_PERMISSIONS
args = check_args_main(args)
if args.multilayer is not None:
phase = SETTINGS[args.phase].name + "_" + SETTINGS[
args.multilayer[0]].name
else:
phase = SETTINGS[args.phase].name
job_name = args.File.job_name(tag=phase + tag)
root_name = args.File.root_name(args.revision)
if not os.path.isdir(args.out_dir):
os.makedirs(args.out_dir, DIR_PERMISSIONS)
out_file = os.path.join(args.out_dir, root_name + phase + '.primary.nc')
if args.clobber >= CLOBBER['main'] or not os.path.isfile(out_file):
# Settings for batch processing
values = {
'job_name': job_name,
'log_file': os.path.join(log_path, job_name + '.log'),
'err_file': os.path.join(log_path, job_name + '.err'),
'duration': args.dur[1],
'ram': args.ram[1]
}
if dependency is not None:
values['depend'] = dependency
driver = build_main_driver(args)
exe = os.path.join(args.orac_dir, 'src', 'orac')
if not os.path.isfile(exe):
exe = os.path.join(args.orac_dir, 'orac')
jid = call_exe(args, exe, driver, values)
else:
jid = None
return jid, out_file
def process_pre(args, log_path, dependency=None, tag='pre'):
"""Call sequence for pre processor"""
from pyorac.drivers import build_preproc_driver
from pyorac.local_defaults import DIR_PERMISSIONS
args = check_args_preproc(args)
driver = build_preproc_driver(args)
# This must be called after building the driver as revision is unknown
job_name = args.File.job_name(args.revision, tag)
root_name = args.File.root_name(args.revision, args.processor,
args.project, args.product_name)
if not os.path.isdir(args.out_dir):
os.makedirs(args.out_dir, DIR_PERMISSIONS)
out_file = os.path.join(args.out_dir, root_name + '.config.nc')
if args.clobber >= CLOBBER['pre'] or not os.path.isfile(out_file):
# Settings for batch processing
values = {
'job_name': job_name,
'log_file': os.path.join(log_path, job_name + '.log'),
'err_file': os.path.join(log_path, job_name + '.err'),
'duration': args.dur[0],
'ram': args.ram[0]
}
if dependency is not None:
values['depend'] = dependency
exe = os.path.join(args.orac_dir, 'pre_processing', 'orac_preproc')
if not os.path.isfile(exe):
exe = os.path.join(args.orac_dir, 'orac_preproc')
jid = call_exe(args, exe, driver, values)
else:
jid = None
return jid, out_file