def setup_computation(satellite):
input_data = {
'HIR1B':
'/mnt/software/flo/hirs_l1b_datalists/{0:}/HIR1B_{0:}_latest'.format(
satellite),
'CFSR':
'/mnt/cephfs_data/geoffc/hirs_data_lists/CFSR.out',
'PTMSX':
'/mnt/software/flo/hirs_l1b_datalists/{0:}/PTMSX_{0:}_latest'.format(
satellite)
}
# Data locations
collection = {'HIR1B': 'ARCDATA', 'CFSR': 'DELTA', 'PTMSX': 'APOLLO'}
input_sources = {'collection': collection, 'input_data': input_data}
# Initialize the hirs2nc module with the data locations
hirs2nc.set_input_sources(input_sources)
# Instantiate the computation
comp = hirs2nc.HIRS2NC()
return comp
def local_execute_example(interval, satellite, hirs2nc_delivery_id, hirs_avhrr_delivery_id, hirs_csrb_daily_delivery_id,
skip_prepare=False, skip_execute=False, verbosity=2):
setup_logging(verbosity)
comp = setup_computation(satellite)
hirs2nc_comp = hirs2nc.HIRS2NC()
hirs_avhrr_comp = hirs_avhrr.HIRS_AVHRR()
# Get the required context...
contexts = comp.find_contexts(interval, satellite, hirs2nc_delivery_id, hirs_avhrr_delivery_id, hirs_csrb_daily_delivery_id)
if len(contexts) != 0:
LOG.info("Candidate contexts in interval...")
for context in contexts:
print("\t{}".format(context))
try:
if not skip_prepare:
LOG.info("Running hirs_csrb_daily local_prepare()...")
LOG.info("Preparing context... {}".format(contexts[0]))
local_prepare(comp, contexts[0], download_onlies=[hirs2nc_comp, hirs_avhrr_comp])
if not skip_execute:
LOG.info("Running hirs_csrb_daily local_execute()...")
LOG.info("Running context... {}".format(contexts[0]))
local_execute(comp, contexts[0], download_onlies=[hirs2nc_comp, hirs_avhrr_comp])
except Exception, err:
LOG.error("{}".format(err))
LOG.debug(traceback.format_exc())
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
global delta_catalog
sensor = 'avhrr'
satellite = context['satellite']
file_type = 'PTMSX'
granule = context['granule']
hirs_context = context.copy()
hirs_context.pop('hirs_avhrr_delivery_id')
LOG.debug("hirs_context: {}".format(hirs_context))
# Initialize the hirs2nc module with the data locations
hirs2nc.delta_catalog = delta_catalog
# Instantiate the hirs2nc computation
hirs2nc_comp = hirs2nc.HIRS2NC()
LOG.debug("hirs input: {}".format(
SPC.exists(hirs2nc_comp.dataset('out').product(hirs_context))))
hirs_file = hirs2nc_comp.dataset('out').product(hirs_context)
ptmsx_file = delta_catalog.file(sensor, satellite, file_type, granule)
LOG.debug('data_interval = {}'.format(ptmsx_file.data_interval))
task.input('HIR1B', hirs_file)
task.input('PTMSX', ptmsx_file)
task.option('data_interval', ptmsx_file.data_interval)
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
global delta_catalog
LOG.debug("Running build_task()")
LOG.debug("context: {}".format(context))
# Initialize the hirs2nc and hirs_avhrr modules with the data locations
hirs2nc.delta_catalog = delta_catalog
hirs_avhrr.delta_catalog = delta_catalog
# Instantiate the hirs and hirs_avhrr computations
hirs2nc_comp = hirs2nc.HIRS2NC()
hirs_avhrr_comp = hirs_avhrr.HIRS_AVHRR()
SPC = StoredProductCatalog()
day = TimeInterval(
context['granule'],
(context['granule'] + timedelta(days=1) - timedelta(seconds=1)))
hirs2nc_contexts = hirs2nc_comp.find_contexts(
day, context['satellite'], context['hirs2nc_delivery_id'])
if len(hirs2nc_contexts) == 0:
raise WorkflowNotReady('NO HIRS Data For {}'.format(
context['granule']))
# Input Counter.
ic = 0
for hirs2nc_context in hirs2nc_contexts:
# Making Input contexts
hirs_avhrr_context = hirs2nc_context.copy()
hirs_avhrr_context['hirs_avhrr_delivery_id'] = context[
'hirs_avhrr_delivery_id']
LOG.debug("HIRS context: {}".format(hirs2nc_context))
LOG.debug("HIRS_AVHRR context: {}".format(hirs_avhrr_context))
# Confirming we have HIRS1B and COLLO products...
hirs2nc_prod = hirs2nc_comp.dataset('out').product(hirs2nc_context)
hirs_avhrr_prod = hirs_avhrr_comp.dataset('out').product(
hirs_avhrr_context)
# If HIRS1B and COLLO products exist, add them and the Patmos-X
# file for this context to the list of input files to be downloaded to
# the workspace...
if SPC.exists(hirs2nc_prod) and SPC.exists(hirs_avhrr_prod):
# Its safe to require all three inputs
task.input('HIR1B-{}'.format(ic), hirs2nc_prod)
task.input('COLLO-{}'.format(ic), hirs_avhrr_prod)
task.input(
'PTMSX-{}'.format(ic),
delta_catalog.file('avhrr', hirs2nc_context['satellite'],
'PTMSX', hirs2nc_context['granule']))
ic += 1
LOG.debug(
"There are {} valid HIR1B/COLLO/PTMSX contexts in ({} -> {})".
format(ic, day.left, day.right))
if ic == 0:
LOG.warn(
"There are no valid HIR1B/COLLO/PTMSX contexts in ({} -> {}), aborting..."
.format(day.left, day.right))
return
interval = TimeInterval(context['granule'],
context['granule'] + timedelta(days=1))
num_cfsr_files = 0
# Search for the old style pgbhnl.gdas.*.grb2 files from the PEATE
if num_cfsr_files == 0:
LOG.debug(
"Trying to retrieve CFSR_PGRBHANL product (pgbhnl.gdas.*.grb2) CFSR files from DAWG..."
)
try:
cfsr_files = dawg_catalog.files('', 'CFSR_PGRBHANL', interval)
num_cfsr_files = len(cfsr_files)
if num_cfsr_files == 0:
LOG.debug("\tpgbhnl.gdas.*.grb2 CFSR files from DAWG : {}".
format(cfsr_files))
except Exception, err:
LOG.error("{}.".format(err))
LOG.warn(
"Retrieval of CFSR_PGRBHANL product (pgbhnl.gdas.*.grb2) CFSR files from DAWG failed"
)
def local_execute_example(interval,
satellite,
hirs2nc_delivery_id,
hirs_avhrr_delivery_id,
hirs_csrb_daily_delivery_id,
hirs_csrb_monthly_delivery_id,
hirs_ctp_orbital_delivery_id,
skip_prepare=False,
skip_execute=False,
single=True,
verbosity=2):
setup_logging(verbosity)
comp = setup_computation(satellite)
hirs2nc_comp = hirs2nc.HIRS2NC()
hirs_avhrr_comp = hirs_avhrr.HIRS_AVHRR()
hirs_csrb_monthly_comp = hirs_csrb_monthly.HIRS_CSRB_MONTHLY()
# Get the required context...
contexts = comp.find_contexts(interval, satellite, hirs2nc_delivery_id,
hirs_avhrr_delivery_id,
hirs_csrb_daily_delivery_id,
hirs_csrb_monthly_delivery_id,
hirs_ctp_orbital_delivery_id)
if len(contexts) != 0:
LOG.info("Candidate contexts in interval...")
for context in contexts:
print("\t{}".format(context))
if not single:
for idx, context in enumerate(contexts):
LOG.info('Current Dir: {} {}'.format(idx, os.getcwd()))
try:
if not skip_prepare:
LOG.info("Running hirs_ctp_orbital local_prepare()...")
LOG.info("Preparing context... {}".format(context))
local_prepare(comp,
context,
download_onlies=[
hirs2nc_comp, hirs_avhrr_comp,
hirs_csrb_monthly_comp
])
if not skip_execute:
LOG.info("Running hirs_ctp_orbital local_execute()...")
LOG.info("Running context... {}".format(context))
local_execute(comp,
context,
download_onlies=[
hirs2nc_comp, hirs_avhrr_comp,
hirs_csrb_monthly_comp
])
if not skip_prepare:
shutil.move('inputs', 'inputs_{}'.format(idx))
if not skip_execute:
shutil.move('outputs', 'outputs_{}'.format(idx))
except Exception, err:
LOG.error("{}".format(err))
LOG.debug(traceback.format_exc())
else:
try:
if not skip_prepare:
LOG.info("Running hirs_ctp_orbital local_prepare()...")
LOG.info("Preparing context... {}".format(contexts[0]))
local_prepare(comp,
contexts[0],
download_onlies=[
hirs2nc_comp, hirs_avhrr_comp,
hirs_csrb_monthly_comp
])
if not skip_execute:
LOG.info("Running hirs_ctp_orbital local_execute()...")
LOG.info("Running context... {}".format(contexts[0]))
local_execute(comp,
contexts[0],
download_onlies=[
hirs2nc_comp, hirs_avhrr_comp,
hirs_csrb_monthly_comp
])
except Exception, err:
LOG.error("{}".format(err))
LOG.debug(traceback.format_exc())
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
global delta_catalog
LOG.debug("Running build_task()")
# Initialize the hirs2nc and hirs_avhrr modules with the data locations
hirs2nc.delta_catalog = delta_catalog
hirs_avhrr.delta_catalog = delta_catalog
# Instantiate the hirs, hirs_avhrr and hirs_csrb_monthly computations
hirs2nc_comp = hirs2nc.HIRS2NC()
hirs_avhrr_comp = hirs_avhrr.HIRS_AVHRR()
hirs_csrb_monthly_comp = hirs_csrb_monthly.HIRS_CSRB_MONTHLY()
SPC = StoredProductCatalog()
# HIRS L1B Input
hirs2nc_context = context.copy()
[
hirs2nc_context.pop(k) for k in [
'hirs_avhrr_delivery_id', 'hirs_csrb_daily_delivery_id',
'hirs_csrb_monthly_delivery_id', 'hirs_ctp_orbital_delivery_id'
]
]
hirs2nc_prod = hirs2nc_comp.dataset('out').product(hirs2nc_context)
if SPC.exists(hirs2nc_prod):
task.input('HIR1B', hirs2nc_prod)
else:
raise WorkflowNotReady('No HIRS inputs available for {}'.format(
hirs2nc_context['granule']))
# PTMSX Input
LOG.debug('Getting PTMSX input...')
sensor = 'avhrr'
satellite = context['satellite']
file_type = 'PTMSX'
granule = context['granule']
try:
ptmsx_file = delta_catalog.file(sensor, satellite, file_type,
granule)
task.input('PTMSX', ptmsx_file)
except WorkflowNotReady:
raise WorkflowNotReady(
'No PTMSX inputs available for {}'.format(granule))
# Collo Input
hirs_avhrr_context = hirs2nc_context
hirs_avhrr_context['hirs_avhrr_delivery_id'] = context[
'hirs_avhrr_delivery_id']
hirs_avhrr_prod = hirs_avhrr_comp.dataset('out').product(
hirs_avhrr_context)
if SPC.exists(hirs_avhrr_prod):
task.input('COLLO', hirs_avhrr_prod)
else:
raise WorkflowNotReady(
'No HIRS_AVHRR inputs available for {}'.format(
hirs_avhrr_context['granule']))
# CSRB Monthly Input
hirs_csrb_monthly_context = context.copy()
[
hirs_csrb_monthly_context.pop(k)
for k in ['hirs_ctp_orbital_delivery_id']
]
hirs_csrb_monthly_context['granule'] = datetime(
context['granule'].year, context['granule'].month, 1)
hirs_csrb_monthly_prod = hirs_csrb_monthly_comp.dataset(
'zonal_means').product(hirs_csrb_monthly_context)
if SPC.exists(hirs_csrb_monthly_prod):
task.input('CSRB', hirs_csrb_monthly_prod)
else:
raise WorkflowNotReady(
'No HIRS_CSRB_MONTHLY inputs available for {}'.format(
hirs_csrb_monthly_context['granule']))
# CFSR Input
LOG.debug('Getting CFSR input...')
cfsr_file = self.get_cfsr(context['granule'])
if cfsr_file is not None:
task.input('CFSR', cfsr_file)
else:
raise WorkflowNotReady(
'No CFSR inputs available for {}'.format(granule))
LOG.debug("Final task.inputs...")
for task_key in task.inputs.keys():
LOG.debug("\t{}: {}".format(task_key, task.inputs[task_key]))
LOG.info("Submitting intervals...")
dt = datetime.utcnow()
log_name = 'hirs_avhrr_{}_s{}_e{}_c{}.log'.format(
satellite, intervals[0].left.strftime('%Y%m%d%H%M'),
intervals[-1].right.strftime('%Y%m%d%H%M'), dt.strftime('%Y%m%d%H%M%S'))
try:
for interval in intervals:
LOG.info("Submitting interval {} -> {}".format(interval.left,
interval.right))
comp = setup_computation(satellite)
hirs2nc_comp = hirs2nc.HIRS2NC()
contexts = comp.find_contexts(interval, satellite, hirs2nc_delivery_id,
hirs_avhrr_delivery_id)
LOG.info("Opening log file {}".format(log_name))
file_obj = open(log_name, 'a')
LOG.info("\tThere are {} contexts in this interval".format(
len(contexts)))
contexts.sort()
if contexts != []:
#for context in contexts:
#LOG.info(context)
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
global delta_catalog
LOG.debug("Running build_task()")
# Initialize the hirs2nc module with the data locations
hirs2nc.delta_catalog = delta_catalog
# Instantiate the hirs2nc and hirs_ctp_orbital computations
hirs2nc_comp = hirs2nc.HIRS2NC()
hirs_ctp_orbital_comp = hirs_ctp_orbital.HIRS_CTP_ORBITAL()
SPC = StoredProductCatalog()
#
# HIRS L1B Input
#
hirs2nc_context = {
'satellite': context['satellite'],
'granule': context['granule'],
'hirs2nc_delivery_id': context['hirs2nc_delivery_id']
}
hirs2nc_prod = hirs2nc_comp.dataset('out').product(hirs2nc_context)
if SPC.exists(hirs2nc_prod):
task.input('HIR1B', hirs2nc_prod)
else:
raise WorkflowNotReady('No HIRS inputs available for {}'.format(
hirs2nc_context['granule']))
#
# CTP Orbital Input
#
hirs_ctp_orbital_context = context.copy()
[
hirs_ctp_orbital_context.pop(k) for k in [
'hirs_ctp_daily_delivery_id', 'hirs_ctp_monthly_delivery_id',
'hirs_tpw_orbital_delivery_id'
]
]
hirs_ctp_orbital_prod = hirs_ctp_orbital_comp.dataset('out').product(
hirs_ctp_orbital_context)
if SPC.exists(hirs_ctp_orbital_prod):
task.input('CTPO', hirs_ctp_orbital_prod)
else:
raise WorkflowNotReady(
'No HIRS CTP Orbital inputs available for {}'.format(
hirs_ctp_orbital_context['granule']))
#
# CFSR Input
#
cfsr_granule = round_datetime(context['granule'], timedelta(hours=6))
cfsr_file = self.get_cfsr(cfsr_granule)
if cfsr_file is not None:
task.input('CFSR', cfsr_file)
else:
raise WorkflowNotReady(
'No CFSR inputs available for {}'.format(cfsr_granule))
LOG.debug("Final task.inputs...") # GPC
for task_key in task.inputs.keys():
LOG.debug("\t{}: {}".format(task_key,
task.inputs[task_key])) # GPC
LOG.debug("Exiting build_task()...") # GPC