def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
LOG.debug("Running build_task()")
# Instantiate the hirs_ctp_daily computation
hirs_ctp_daily_comp = hirs_ctp_daily.HIRS_CTP_DAILY()
num_days = monthrange(context['granule'].year,
context['granule'].month)[1]
interval = TimeInterval(context['granule'],
context['granule'] + timedelta(num_days),
False, True)
daily_contexts = hirs_ctp_daily_comp.find_contexts(
interval, context['satellite'], context['hirs2nc_delivery_id'],
context['hirs_avhrr_delivery_id'],
context['hirs_csrb_daily_delivery_id'],
context['hirs_csrb_monthly_delivery_id'],
context['hirs_ctp_orbital_delivery_id'],
context['hirs_ctp_daily_delivery_id'])
if len(daily_contexts) == 0:
raise WorkflowNotReady(
'No HIRS_CTP_DAILY inputs available for {}'.format(
context['granule']))
for (idx, daily_context) in enumerate(daily_contexts):
hirs_ctp_daily_prod = hirs_ctp_daily_comp.dataset('out').product(
daily_context)
if SPC.exists(hirs_ctp_daily_prod):
task.input('CTPD-{}'.format(idx), hirs_ctp_daily_prod, True)
def files(self, sensor, sat, file_type, target_interval):
LOG.debug('sensor = {}'.format(sensor))
LOG.debug('sat = {}'.format(sat))
LOG.debug('file_type = {}'.format(file_type))
LOG.debug('target_interval = {}'.format(target_interval))
# Loading files of type file_type before searching
self.check_file_index(file_type)
# Get the dict of the file metadata for the correct sensor, satellite and
# file type...
if file_type not in self.file_data[sensor][sat]:
raise WorkflowNotReady('No files for {} {} {} {}'.format(
sensor, sat, file_type, target_interval.left))
else:
file_search = self.file_data[sensor][sat][file_type]
fs_keys = file_search.keys()
fs_keys.sort()
#print('file_search[{}] = {}'.format(fs_keys[0],file_search[fs_keys[0]]))
#print('file_search = {}'.format(file_search))
# Create a list of all of the files which overlap the desired time interval.
files = [
self.file_info(file_search[name], file_type) for name in fs_keys
if target_interval.overlaps(file_search[name]['data_interval'])
]
# Remove any duplicates from the file list.
return self.remove_duplicates(files)
def file(self, sensor, sat, file_type, begin_time):
file_list = self.files(sensor, sat, file_type,
TimeInterval(begin_time, begin_time))
LOG.debug("file_list: {}".format(file_list))
# Making sure we have the right begin time as some inputs overlap
for (i, file) in enumerate(file_list):
if file.data_interval.left == begin_time:
return file_list[i]
raise WorkflowNotReady('No files for {} {} {} {}'.format(
sensor, sat, file_type, begin_time))
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
global delta_catalog
LOG.debug("Running build_task()")
LOG.debug("context: {}".format(context))
# Initialize the hirs2nc and hirs_avhrr modules with the data locations
hirs2nc.delta_catalog = delta_catalog
hirs_avhrr.delta_catalog = delta_catalog
# Instantiate the hirs and hirs_avhrr computations
hirs2nc_comp = hirs2nc.HIRS2NC()
hirs_avhrr_comp = hirs_avhrr.HIRS_AVHRR()
SPC = StoredProductCatalog()
day = TimeInterval(
context['granule'],
(context['granule'] + timedelta(days=1) - timedelta(seconds=1)))
hirs2nc_contexts = hirs2nc_comp.find_contexts(
day, context['satellite'], context['hirs2nc_delivery_id'])
if len(hirs2nc_contexts) == 0:
raise WorkflowNotReady('NO HIRS Data For {}'.format(
context['granule']))
# Input Counter.
ic = 0
for hirs2nc_context in hirs2nc_contexts:
# Making Input contexts
hirs_avhrr_context = hirs2nc_context.copy()
hirs_avhrr_context['hirs_avhrr_delivery_id'] = context[
'hirs_avhrr_delivery_id']
LOG.debug("HIRS context: {}".format(hirs2nc_context))
LOG.debug("HIRS_AVHRR context: {}".format(hirs_avhrr_context))
# Confirming we have HIRS1B and COLLO products...
hirs2nc_prod = hirs2nc_comp.dataset('out').product(hirs2nc_context)
hirs_avhrr_prod = hirs_avhrr_comp.dataset('out').product(
hirs_avhrr_context)
# If HIRS1B and COLLO products exist, add them and the Patmos-X
# file for this context to the list of input files to be downloaded to
# the workspace...
if SPC.exists(hirs2nc_prod) and SPC.exists(hirs_avhrr_prod):
# Its safe to require all three inputs
task.input('HIR1B-{}'.format(ic), hirs2nc_prod)
task.input('COLLO-{}'.format(ic), hirs_avhrr_prod)
task.input(
'PTMSX-{}'.format(ic),
delta_catalog.file('avhrr', hirs2nc_context['satellite'],
'PTMSX', hirs2nc_context['granule']))
ic += 1
LOG.debug(
"There are {} valid HIR1B/COLLO/PTMSX contexts in ({} -> {})".
format(ic, day.left, day.right))
if ic == 0:
LOG.warn(
"There are no valid HIR1B/COLLO/PTMSX contexts in ({} -> {}), aborting..."
.format(day.left, day.right))
return
interval = TimeInterval(context['granule'],
context['granule'] + timedelta(days=1))
num_cfsr_files = 0
# Search for the old style pgbhnl.gdas.*.grb2 files from the PEATE
if num_cfsr_files == 0:
LOG.debug(
"Trying to retrieve CFSR_PGRBHANL product (pgbhnl.gdas.*.grb2) CFSR files from DAWG..."
)
try:
cfsr_files = dawg_catalog.files('', 'CFSR_PGRBHANL', interval)
num_cfsr_files = len(cfsr_files)
if num_cfsr_files == 0:
LOG.debug("\tpgbhnl.gdas.*.grb2 CFSR files from DAWG : {}".
format(cfsr_files))
except Exception, err:
LOG.error("{}.".format(err))
LOG.warn(
"Retrieval of CFSR_PGRBHANL product (pgbhnl.gdas.*.grb2) CFSR files from DAWG failed"
)
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
global delta_catalog
LOG.debug("Running build_task()")
# Initialize the hirs_tpw_orbital module with the data locations
hirs_tpw_orbital.delta_catalog = delta_catalog
# Instantiate the hirs_tpw_orbital computation
hirs_tpw_orbital_comp = hirs_tpw_orbital.HIRS_TPW_ORBITAL()
SPC = StoredProductCatalog()
# TPW Orbital Input
granule = context['granule']
wedge = timedelta(seconds=1)
hour = timedelta(hours=1)
day = timedelta(days=1)
# Add an hour to each end of the day to make sure the day is completely covered
interval = TimeInterval(context['granule'] - 1 * hour,
(context['granule'] + day + 1 * hour))
hirs_tpw_orbital_contexts = hirs_tpw_orbital_comp.find_contexts(
interval, context['satellite'], context['hirs2nc_delivery_id'],
context['hirs_avhrr_delivery_id'],
context['hirs_csrb_daily_delivery_id'],
context['hirs_csrb_monthly_delivery_id'],
context['hirs_ctp_orbital_delivery_id'],
context['hirs_ctp_daily_delivery_id'],
context['hirs_ctp_monthly_delivery_id'],
context['hirs_tpw_orbital_delivery_id'])
if len(hirs_tpw_orbital_contexts) == 0:
raise WorkflowNotReady(
'No HIRS_TPW_ORBITAL inputs available for {}'.format(
context['granule']))
LOG.debug("There are {} TPW Orbital contexts for {}.".format(
len(hirs_tpw_orbital_contexts), interval))
for context in hirs_tpw_orbital_contexts:
LOG.debug(context)
# Knock off all but the last of the "previous" day's contexts
this_day = granule.day
previous_day = (granule - day + wedge).day
next_day = (granule + day + wedge).day
LOG.debug("previous_day: {}".format(previous_day))
LOG.debug("this_day: {}".format(this_day))
LOG.debug("next_day: {}".format(next_day))
start_idx = 0
end_idx = -1
num_contexts = len(hirs_tpw_orbital_contexts)
indices = np.arange(num_contexts)
reverse_indices = np.flip(np.arange(num_contexts) - num_contexts,
axis=0)
# have this set to zero unless we need to set it otherwise (say for Metop-B)
interval_pad = 0
# Pruning all but the last of the previous day's contexts
for idx in indices:
if hirs_tpw_orbital_contexts[
idx + interval_pad]['granule'].day == this_day:
start_idx = idx
LOG.debug("Breaking: start_idx = {}, granule = {}".format(
start_idx,
hirs_tpw_orbital_contexts[start_idx]['granule']))
break
# Pruning all but the first of the next day's contexts
for idx in reverse_indices:
if hirs_tpw_orbital_contexts[
idx - interval_pad]['granule'].day == this_day:
end_idx = idx
LOG.debug("Breaking: end_idx = {}, granule = {}".format(
end_idx, hirs_tpw_orbital_contexts[end_idx]['granule']))
break
hirs_tpw_orbital_contexts = hirs_tpw_orbital_contexts[
start_idx:end_idx + 1]
#hirs_tpw_orbital_contexts = hirs_tpw_orbital_contexts[start_idx:end_idx]
for context in hirs_tpw_orbital_contexts:
LOG.debug("{}".format(context))
for idx, context in enumerate(hirs_tpw_orbital_contexts):
hirs_tpw_orbital_prod = hirs_tpw_orbital_comp.dataset(
'shift').product(context)
if SPC.exists(hirs_tpw_orbital_prod):
task.input('TPWO_shift-{}'.format(str(idx).zfill(2)),
hirs_tpw_orbital_prod)
for idx, context in enumerate(hirs_tpw_orbital_contexts):
hirs_tpw_orbital_prod = hirs_tpw_orbital_comp.dataset(
'noshift').product(context)
if SPC.exists(hirs_tpw_orbital_prod):
task.input('TPWO_noshift-{}'.format(str(idx).zfill(2)),
hirs_tpw_orbital_prod)
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
global delta_catalog
LOG.debug("Running build_task()")
# Initialize the hirs2nc and hirs_avhrr modules with the data locations
hirs2nc.delta_catalog = delta_catalog
hirs_avhrr.delta_catalog = delta_catalog
# Instantiate the hirs, hirs_avhrr and hirs_csrb_monthly computations
hirs2nc_comp = hirs2nc.HIRS2NC()
hirs_avhrr_comp = hirs_avhrr.HIRS_AVHRR()
hirs_csrb_monthly_comp = hirs_csrb_monthly.HIRS_CSRB_MONTHLY()
SPC = StoredProductCatalog()
# HIRS L1B Input
hirs2nc_context = context.copy()
[
hirs2nc_context.pop(k) for k in [
'hirs_avhrr_delivery_id', 'hirs_csrb_daily_delivery_id',
'hirs_csrb_monthly_delivery_id', 'hirs_ctp_orbital_delivery_id'
]
]
hirs2nc_prod = hirs2nc_comp.dataset('out').product(hirs2nc_context)
if SPC.exists(hirs2nc_prod):
task.input('HIR1B', hirs2nc_prod)
else:
raise WorkflowNotReady('No HIRS inputs available for {}'.format(
hirs2nc_context['granule']))
# PTMSX Input
LOG.debug('Getting PTMSX input...')
sensor = 'avhrr'
satellite = context['satellite']
file_type = 'PTMSX'
granule = context['granule']
try:
ptmsx_file = delta_catalog.file(sensor, satellite, file_type,
granule)
task.input('PTMSX', ptmsx_file)
except WorkflowNotReady:
raise WorkflowNotReady(
'No PTMSX inputs available for {}'.format(granule))
# Collo Input
hirs_avhrr_context = hirs2nc_context
hirs_avhrr_context['hirs_avhrr_delivery_id'] = context[
'hirs_avhrr_delivery_id']
hirs_avhrr_prod = hirs_avhrr_comp.dataset('out').product(
hirs_avhrr_context)
if SPC.exists(hirs_avhrr_prod):
task.input('COLLO', hirs_avhrr_prod)
else:
raise WorkflowNotReady(
'No HIRS_AVHRR inputs available for {}'.format(
hirs_avhrr_context['granule']))
# CSRB Monthly Input
hirs_csrb_monthly_context = context.copy()
[
hirs_csrb_monthly_context.pop(k)
for k in ['hirs_ctp_orbital_delivery_id']
]
hirs_csrb_monthly_context['granule'] = datetime(
context['granule'].year, context['granule'].month, 1)
hirs_csrb_monthly_prod = hirs_csrb_monthly_comp.dataset(
'zonal_means').product(hirs_csrb_monthly_context)
if SPC.exists(hirs_csrb_monthly_prod):
task.input('CSRB', hirs_csrb_monthly_prod)
else:
raise WorkflowNotReady(
'No HIRS_CSRB_MONTHLY inputs available for {}'.format(
hirs_csrb_monthly_context['granule']))
# CFSR Input
LOG.debug('Getting CFSR input...')
cfsr_file = self.get_cfsr(context['granule'])
if cfsr_file is not None:
task.input('CFSR', cfsr_file)
else:
raise WorkflowNotReady(
'No CFSR inputs available for {}'.format(granule))
LOG.debug("Final task.inputs...")
for task_key in task.inputs.keys():
LOG.debug("\t{}: {}".format(task_key, task.inputs[task_key]))
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
global delta_catalog
LOG.debug("Running build_task()")
# Initialize the hirs2nc module with the data locations
hirs2nc.delta_catalog = delta_catalog
# Instantiate the hirs2nc and hirs_ctp_orbital computations
hirs2nc_comp = hirs2nc.HIRS2NC()
hirs_ctp_orbital_comp = hirs_ctp_orbital.HIRS_CTP_ORBITAL()
SPC = StoredProductCatalog()
#
# HIRS L1B Input
#
hirs2nc_context = {
'satellite': context['satellite'],
'granule': context['granule'],
'hirs2nc_delivery_id': context['hirs2nc_delivery_id']
}
hirs2nc_prod = hirs2nc_comp.dataset('out').product(hirs2nc_context)
if SPC.exists(hirs2nc_prod):
task.input('HIR1B', hirs2nc_prod)
else:
raise WorkflowNotReady('No HIRS inputs available for {}'.format(
hirs2nc_context['granule']))
#
# CTP Orbital Input
#
hirs_ctp_orbital_context = context.copy()
[
hirs_ctp_orbital_context.pop(k) for k in [
'hirs_ctp_daily_delivery_id', 'hirs_ctp_monthly_delivery_id',
'hirs_tpw_orbital_delivery_id'
]
]
hirs_ctp_orbital_prod = hirs_ctp_orbital_comp.dataset('out').product(
hirs_ctp_orbital_context)
if SPC.exists(hirs_ctp_orbital_prod):
task.input('CTPO', hirs_ctp_orbital_prod)
else:
raise WorkflowNotReady(
'No HIRS CTP Orbital inputs available for {}'.format(
hirs_ctp_orbital_context['granule']))
#
# CFSR Input
#
cfsr_granule = round_datetime(context['granule'], timedelta(hours=6))
cfsr_file = self.get_cfsr(cfsr_granule)
if cfsr_file is not None:
task.input('CFSR', cfsr_file)
else:
raise WorkflowNotReady(
'No CFSR inputs available for {}'.format(cfsr_granule))
LOG.debug("Final task.inputs...") # GPC
for task_key in task.inputs.keys():
LOG.debug("\t{}: {}".format(task_key,
task.inputs[task_key])) # GPC
LOG.debug("Exiting build_task()...") # GPC