def setup_computation(satellite):
input_data = {
'HIR1B':
'/mnt/software/flo/hirs_l1b_datalists/{0:}/HIR1B_{0:}_latest'.format(
satellite),
'CFSR':
'/mnt/cephfs_data/geoffc/hirs_data_lists/CFSR.out',
'PTMSX':
'/mnt/software/flo/hirs_l1b_datalists/{0:}/PTMSX_{0:}_latest'.format(
satellite)
}
# Data locations
collection = {'HIR1B': 'ILIAD', 'CFSR': 'DELTA', 'PTMSX': 'FJORD'}
input_sources = {'collection': collection, 'input_data': input_data}
# Initialize the hirs_csrb_daily module with the data locations
hirs_ctp_daily.set_input_sources(input_sources)
# Instantiate the computation
comp = hirs_ctp_daily.HIRS_CTP_DAILY()
return comp
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
LOG.debug("Running build_task()")
# Instantiate the hirs_ctp_daily computation
hirs_ctp_daily_comp = hirs_ctp_daily.HIRS_CTP_DAILY()
num_days = monthrange(context['granule'].year,
context['granule'].month)[1]
interval = TimeInterval(context['granule'],
context['granule'] + timedelta(num_days),
False, True)
daily_contexts = hirs_ctp_daily_comp.find_contexts(
interval, context['satellite'], context['hirs2nc_delivery_id'],
context['hirs_avhrr_delivery_id'],
context['hirs_csrb_daily_delivery_id'],
context['hirs_csrb_monthly_delivery_id'],
context['hirs_ctp_orbital_delivery_id'],
context['hirs_ctp_daily_delivery_id'])
if len(daily_contexts) == 0:
raise WorkflowNotReady(
'No HIRS_CTP_DAILY inputs available for {}'.format(
context['granule']))
for (idx, daily_context) in enumerate(daily_contexts):
hirs_ctp_daily_prod = hirs_ctp_daily_comp.dataset('out').product(
daily_context)
if SPC.exists(hirs_ctp_daily_prod):
task.input('CTPD-{}'.format(idx), hirs_ctp_daily_prod, True)
#for years in range(2010, 2018):
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,13) ]
#years = 2018
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,2) ]
satellite = 'metop-b'
intervals = []
years = 2017
#intervals += [TimeInterval(datetime(years, 1, 1), datetime(years, 2, 1)-wedge)]
intervals += [TimeInterval(datetime(years, 1, 1), datetime(years+1, 1, 1)-wedge)]
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,13) ]
#for years in range(2013,2018):
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,13) ]
# Instantiate the computations
hirs_ctp_daily_comp = hirs_ctp_daily.HIRS_CTP_DAILY()
comp = hirs_ctp_monthly.HIRS_CTP_MONTHLY()
satellite_choices = ['noaa-06', 'noaa-07', 'noaa-08', 'noaa-09', 'noaa-10', 'noaa-11',
'noaa-12', 'noaa-14', 'noaa-15', 'noaa-16', 'noaa-17', 'noaa-18',
'noaa-19', 'metop-a', 'metop-b']
LOG.info("Submitting intervals...")
dt = datetime.utcnow()
log_name = 'hirs_ctp_monthly_{}_s{}_e{}_c{}.log'.format(
satellite,
intervals[0].left.strftime('%Y%m'),
intervals[-1].right.strftime('%Y%m'),
dt.strftime('%Y%m%d%H%M%S'))
def local_execute_example(interval,
satellite,
hirs2nc_delivery_id,
hirs_avhrr_delivery_id,
hirs_csrb_daily_delivery_id,
hirs_csrb_monthly_delivery_id,
hirs_ctp_orbital_delivery_id,
hirs_ctp_daily_delivery_id,
hirs_ctp_monthly_delivery_id,
skip_prepare=False,
skip_execute=False,
single=True,
verbosity=2):
setup_logging(verbosity)
hirs_ctp_daily_comp = hirs_ctp_daily.HIRS_CTP_DAILY()
comp = hirs_ctp_monthly.HIRS_CTP_MONTHLY()
# Get the required context...
contexts = comp.find_contexts(
interval, satellite, hirs2nc_delivery_id, hirs_avhrr_delivery_id,
hirs_csrb_daily_delivery_id, hirs_csrb_monthly_delivery_id,
hirs_ctp_orbital_delivery_id, hirs_ctp_daily_delivery_id,
hirs_ctp_monthly_delivery_id)
if len(contexts) != 0:
LOG.info("Candidate contexts in interval...")
for context in contexts:
print("\t{}".format(context))
if not single:
for idx, context in enumerate(contexts):
LOG.info('Current Dir: {} {}'.format(idx, os.getcwd()))
try:
if not skip_prepare:
LOG.info("Running hirs_ctp_monthly local_prepare()...")
LOG.info("Preparing context... {}".format(context))
local_prepare(comp,
context,
download_onlies=[hirs_ctp_daily_comp])
if not skip_execute:
LOG.info("Running hirs_ctp_monthly local_execute()...")
LOG.info("Running context... {}".format(context))
local_execute(comp,
context,
download_onlies=[hirs_ctp_daily_comp])
if not skip_prepare:
shutil.move('inputs', 'inputs_{}'.format(idx))
if not skip_execute:
shutil.move('outputs', 'outputs_{}'.format(idx))
except Exception, err:
LOG.error("{}".format(err))
LOG.debug(traceback.format_exc())
else:
LOG.info("Single context!")
try:
if not skip_prepare:
LOG.info("Running hirs_ctp_monthly local_prepare()...")
LOG.info("Preparing context... {}".format(contexts[0]))
local_prepare(comp,
contexts[0],
download_onlies=[hirs_ctp_daily_comp])
if not skip_execute:
LOG.info("Running hirs_ctp_monthly local_execute()...")
LOG.info("Running context... {}".format(contexts[0]))
local_execute(comp,
contexts[0],
download_onlies=[hirs_ctp_daily_comp])
except Exception, err:
LOG.error("{}".format(err))
LOG.debug(traceback.format_exc())