def find_contexts(self, time_interval, satellite, hirs2nc_delivery_id,
hirs_avhrr_delivery_id, hirs_csrb_daily_delivery_id,
hirs_csrb_monthly_delivery_id,
hirs_ctp_orbital_delivery_id, hirs_ctp_daily_delivery_id,
hirs_ctp_monthly_delivery_id):
granules = []
start = datetime(time_interval.left.year, time_interval.left.month, 1)
end = datetime(time_interval.right.year, time_interval.right.month, 1)
date = start
while date <= end:
granules.append(date)
date = date + timedelta(days=monthrange(date.year, date.month)[1])
return [{
'granule': g,
'satellite': satellite,
'hirs2nc_delivery_id': hirs2nc_delivery_id,
'hirs_avhrr_delivery_id': hirs_avhrr_delivery_id,
'hirs_csrb_daily_delivery_id': hirs_csrb_daily_delivery_id,
'hirs_csrb_monthly_delivery_id': hirs_csrb_monthly_delivery_id,
'hirs_ctp_orbital_delivery_id': hirs_ctp_orbital_delivery_id,
'hirs_ctp_daily_delivery_id': hirs_ctp_daily_delivery_id,
'hirs_ctp_monthly_delivery_id': hirs_ctp_monthly_delivery_id
} for g in granules]
def satellite_version(self, satellite, granule):
# 4 for everything post April 28, 2005
if granule > datetime(2005, 4, 28):
return 4
# 3 for noaa-15,16,17 until April 28, 2005
elif satellite in ['noaa-15', 'noaa-16', 'noaa-17']:
return 3
# 2 for noaa <= 14
else:
return 2
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(4,13) ]
#for years in range(2010, 2018):
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,13) ]
#years = 2018
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,2) ]
satellite = 'metop-b'
#granule = datetime(2015,5,14, 9, 23)
#intervals = [TimeInterval(granule, granule + hour - wedge)]
# NSS.GHRR.M1.D13140.S0029.E0127.B0347172.SV.level2.hdf --> NSS.GHRR.M1.D15365.S2307.E0004.B1705253.SV.level2.hdf
# datetime(2013, 5, 20, 0, 0) --> datetime(2017, 12, 31, 0, 0)
intervals = []
years = 2013
intervals += [
TimeInterval(
datetime(years, month, 1),
datetime(years, month,
calendar.monthrange(years, month)[1]) + day - wedge)
for month in range(5, 13)
]
for years in range(2014, 2018):
intervals += [
TimeInterval(
datetime(years, month, 1),
datetime(years, month,
calendar.monthrange(years, month)[1]) + day - wedge)
for month in range(1, 13)
]
satellite_choices = [
'noaa-06', 'noaa-07', 'noaa-08', 'noaa-09', 'noaa-10', 'noaa-11',
def build_task(self, context, task):
'''
Build up a set of inputs for a single context
'''
global delta_catalog
LOG.debug("Running build_task()")
# Initialize the hirs2nc and hirs_avhrr modules with the data locations
hirs2nc.delta_catalog = delta_catalog
hirs_avhrr.delta_catalog = delta_catalog
# Instantiate the hirs, hirs_avhrr and hirs_csrb_monthly computations
hirs2nc_comp = hirs2nc.HIRS2NC()
hirs_avhrr_comp = hirs_avhrr.HIRS_AVHRR()
hirs_csrb_monthly_comp = hirs_csrb_monthly.HIRS_CSRB_MONTHLY()
SPC = StoredProductCatalog()
# HIRS L1B Input
hirs2nc_context = context.copy()
[
hirs2nc_context.pop(k) for k in [
'hirs_avhrr_delivery_id', 'hirs_csrb_daily_delivery_id',
'hirs_csrb_monthly_delivery_id', 'hirs_ctp_orbital_delivery_id'
]
]
hirs2nc_prod = hirs2nc_comp.dataset('out').product(hirs2nc_context)
if SPC.exists(hirs2nc_prod):
task.input('HIR1B', hirs2nc_prod)
else:
raise WorkflowNotReady('No HIRS inputs available for {}'.format(
hirs2nc_context['granule']))
# PTMSX Input
LOG.debug('Getting PTMSX input...')
sensor = 'avhrr'
satellite = context['satellite']
file_type = 'PTMSX'
granule = context['granule']
try:
ptmsx_file = delta_catalog.file(sensor, satellite, file_type,
granule)
task.input('PTMSX', ptmsx_file)
except WorkflowNotReady:
raise WorkflowNotReady(
'No PTMSX inputs available for {}'.format(granule))
# Collo Input
hirs_avhrr_context = hirs2nc_context
hirs_avhrr_context['hirs_avhrr_delivery_id'] = context[
'hirs_avhrr_delivery_id']
hirs_avhrr_prod = hirs_avhrr_comp.dataset('out').product(
hirs_avhrr_context)
if SPC.exists(hirs_avhrr_prod):
task.input('COLLO', hirs_avhrr_prod)
else:
raise WorkflowNotReady(
'No HIRS_AVHRR inputs available for {}'.format(
hirs_avhrr_context['granule']))
# CSRB Monthly Input
hirs_csrb_monthly_context = context.copy()
[
hirs_csrb_monthly_context.pop(k)
for k in ['hirs_ctp_orbital_delivery_id']
]
hirs_csrb_monthly_context['granule'] = datetime(
context['granule'].year, context['granule'].month, 1)
hirs_csrb_monthly_prod = hirs_csrb_monthly_comp.dataset(
'zonal_means').product(hirs_csrb_monthly_context)
if SPC.exists(hirs_csrb_monthly_prod):
task.input('CSRB', hirs_csrb_monthly_prod)
else:
raise WorkflowNotReady(
'No HIRS_CSRB_MONTHLY inputs available for {}'.format(
hirs_csrb_monthly_context['granule']))
# CFSR Input
LOG.debug('Getting CFSR input...')
cfsr_file = self.get_cfsr(context['granule'])
if cfsr_file is not None:
task.input('CFSR', cfsr_file)
else:
raise WorkflowNotReady(
'No CFSR inputs available for {}'.format(granule))
LOG.debug("Final task.inputs...")
for task_key in task.inputs.keys():
LOG.debug("\t{}: {}".format(task_key, task.inputs[task_key]))
# Satellite specific information
#satellite = 'noaa-19'
#granule = datetime(2015, 4, 17, 0, 20)
#intervals = [TimeInterval(granule, granule + day - wedge)]
# NSS.GHRR.NP.D09108.S2301.E0050.B0100809.GC.level2.hdf --> NSS.GHRR.NP.D17365.S2238.E2359.B4585757.GC.level2.hdf
# datetime(2009, 4, 18, 0, 0) --> datetime(2017, 12, 31, 0, 0)
#intervals = [TimeInterval(datetime(years,1,1), datetime(years+1,1,1)-wedge) for years in range(2009, 2018) ]
satellite = 'metop-b'
#intervals = [TimeInterval(datetime(2015,1,1), datetime(2015,2,1)-wedge)]
# NSS.GHRR.M1.D13140.S0029.E0127.B0347172.SV.level2.hdf --> NSS.GHRR.M1.D15365.S2307.E0004.B1705253.SV.level2.hdf
# datetime(2013, 5, 20, 0, 0) --> datetime(2017, 12, 31, 0, 0)
intervals = [
TimeInterval(datetime(years, 1, 1),
datetime(years + 1, 1, 1) - wedge)
for years in range(2013, 2018)
]
satellite_choices = [
'noaa-06', 'noaa-07', 'noaa-08', 'noaa-09', 'noaa-10', 'noaa-11',
'noaa-12', 'noaa-14', 'noaa-15', 'noaa-16', 'noaa-17', 'noaa-18',
'noaa-19', 'metop-a', 'metop-b'
]
# Instantiate the computation
hirs_csrb_daily_comp = hirs_csrb_daily.HIRS_CSRB_DAILY()
comp = hirs_csrb_monthly.HIRS_CSRB_MONTHLY()
LOG.info("Submitting intervals...")
#intervals = []
#years = 2009
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(4,13) ]
#for years in range(2010, 2018):
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,13) ]
#years = 2018
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,2) ]
satellite = 'metop-b'
#granule = datetime(2015,5,14, 9, 23)
#intervals = [TimeInterval(granule, granule + hour - wedge)]
# NSS.GHRR.M1.D13140.S0029.E0127.B0347172.SV.level2.hdf --> NSS.GHRR.M1.D15365.S2307.E0004.B1705253.SV.level2.hdf
# datetime(2013, 5, 20, 0, 0) --> datetime(2017, 12, 31, 0, 0)
intervals = []
years = 2013
intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(5,13) ]
for years in range(2014,2018):
intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,13) ]
satellite_choices = ['noaa-06', 'noaa-07', 'noaa-08', 'noaa-09', 'noaa-10', 'noaa-11',
'noaa-12', 'noaa-14', 'noaa-15', 'noaa-16', 'noaa-17', 'noaa-18',
'noaa-19', 'metop-a', 'metop-b']
def setup_computation(satellite):
input_data = {'HIR1B': '/mnt/software/flo/hirs_l1b_datalists/{0:}/HIR1B_{0:}_latest'.format(satellite),
'CFSR': '/mnt/cephfs_data/geoffc/hirs_data_lists/CFSR.out',
'PTMSX': '/mnt/software/flo/hirs_l1b_datalists/{0:}/PTMSX_{0:}_latest'.format(satellite)}
# Data locations
collection = {'HIR1B': 'ILIAD',
