def find_contexts(self, time_interval, satellite, hirs2nc_delivery_id, hirs_avhrr_delivery_id, hirs_csrb_daily_delivery_id, hirs_csrb_monthly_delivery_id, hirs_ctp_orbital_delivery_id, hirs_ctp_daily_delivery_id, hirs_ctp_monthly_delivery_id): granules = [] start = datetime(time_interval.left.year, time_interval.left.month, 1) end = datetime(time_interval.right.year, time_interval.right.month, 1) date = start while date <= end: granules.append(date) date = date + timedelta(days=monthrange(date.year, date.month)[1]) return [{ 'granule': g, 'satellite': satellite, 'hirs2nc_delivery_id': hirs2nc_delivery_id, 'hirs_avhrr_delivery_id': hirs_avhrr_delivery_id, 'hirs_csrb_daily_delivery_id': hirs_csrb_daily_delivery_id, 'hirs_csrb_monthly_delivery_id': hirs_csrb_monthly_delivery_id, 'hirs_ctp_orbital_delivery_id': hirs_ctp_orbital_delivery_id, 'hirs_ctp_daily_delivery_id': hirs_ctp_daily_delivery_id, 'hirs_ctp_monthly_delivery_id': hirs_ctp_monthly_delivery_id } for g in granules]
def satellite_version(self, satellite, granule): # 4 for everything post April 28, 2005 if granule > datetime(2005, 4, 28): return 4 # 3 for noaa-15,16,17 until April 28, 2005 elif satellite in ['noaa-15', 'noaa-16', 'noaa-17']: return 3 # 2 for noaa <= 14 else: return 2
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(4,13) ] #for years in range(2010, 2018): #intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,13) ] #years = 2018 #intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,2) ] satellite = 'metop-b' #granule = datetime(2015,5,14, 9, 23) #intervals = [TimeInterval(granule, granule + hour - wedge)] # NSS.GHRR.M1.D13140.S0029.E0127.B0347172.SV.level2.hdf --> NSS.GHRR.M1.D15365.S2307.E0004.B1705253.SV.level2.hdf # datetime(2013, 5, 20, 0, 0) --> datetime(2017, 12, 31, 0, 0) intervals = [] years = 2013 intervals += [ TimeInterval( datetime(years, month, 1), datetime(years, month, calendar.monthrange(years, month)[1]) + day - wedge) for month in range(5, 13) ] for years in range(2014, 2018): intervals += [ TimeInterval( datetime(years, month, 1), datetime(years, month, calendar.monthrange(years, month)[1]) + day - wedge) for month in range(1, 13) ] satellite_choices = [ 'noaa-06', 'noaa-07', 'noaa-08', 'noaa-09', 'noaa-10', 'noaa-11',
def build_task(self, context, task): ''' Build up a set of inputs for a single context ''' global delta_catalog LOG.debug("Running build_task()") # Initialize the hirs2nc and hirs_avhrr modules with the data locations hirs2nc.delta_catalog = delta_catalog hirs_avhrr.delta_catalog = delta_catalog # Instantiate the hirs, hirs_avhrr and hirs_csrb_monthly computations hirs2nc_comp = hirs2nc.HIRS2NC() hirs_avhrr_comp = hirs_avhrr.HIRS_AVHRR() hirs_csrb_monthly_comp = hirs_csrb_monthly.HIRS_CSRB_MONTHLY() SPC = StoredProductCatalog() # HIRS L1B Input hirs2nc_context = context.copy() [ hirs2nc_context.pop(k) for k in [ 'hirs_avhrr_delivery_id', 'hirs_csrb_daily_delivery_id', 'hirs_csrb_monthly_delivery_id', 'hirs_ctp_orbital_delivery_id' ] ] hirs2nc_prod = hirs2nc_comp.dataset('out').product(hirs2nc_context) if SPC.exists(hirs2nc_prod): task.input('HIR1B', hirs2nc_prod) else: raise WorkflowNotReady('No HIRS inputs available for {}'.format( hirs2nc_context['granule'])) # PTMSX Input LOG.debug('Getting PTMSX input...') sensor = 'avhrr' satellite = context['satellite'] file_type = 'PTMSX' granule = context['granule'] try: ptmsx_file = delta_catalog.file(sensor, satellite, file_type, granule) task.input('PTMSX', ptmsx_file) except WorkflowNotReady: raise WorkflowNotReady( 'No PTMSX inputs available for {}'.format(granule)) # Collo Input hirs_avhrr_context = hirs2nc_context hirs_avhrr_context['hirs_avhrr_delivery_id'] = context[ 'hirs_avhrr_delivery_id'] hirs_avhrr_prod = hirs_avhrr_comp.dataset('out').product( hirs_avhrr_context) if SPC.exists(hirs_avhrr_prod): task.input('COLLO', hirs_avhrr_prod) else: raise WorkflowNotReady( 'No HIRS_AVHRR inputs available for {}'.format( hirs_avhrr_context['granule'])) # CSRB Monthly Input hirs_csrb_monthly_context = context.copy() [ hirs_csrb_monthly_context.pop(k) for k in ['hirs_ctp_orbital_delivery_id'] ] hirs_csrb_monthly_context['granule'] = datetime( context['granule'].year, context['granule'].month, 1) hirs_csrb_monthly_prod = hirs_csrb_monthly_comp.dataset( 'zonal_means').product(hirs_csrb_monthly_context) if SPC.exists(hirs_csrb_monthly_prod): task.input('CSRB', hirs_csrb_monthly_prod) else: raise WorkflowNotReady( 'No HIRS_CSRB_MONTHLY inputs available for {}'.format( hirs_csrb_monthly_context['granule'])) # CFSR Input LOG.debug('Getting CFSR input...') cfsr_file = self.get_cfsr(context['granule']) if cfsr_file is not None: task.input('CFSR', cfsr_file) else: raise WorkflowNotReady( 'No CFSR inputs available for {}'.format(granule)) LOG.debug("Final task.inputs...") for task_key in task.inputs.keys(): LOG.debug("\t{}: {}".format(task_key, task.inputs[task_key]))
# Satellite specific information #satellite = 'noaa-19' #granule = datetime(2015, 4, 17, 0, 20) #intervals = [TimeInterval(granule, granule + day - wedge)] # NSS.GHRR.NP.D09108.S2301.E0050.B0100809.GC.level2.hdf --> NSS.GHRR.NP.D17365.S2238.E2359.B4585757.GC.level2.hdf # datetime(2009, 4, 18, 0, 0) --> datetime(2017, 12, 31, 0, 0) #intervals = [TimeInterval(datetime(years,1,1), datetime(years+1,1,1)-wedge) for years in range(2009, 2018) ] satellite = 'metop-b' #intervals = [TimeInterval(datetime(2015,1,1), datetime(2015,2,1)-wedge)] # NSS.GHRR.M1.D13140.S0029.E0127.B0347172.SV.level2.hdf --> NSS.GHRR.M1.D15365.S2307.E0004.B1705253.SV.level2.hdf # datetime(2013, 5, 20, 0, 0) --> datetime(2017, 12, 31, 0, 0) intervals = [ TimeInterval(datetime(years, 1, 1), datetime(years + 1, 1, 1) - wedge) for years in range(2013, 2018) ] satellite_choices = [ 'noaa-06', 'noaa-07', 'noaa-08', 'noaa-09', 'noaa-10', 'noaa-11', 'noaa-12', 'noaa-14', 'noaa-15', 'noaa-16', 'noaa-17', 'noaa-18', 'noaa-19', 'metop-a', 'metop-b' ] # Instantiate the computation hirs_csrb_daily_comp = hirs_csrb_daily.HIRS_CSRB_DAILY() comp = hirs_csrb_monthly.HIRS_CSRB_MONTHLY() LOG.info("Submitting intervals...")
#intervals = [] #years = 2009 #intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(4,13) ] #for years in range(2010, 2018): #intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,13) ] #years = 2018 #intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,2) ] satellite = 'metop-b' #granule = datetime(2015,5,14, 9, 23) #intervals = [TimeInterval(granule, granule + hour - wedge)] # NSS.GHRR.M1.D13140.S0029.E0127.B0347172.SV.level2.hdf --> NSS.GHRR.M1.D15365.S2307.E0004.B1705253.SV.level2.hdf # datetime(2013, 5, 20, 0, 0) --> datetime(2017, 12, 31, 0, 0) intervals = [] years = 2013 intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(5,13) ] for years in range(2014,2018): intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,13) ] satellite_choices = ['noaa-06', 'noaa-07', 'noaa-08', 'noaa-09', 'noaa-10', 'noaa-11', 'noaa-12', 'noaa-14', 'noaa-15', 'noaa-16', 'noaa-17', 'noaa-18', 'noaa-19', 'metop-a', 'metop-b'] def setup_computation(satellite): input_data = {'HIR1B': '/mnt/software/flo/hirs_l1b_datalists/{0:}/HIR1B_{0:}_latest'.format(satellite), 'CFSR': '/mnt/cephfs_data/geoffc/hirs_data_lists/CFSR.out', 'PTMSX': '/mnt/software/flo/hirs_l1b_datalists/{0:}/PTMSX_{0:}_latest'.format(satellite)} # Data locations collection = {'HIR1B': 'ILIAD',