def create_ctp_monthly(self, inputs, context): ''' Create the CTP monthly mean current month. ''' rc = 0 # Create the output directory current_dir = os.getcwd() # Get the required CTP script locations hirs_ctp_monthly_delivery_id = context['hirs_ctp_monthly_delivery_id'] delivery = delivered_software.lookup( 'hirs_ctp_monthly', delivery_id=hirs_ctp_monthly_delivery_id) dist_root = pjoin(delivery.path, 'dist') version = delivery.version # Determine the output filenames output_file = 'hirs_ctp_monthly_{}_{}.nc'.format( context['satellite'], context['granule'].strftime('D%Y%m')) LOG.info("output_file: {}".format(output_file)) # Generating CTP Daily Input List ctp_daily_file = 'ctp_daily_list' with open(ctp_daily_file, 'w') as f: [ f.write('{}\n'.format(basename(input))) for input in inputs.values() ] # Run the CTP monthly binary ctp_monthly_bin = pjoin(dist_root, 'bin/create_monthly_daynight_ctps.exe') cmd = '{} {} {}'.format(ctp_monthly_bin, ctp_daily_file, output_file) #cmd = 'sleep 0.5; touch {}'.format(output_file) try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ', ' \\\n\t'))) rc_ctp = 0 runscript(cmd, [delivery]) except CalledProcessError as err: rc_ctp = err.returncode LOG.error(" CTP monthly binary {} returned a value of {}".format( ctp_monthly_bin, rc_ctp)) return rc_ctp, None # Verify output file output_file = glob(output_file) if len(output_file) != 0: output_file = output_file[0] LOG.debug('Found output CTP monthly file "{}"'.format(output_file)) else: LOG.error('Failed to generate "{}", aborting'.format(output_file)) rc = 1 return rc, None return rc, output_file
def create_cfsr_means(self, inputs, context, stats_file): ''' Create the CFSR means for the current day. ''' rc = 0 # Create the output directory current_dir = os.getcwd() # Get the required CFSR and wgrib2 script locations hirs_csrb_daily_delivery_id = context['hirs_csrb_daily_delivery_id'] delivery = delivered_software.lookup( 'hirs_csrb_daily', delivery_id=hirs_csrb_daily_delivery_id) dist_root = pjoin(delivery.path, 'dist') lut_dir = pjoin(dist_root, 'luts') version = delivery.version # Determine the output filenames output_means = basename(stats_file).replace('stats', 'means') LOG.info("output_means: {}".format(output_means)) csrb_daily_means_bin = pjoin( dist_root, 'bin/create_daily_global_csrbs_netcdf.exe') shifted_FM_opt = 2 cmd = '{} {} {} {}'.format(csrb_daily_means_bin, basename(stats_file), output_means, shifted_FM_opt) #cmd = 'sleep 1; touch {}'.format(output_means) LOG.info('\n' + cmd + '\n') try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ', ' \\\n\t'))) rc_cfsr_means = 0 runscript(cmd, [delivery]) except CalledProcessError as err: rc_cfsr_means = err.returncode LOG.error("csrb_cfsr binary {} returned a value of {}".format( csrb_daily_means_bin, rc_cfsr_means)) return rc_cfsr_means, None # Verify output file output_means = glob(output_means) if len(output_means) != 0: output_means = output_means[0] LOG.info('Found output CFSR means file "{}"'.format(output_means)) else: LOG.error( 'Failed to genereate "{}", aborting'.format(output_means)) rc = 1 return rc, None return rc, output_means
def extract_bin_from_cfsr(self, inputs, context): ''' Run wgrib2 on the input CFSR grib files, to create flat binary files containing the desired data. ''' # Where are we running the package work_dir = abspath(curdir) LOG.debug("working dir = {}".format(work_dir)) # Get the required CFSR and wgrib2 script locations hirs_ctp_orbital_delivery_id = context['hirs_ctp_orbital_delivery_id'] delivery = delivered_software.lookup( 'hirs_ctp_orbital', delivery_id=hirs_ctp_orbital_delivery_id) dist_root = pjoin(delivery.path, 'dist') extract_cfsr_bin = pjoin(dist_root, 'bin/extract_cfsr.csh') version = delivery.version # Get the CFSR input cfsr_file = inputs['CFSR'] LOG.debug("CFSR file: {}".format(cfsr_file)) # Extract the desired datasets for each CFSR file rc = 0 new_cfsr_files = [] output_cfsr_file = '{}.bin'.format(basename(cfsr_file)) cmd = '{} {} {} {}'.format(extract_cfsr_bin, cfsr_file, output_cfsr_file, dirname(extract_cfsr_bin)) #cmd = 'sleep 0; touch {}'.format(output_cfsr_file) # DEBUG try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ', ' \\\n\t'))) rc_extract_cfsr = 0 runscript(cmd, [delivery]) except CalledProcessError as err: rc_extract_cfsr = err.returncode LOG.error("extract_cfsr binary {} returned a value of {}".format( extract_cfsr_bin, rc_extract_cfsr)) return rc_extract_cfsr, [] # Verify output file output_cfsr_file = glob(output_cfsr_file) if len(output_cfsr_file) != 0: output_cfsr_file = output_cfsr_file[0] LOG.debug('Found flat CFSR file "{}"'.format(output_cfsr_file)) else: LOG.error( 'Failed to generate "{}", aborting'.format(output_cfsr_file)) rc = 1 return rc, None return rc, output_cfsr_file
def create_monthly_zonal_means(self, stats_file, context): ''' Create the CFSR statistics for the current month. ''' rc = 0 # Create the output directory current_dir = os.getcwd() # Get the required CFSR and wgrib2 script locations hirs_csrb_monthly_delivery_id = context['hirs_csrb_monthly_delivery_id'] delivery = delivered_software.lookup('hirs_csrb_monthly', delivery_id=hirs_csrb_monthly_delivery_id) dist_root = pjoin(delivery.path, 'dist') version = delivery.version # Determine the output filenames output_zonal_means = basename(stats_file).replace('stats', 'zmeans') LOG.debug("output_zonal_means: {}".format(output_zonal_means)) csrb_zonal_means_bin = pjoin(dist_root, 'bin/create_monthly_zonal_csrbs_netcdf.exe') cmd = '{0:} {1:} {2:} > {2:}.log'.format( csrb_zonal_means_bin, stats_file, output_zonal_means ) #cmd = 'sleep 1; touch {}'.format(output_zonal_means) LOG.debug('\n'+cmd+'\n') try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ',' \\\n\t'))) rc_csrb_cfsr = 0 runscript(cmd, [delivery]) except CalledProcessError as err: rc_csrb_cfsr = err.returncode LOG.error("CSRB monthly binary {} returned a value of {}".format(csrb_zonal_means_bin, rc_csrb_cfsr)) return rc_csrb_cfsr, None # Verify output file output_zonal_means = glob(output_zonal_means) if len(output_zonal_means) != 0: output_zonal_means = output_zonal_means[0] LOG.info('Found output CFSR monthly zonal means file "{}"'.format(output_zonal_means)) else: LOG.error('Failed to generate "{}", aborting'.format(output_zonal_means)) rc = 1 return rc, None return rc, output_zonal_means
def create_cfsr_statistics(self, inputs, context, cfsr_bin_files): ''' Create the CFSR statistics for the current day. ''' rc = 0 # Create the output directory current_dir = os.getcwd() # Get the required CFSR and wgrib2 script locations hirs_csrb_daily_delivery_id = context['hirs_csrb_daily_delivery_id'] delivery = delivered_software.lookup( 'hirs_csrb_daily', delivery_id=hirs_csrb_daily_delivery_id) dist_root = pjoin(delivery.path, 'dist') lut_dir = pjoin(dist_root, 'luts') version = delivery.version # Compile a dictionary of the input orbital data files rc, intervals = self.get_orbital_intervals(inputs, context) # Determine the output filenames output_stats = 'csrb_daily_stats_{}_{}.nc'.format( context['satellite'], context['granule'].strftime('D%y%j')) LOG.info("output_stats: {}".format(output_stats)) # Link the coefficient files into the working directory shifted_coeffs = [ abspath(x) for x in glob(pjoin(lut_dir, 'shifted_hirs_FM_coeff/*')) ] unshifted_coeffs = [ abspath(x) for x in glob(pjoin(lut_dir, 'unshifted_hirs_FM_coeff/*')) ] linked_coeffs = link_files( current_dir, shifted_coeffs + unshifted_coeffs + [ abspath(pjoin(lut_dir, 'CFSR_lst.bin')), abspath(pjoin(lut_dir, 'CO2_1979-2017_monthly_181_lat.dat')) ]) LOG.debug("Linked coeffs: {}".format(linked_coeffs)) csrb_daily_stats_bin = pjoin(dist_root, 'bin/process_csrb_cfsr.exe') debug = 0 shifted_FM_opt = 2 # Loop through the intervals indicies = xrange(len(intervals)) for idx in indicies: interval = intervals['interval-{}'.format(idx)] LOG.debug("HIRS interval {}: {} -> {}".format( idx, interval.left, interval.right)) cfsr_bin = self.cfsr_input(cfsr_bin_files, interval) LOG.debug("cfsr_bin ({}:{}): {}".format(idx, interval, cfsr_bin)) if cfsr_bin == None: LOG.warn( "Could not find cfsr_bin file to match HIRS interval: {} -> {}" .format(interval.left, interval.right)) continue cmd = '{} {} {} {} {} {} {} {} {} {}'.format( csrb_daily_stats_bin, inputs['HIR1B-{}'.format(idx)], cfsr_bin, inputs['COLLO-{}'.format(idx)], inputs['PTMSX-{}'.format(idx)], 'CFSR_lst.bin', 'CO2_1979-2017_monthly_181_lat.dat', debug, shifted_FM_opt, output_stats) #cmd = 'sleep 1; touch {}'.format(output_stats) # DEBUG try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ', ' \\\n\t'))) rc_csrb_cfsr = 0 runscript(cmd, [delivery]) except CalledProcessError as err: rc_csrb_cfsr = err.returncode LOG.error("csrb_cfsr binary {} returned a value of {}".format( csrb_daily_stats_bin, rc_csrb_cfsr)) return rc_csrb_cfsr, None # Verify output file output_stats_file = glob(output_stats) if len(output_stats_file) != 0: output_stats_file = output_stats_file[0] LOG.info('Found output CFSR statistics file "{}"'.format( output_stats_file)) else: LOG.error('Failed to generate "{}", aborting'.format(output_stats)) rc = 1 return rc, None return rc, output_stats
def create_tpw_daily(self, inputs, context, shifted=False): ''' Create the TPW statistics for the current day. ''' rc = 0 # Create the output directory current_dir = os.getcwd() # Get the required TPW script locations hirs_tpw_daily_delivery_id = context['hirs_tpw_daily_delivery_id'] delivery = delivered_software.lookup( 'hirstpw_daily', delivery_id=hirs_tpw_daily_delivery_id) dist_root = pjoin(delivery.path, 'dist') version = delivery.version shifted_str = '_shift' if shifted else '_noshift' # Determine the output filenames output_file = 'hirs_tpw_daily_{}{}_{}.nc'.format( context['satellite'], shifted_str, context['granule'].strftime('D%y%j')) LOG.info("output_file: {}".format(output_file)) # Generating TPW Orbital Input List tpw_orbital_file_list = 'tpw_orbital{}_list.txt'.format(shifted_str) with open(tpw_orbital_file_list, 'w') as f: [ f.write('{}\n'.format(basename(inputs[key]))) for key in sorted(inputs.keys()) if shifted_str in key ] # Run the TPW daily binary tpw_daily_bin = pjoin(dist_root, 'create_daily_daynight_tpw.exe') cmd = '{} {} {} {} {} {} &> {}'.format( tpw_daily_bin, tpw_orbital_file_list, output_file, context['granule'].strftime('%Y'), context['granule'].strftime('%m'), context['granule'].strftime('%j'), '{}.log'.format(splitext(output_file)[0])) #cmd = 'sleep 0.5; touch {}'.format(output_file) try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ', ' \\\n\t'))) rc_tpw = 0 runscript(cmd, [delivery]) except CalledProcessError as err: rc_tpw = err.returncode LOG.error(" TPW daily binary {} returned a value of {}".format( tpw_daily_bin, rc_tpw)) return rc_tpw, None # Verify output file output_file = glob(output_file) if len(output_file) != 0: output_file = output_file[0] LOG.debug('Found output TPW daily file "{}"'.format(output_file)) else: LOG.error('Failed to generate "{}", aborting'.format(output_file)) rc = 1 return rc, None return rc, output_file
def run_task(self, inputs, context): LOG.debug("Running run_task()...") for key in context.keys(): LOG.debug("run_task() context['{}'] = {}".format( key, context[key])) granule = context['granule'] satellite = context['satellite'] hirs2nc_delivery_id = context['hirs2nc_delivery_id'] hirs_version = self.satellite_version(context['satellite'], context['granule']) # Get the location of the binary package delivery = delivered_software.lookup('hirs2nc', delivery_id=hirs2nc_delivery_id) dist_root = pjoin(delivery.path, 'dist') envroot = pjoin(dist_root, 'env') # Get the required environment variables env = prepare_env([delivery]) LOG.debug(env) # What is the path of the python interpreter py_interp = "{}/bin/python".format(envroot) LOG.debug("py_interp = '{}'".format(py_interp)) # Path of the hirs2nc binary hirs2nc_bin = pjoin(envroot, 'bin', 'hirs2nc') # Where are we running the package work_dir = abspath(curdir) LOG.debug("working dir = {}".format(work_dir)) input_file = inputs['HIR1B'] output_file = pjoin(work_dir, basename('{}.nc'.format(inputs['HIR1B']))) LOG.debug("Input file = {}".format(input_file)) LOG.debug("Output file = {}".format(output_file)) # What are our inputs? for input in inputs.keys(): inputs_dir = dirname(inputs[input]) LOG.debug("inputs['{}'] = {}".format(input, inputs[input])) LOG.debug("Inputs dir = {}".format(inputs_dir)) # Convert the flat HIRS file to NetCDF4 cmd = '{} {} {} {} {}'.format(py_interp, hirs2nc_bin, hirs_version, input_file, output_file) try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ', ' \\\n\t'))) rc_hirs2nc = 0 runscript(cmd, requirements=[], env=env) except CalledProcessError as err: rc_hirs2nc = err.returncode LOG.error("hirs2nc binary {} returned a value of {}".format( hirs2nc_bin, rc_hirs2nc)) return rc_hirs2nc, [] # The staging routine assumes that the output file is located in the work directory # "tmp******", and that the output path is to be prepended, so return the basename. output = basename('{}.nc'.format(inputs['HIR1B'])) data_interval = context['data_interval'] extra_attrs = { 'begin_time': data_interval.left, 'end_time': data_interval.right } return { 'out': { 'file': nc_compress(output), 'extra_attrs': extra_attrs } }
def create_ctp_orbital(self, inputs, context): ''' Create the the CTP Orbital for the current granule. ''' rc = 0 # Create the output directory current_dir = os.getcwd() # Get the required CFSR and wgrib2 script locations hirs_ctp_orbital_delivery_id = context['hirs_ctp_orbital_delivery_id'] delivery = delivered_software.lookup( 'hirs_ctp_orbital', delivery_id=hirs_ctp_orbital_delivery_id) dist_root = pjoin(delivery.path, 'dist') lut_dir = pjoin(dist_root, 'luts') version = delivery.version # Compile a dictionary of the input orbital data files interval = self.hirs_to_time_interval(inputs['HIR1B']) LOG.debug("HIRS interval {} -> {}".format(interval.left, interval.right)) # Determine the output filenames output_file = 'hirs_ctp_orbital_{}_{}{}.nc'.format( context['satellite'], interval.left.strftime('D%y%j.S%H%M'), interval.right.strftime('.E%H%M')) LOG.info("output_file: {}".format(output_file)) # Link the coefficient files into the working directory shifted_coeffs = [ abspath(x) for x in glob(pjoin(lut_dir, 'shifted_hirs_FM_coeff/*')) ] unshifted_coeffs = [ abspath(x) for x in glob(pjoin(lut_dir, 'unshifted_hirs_FM_coeff/*')) ] linked_coeffs = link_files( current_dir, shifted_coeffs + unshifted_coeffs + [ abspath(pjoin(lut_dir, 'CFSR_lst.bin')), abspath(pjoin(lut_dir, 'CO2_1979-2017_monthly_181_lat.dat')) ]) LOG.debug("Linked coeffs: {}".format(linked_coeffs)) ctp_orbital_bin = pjoin(dist_root, 'bin/process_hirs_cfsr.exe') debug = 0 shifted_FM_opt = 2 cmd = '{} {} {} {} {} {} {} {} {} {} {}'.format( ctp_orbital_bin, inputs['HIR1B'], inputs['CFSR'], inputs['COLLO'], inputs['PTMSX'], inputs['CSRB'], 'CFSR_lst.bin', 'CO2_1979-2017_monthly_181_lat.dat', debug, shifted_FM_opt, output_file) #cmd = 'sleep 1; touch {}'.format(output_file) # DEBUG try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ', ' \\\n\t'))) rc_ctp = 0 runscript(cmd, [delivery]) except CalledProcessError as err: rc_ctp = err.returncode LOG.error(" CTP orbital binary {} returned a value of {}".format( ctp_orbital_bin, rc_ctp)) return rc_ctp, None # Verify output file output_file = glob(output_file) if len(output_file) != 0: output_file = output_file[0] LOG.debug( 'Found output CTP orbital file "{}"'.format(output_file)) else: LOG.error('Failed to generate "{}", aborting'.format(output_file)) rc = 1 return rc, None return rc, output_file
def create_tpw_orbital(self, inputs, context, shifted=False): ''' Create the the TPW Orbital for the current granule. ''' rc = 0 # Create the output directory current_dir = os.getcwd() # Get the required CFSR and wgrib2 script locations hirs_tpw_orbital_delivery_id = context['hirs_tpw_orbital_delivery_id'] delivery = delivered_software.lookup( 'hirstpw_L2', delivery_id=hirs_tpw_orbital_delivery_id) dist_root = pjoin(delivery.path, 'dist') lut_dir = dist_root version = delivery.version # Compile a dictionary of the input orbital data files interval = self.hirs_to_time_interval(inputs['HIR1B']) LOG.debug("HIRS interval {} -> {}".format(interval.left, interval.right)) # Determine the output filenames output_file = 'hirs_tpw_orbital_{}_{}_{}{}.nc'.format( context['satellite'], 'shift' if shifted else 'noshift', interval.left.strftime('D%y%j.S%H%M'), interval.right.strftime('.E%H%M')) LOG.info("output_file: {}".format(output_file)) # No shift for NOAA-8 if (not shifted) or (context['satellite'] == 'noaa-08'): shifted_FM_opt = 1 else: shifted_FM_opt = 2 tpw_orbital_bin = pjoin(dist_root, 'hirs_regrtvl_main_cdf.exe') cmd = '{} {} {} {} {} {} {} &> {}'.format( tpw_orbital_bin, inputs['HIR1B'], inputs['CTPO'], inputs['CFSR'], output_file, '{}_QC.nc'.format(splitext(output_file)[0]), shifted_FM_opt, '{}.log'.format(splitext(output_file)[0])) #cmd = 'sleep 1; touch {}'.format(output_file) # DEBUG try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ', ' \\\n\t'))) rc_tpw = 0 runscript(cmd, [delivery]) except CalledProcessError as err: rc_tpw = err.returncode LOG.error(" TPW orbital binary {} returned a value of {}".format( tpw_orbital_bin, rc_tpw)) return rc_tpw, None # Verify output file output_file = glob(output_file) if len(output_file) != 0: output_file = output_file[0] LOG.debug('Found output TPW orbital file "{}"'.format(output_file)) else: LOG.error('Failed to generate "{}", aborting'.format(output_file)) rc = 1 return rc, None return rc, output_file
def create_monthly_statistics(self, inputs, context): ''' Create the CFSR statistics for the current month. ''' rc = 0 # Create the output directory current_dir = os.getcwd() # Check that we actually have some inputs... num_inputs = len(inputs.keys()) if num_inputs == 0: LOG.warn('No HIRS_CSRB_DAILY inputs for {}, aborting...'.format(context['granule'])) rc = 1 return rc, None # Get the required binary locations hirs_csrb_monthly_delivery_id = context['hirs_csrb_monthly_delivery_id'] delivery = delivered_software.lookup('hirs_csrb_monthly', delivery_id=hirs_csrb_monthly_delivery_id) dist_root = pjoin(delivery.path, 'dist') version = delivery.version # Link the input files into the working directory _ = link_files(current_dir, inputs.values()) # Create a text file containing the input daily files. rc, inputs_filelist = self.create_inputs_filelist(inputs) if rc != 0: return rc, None # Determine the output filenames output_stats = 'csrb_monthly_stats_{}_{}.nc'.format(context['satellite'], context['granule'].strftime('d%Y%m')) LOG.debug("output_stats: {}".format(output_stats)) csrb_monthly_stats_bin = pjoin(dist_root, 'bin/create_monthly_global_csrbs_netcdf.exe') cmd = '{0:} {1:} {2:} > {2:}.log'.format( csrb_monthly_stats_bin, inputs_filelist, output_stats ) #cmd = 'sleep 1; touch {}'.format(output_stats) try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ',' \\\n\t'))) rc_csrb_cfsr = 0 runscript(cmd, [delivery]) except CalledProcessError as err: rc_csrb_cfsr = err.returncode LOG.error("CSRB monthly binary {} returned a value of {}".format(csrb_monthly_stats_bin, rc_csrb_cfsr)) return rc_csrb_cfsr, None # Verify output file output_stats = glob(output_stats) if len(output_stats) != 0: output_stats = output_stats[0] LOG.info('Found output CFSR monthly statistics file "{}"'.format(output_stats)) else: LOG.error('Failed to generate "{}", aborting'.format(output_stats)) rc = 1 return rc, None return rc, output_stats
def hirs_avhrr_collocation(self, inputs, context): ''' Run the collocation executable. ''' rc = 0 LOG.info('inputs = {}'.format(inputs)) satellite = context['satellite'] hirs_avhrr_delivery_id = context['hirs_avhrr_delivery_id'] hirs_file = inputs['HIR1B'] patmosx_file = inputs['PTMSX'] # Where are we running the package work_dir = abspath(curdir) LOG.debug("working dir = {}".format(work_dir)) # Get the required collocation exe delivery = delivered_software.lookup( 'hirs_avhrr', delivery_id=hirs_avhrr_delivery_id) dist_root = pjoin(delivery.path, 'dist') hirs_avhrr_bin = pjoin(dist_root, 'bin/hirs_avhrr_v4.exe') version = delivery.version # Get the required environment variables env = prepare_env([delivery]) LOG.debug(env) # Removing the output file if it exists granule_datestamp = '.'.join(basename(hirs_file).split('.')[3:8]) output_file = 'colloc.hirs.avhrr.{}.{}.v{}.hdf'.format( satellite, granule_datestamp, version) if os.path.exists(output_file): LOG.info('{} exists, removing...'.format(output_file)) os.remove(output_file) cmd = '{} {} {} {}'.format(hirs_avhrr_bin, hirs_file, patmosx_file, output_file) #cmd = 'sleep 1; touch {}'.format(output_file) try: LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ', ' \\\n\t'))) rc_hirs_avhrr = 0 runscript(cmd, requirements=[], env=env) except CalledProcessError as err: rc_hirs_avhrr = err.returncode LOG.error("hirs_avhrr binary {} returned a value of {}".format( hirs_avhrr_bin, rc_hirs_avhrr)) return rc_hirs_avhrr, [] # Verify output file output_file = glob(output_file) if len(output_file) != 0: output_file = output_file[0] LOG.info('Found collocation file "{}"'.format(output_file)) else: LOG.error( 'There are no output collocation file "{}", aborting'.format( output_file)) rc = 1 return rc, [] LOG.info('Finished collocation routine') return rc, output_file