def create_process_output_script(job_id): cfg = load_sys_cfg() script_path = osp.join(cfg.workspace_path, job_id, 'job_process_output.sh') log_path = osp.join(cfg.workspace_path, job_id, 'job_process_output.log') process_script = osp.join(cfg.sys_install_path, 'process_output.sh') with open(script_path, 'w') as f: f.write('#!/usr/bin/env bash\n') f.write('cd ' + cfg.sys_install_path + '\n') f.write('LOG=' + log_path + '\n') f.write(process_script + ' ' + job_id + ' &> $LOG \n') # make it executable st = os.stat(script_path) os.chmod(script_path, st.st_mode | stat.S_IEXEC)
def execute(self): """ Execute real.exe in given working directory. This method is redefined here as real.exe needs special treatment. DMPAR compilation of WRF causes real.exe to output stdout and stderr into rsl.out.0000 and rsl.error.0000 respectively. We don't redirect these (we can't) but we rename the output files after the fact. NOTE: on some machines it is OK to run real.exe from command line, but generally mpirun is required! If cfg.wrf_serial_install_path is not None, run real.exe as serial, This is to support systems that do not allow executing mpi binary from command line. We do not run real.exe by mpirun because mpirun on head node may not be allowed. :return: raises OutputCheckFailed if return code is non-zero """ exec_name = self.exec_name stdout_path = self.stdout_path stderr_path = self.stderr_path # first verify if we have already done our job try: self.check_output() return self except OutputCheckFailed: pass cfg = load_sys_cfg() if "wrf_serial_install_path" in cfg: logging.info("Executing serial %s" % exec_name) stdout_file = open(stdout_path, 'w') stderr_file = open(stderr_path, 'w') check_call(exec_name, cwd=self.work_dir, stdout=stdout_file, stderr=stderr_file) else: logging.info("Executing MPI %s directly without mpirun" % exec_name) check_call(exec_name, cwd=self.work_dir) os.rename(osp.join(self.work_dir, "rsl.out.0000"), stdout_path) os.rename(osp.join(self.work_dir, "rsl.error.0000"), stderr_path) return self
def send_products_to_server(job_id): args = load_sys_cfg() jobfile = osp.abspath(osp.join(args.workspace_path, job_id, 'job.json')) logging.info('sent_products_to_server: loading job description from %s' % jobfile) try: js = Dict(json.load(open(jobfile, 'r'))) except Exception as e: logging.error('Cannot load the job description file %s' % jobfile) logging.error('%s' % e) sys.exit(1) desc = js.postproc[ 'description'] if 'description' in js.postproc else js.job_id pp_dir = js.get( 'pp_dir', osp.abspath(osp.join(args.workspace_path, job_id, "products"))) manifest_filename = js.get('manifest_filename', 'wfc-' + js.grid_code + '.json') send_product_to_server(args, pp_dir, job_id, job_id, manifest_filename, desc)
def test_time(): cfg = load_sys_cfg() g = NAM218(cfg) cycle_start_esmf = "2005-01-05_00:00:00" cycle_start = esmf_to_utc(cycle_start_esmf) from_utc = esmf_to_utc("2005-01-05_00:00:00") to_utc = esmf_to_utc("2005-01-06_22:00:0") print(cycle_start.year, cycle_start.month, cycle_start.day, cycle_start.hour) fc_hours = int((to_utc - cycle_start).total_seconds()) / 3600 delta = from_utc - cycle_start print(str(delta)) print(delta.days, delta.seconds, delta.total_seconds()) print((to_utc - cycle_start).total_seconds()) print(timedelta_hours(to_utc - cycle_start)) print(timedelta_hours(to_utc - cycle_start, False)) fc_start, fc_hours = g.forecast_times(cycle_start, from_utc, to_utc) print('fc_start = ', fc_start) print('fc_hours = ', fc_hours) fc_list, colmet_list_utc = g.file_times(cycle_start, fc_start, fc_hours) grib_files, colmet_prefix, colmet_files = g.file_names( cycle_start, fc_list, colmet_list_utc) print('fc_list = ', fc_list) print('colmet_list_utc = ') for x in colmet_list_utc: print(x) print('grib_files = ') for x in grib_files: print(x) print('colmet_files = ') for x in colmet_files: print(colmet_prefix + '/' + x)
from utils import esmf_to_utc, load_sys_cfg import requests from datetime import datetime, timedelta import pytz import logging import os.path as osp from utils import readhead import time from .grib_file import grib_messages from six.moves import range # global parameter min_content_size = 10000 cfg = load_sys_cfg() sleep_seconds = cfg.get('sleep_seconds', 20) max_retries = cfg.get('max_retries', 3) class RTMA(object): """ This class supports the ingest of NOAA RTMA (Real-time Mesoscale Analysis) data. RTMA is different since there is only one set of files for 24 hours (1 fileset per hour), which are continuously overwritten. This RTMA class thus checks the filetime for each file to verify it belongs to 'today' instead of the day before. Note that RTMA operates in the UTC time zone. """
def ssh_command(command): cfg = load_sys_cfg() s = SSHShuttle(cfg) s.connect() s.simple_command(command) s.disconnect()
def process_output(job_id): args = load_sys_cfg() jobfile = osp.abspath(osp.join(args.workspace_path, job_id, 'job.json')) logging.info('process_output: loading job description from %s' % jobfile) try: js = Dict(json.load(open(jobfile, 'r'))) except Exception as e: logging.error('Cannot load the job description file %s' % jobfile) logging.error('%s' % e) sys.exit(1) js.old_pid = js.pid js.pid = os.getpid() js.state = 'Processing' json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': ')) js.wrf_dir = osp.abspath(osp.join(args.workspace_path, js.job_id, 'wrf')) pp = None already_sent_files, max_pp_dom = [], -1 if js.postproc is None: logging.info('No postprocessing specified, exiting.') return # set up postprocessing delete_visualization(js.job_id) js.pp_dir = osp.join(args.workspace_path, js.job_id, "products") make_clean_dir(js.pp_dir) pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code) js.manifest_filename = 'wfc-' + js.grid_code + '.json' logging.debug('Postprocessor created manifest %s', js.manifest_filename) max_pp_dom = max( [int(x) for x in filter(lambda x: len(x) == 1, js.postproc)]) if js.postproc.get('from', None) == 'wrfout': logging.info('Postprocessing all wrfout files.') # postprocess all wrfouts for wrfout_path in sorted( glob.glob( osp.join(js.wrf_dir, 'wrfout_d??_????-??-??_??:??:??'))): logging.info("Found %s" % wrfout_path) domain_str, wrfout_esmf_time = re.match( r'.*wrfout_d(0[0-9])_([0-9_\-:]{19})', wrfout_path).groups() dom_id = int(domain_str) d = nc4.Dataset(wrfout_path) # extract ESMF string times times = [''.join(x) for x in d.variables['Times'][:]] d.close() for esmf_time in sorted(times): logging.info("Processing domain %d for time %s." % (dom_id, esmf_time)) if js.postproc is not None and str(dom_id) in js.postproc: var_list = [str(x) for x in js.postproc[str(dom_id)]] logging.info( "Executing postproc instructions for vars %s for domain %d." % (str(var_list), dom_id)) try: pp.process_vars(osp.join(js.wrf_dir, wrfout_path), dom_id, esmf_time, var_list) # in incremental mode, upload to server if js.postproc.get('shuttle', None) == 'incremental': desc = js.postproc[ 'description'] if 'description' in js.postproc else js.job_id sent_files_1 = send_product_to_server( args, js.pp_dir, js.job_id, js.job_id, js.manifest_filename, desc, already_sent_files) already_sent_files = filter( lambda x: not x.endswith('json'), already_sent_files + sent_files_1) except Exception as e: logging.warning( 'Failed to postprocess for time %s with error %s.' % (esmf_time, str(e))) # if we are to send out the postprocessed files after completion, this is the time if js.postproc.get('shuttle', None) == 'on_completion': desc = js.postproc[ 'description'] if 'description' in js.postproc else js.job_id send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, js.manifest_filename, desc) json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': ')) return # step 9: wait for appearance of rsl.error.0000 and open it wrf_out = None rsl_path = osp.join(js.wrf_dir, 'rsl.error.0000') while wrf_out is None: try: wrf_out = open(rsl_path) break except IOError: logging.info( 'process_output: waiting 5 seconds for rsl.error.0000 file') time.sleep(5) logging.info('process_output: Detected rsl.error.0000') js.run_utc = time.ctime(os.path.getmtime(rsl_path)) js.processed_utc = time.asctime(time.gmtime()) # step 10: track log output and check for history writes fro WRF wait_lines = 0 wait_wrfout = 0 while True: line = wrf_out.readline().strip() if not line: if not parallel_job_running(js): logging.warning('WRF did not run to completion.') break if not wait_lines: logging.info('Waiting for more output lines') wait_lines = wait_lines + 1 time.sleep(5) continue wait_lines = 0 if "SUCCESS COMPLETE WRF" in line: # send_email(js, 'complete', 'Job %s - wrf job complete SUCCESS.' % js.job_id) logging.info("WRF completion detected.") js.old_job_num = js.job_num js.job_num = None json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': ')) break if "Timing for Writing wrfout" in line: wait_wrfout = 0 esmf_time, domain_str = re.match( r'.*wrfout_d.._([0-9_\-:]{19}) for domain\ +(\d+):', line).groups() wrfout_path, domain_str = re.match( r'.*(wrfout_d.._[0-9_\-:]{19}) for domain\ +(\d+):', line).groups() dom_id = int(domain_str) logging.info("Detected history write for domain %d for time %s." % (dom_id, esmf_time)) if js.postproc is not None and str(dom_id) in js.postproc: var_list = [str(x) for x in js.postproc[str(dom_id)]] logging.info( "Executing postproc instructions for vars %s for domain %d." % (str(var_list), dom_id)) wrfout_path = find_wrfout(js.wrf_dir, dom_id, esmf_time) try: pp.process_vars(osp.join(js.wrf_dir, wrfout_path), dom_id, esmf_time, var_list) except Exception as e: logging.warning( 'Failed to postprocess for time %s with error %s.' % (esmf_time, str(e))) else: # in incremental mode, upload to server if js.postproc.get('shuttle', None) == 'incremental': desc = js.postproc[ 'description'] if 'description' in js.postproc else js.job_id sent_files_1 = send_product_to_server( args, js.pp_dir, js.job_id, js.job_id, js.manifest_filename, desc, already_sent_files) already_sent_files = filter( lambda x: not x.endswith('json'), already_sent_files + sent_files_1) else: if not wait_wrfout: logging.info('Waiting for wrfout') wait_wrfout = wait_wrfout + 1 # if we are to send out the postprocessed files after completion, this is the time if js.postproc.get('shuttle', None) == 'on_completion': desc = js.postproc[ 'description'] if 'description' in js.postproc else js.job_id send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, js.manifest_filename, desc) if js.postproc.get('shuttle', None) is not None: make_kmz(js.job_id) # arguments can be added to the job id string js.old_pid = js.pid js.pid = None js.state = 'Completed' json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': '))
def process_outputs_tiff(job_id): args = load_sys_cfg() jobfile = osp.abspath(osp.join(args.workspace_path, job_id,'job.json')) satfile = osp.abspath(osp.join(args.workspace_path, job_id,'sat.json')) logging.info('process_tiffs: loading job description from %s' % jobfile) try: js = Dict(json.load(open(jobfile,'r'))) except Exception as e: logging.error('Cannot load the job description file %s' % jobfile) logging.error('%s' % e) sys.exit(1) logging.info('process_tiffs: loading satellite description from %s' % satfile) try: jsat = Dict(json.load(open(satfile,'r'))) available_sats = [sat.upper()+'_AF' for sat in jsat.granules.keys()] not_empty_sats = [sat.upper()+'_AF' for sat in jsat.granules.keys() if jsat.granules[sat]] except: logging.warning('Cannot load the satellite data in satellite description file %s' % satfile) available_sats = [] not_empty_sats = [] pass logging.info('process_tiffs: available satellite data %s' % available_sats) logging.info('process_tiffs: not empty satellite data %s' % not_empty_sats) js.old_pid = js.pid js.pid = os.getpid() js.state = 'Processing' json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': ')) js.wrf_dir = osp.abspath(osp.join(args.workspace_path, js.job_id, 'wrf')) # set up postprocessing pp = None js.pp_dir = osp.join(args.workspace_path, js.job_id, "products", "tiffs") make_clean_dir(js.pp_dir) pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code) js.manifest_filename= 'wfc-' + js.grid_code + '.json' logging.debug('Postprocessor created manifest %s',js.manifest_filename) logging.info('Postprocessing all wrfout files.') # postprocess all wrfouts for wrfout_path in sorted(glob.glob(osp.join(js.wrf_dir,'wrfout_d??_????-??-??_??:??:??'))): logging.info("Found %s" % wrfout_path) domain_str,wrfout_esmf_time = re.match(r'.*wrfout_d(0[0-9])_([0-9_\-:]{19})',wrfout_path).groups() dom_id = int(domain_str) d = nc4.Dataset(wrfout_path) # extract ESMF string times times = [''.join(x) for x in d.variables['Times'][:].astype(str)] if js.postproc is not None and str(dom_id) in js.postproc: if available_sats: sat_list = [sat for sat in available_sats if sat in js.postproc[str(dom_id)]] var_list = [str(x) for x in js.postproc[str(dom_id)] if not str(x) in sat_list] sat_list = [sat for sat in sat_list if sat in not_empty_sats] logging.info("Executing postproc instructions for sats %s for domain %d." % (str(sat_list), dom_id)) else: sat_list = [] var_list = [str(x) for x in js.postproc[str(dom_id)]] logging.info("Executing postproc tiff instructions for vars %s for domain %d." % (str(var_list), dom_id)) try: if sat_list: pass #process_sats_tiff() process_vars_tiff(pp, d, wrfout_path, dom_id, times, var_list) except Exception as e: logging.warning('Failed to postprocess with error %s.' % str(e)) d.close() js.old_pid = js.pid js.pid = None js.state = 'Completed' json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
from simulation import cancel_simulation, delete_simulation, delete_simulation_files, load_simulations, cleanup_sim_output, cleanup_sim_workspace import json import sys import logging from utils import load_sys_cfg conf = load_sys_cfg() sims_path = conf['sims_path'] def cleanup_delete(sim_id): simulations = load_simulations(sims_path) try: logging.info('Deleting simulation %s' % sim_id) sim_info = simulations[sim_id] delete_simulation(sim_info, conf) except KeyError: logging.error('Simulation %s not found.' % sim_id) delete_simulation_files(sim_id, conf) # rm any stray files def cleanup_cancel(sim_id): simulations = load_simulations(sims_path) try: logging.info('Canceling simulation %s' % sim_id) sim_info = simulations[sim_id] cancel_simulation(sim_info, conf) except KeyError: logging.error('Simulation %s not found.' % sim_id)
from simulation import cancel_simulation, delete_simulation, delete_simulation_files, load_simulations, cleanup_sim_output, cleanup_sim_workspace import json import sys import logging from utils import load_sys_cfg conf = load_sys_cfg() sims_path = conf['sims_path'] def cleanup_delete(sim_id): simulations = load_simulations(sims_path) try: logging.info('Deleting simulation %s' % sim_id) sim_info = simulations[sim_id] delete_simulation(sim_info,conf) except KeyError: logging.error('Simulation %s not found.' % sim_id) delete_simulation_files(sim_id,conf) # rm any stray files def cleanup_cancel(sim_id): simulations = load_simulations(sims_path) try: logging.info('Canceling simulation %s' % sim_id) sim_info = simulations[sim_id] cancel_simulation(sim_info,conf) except KeyError: logging.error('Simulation %s not found.' % sim_id) def cleanup_output(sim_id): simulations = load_simulations(sims_path) try:
import os.path as osp ## Standalone script that can be used to simply download files if __name__ == '__main__': if len(sys.argv) != 5: print( 'Usage: %s <grib_source_name> <esmf_from_utc> <esmf_to_utc> <target_directory>' % sys.argv[0]) print(' supported GRIB sources: HRRR, NAM, CFSR_P, CFSR_S, NARR') sys.exit(-1) # configure the basic logger logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') js = load_sys_cfg() grib_src_name = sys.argv[1] from_utc = esmf_to_utc(sys.argv[2]) to_utc = esmf_to_utc(sys.argv[3]) ingest_dir = sys.argv[4] js.ingest_dir = ingest_dir grib_src = None if grib_src_name == 'HRRR': grib_src = HRRR(js) elif grib_src_name == 'NAM': grib_src = NAM218(js) elif grib_src_name == 'NAM227': grib_src = NAM227(js) elif grib_src_name == 'CFSR_P': grib_src = CFSR_P(js)