def process_output(job_id):
args = load_sys_cfg()
jobfile = osp.abspath(osp.join(args.workspace_path, job_id, 'job.json'))
logging.info('process_output: loading job description from %s' % jobfile)
try:
js = Dict(json.load(open(jobfile, 'r')))
except Exception as e:
logging.error('Cannot load the job description file %s' % jobfile)
logging.error('%s' % e)
sys.exit(1)
js.old_pid = js.pid
js.pid = os.getpid()
js.state = 'Processing'
json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': '))
js.wrf_dir = osp.abspath(osp.join(args.workspace_path, js.job_id, 'wrf'))
pp = None
already_sent_files, max_pp_dom = [], -1
if js.postproc is None:
logging.info('No postprocessing specified, exiting.')
return
# set up postprocessing
delete_visualization(js.job_id)
js.pp_dir = osp.join(args.workspace_path, js.job_id, "products")
make_clean_dir(js.pp_dir)
pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code)
js.manifest_filename = 'wfc-' + js.grid_code + '.json'
logging.debug('Postprocessor created manifest %s', js.manifest_filename)
max_pp_dom = max(
[int(x) for x in filter(lambda x: len(x) == 1, js.postproc)])
if js.postproc.get('from', None) == 'wrfout':
logging.info('Postprocessing all wrfout files.')
# postprocess all wrfouts
for wrfout_path in sorted(
glob.glob(
osp.join(js.wrf_dir, 'wrfout_d??_????-??-??_??:??:??'))):
logging.info("Found %s" % wrfout_path)
domain_str, wrfout_esmf_time = re.match(
r'.*wrfout_d(0[0-9])_([0-9_\-:]{19})', wrfout_path).groups()
dom_id = int(domain_str)
d = nc4.Dataset(wrfout_path)
# extract ESMF string times
times = [''.join(x) for x in d.variables['Times'][:]]
d.close()
for esmf_time in sorted(times):
logging.info("Processing domain %d for time %s." %
(dom_id, esmf_time))
if js.postproc is not None and str(dom_id) in js.postproc:
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info(
"Executing postproc instructions for vars %s for domain %d."
% (str(var_list), dom_id))
try:
pp.process_vars(osp.join(js.wrf_dir, wrfout_path),
dom_id, esmf_time, var_list)
# in incremental mode, upload to server
if js.postproc.get('shuttle', None) == 'incremental':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
sent_files_1 = send_product_to_server(
args, js.pp_dir, js.job_id, js.job_id,
js.manifest_filename, desc, already_sent_files)
already_sent_files = filter(
lambda x: not x.endswith('json'),
already_sent_files + sent_files_1)
except Exception as e:
logging.warning(
'Failed to postprocess for time %s with error %s.'
% (esmf_time, str(e)))
# if we are to send out the postprocessed files after completion, this is the time
if js.postproc.get('shuttle', None) == 'on_completion':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
send_product_to_server(args, js.pp_dir, js.job_id, js.job_id,
js.manifest_filename, desc)
json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': '))
return
# step 9: wait for appearance of rsl.error.0000 and open it
wrf_out = None
rsl_path = osp.join(js.wrf_dir, 'rsl.error.0000')
while wrf_out is None:
try:
wrf_out = open(rsl_path)
break
except IOError:
logging.info(
'process_output: waiting 5 seconds for rsl.error.0000 file')
time.sleep(5)
logging.info('process_output: Detected rsl.error.0000')
js.run_utc = time.ctime(os.path.getmtime(rsl_path))
js.processed_utc = time.asctime(time.gmtime())
# step 10: track log output and check for history writes fro WRF
wait_lines = 0
wait_wrfout = 0
while True:
line = wrf_out.readline().strip()
if not line:
if not parallel_job_running(js):
logging.warning('WRF did not run to completion.')
break
if not wait_lines:
logging.info('Waiting for more output lines')
wait_lines = wait_lines + 1
time.sleep(5)
continue
wait_lines = 0
if "SUCCESS COMPLETE WRF" in line:
# send_email(js, 'complete', 'Job %s - wrf job complete SUCCESS.' % js.job_id)
logging.info("WRF completion detected.")
js.old_job_num = js.job_num
js.job_num = None
json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': '))
break
if "Timing for Writing wrfout" in line:
wait_wrfout = 0
esmf_time, domain_str = re.match(
r'.*wrfout_d.._([0-9_\-:]{19}) for domain\ +(\d+):',
line).groups()
wrfout_path, domain_str = re.match(
r'.*(wrfout_d.._[0-9_\-:]{19}) for domain\ +(\d+):',
line).groups()
dom_id = int(domain_str)
logging.info("Detected history write for domain %d for time %s." %
(dom_id, esmf_time))
if js.postproc is not None and str(dom_id) in js.postproc:
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info(
"Executing postproc instructions for vars %s for domain %d."
% (str(var_list), dom_id))
wrfout_path = find_wrfout(js.wrf_dir, dom_id, esmf_time)
try:
pp.process_vars(osp.join(js.wrf_dir, wrfout_path), dom_id,
esmf_time, var_list)
except Exception as e:
logging.warning(
'Failed to postprocess for time %s with error %s.' %
(esmf_time, str(e)))
else:
# in incremental mode, upload to server
if js.postproc.get('shuttle', None) == 'incremental':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
sent_files_1 = send_product_to_server(
args, js.pp_dir, js.job_id, js.job_id,
js.manifest_filename, desc, already_sent_files)
already_sent_files = filter(
lambda x: not x.endswith('json'),
already_sent_files + sent_files_1)
else:
if not wait_wrfout:
logging.info('Waiting for wrfout')
wait_wrfout = wait_wrfout + 1
# if we are to send out the postprocessed files after completion, this is the time
if js.postproc.get('shuttle', None) == 'on_completion':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
send_product_to_server(args, js.pp_dir, js.job_id, js.job_id,
js.manifest_filename, desc)
if js.postproc.get('shuttle', None) is not None:
make_kmz(js.job_id) # arguments can be added to the job id string
js.old_pid = js.pid
js.pid = None
js.state = 'Completed'
json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': '))
def execute(args, job_args):
"""
Executes a weather/fire simulation.
:param args: a dictionary with all to start the simulationfollowing keys
:param job_args: a the original json given the forecast
Keys in args:
:param grid_code: the (unique) code of the grid that is used
:param sys_install_path: system installation directory
:param start_utc: start time of simulation in UTC
:param end_utc: end time of simulation in UTC
:param workspace_path: workspace directory
:param wps_install_path: installation directory of WPS that will be used
:param wrf_install_path: installation directory of WRF that will be used
:param grib_source: a string identifying a valid GRIB2 source
:param wps_namelist_path: the path to the namelist.wps file that will be used as template
:param wrf_namelist_path: the path to the namelist.input file that will be used as template
:param fire_namelist_path: the path to the namelist.fire file that will be used as template
:param wps_geog_path: the path to the geogrid data directory providing terrain/fuel data
:param email_notification: dictionary containing keys address and events indicating when a mail should be fired off
"""
logging.info('step 0 initialize the job state from the arguments')
## logging.info('args = %s' % json.dumps(jargs, open(osp.join(jobdir,'input.json'),'w'), indent=4, separators=(',', ': ')))
js = JobState(args)
## logging.info('js = %s' % json.dumps(js, open(osp.join(jobdir,'input.json'),'w'), indent=4, separators=(',', ': ')))
jobdir = osp.abspath(osp.join(js.workspace_path, js.job_id))
make_clean_dir(jobdir)
json.dump(job_args,
open(osp.join(jobdir, 'input.json'), 'w'),
indent=4,
separators=(',', ': '))
jsub = make_job_file(js)
json.dump(jsub, open(jsub.jobfile, 'w'), indent=4, separators=(',', ': '))
logging.info("job %s starting [%d hours to forecast]." %
(js.job_id, js.fc_hrs))
sys.stdout.flush()
send_email(js, 'start', 'Job %s started.' % js.job_id)
# read in all namelists
js.wps_nml = read_namelist(js.args['wps_namelist_path'])
js.wrf_nml = read_namelist(js.args['wrf_namelist_path'])
js.fire_nml = read_namelist(js.args['fire_namelist_path'])
js.ems_nml = None
if 'emissions_namelist_path' in js.args:
js.ems_nml = read_namelist(js.args['emissions_namelist_path'])
# Parse and setup the domain configuration
js.domain_conf = WPSDomainConf(js.domains)
js.num_doms = len(js.domain_conf)
js.wps_nml['share']['interval_seconds'] = js.grib_source[
0].interval_seconds
logging.info("number of domains defined is %d." % js.num_doms)
# build directories in workspace
js.wps_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wps'))
js.wrf_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wrf'))
#check_obj(args,'args')
#check_obj(js,'Initial job state')
logging.info("step 1: clone WPS and WRF directories")
logging.info("cloning WPS into %s" % js.wps_dir)
cln = WRFCloner(js.args)
cln.clone_wps(js.wps_dir, [])
js.grib_source[0].clone_vtables(js.wps_dir)
logging.info(
"step 2: process domain information and patch namelist for geogrid")
js.wps_nml['share']['start_date'] = [utc_to_esmf(js.start_utc)
] * js.num_doms
js.wps_nml['share']['end_date'] = [utc_to_esmf(js.end_utc)] * js.num_doms
js.wps_nml['geogrid']['geog_data_path'] = js.args['wps_geog_path']
js.domain_conf.prepare_for_geogrid(js.wps_nml, js.wrf_nml, js.wrfxpy_dir,
js.wps_dir)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
# do steps 2 & 3 & 4 in parallel (two execution streams)
# -> GEOGRID ->
# -> GRIB2 download -> UNGRIB ->
proc_q = Queue()
geogrid_proc = Process(target=run_geogrid, args=(js, proc_q))
# grib_proc = Process(target=retrieve_gribs_and_run_ungrib_all, args=(js, proc_q, ref_utc))
grib_proc = {}
for grib_source in js.grib_source:
grib_proc[grib_source.id] = Process(
target=retrieve_gribs_and_run_ungrib,
args=(js, grib_source, proc_q))
logging.info('starting GEOGRID and GRIB2/UNGRIB')
if js.ungrib_only:
logging.info(
'ungrib_only set, skipping GEOGRID, will exit after UNGRIB')
else:
geogrid_proc.start()
for grib_source in js.grib_source:
grib_proc[grib_source.id].start()
# wait until all tasks are done
logging.info('waiting until all tasks are done')
for grib_source in js.grib_source:
grib_proc[grib_source.id].join()
if js.ungrib_only:
return
else:
geogrid_proc.join()
for grib_source in js.grib_source:
if proc_q.get() != 'SUCCESS':
return
if proc_q.get() != 'SUCCESS':
return
proc_q.close()
logging.info(
"step 5: execute metgrid after ensuring all grids will be processed")
update_namelist(js.wps_nml, js.grib_source[0].namelist_wps_keys())
js.domain_conf.prepare_for_metgrid(js.wps_nml)
logging.info("namelist.wps for METGRID: %s" %
json.dumps(js.wps_nml, indent=4, separators=(',', ': ')))
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
logging.info("running METGRID")
Metgrid(js.wps_dir).execute().check_output()
send_email(js, 'metgrid', 'Job %s - metgrid complete.' % js.job_id)
logging.info("METGRID complete")
logging.info("cloning WRF into %s" % js.wrf_dir)
logging.info(
"step 6: clone wrf directory, symlink all met_em* files, make namelists"
)
cln.clone_wrf(js.wrf_dir, [])
symlink_matching_files(js.wrf_dir, js.wps_dir, "met_em*")
time_ctrl = update_time_control(js.start_utc, js.end_utc, js.num_doms)
js.wrf_nml['time_control'].update(time_ctrl)
js.wrf_nml['time_control']['interval_seconds'] = js.grib_source[
0].interval_seconds
update_namelist(js.wrf_nml, js.grib_source[0].namelist_keys())
if 'ignitions' in js.args:
update_namelist(js.wrf_nml, render_ignitions(js, js.num_doms))
# if we have an emissions namelist, automatically turn on the tracers
if js.ems_nml is not None:
logging.debug('namelist.fire_emissions given, turning on tracers')
f90nml.write(js.ems_nml,
osp.join(js.wrf_dir, 'namelist.fire_emissions'),
force=True)
js.wrf_nml['dynamics']['tracer_opt'] = [2] * js.num_doms
f90nml.write(js.wrf_nml,
osp.join(js.wrf_dir, 'namelist.input'),
force=True)
f90nml.write(js.fire_nml,
osp.join(js.wrf_dir, 'namelist.fire'),
force=True)
# step 7: execute real.exe
logging.info("running REAL")
# try to run Real twice as it sometimes fails the first time
# it's not clear why this error happens
try:
Real(js.wrf_dir).execute().check_output()
except Exception as e:
logging.error('Real step failed with exception %s, retrying ...' %
str(e))
Real(js.wrf_dir).execute().check_output()
logging.info('step 7b: if requested, do fuel moisture DA')
logging.info('fmda = %s' % js.fmda)
if js.fmda is not None:
logging.info('running fuel moisture data assimilation')
for dom in js.fmda.domains:
logging.info('assimilate_fm10_observations for domain %s' % dom)
assimilate_fm10_observations(
osp.join(js.wrf_dir, 'wrfinput_d%02d' % int(dom)), None,
js.fmda.token)
# step 8: execute wrf.exe on parallel backend
logging.info('submitting WRF job')
send_email(js, 'wrf_submit', 'Job %s - wrf job submitted.' % js.job_id)
js.task_id = "sim-" + js.grid_code + "-" + utc_to_esmf(js.start_utc)[:10]
jsub.job_num = WRF(js.wrf_dir, js.qsys).submit(js.task_id, js.num_nodes,
js.ppn, js.wall_time_hrs)
send_email(
js, 'wrf_exec',
'Job %s - wrf job starting now with id %s.' % (js.job_id, js.task_id))
logging.info(
"WRF job %s submitted with id %s, waiting for rsl.error.0000" %
(jsub.job_num, js.task_id))
jobfile = osp.abspath(osp.join(js.workspace_path, js.job_id, 'job.json'))
json.dump(jsub, open(jobfile, 'w'), indent=4, separators=(',', ': '))
process_output(js.job_id)
def retrieve_gribs_and_run_ungrib(js, grib_source, q):
"""
This function retrieves required GRIB files and runs ungrib.
It returns either 'SUCCESS' or 'FAILURE' on completion.
:param js: the JobState object containing the forecast configuration
:param grib_source: the GribSource object containing ungrib configuration
:param q: the multiprocessing Queue into which we will send either 'SUCCESS' or 'FAILURE'
"""
wps_dir = osp.abspath(js.wps_dir)
grib_dir = osp.join(wps_dir, grib_source.id)
make_clean_dir(grib_dir)
wps_nml = js.wps_nml
try:
logging.info("retrieving GRIB files from %s" % grib_source.id)
download_whole_cycle = js.get('download_whole_cycle', False)
manifest = grib_source.retrieve_gribs(js.start_utc, js.end_utc,
js.ref_utc, js.cycle_start_utc,
download_whole_cycle)
# logging.info('manifest: ' + str(manifest))
cache_colmet = len(manifest) > 1
have_all_colmet = False
if cache_colmet:
have_all_colmet = len(manifest.colmet_missing) == 0
colmet_dir = osp.join(grib_source.cache_dir,
manifest.colmet_prefix)
logging.info('cache colmet %s, have all colmet %s' %
(cache_colmet, have_all_colmet))
if not have_all_colmet:
# this is also if we do not cache
grib_source.symlink_gribs(manifest.grib_files, grib_dir)
send_email(
js, 'grib2', 'Job %s - %d GRIB2 files downloaded.' %
(js.job_id, len(manifest)))
logging.info("running UNGRIB for %s" % grib_source.id)
logging.info(
"step 4: patch namelist for ungrib end execute ungrib on %s files"
% grib_source.id)
update_namelist(wps_nml, grib_source.namelist_wps_keys())
if cache_colmet:
wps_nml['share']['start_date'] = [
utc_to_esmf(manifest.colmet_files_utc[0])
] * js.num_doms
wps_nml['share']['end_date'] = [
utc_to_esmf(manifest.colmet_files_utc[-1])
] * js.num_doms
# logging.info("namelist.wps for UNGRIB: %s" % json.dumps(wps_nml, indent=4, separators=(',', ': ')))
f90nml.write(wps_nml,
osp.join(grib_dir, 'namelist.wps'),
force=True)
grib_source.clone_vtables(grib_dir)
symlink_unless_exists(osp.join(wps_dir, 'ungrib.exe'),
osp.join(grib_dir, 'ungrib.exe'))
print(grib_dir + ':')
os.system('ls -l %s' % grib_dir)
Ungrib(grib_dir).execute().check_output()
print(grib_dir + ':')
os.system('ls -l %s' % grib_dir)
if cache_colmet:
# move output to cache directory
make_dir(colmet_dir)
for f in manifest.colmet_files:
move(osp.join(grib_dir, f), osp.join(colmet_dir, f))
# now all colmet files should be in the cache
if cache_colmet:
for f in manifest.colmet_files:
symlink_unless_exists(osp.join(colmet_dir, f),
osp.join(wps_dir, f))
else:
# move output
for f in glob.glob(osp.join(grib_dir, grib_source.prefix() + '*')):
move(f, wps_dir)
send_email(js, 'ungrib', 'Job %s - ungrib complete.' % js.job_id)
logging.info('UNGRIB complete for %s' % grib_source.id)
q.put('SUCCESS')
except Exception as e:
logging.error('GRIB2/UNGRIB step failed with exception %s' % repr(e))
traceback.print_exc()
q.put('FAILURE')
def process_outputs_tiff(job_id):
args = load_sys_cfg()
jobfile = osp.abspath(osp.join(args.workspace_path, job_id,'job.json'))
satfile = osp.abspath(osp.join(args.workspace_path, job_id,'sat.json'))
logging.info('process_tiffs: loading job description from %s' % jobfile)
try:
js = Dict(json.load(open(jobfile,'r')))
except Exception as e:
logging.error('Cannot load the job description file %s' % jobfile)
logging.error('%s' % e)
sys.exit(1)
logging.info('process_tiffs: loading satellite description from %s' % satfile)
try:
jsat = Dict(json.load(open(satfile,'r')))
available_sats = [sat.upper()+'_AF' for sat in jsat.granules.keys()]
not_empty_sats = [sat.upper()+'_AF' for sat in jsat.granules.keys() if jsat.granules[sat]]
except:
logging.warning('Cannot load the satellite data in satellite description file %s' % satfile)
available_sats = []
not_empty_sats = []
pass
logging.info('process_tiffs: available satellite data %s' % available_sats)
logging.info('process_tiffs: not empty satellite data %s' % not_empty_sats)
js.old_pid = js.pid
js.pid = os.getpid()
js.state = 'Processing'
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
js.wrf_dir = osp.abspath(osp.join(args.workspace_path, js.job_id, 'wrf'))
# set up postprocessing
pp = None
js.pp_dir = osp.join(args.workspace_path, js.job_id, "products", "tiffs")
make_clean_dir(js.pp_dir)
pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code)
js.manifest_filename= 'wfc-' + js.grid_code + '.json'
logging.debug('Postprocessor created manifest %s',js.manifest_filename)
logging.info('Postprocessing all wrfout files.')
# postprocess all wrfouts
for wrfout_path in sorted(glob.glob(osp.join(js.wrf_dir,'wrfout_d??_????-??-??_??:??:??'))):
logging.info("Found %s" % wrfout_path)
domain_str,wrfout_esmf_time = re.match(r'.*wrfout_d(0[0-9])_([0-9_\-:]{19})',wrfout_path).groups()
dom_id = int(domain_str)
d = nc4.Dataset(wrfout_path)
# extract ESMF string times
times = [''.join(x) for x in d.variables['Times'][:].astype(str)]
if js.postproc is not None and str(dom_id) in js.postproc:
if available_sats:
sat_list = [sat for sat in available_sats if sat in js.postproc[str(dom_id)]]
var_list = [str(x) for x in js.postproc[str(dom_id)] if not str(x) in sat_list]
sat_list = [sat for sat in sat_list if sat in not_empty_sats]
logging.info("Executing postproc instructions for sats %s for domain %d." % (str(sat_list), dom_id))
else:
sat_list = []
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info("Executing postproc tiff instructions for vars %s for domain %d." % (str(var_list), dom_id))
try:
if sat_list:
pass
#process_sats_tiff()
process_vars_tiff(pp, d, wrfout_path, dom_id, times, var_list)
except Exception as e:
logging.warning('Failed to postprocess with error %s.' % str(e))
d.close()
js.old_pid = js.pid
js.pid = None
js.state = 'Completed'
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
def process_output(job_id):
args = load_sys_cfg()
jobfile = osp.abspath(osp.join(args.workspace_path, job_id,'job.json'))
logging.info('process_output: loading job description from %s' % jobfile)
try:
js = Dict(json.load(open(jobfile,'r')))
except Exception as e:
logging.error('Cannot load the job description file %s' % jobfile)
logging.error('%s' % e)
sys.exit(1)
js.old_pid = js.pid
js.pid = os.getpid()
js.state = 'Processing'
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
js.wrf_dir = osp.abspath(osp.join(args.workspace_path, js.job_id, 'wrf'))
# step 9: wait for appearance of rsl.error.0000 and open it
wrf_out = None
while wrf_out is None:
try:
wrf_out = open(osp.join(js.wrf_dir, 'rsl.error.0000'))
break
except IOError:
logging.info('process_output: waiting 5 seconds for rsl.error.0000 file')
time.sleep(5)
logging.info('process_output: Detected rsl.error.0000')
# step 10: track log output and check for history writes fro WRF
pp = None
already_sent_files, max_pp_dom = [], -1
if js.postproc is not None:
js.pp_dir = osp.join(args.workspace_path, js.job_id, "products")
make_clean_dir(js.pp_dir)
pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code)
max_pp_dom = max([int(x) for x in filter(lambda x: len(x) == 1, js.postproc)])
wait_lines = 0
wait_wrfout = 0
while True:
line = wrf_out.readline().strip()
if not line:
if not parallel_job_running(js):
logging.warning('WRF did not run to completion.')
break
if not wait_lines:
logging.info('Waiting for more output lines')
wait_lines = wait_lines + 1
time.sleep(0.5)
continue
wait_lines = 0
if "SUCCESS COMPLETE WRF" in line:
# send_email(js, 'complete', 'Job %s - wrf job complete SUCCESS.' % js.job_id)
logging.info("WRF completion detected.")
js.old_job_num = js.job_num
js.job_num = None
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
break
if "Timing for Writing wrfout" in line:
esmf_time,domain_str = re.match(r'.*wrfout_d.._([0-9_\-:]{19}) for domain\ +(\d+):' ,line).groups()
wrfout_path,domain_str = re.match(r'.*(wrfout_d.._[0-9_\-:]{19}) for domain\ +(\d+):' ,line).groups()
dom_id = int(domain_str)
logging.info("Detected history write for domain %d for time %s." % (dom_id, esmf_time))
if js.postproc is not None and str(dom_id) in js.postproc:
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info("Executing postproc instructions for vars %s for domain %d." % (str(var_list), dom_id))
wrfout_path = find_wrfout(js.wrf_dir, dom_id, esmf_time)
try:
pp.process_vars(osp.join(js.wrf_dir,wrfout_path), dom_id, esmf_time, var_list)
except Exception as e:
logging.warning('Failed to postprocess for time %s with error %s.' % (esmf_time, str(e)))
# in incremental mode, upload to server
if js.postproc.get('shuttle', None) == 'incremental':
desc = js.postproc['description'] if 'description' in js.postproc else js.job_id
sent_files_1 = send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, desc, already_sent_files)
already_sent_files = filter(lambda x: not x.endswith('json'), already_sent_files + sent_files_1)
wait_wrfout = 0
else:
if not wait_wrfout:
logging.info('Waiting for wrfout')
wait_wrfout = wait_wrfout + 1
# if we are to send out the postprocessed files after completion, this is the time
if js.postproc.get('shuttle', None) == 'on_completion':
desc = js.postproc['description'] if 'description' in js.postproc else js.job_id
send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, desc)
js.old_pid = js.pid
js.pid = None
js.state = 'Completed'
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
def execute(args,job_args):
"""
Executes a weather/fire simulation.
:param args: a dictionary with all to start the simulationfollowing keys
:param job_args: a the original json given the forecast
Keys in args:
:param grid_code: the (unique) code of the grid that is used
:param sys_install_path: system installation directory
:param start_utc: start time of simulation in UTC
:param end_utc: end time of simulation in UTC
:param workspace_path: workspace directory
:param wps_install_path: installation directory of WPS that will be used
:param wrf_install_path: installation directory of WRF that will be used
:param grib_source: a string identifying a valid GRIB2 source
:param wps_namelist_path: the path to the namelist.wps file that will be used as template
:param wrf_namelist_path: the path to the namelist.input file that will be used as template
:param fire_namelist_path: the path to the namelist.fire file that will be used as template
:param wps_geog_path: the path to the geogrid data directory providing terrain/fuel data
:param email_notification: dictionary containing keys address and events indicating when a mail should be fired off
"""
# step 0 initialize the job state from the arguments
js = JobState(args)
jobdir = osp.abspath(osp.join(js.workspace_path, js.job_id))
make_clean_dir(jobdir)
json.dump(job_args, open(osp.join(jobdir,'input.json'),'w'), indent=4, separators=(',', ': '))
jsub = make_job_file(js)
json.dump(jsub, open(jsub.jobfile,'w'), indent=4, separators=(',', ': '))
logging.info("job %s starting [%d hours to forecast]." % (js.job_id, js.fc_hrs))
sys.stdout.flush()
send_email(js, 'start', 'Job %s started.' % js.job_id)
# read in all namelists
js.wps_nml = f90nml.read(js.args['wps_namelist_path'])
js.wrf_nml = f90nml.read(js.args['wrf_namelist_path'])
js.fire_nml = f90nml.read(js.args['fire_namelist_path'])
js.ems_nml = None
if 'emissions_namelist_path' in js.args:
js.ems_nml = f90nml.read(js.args['emissions_namelist_path'])
# Parse and setup the domain configuration
js.domain_conf = WPSDomainConf(js.domains)
num_doms = len(js.domain_conf)
js.wps_nml['share']['start_date'] = [utc_to_esmf(js.start_utc)] * num_doms
js.wps_nml['share']['end_date'] = [utc_to_esmf(js.end_utc)] * num_doms
js.wps_nml['share']['interval_seconds'] = 3600
logging.info("number of domains defined is %d." % num_doms)
# build directories in workspace
js.wps_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wps'))
js.wrf_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wrf'))
#check_obj(args,'args')
#check_obj(js,'Initial job state')
# step 1: clone WPS and WRF directories
logging.info("cloning WPS into %s" % js.wps_dir)
cln = WRFCloner(js.args)
cln.clone_wps(js.wps_dir, js.grib_source.vtables(), [])
# step 2: process domain information and patch namelist for geogrid
js.wps_nml['geogrid']['geog_data_path'] = js.args['wps_geog_path']
js.domain_conf.prepare_for_geogrid(js.wps_nml, js.wrf_nml, js.wrfxpy_dir, js.wps_dir)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
# do steps 2 & 3 & 4 in parallel (two execution streams)
# -> GEOGRID ->
# -> GRIB2 download -> UNGRIB ->
proc_q = Queue()
geogrid_proc = Process(target=run_geogrid, args=(js, proc_q))
grib_proc = Process(target=retrieve_gribs_and_run_ungrib, args=(js, proc_q))
logging.info('starting GEOGRID and GRIB2/UNGRIB')
geogrid_proc.start()
grib_proc.start()
# wait until both tasks are done
logging.info('waiting until both tasks are done')
grib_proc.join()
geogrid_proc.join()
if proc_q.get() != 'SUCCESS':
return
if proc_q.get() != 'SUCCESS':
return
proc_q.close()
# step 5: execute metgrid after ensuring all grids will be processed
js.domain_conf.prepare_for_metgrid(js.wps_nml)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
logging.info("running METGRID")
Metgrid(js.wps_dir).execute().check_output()
send_email(js, 'metgrid', 'Job %s - metgrid complete.' % js.job_id)
logging.info("cloning WRF into %s" % js.wrf_dir)
# step 6: clone wrf directory, symlink all met_em* files, make namelists
cln.clone_wrf(js.wrf_dir, [])
symlink_matching_files(js.wrf_dir, js.wps_dir, "met_em*")
time_ctrl = update_time_control(js.start_utc, js.end_utc, num_doms)
js.wrf_nml['time_control'].update(time_ctrl)
update_namelist(js.wrf_nml, js.grib_source.namelist_keys())
if 'ignitions' in js.args:
update_namelist(js.wrf_nml, render_ignitions(js, num_doms))
# if we have an emissions namelist, automatically turn on the tracers
if js.ems_nml is not None:
logging.debug('namelist.fire_emissions given, turning on tracers')
f90nml.write(js.ems_nml, osp.join(js.wrf_dir, 'namelist.fire_emissions'), force=True)
js.wrf_nml['dynamics']['tracer_opt'] = [2] * num_doms
f90nml.write(js.wrf_nml, osp.join(js.wrf_dir, 'namelist.input'), force=True)
f90nml.write(js.fire_nml, osp.join(js.wrf_dir, 'namelist.fire'), force=True)
# step 7: execute real.exe
logging.info("running REAL")
# try to run Real twice as it sometimes fails the first time
# it's not clear why this error happens
try:
Real(js.wrf_dir).execute().check_output()
except Exception as e:
logging.error('Real step failed with exception %s, retrying ...' % str(e))
Real(js.wrf_dir).execute().check_output()
# step 7b: if requested, do fuel moisture DA
if js.fmda is not None:
logging.info('running fuel moisture data assimilation')
for dom in js.fmda.domains:
assimilate_fm10_observations(osp.join(wrf_dir, 'wrfinput_d%02d' % dom), None, js.fmda.token)
# step 8: execute wrf.exe on parallel backend
logging.info('submitting WRF job')
send_email(js, 'wrf_submit', 'Job %s - wrf job submitted.' % js.job_id)
js.task_id = "sim-" + js.grid_code + "-" + utc_to_esmf(js.start_utc)[:10]
jsub.job_num=WRF(js.wrf_dir, js.qsys).submit(js.task_id, js.num_nodes, js.ppn, js.wall_time_hrs)
send_email(js, 'wrf_exec', 'Job %s - wrf job starting now with id %s.' % (js.job_id, js.task_id))
logging.info("WRF job %s submitted with id %s, waiting for rsl.error.0000" % (jsub.job_num, js.task_id))
jobfile = osp.abspath(osp.join(js.workspace_path, js.job_id,'job.json'))
json.dump(jsub, open(jobfile,'w'), indent=4, separators=(',', ': '))
process_output(js.job_id)