def process_output(job_id):
args = load_sys_cfg()
jobfile = osp.abspath(osp.join(args.workspace_path, job_id, 'job.json'))
logging.info('process_output: loading job description from %s' % jobfile)
try:
js = Dict(json.load(open(jobfile, 'r')))
except Exception as e:
logging.error('Cannot load the job description file %s' % jobfile)
logging.error('%s' % e)
sys.exit(1)
js.old_pid = js.pid
js.pid = os.getpid()
js.state = 'Processing'
json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': '))
js.wrf_dir = osp.abspath(osp.join(args.workspace_path, js.job_id, 'wrf'))
pp = None
already_sent_files, max_pp_dom = [], -1
if js.postproc is None:
logging.info('No postprocessing specified, exiting.')
return
# set up postprocessing
delete_visualization(js.job_id)
js.pp_dir = osp.join(args.workspace_path, js.job_id, "products")
make_clean_dir(js.pp_dir)
pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code)
js.manifest_filename = 'wfc-' + js.grid_code + '.json'
logging.debug('Postprocessor created manifest %s', js.manifest_filename)
max_pp_dom = max(
[int(x) for x in filter(lambda x: len(x) == 1, js.postproc)])
if js.postproc.get('from', None) == 'wrfout':
logging.info('Postprocessing all wrfout files.')
# postprocess all wrfouts
for wrfout_path in sorted(
glob.glob(
osp.join(js.wrf_dir, 'wrfout_d??_????-??-??_??:??:??'))):
logging.info("Found %s" % wrfout_path)
domain_str, wrfout_esmf_time = re.match(
r'.*wrfout_d(0[0-9])_([0-9_\-:]{19})', wrfout_path).groups()
dom_id = int(domain_str)
d = nc4.Dataset(wrfout_path)
# extract ESMF string times
times = [''.join(x) for x in d.variables['Times'][:]]
d.close()
for esmf_time in sorted(times):
logging.info("Processing domain %d for time %s." %
(dom_id, esmf_time))
if js.postproc is not None and str(dom_id) in js.postproc:
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info(
"Executing postproc instructions for vars %s for domain %d."
% (str(var_list), dom_id))
try:
pp.process_vars(osp.join(js.wrf_dir, wrfout_path),
dom_id, esmf_time, var_list)
# in incremental mode, upload to server
if js.postproc.get('shuttle', None) == 'incremental':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
sent_files_1 = send_product_to_server(
args, js.pp_dir, js.job_id, js.job_id,
js.manifest_filename, desc, already_sent_files)
already_sent_files = filter(
lambda x: not x.endswith('json'),
already_sent_files + sent_files_1)
except Exception as e:
logging.warning(
'Failed to postprocess for time %s with error %s.'
% (esmf_time, str(e)))
# if we are to send out the postprocessed files after completion, this is the time
if js.postproc.get('shuttle', None) == 'on_completion':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
send_product_to_server(args, js.pp_dir, js.job_id, js.job_id,
js.manifest_filename, desc)
json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': '))
return
# step 9: wait for appearance of rsl.error.0000 and open it
wrf_out = None
rsl_path = osp.join(js.wrf_dir, 'rsl.error.0000')
while wrf_out is None:
try:
wrf_out = open(rsl_path)
break
except IOError:
logging.info(
'process_output: waiting 5 seconds for rsl.error.0000 file')
time.sleep(5)
logging.info('process_output: Detected rsl.error.0000')
js.run_utc = time.ctime(os.path.getmtime(rsl_path))
js.processed_utc = time.asctime(time.gmtime())
# step 10: track log output and check for history writes fro WRF
wait_lines = 0
wait_wrfout = 0
while True:
line = wrf_out.readline().strip()
if not line:
if not parallel_job_running(js):
logging.warning('WRF did not run to completion.')
break
if not wait_lines:
logging.info('Waiting for more output lines')
wait_lines = wait_lines + 1
time.sleep(5)
continue
wait_lines = 0
if "SUCCESS COMPLETE WRF" in line:
# send_email(js, 'complete', 'Job %s - wrf job complete SUCCESS.' % js.job_id)
logging.info("WRF completion detected.")
js.old_job_num = js.job_num
js.job_num = None
json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': '))
break
if "Timing for Writing wrfout" in line:
wait_wrfout = 0
esmf_time, domain_str = re.match(
r'.*wrfout_d.._([0-9_\-:]{19}) for domain\ +(\d+):',
line).groups()
wrfout_path, domain_str = re.match(
r'.*(wrfout_d.._[0-9_\-:]{19}) for domain\ +(\d+):',
line).groups()
dom_id = int(domain_str)
logging.info("Detected history write for domain %d for time %s." %
(dom_id, esmf_time))
if js.postproc is not None and str(dom_id) in js.postproc:
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info(
"Executing postproc instructions for vars %s for domain %d."
% (str(var_list), dom_id))
wrfout_path = find_wrfout(js.wrf_dir, dom_id, esmf_time)
try:
pp.process_vars(osp.join(js.wrf_dir, wrfout_path), dom_id,
esmf_time, var_list)
except Exception as e:
logging.warning(
'Failed to postprocess for time %s with error %s.' %
(esmf_time, str(e)))
else:
# in incremental mode, upload to server
if js.postproc.get('shuttle', None) == 'incremental':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
sent_files_1 = send_product_to_server(
args, js.pp_dir, js.job_id, js.job_id,
js.manifest_filename, desc, already_sent_files)
already_sent_files = filter(
lambda x: not x.endswith('json'),
already_sent_files + sent_files_1)
else:
if not wait_wrfout:
logging.info('Waiting for wrfout')
wait_wrfout = wait_wrfout + 1
# if we are to send out the postprocessed files after completion, this is the time
if js.postproc.get('shuttle', None) == 'on_completion':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
send_product_to_server(args, js.pp_dir, js.job_id, js.job_id,
js.manifest_filename, desc)
if js.postproc.get('shuttle', None) is not None:
make_kmz(js.job_id) # arguments can be added to the job id string
js.old_pid = js.pid
js.pid = None
js.state = 'Completed'
json.dump(js, open(jobfile, 'w'), indent=4, separators=(',', ': '))
def execute(args):
"""
Executes a weather/fire simulation.
The args dictionary contains
:param args: a dictionary with the following keys
:param grid_code: the (unique) code of the grid that is used
:param sys_install_path: system installation directory
:param start_utc: start time of simulation in UTC
:param end_utc: end time of simulation in UTC
:param workspace_path: workspace directory
:param wps_install_path: installation directory of WPS that will be used
:param wrf_install_path: installation directory of WRF that will be used
:param grib_source: a string identifying a valid GRIB2 source
:param wps_namelist_path: the path to the namelist.wps file that will be used as template
:param wrf_namelist_path: the path to the namelist.input file that will be used as template
:param fire_namelist_path: the path to the namelist.fire file that will be used as template
:param wps_geog_path: the path to the geogrid data directory providing terrain/fuel data
:param email_notification: dictionary containing keys address and events indicating when a mail should be fired off
"""
logging.basicConfig(level=logging.INFO)
# initialize the job state from the arguments
js = JobState(args)
logging.info("job %s starting [%d hours to forecast]." % (js.job_id, js.fc_hrs))
send_email(js, 'start', 'Job %s started.' % js.job_id)
# read in all namelists
js.wps_nml = f90nml.read(args['wps_namelist_path'])
js.wrf_nml = f90nml.read(args['wrf_namelist_path'])
js.fire_nml = f90nml.read(args['fire_namelist_path'])
js.ems_nml = None
if 'emissions_namelist_path' in args:
js.ems_nml = f90nml.read(args['emissions_namelist_path'])
# Parse and setup the domain configuration
js.domain_conf = WPSDomainConf(js.domains)
num_doms = len(js.domain_conf)
js.wps_nml['share']['start_date'] = [utc_to_esmf(js.start_utc)] * num_doms
js.wps_nml['share']['end_date'] = [utc_to_esmf(js.end_utc)] * num_doms
js.wps_nml['share']['interval_seconds'] = 3600
logging.info("number of domains defined is %d." % num_doms)
# build directories in workspace
js.wps_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wps'))
js.wrf_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wrf'))
logging.info("cloning WPS into %s" % js.wps_dir)
# step 1: clone WPS and WRF directories
cln = WRFCloner(args)
cln.clone_wps(js.wps_dir, js.grib_source.vtables(), [])
# step 2: process domain information and patch namelist for geogrid
js.wps_nml['geogrid']['geog_data_path'] = args['wps_geog_path']
js.domain_conf.prepare_for_geogrid(js.wps_nml, js.wrf_nml, js.wrfxpy_dir, js.wps_dir)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
# do steps 2 & 3 & 4 in parallel (two execution streams)
# -> GEOGRID ->
# -> GRIB2 download -> UNGRIB ->
proc_q = Queue()
geogrid_proc = Process(target=run_geogrid, args=(js, proc_q))
grib_proc = Process(target=retrieve_gribs_and_run_ungrib, args=(js, proc_q))
geogrid_proc.start()
grib_proc.start()
# wait until both tasks are done
geogrid_proc.join()
grib_proc.join()
if proc_q.get() != 'SUCCESS':
return
if proc_q.get() != 'SUCCESS':
return
proc_q.close()
# step 5: execute metgrid after ensuring all grids will be processed
js.domain_conf.prepare_for_metgrid(js.wps_nml)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
logging.info("running METGRID")
Metgrid(js.wps_dir).execute().check_output()
send_email(js, 'metgrid', 'Job %s - metgrid complete.' % js.job_id)
logging.info("cloning WRF into %s" % js.wrf_dir)
# step 6: clone wrf directory, symlink all met_em* files
cln.clone_wrf(js.wrf_dir, [])
symlink_matching_files(js.wrf_dir, js.wps_dir, "met_em*")
logging.info("running REAL")
# step 7: patch input namelist, fire namelist, emissions namelist (if required)
# and execute real.exe
time_ctrl = update_time_control(js.start_utc, js.end_utc, num_doms)
js.wrf_nml['time_control'].update(time_ctrl)
update_namelist(js.wrf_nml, js.grib_source.namelist_keys())
if 'ignitions' in args:
update_namelist(js.wrf_nml, render_ignitions(js, num_doms))
# if we have an emissions namelist, automatically turn on the tracers
if js.ems_nml is not None:
f90nml.write(js.ems_nml, osp.join(js.wrf_dir, 'namelist.fire_emissions'), force=True)
js.wrf_nml['dynamics']['tracer_opt'] = [2] * num_doms
f90nml.write(js.wrf_nml, osp.join(js.wrf_dir, 'namelist.input'), force=True)
f90nml.write(js.fire_nml, osp.join(js.wrf_dir, 'namelist.fire'), force=True)
# try to run Real twice as it sometimes fails the first time
# it's not clear why this error happens
try:
Real(js.wrf_dir).execute().check_output()
except Exception as e:
logging.error('Real step failed with exception %s, retrying ...' % str(e))
Real(js.wrf_dir).execute().check_output()
# step 8: if requested, do fuel moisture DA
if js.fmda is not None:
logging.info('running fuel moisture data assimilation')
for dom in js.fmda.domains:
assimilate_fm10_observations(osp.join(wrf_dir, 'wrfinput_d%02d' % dom), None, js.fmda.token)
logging.info('submitting WRF job')
send_email(js, 'wrf_submit', 'Job %s - wrf job submitted.' % js.job_id)
# step 8: execute wrf.exe on parallel backend
js.task_id = "sim-" + js.grid_code + "-" + utc_to_esmf(js.start_utc)[:10]
WRF(js.wrf_dir, js.qsys).submit(js.task_id, js.num_nodes, js.ppn, js.wall_time_hrs)
send_email(js, 'wrf_exec', 'Job %s - wrf job starting now with id %s.' % (js.job_id, js.task_id))
logging.info("WRF job submitted with id %s, waiting for rsl.error.0000" % js.task_id)
# step 9: wait for appearance of rsl.error.0000 and open it
wrf_out = None
while wrf_out is None:
try:
wrf_out = open(osp.join(js.wrf_dir, 'rsl.error.0000'))
break
except IOError:
logging.info('forecast: waiting 10 seconds for rsl.error.0000 file')
time.sleep(5)
logging.info('Detected rsl.error.0000')
# step 10: track log output and check for history writes fro WRF
pp = None
already_sent_files, max_pp_dom = [], -1
if js.postproc is not None:
js.pp_dir = osp.join(js.workspace_path, js.job_id, "products")
make_dir(js.pp_dir)
pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code)
max_pp_dom = max([int(x) for x in filter(lambda x: len(x) == 1, js.postproc)])
while True:
line = wrf_out.readline().strip()
if not line:
time.sleep(0.2)
continue
if "SUCCESS COMPLETE WRF" in line:
send_email(js, 'complete', 'Job %s - wrf job complete SUCCESS.' % js.job_id)
logging.info("WRF completion detected.")
break
if "Timing for Writing wrfout" in line:
esmf_time,domain_str = re.match(r'.*wrfout_d.._([0-9_\-:]{19}) for domain\ +(\d+):' ,line).groups()
dom_id = int(domain_str)
logging.info("Detected history write for domain %d for time %s." % (dom_id, esmf_time))
if js.postproc is not None and str(dom_id) in js.postproc:
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info("Executing postproc instructions for vars %s for domain %d." % (str(var_list), dom_id))
wrfout_path = find_fresh_wrfout(js.wrf_dir, dom_id)
try:
pp.process_vars(wrfout_path, dom_id, esmf_time, var_list)
except Exception as e:
logging.warning('Failed to postprocess for time %s with error %s.' % (esmf_time, str(e)))
# if this is the last processed domain for this timestamp in incremental mode, upload to server
if dom_id == max_pp_dom and js.postproc.get('shuttle', None) == 'incremental':
desc = js.postproc['description'] if 'description' in js.postproc else js.job_id
sent_files_1 = send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, desc, already_sent_files)
logging.info('sent %d files to visualization server.' % len(sent_files_1))
already_sent_files = filter(lambda x: not x.endswith('json'), already_sent_files + sent_files_1)
# if we are to send out the postprocessed files after completion, this is the time
if js.postproc.get('shuttle', None) == 'on_completion':
desc = js.postproc['description'] if 'description' in js.postproc else js.job_id
send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, desc)
def process_outputs_tiff(job_id):
args = load_sys_cfg()
jobfile = osp.abspath(osp.join(args.workspace_path, job_id,'job.json'))
satfile = osp.abspath(osp.join(args.workspace_path, job_id,'sat.json'))
logging.info('process_tiffs: loading job description from %s' % jobfile)
try:
js = Dict(json.load(open(jobfile,'r')))
except Exception as e:
logging.error('Cannot load the job description file %s' % jobfile)
logging.error('%s' % e)
sys.exit(1)
logging.info('process_tiffs: loading satellite description from %s' % satfile)
try:
jsat = Dict(json.load(open(satfile,'r')))
available_sats = [sat.upper()+'_AF' for sat in jsat.granules.keys()]
not_empty_sats = [sat.upper()+'_AF' for sat in jsat.granules.keys() if jsat.granules[sat]]
except:
logging.warning('Cannot load the satellite data in satellite description file %s' % satfile)
available_sats = []
not_empty_sats = []
pass
logging.info('process_tiffs: available satellite data %s' % available_sats)
logging.info('process_tiffs: not empty satellite data %s' % not_empty_sats)
js.old_pid = js.pid
js.pid = os.getpid()
js.state = 'Processing'
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
js.wrf_dir = osp.abspath(osp.join(args.workspace_path, js.job_id, 'wrf'))
# set up postprocessing
pp = None
js.pp_dir = osp.join(args.workspace_path, js.job_id, "products", "tiffs")
make_clean_dir(js.pp_dir)
pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code)
js.manifest_filename= 'wfc-' + js.grid_code + '.json'
logging.debug('Postprocessor created manifest %s',js.manifest_filename)
logging.info('Postprocessing all wrfout files.')
# postprocess all wrfouts
for wrfout_path in sorted(glob.glob(osp.join(js.wrf_dir,'wrfout_d??_????-??-??_??:??:??'))):
logging.info("Found %s" % wrfout_path)
domain_str,wrfout_esmf_time = re.match(r'.*wrfout_d(0[0-9])_([0-9_\-:]{19})',wrfout_path).groups()
dom_id = int(domain_str)
d = nc4.Dataset(wrfout_path)
# extract ESMF string times
times = [''.join(x) for x in d.variables['Times'][:].astype(str)]
if js.postproc is not None and str(dom_id) in js.postproc:
if available_sats:
sat_list = [sat for sat in available_sats if sat in js.postproc[str(dom_id)]]
var_list = [str(x) for x in js.postproc[str(dom_id)] if not str(x) in sat_list]
sat_list = [sat for sat in sat_list if sat in not_empty_sats]
logging.info("Executing postproc instructions for sats %s for domain %d." % (str(sat_list), dom_id))
else:
sat_list = []
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info("Executing postproc tiff instructions for vars %s for domain %d." % (str(var_list), dom_id))
try:
if sat_list:
pass
#process_sats_tiff()
process_vars_tiff(pp, d, wrfout_path, dom_id, times, var_list)
except Exception as e:
logging.warning('Failed to postprocess with error %s.' % str(e))
d.close()
js.old_pid = js.pid
js.pid = None
js.state = 'Completed'
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
# Postprocessor must be imported first since it selects the Agg backend
from vis.postprocessor import Postprocessor
from vis.rasterizer import make_colorbar, basemap_raster_mercator, basemap_barbs_mercator
from vis.var_wisdom import convert_value, get_wisdom
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
if len(sys.argv) != 4 and len(sys.argv) != 5:
print('usage: %s <wrfout_path> <var_instr> <prefix> [skip]' % sys.argv[0])
sys.exit(1)
wrf_path = sys.argv[1]
var_instr = None
if sys.argv[2][0] == '@':
var_instr = json.load(open(sys.argv[2][1:]))
else:
var_instr = {x:{} for x in sys.argv[2].split(',')}
prefix = sys.argv[3]
skip = 1
if len(sys.argv) == 5:
skip = int(sys.argv[4])
p = Postprocessor(os.path.dirname(prefix), os.path.basename(prefix), var_instr)
p.process_file(wrf_path, var_instr.keys(), skip)
# Postprocessor must be imported first since it selects the Agg backend
from vis.postprocessor import Postprocessor
from vis.rasterizer import make_colorbar, basemap_raster_mercator, basemap_barbs_mercator
from vis.var_wisdom import convert_value, get_wisdom
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s')
if len(sys.argv) != 4 and len(sys.argv) != 5:
print(('usage: %s <wrfout_path> <var_instr> <prefix> [skip]' %
sys.argv[0]))
sys.exit(1)
wrf_path = sys.argv[1]
var_instr = None
if sys.argv[2][0] == '@':
var_instr = json.load(open(sys.argv[2][1:]))
else:
var_instr = {x: {} for x in sys.argv[2].split(',')}
prefix = sys.argv[3]
skip = 1
if len(sys.argv) == 5:
skip = int(sys.argv[4])
p = Postprocessor(os.path.dirname(prefix), os.path.basename(prefix),
var_instr)
p.process_file(wrf_path, list(var_instr.keys()), skip)
def process_output(job_id):
args = load_sys_cfg()
jobfile = osp.abspath(osp.join(args.workspace_path, job_id,'job.json'))
logging.info('process_output: loading job description from %s' % jobfile)
try:
js = Dict(json.load(open(jobfile,'r')))
except Exception as e:
logging.error('Cannot load the job description file %s' % jobfile)
logging.error('%s' % e)
sys.exit(1)
js.old_pid = js.pid
js.pid = os.getpid()
js.state = 'Processing'
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
js.wrf_dir = osp.abspath(osp.join(args.workspace_path, js.job_id, 'wrf'))
# step 9: wait for appearance of rsl.error.0000 and open it
wrf_out = None
while wrf_out is None:
try:
wrf_out = open(osp.join(js.wrf_dir, 'rsl.error.0000'))
break
except IOError:
logging.info('process_output: waiting 5 seconds for rsl.error.0000 file')
time.sleep(5)
logging.info('process_output: Detected rsl.error.0000')
# step 10: track log output and check for history writes fro WRF
pp = None
already_sent_files, max_pp_dom = [], -1
if js.postproc is not None:
js.pp_dir = osp.join(args.workspace_path, js.job_id, "products")
make_clean_dir(js.pp_dir)
pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code)
max_pp_dom = max([int(x) for x in filter(lambda x: len(x) == 1, js.postproc)])
wait_lines = 0
wait_wrfout = 0
while True:
line = wrf_out.readline().strip()
if not line:
if not parallel_job_running(js):
logging.warning('WRF did not run to completion.')
break
if not wait_lines:
logging.info('Waiting for more output lines')
wait_lines = wait_lines + 1
time.sleep(0.5)
continue
wait_lines = 0
if "SUCCESS COMPLETE WRF" in line:
# send_email(js, 'complete', 'Job %s - wrf job complete SUCCESS.' % js.job_id)
logging.info("WRF completion detected.")
js.old_job_num = js.job_num
js.job_num = None
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
break
if "Timing for Writing wrfout" in line:
esmf_time,domain_str = re.match(r'.*wrfout_d.._([0-9_\-:]{19}) for domain\ +(\d+):' ,line).groups()
wrfout_path,domain_str = re.match(r'.*(wrfout_d.._[0-9_\-:]{19}) for domain\ +(\d+):' ,line).groups()
dom_id = int(domain_str)
logging.info("Detected history write for domain %d for time %s." % (dom_id, esmf_time))
if js.postproc is not None and str(dom_id) in js.postproc:
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info("Executing postproc instructions for vars %s for domain %d." % (str(var_list), dom_id))
wrfout_path = find_wrfout(js.wrf_dir, dom_id, esmf_time)
try:
pp.process_vars(osp.join(js.wrf_dir,wrfout_path), dom_id, esmf_time, var_list)
except Exception as e:
logging.warning('Failed to postprocess for time %s with error %s.' % (esmf_time, str(e)))
# in incremental mode, upload to server
if js.postproc.get('shuttle', None) == 'incremental':
desc = js.postproc['description'] if 'description' in js.postproc else js.job_id
sent_files_1 = send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, desc, already_sent_files)
already_sent_files = filter(lambda x: not x.endswith('json'), already_sent_files + sent_files_1)
wait_wrfout = 0
else:
if not wait_wrfout:
logging.info('Waiting for wrfout')
wait_wrfout = wait_wrfout + 1
# if we are to send out the postprocessed files after completion, this is the time
if js.postproc.get('shuttle', None) == 'on_completion':
desc = js.postproc['description'] if 'description' in js.postproc else js.job_id
send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, desc)
js.old_pid = js.pid
js.pid = None
js.state = 'Completed'
json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))
def execute(args):
"""
Executes a weather/fire simulation.
The args dictionary contains
:param args: a dictionary with the following keys
:param grid_code: the (unique) code of the grid that is used
:param sys_install_path: system installation directory
:param start_utc: start time of simulation in UTC
:param end_utc: end time of simulation in UTC
:param workspace_path: workspace directory
:param wps_install_path: installation directory of WPS that will be used
:param wrf_install_path: installation directory of WRF that will be used
:param grib_source: a string identifying a valid GRIB2 source
:param wps_namelist_path: the path to the namelist.wps file that will be used as template
:param wrf_namelist_path: the path to the namelist.input file that will be used as template
:param fire_namelist_path: the path to the namelist.fire file that will be used as template
:param wps_geog_path: the path to the geogrid data directory providing terrain/fuel data
:param email_notification: dictionary containing keys address and events indicating when a mail should be fired off
"""
logging.basicConfig(level=logging.INFO)
# initialize the job state from the arguments
js = JobState(args)
logging.info("job %s starting [%d hours to forecast]." %
(js.job_id, js.fc_hrs))
send_email(js, 'start', 'Job %s started.' % js.job_id)
# read in all namelists
js.wps_nml = f90nml.read(args['wps_namelist_path'])
js.wrf_nml = f90nml.read(args['wrf_namelist_path'])
js.fire_nml = f90nml.read(args['fire_namelist_path'])
js.ems_nml = None
if 'emissions_namelist_path' in args:
js.ems_nml = f90nml.read(args['emissions_namelist_path'])
# Parse and setup the domain configuration
js.domain_conf = WPSDomainConf(js.domains)
num_doms = len(js.domain_conf)
js.wps_nml['share']['start_date'] = [utc_to_esmf(js.start_utc)] * num_doms
js.wps_nml['share']['end_date'] = [utc_to_esmf(js.end_utc)] * num_doms
js.wps_nml['share']['interval_seconds'] = 3600
logging.info("number of domains defined is %d." % num_doms)
# build directories in workspace
js.wps_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wps'))
js.wrf_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wrf'))
logging.info("cloning WPS into %s" % js.wps_dir)
# step 1: clone WPS and WRF directories
cln = WRFCloner(args)
cln.clone_wps(js.wps_dir, js.grib_source.vtables(), [])
# step 2: process domain information and patch namelist for geogrid
js.wps_nml['geogrid']['geog_data_path'] = args['wps_geog_path']
js.domain_conf.prepare_for_geogrid(js.wps_nml, js.wrf_nml, js.wrfxpy_dir,
js.wps_dir)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
# do steps 2 & 3 & 4 in parallel (two execution streams)
# -> GEOGRID ->
# -> GRIB2 download -> UNGRIB ->
proc_q = Queue()
geogrid_proc = Process(target=run_geogrid, args=(js, proc_q))
grib_proc = Process(target=retrieve_gribs_and_run_ungrib,
args=(js, proc_q))
geogrid_proc.start()
grib_proc.start()
# wait until both tasks are done
geogrid_proc.join()
grib_proc.join()
if proc_q.get() != 'SUCCESS':
return
if proc_q.get() != 'SUCCESS':
return
proc_q.close()
# step 5: execute metgrid after ensuring all grids will be processed
js.domain_conf.prepare_for_metgrid(js.wps_nml)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
logging.info("running METGRID")
Metgrid(js.wps_dir).execute().check_output()
send_email(js, 'metgrid', 'Job %s - metgrid complete.' % js.job_id)
logging.info("cloning WRF into %s" % js.wrf_dir)
# step 6: clone wrf directory, symlink all met_em* files
cln.clone_wrf(js.wrf_dir, [])
symlink_matching_files(js.wrf_dir, js.wps_dir, "met_em*")
logging.info("running REAL")
# step 7: patch input namelist, fire namelist, emissions namelist (if required)
# and execute real.exe
time_ctrl = update_time_control(js.start_utc, js.end_utc, num_doms)
js.wrf_nml['time_control'].update(time_ctrl)
update_namelist(js.wrf_nml, js.grib_source.namelist_keys())
if 'ignitions' in args:
update_namelist(js.wrf_nml, render_ignitions(js, num_doms))
# if we have an emissions namelist, automatically turn on the tracers
if js.ems_nml is not None:
f90nml.write(js.ems_nml,
osp.join(js.wrf_dir, 'namelist.fire_emissions'),
force=True)
js.wrf_nml['dynamics']['tracer_opt'] = [2] * num_doms
f90nml.write(js.wrf_nml,
osp.join(js.wrf_dir, 'namelist.input'),
force=True)
f90nml.write(js.fire_nml,
osp.join(js.wrf_dir, 'namelist.fire'),
force=True)
# try to run Real twice as it sometimes fails the first time
# it's not clear why this error happens
try:
Real(js.wrf_dir).execute().check_output()
except Exception as e:
logging.error('Real step failed with exception %s, retrying ...' %
str(e))
Real(js.wrf_dir).execute().check_output()
# step 8: if requested, do fuel moisture DA
if js.fmda is not None:
logging.info('running fuel moisture data assimilation')
for dom in js.fmda.domains:
assimilate_fm10_observations(
osp.join(wrf_dir, 'wrfinput_d%02d' % dom), None, js.fmda.token)
logging.info('submitting WRF job')
send_email(js, 'wrf_submit', 'Job %s - wrf job submitted.' % js.job_id)
# step 8: execute wrf.exe on parallel backend
js.task_id = "sim-" + js.grid_code + "-" + utc_to_esmf(js.start_utc)[:10]
WRF(js.wrf_dir, js.qsys).submit(js.task_id, js.num_nodes, js.ppn,
js.wall_time_hrs)
send_email(
js, 'wrf_exec',
'Job %s - wrf job starting now with id %s.' % (js.job_id, js.task_id))
logging.info("WRF job submitted with id %s, waiting for rsl.error.0000" %
js.task_id)
# step 9: wait for appearance of rsl.error.0000 and open it
wrf_out = None
while wrf_out is None:
try:
wrf_out = open(osp.join(js.wrf_dir, 'rsl.error.0000'))
break
except IOError:
logging.info(
'forecast: waiting 10 seconds for rsl.error.0000 file')
time.sleep(5)
logging.info('Detected rsl.error.0000')
# step 10: track log output and check for history writes fro WRF
pp = None
already_sent_files, max_pp_dom = [], -1
if js.postproc is not None:
js.pp_dir = osp.join(js.workspace_path, js.job_id, "products")
make_dir(js.pp_dir)
pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code)
max_pp_dom = max(
[int(x) for x in filter(lambda x: len(x) == 1, js.postproc)])
while True:
line = wrf_out.readline().strip()
if not line:
time.sleep(0.2)
continue
if "SUCCESS COMPLETE WRF" in line:
send_email(js, 'complete',
'Job %s - wrf job complete SUCCESS.' % js.job_id)
logging.info("WRF completion detected.")
break
if "Timing for Writing wrfout" in line:
esmf_time, domain_str = re.match(
r'.*wrfout_d.._([0-9_\-:]{19}) for domain\ +(\d+):',
line).groups()
dom_id = int(domain_str)
logging.info("Detected history write for domain %d for time %s." %
(dom_id, esmf_time))
if js.postproc is not None and str(dom_id) in js.postproc:
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info(
"Executing postproc instructions for vars %s for domain %d."
% (str(var_list), dom_id))
wrfout_path = find_fresh_wrfout(js.wrf_dir, dom_id)
try:
pp.process_vars(wrfout_path, dom_id, esmf_time, var_list)
except Exception as e:
logging.warning(
'Failed to postprocess for time %s with error %s.' %
(esmf_time, str(e)))
# if this is the last processed domain for this timestamp in incremental mode, upload to server
if dom_id == max_pp_dom and js.postproc.get('shuttle',
None) == 'incremental':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
sent_files_1 = send_product_to_server(args, js.pp_dir,
js.job_id, js.job_id,
desc, already_sent_files)
logging.info('sent %d files to visualization server.' %
len(sent_files_1))
already_sent_files = filter(lambda x: not x.endswith('json'),
already_sent_files + sent_files_1)
# if we are to send out the postprocessed files after completion, this is the time
if js.postproc.get('shuttle', None) == 'on_completion':
desc = js.postproc[
'description'] if 'description' in js.postproc else js.job_id
send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, desc)
