def overview():
deadline = to_esmf(datetime.now() - timedelta(seconds=5))
# only update stale & running simulations in overview
for sim_id,sim in simulations.iteritems():
if sim['state']['wrf'] != 'complete':
last_upd = sim.get('last_updated', '2000-01-01_00:00:00')
if last_upd < deadline:
sim['state'] = get_simulation_state(sim['log_file'])
sim['last_updated'] = to_esmf(datetime.now())
json.dump(sim, open('simulations/' + sim_id + '.json', 'w'), indent=4, separators=(',', ': '))
return render_template('overview.html', simulations = simulations)
def overview():
deadline = to_esmf(datetime.now() - timedelta(seconds=5))
# only update stale & running simulations in overview
for sim_id, sim in simulations.iteritems():
if sim['state']['wrf'] != 'complete':
last_upd = sim.get('last_updated', '2000-01-01_00:00:00')
if last_upd < deadline:
sim['state'] = get_simulation_state(sim['log_file'])
sim['last_updated'] = to_esmf(datetime.now())
json.dump(sim,
open('simulations/' + sim_id + '.json', 'w'),
indent=4,
separators=(',', ': '))
return render_template('overview.html', simulations=simulations)
def overview():
if request.method == 'GET':
# reload, cleanup might delete jsons while webserver is running
simulations = load_simulations(sims_path)
deadline = to_esmf(datetime.now() - timedelta(seconds=5))
# only update stale & running simulations in overview
kk = simulations.keys()
for sim_id in kk:
sim = simulations[sim_id]
if sim['state']['wrf'] != 'complete':
last_upd = sim.get('last_updated', '2000-01-01_00:00:00')
if last_upd < deadline:
sim['state'] = get_simulation_state(sim['log_file'])
sim['last_updated'] = to_esmf(datetime.now())
f = sims_path + '/' + sim_id + '.json'
if osp.isfile(f):
json.dump(sim,
open(f, 'w'),
indent=4,
separators=(',', ': '))
simulations[sim_id] = sim
else:
print('File %s no longer exists, deleting simulation' %
f)
del simulations[sim_id]
return render_template('overview.html',
simulations=simulations,
urls=urls)
elif request.method == 'POST':
print 'Values returned by overview page:'
sims_checked = request.form.getlist('sim_chk')
print(sims_checked)
for sim_id in sims_checked: # Only the simulation(s) checked in checkbox.
if 'RemoveB' in request.form:
print('Remove Sim: box checked= %s' % (sim_id))
cleanup_delete(sim_id)
else:
if 'CancelB' in request.form:
print('Cancel Sim: box checked= %s' % (sim_id))
cleanup_cancel(sim_id)
else:
print('Error-No button push detected: box checked= %s' %
(sim_id))
simulations = load_simulations(sims_path)
return render_template('overview.html',
simulations=simulations,
urls=urls)
def get_state(sim_id=None):
sim_info = simulations.get(sim_id, None)
if sim_info is None:
return "{}"
else:
sim_state = None
# always update during get_state()
if sim_info['state']['wrf'] != 'completed':
sim_state = get_simulation_state(sim_info['log_file'])
sim_info['state'] = sim_state
sim_info['last_updated'] = to_esmf(datetime.now())
json.dump(sim_info, open('simulations/' + sim_id + '.json', 'w'))
return json.dumps(sim_state)
def get_state(sim_id=None):
sim_info = simulations.get(sim_id, None)
if sim_info is None:
return "{}"
else:
sim_state = None
# always update during get_state()
if sim_info['state']['wrf'] != 'completed':
sim_state = get_simulation_state(sim_info['log_file'])
sim_info['state'] = sim_state
sim_info['last_updated'] = to_esmf(datetime.now())
json.dump(sim_info, open('simulations/' + sim_id + '.json', 'w'))
return json.dumps(sim_state)
def overview():
if request.method == 'GET':
# reload, cleanup might delete jsons while webserver is running
simulations = load_simulations(sims_path)
deadline = to_esmf(datetime.now() - timedelta(seconds=5))
# only update stale & running simulations in overview
kk = simulations.keys()
for sim_id in kk:
sim = simulations[sim_id]
if sim['state']['wrf'] != 'complete':
last_upd = sim.get('last_updated', '2000-01-01_00:00:00')
if last_upd < deadline:
sim['state'] = get_simulation_state(sim['log_file'])
sim['last_updated'] = to_esmf(datetime.now())
f = sims_path + '/' + sim_id + '.json'
if osp.isfile(f):
json.dump(sim, open(f,'w'), indent=4, separators=(',', ': '))
simulations[sim_id]=sim
else:
print('File %s no longer exists, deleting simulation' % f)
del simulations[sim_id]
return render_template('overview.html', simulations = simulations, urls=urls)
elif request.method == 'POST':
print 'Values returned by overview page:'
sims_checked= request.form.getlist('sim_chk')
print (sims_checked)
for sim_id in sims_checked: # Only the simulation(s) checked in checkbox.
if 'RemoveB' in request.form:
print ('Remove Sim: box checked= %s' % (sim_id))
cleanup_delete(sim_id)
else:
if 'CancelB' in request.form:
print ('Cancel Sim: box checked= %s' % (sim_id))
cleanup_cancel(sim_id)
else:
print ('Error-No button push detected: box checked= %s' % (sim_id))
simulations = load_simulations(sims_path)
return render_template('overview.html', simulations = simulations, urls=urls)
def create_simulation(info, conf, cluster):
"""
Build a simulation JSON configuration based on profiles and execute
the simulation using wrfxpy.
:param info: the simulation info gathered from the build page
:param conf: configuration
:param cluster: a cluster object that conveys information about the computing environment
:return: the simulation info object
"""
now = datetime.utcnow()
sim_id = 'from-web-%04d-%02d-%02d_%02d-%02d-%02d' % (
now.year, now.month, now.day, now.hour, now.minute, now.second)
# get paths of all files created
path = simulation_paths(sim_id, conf)
log_path = path['log_path']
json_path = path['json_path']
run_script = path['run_script']
# store simulation configuration
profile = info['profile']
print('profile = %s' %
json.dumps(profile, indent=1, separators=(',', ':')))
ign_lat, ign_lon = float(info['ignition_latitude']), float(
info['ignition_longitude'])
# example of ignition time: Apr 10, 1975 9:45 PM
ign_time_esmf = to_esmf(
datetime.strptime(info['ignition_time'], '%b %d, %Y %I:%M %p'))
sim_descr = info['description']
fc_hours = int(info['fc_hours'])
sim_info = {
'id': sim_id,
'started_at': to_esmf(datetime.now()),
'description': sim_descr,
'ign_latitude': ign_lat,
'ign_longitude': ign_lon,
'ign_time_esmf': ign_time_esmf,
'fc_hours': fc_hours,
'profile': info['profile'],
'log_file': log_path,
'state': make_initial_state()
}
# build a new job template
template = osp.abspath(profile['template'])
cfg = json.load(open(template))
print('Job template %s:' % template)
print json.dumps(cfg, indent=4, separators=(',', ': '))
cfg['template'] = template
cfg['profile'] = profile
cfg['grid_code'] = sim_id
# cfg['qsys'] = cluster.qsys
cfg['num_nodes'] = 6
cfg['ppn'] = cluster.ppn
ign_time = to_utc(ign_time_esmf)
sim_start = (ign_time - timedelta(minutes=30)).replace(minute=0, second=0)
sim_end = sim_start + timedelta(hours=fc_hours)
sim_info['start_utc'] = to_esmf(sim_start)
sim_info['end_utc'] = to_esmf(sim_end)
cfg['start_utc'] = to_esmf(sim_start)
cfg['end_utc'] = to_esmf(sim_end)
if not cfg.has_key('grib_source') or cfg['grib_source'] == 'auto':
cfg['grib_source'] = select_grib_source(sim_start)
print 'GRIB source not specified, selected %s from sim start time' % cfg[
'grib_source']
else:
print 'Using GRIB source %s from %s' % (cfg['grib_source'],
profile['template'])
# build wrfpy_id and the visualization link
job_id = 'wfc-%s-%s-%02d' % (sim_id, to_esmf(sim_start), fc_hours)
sim_info['job_id'] = job_id
sim_info['visualization_link'] = conf[
'wrfxweb_url'] + '/#/view1?sim_id=' + job_id
cfg['job_id'] = job_id
# place top-level domain
cfg['domains']['1']['truelats'] = [ign_lat, ign_lat]
cfg['domains']['1']['stand_lon'] = ign_lon
cfg['domains']['1']['center_latlon'] = [ign_lat, ign_lon]
# all templates have exactly one ignition
domain = cfg['ignitions'].keys()[0]
cfg['ignitions'][domain][0]['time_utc'] = ign_time_esmf
# example: "latlon" : [39.894264, -103.903222]
cfg['ignitions'][domain][0]['latlon'] = [ign_lat, ign_lon]
# switch on sending results to visualization server
cfg['postproc']['shuttle'] = 'incremental'
cfg['postproc']['description'] = sim_descr
json.dump(cfg, open(json_path, 'w'), indent=1, separators=(',', ':'))
print('Job configuration %s:' % json_path)
print json.dumps(cfg, indent=4, separators=(',', ': '))
# drop a shell script that will run the file
with open(run_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
#f.write('/usr/bin/env\n')
f.write('export PYTHONPATH=src\n')
f.write('cd ' + conf['wrfxpy_path'] + '\n')
f.write('LOG=' + osp.abspath(log_path) + '\n')
f.write('./forecast.sh ' + osp.abspath(json_path) + ' &> $LOG \n')
# make it executable
st = os.stat(run_script)
os.chmod(run_script, st.st_mode | stat.S_IEXEC)
# execute the fire forecast and reroute into the log file provided
print('Running %s' % run_script)
proc = Popen(run_script,
shell=True,
stdin=None,
stdout=None,
stderr=None,
close_fds=True)
print('Ready')
return sim_info
def create_simulation(info, conf, cluster):
"""
Build a simulation JSON configuration based on profiles and execute
the simulation using wrfxpy.
:param info: the simulation info gathered from the build page
:param conf: configuration
:param cluster: a cluster object that conveys information about the computing environment
:return: the simulation info object
"""
now = datetime.utcnow()
sim_id = 'from-web-%04d-%02d-%02d_%02d-%02d-%02d' % (now.year, now.month, now.day, now.hour, now.minute, now.second)
# get paths of all files created
path= simulation_paths(sim_id,conf)
log_path = path['log_path']
json_path = path['json_path']
run_script = path['run_script']
# store simulation configuration
profile = info['profile']
print ('profile = %s' % json.dumps(profile,indent=1, separators=(',',':')))
ign_lat, ign_lon = float(info['ignition_latitude']), float(info['ignition_longitude'])
# example of ignition time: Apr 10, 1975 9:45 PM
ign_time_esmf = to_esmf(datetime.strptime(info['ignition_time'], '%b %d, %Y %I:%M %p'))
sim_descr = info['description']
fc_hours = int(info['fc_hours'])
sim_info = {
'id' : sim_id,
'started_at' : to_esmf(datetime.now()),
'description' : sim_descr,
'ign_latitude' : ign_lat,
'ign_longitude' : ign_lon,
'ign_time_esmf' : ign_time_esmf,
'fc_hours' : fc_hours,
'profile' : info['profile'],
'log_file' : log_path,
'state' : make_initial_state()
}
# build a new job template
template = osp.abspath(profile['template'])
cfg = json.load(open(template))
print ('Job template %s:' % template)
print json.dumps(cfg, indent=4, separators=(',', ': '))
cfg['template'] = template
cfg['profile'] = profile
cfg['grid_code'] = sim_id
# cfg['qsys'] = cluster.qsys
cfg['num_nodes'] = 6
cfg['ppn'] = cluster.ppn
ign_time = to_utc(ign_time_esmf)
sim_start = (ign_time - timedelta(minutes=30)).replace(minute=0, second=0)
sim_end = sim_start + timedelta(hours=fc_hours)
sim_info['start_utc'] = to_esmf(sim_start)
sim_info['end_utc'] = to_esmf(sim_end)
cfg['start_utc'] = to_esmf(sim_start)
cfg['end_utc'] = to_esmf(sim_end)
if not cfg.has_key('grib_source') or cfg['grib_source'] == 'auto':
cfg['grib_source'] = select_grib_source(sim_start)
print 'GRIB source not specified, selected %s from sim start time' % cfg['grib_source']
else:
print 'Using GRIB source %s from %s' % (cfg['grib_source'], profile['template'])
# build wrfpy_id and the visualization link
job_id = 'wfc-%s-%s-%02d' % (sim_id, to_esmf(sim_start), fc_hours)
sim_info['job_id']=job_id
sim_info['visualization_link'] = conf['wrfxweb_url'] + '/#/view1?sim_id=' + job_id
cfg['job_id']=job_id
# place top-level domain
cfg['domains']['1']['truelats'] = [ign_lat, ign_lat]
cfg['domains']['1']['stand_lon'] = ign_lon
cfg['domains']['1']['center_latlon'] = [ign_lat, ign_lon]
# all templates have exactly one ignition
domain = cfg['ignitions'].keys()[0]
cfg['ignitions'][domain][0]['time_utc'] = ign_time_esmf
# example: "latlon" : [39.894264, -103.903222]
cfg['ignitions'][domain][0]['latlon'] = [ign_lat, ign_lon]
# switch on sending results to visualization server
cfg['postproc']['shuttle'] = 'incremental'
cfg['postproc']['description'] = sim_descr
json.dump(cfg, open(json_path, 'w'),indent=1, separators=(',',':'))
print ('Job configuration %s:' % json_path)
print json.dumps(cfg, indent=4, separators=(',', ': '))
# drop a shell script that will run the file
with open(run_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
#f.write('/usr/bin/env\n')
f.write('export PYTHONPATH=src\n')
f.write('cd ' + conf['wrfxpy_path'] + '\n')
f.write('LOG=' + osp.abspath(log_path) + '\n')
f.write('./forecast.sh ' + osp.abspath(json_path) + ' &> $LOG \n')
# make it executable
st = os.stat(run_script)
os.chmod(run_script, st.st_mode | stat.S_IEXEC)
# execute the fire forecast and reroute into the log file provided
print('Running %s' % run_script)
proc = Popen(run_script, shell=True, stdin=None, stdout=None, stderr=None, close_fds=True)
print('Ready')
return sim_info
def create_simulation(info, wrfxpy_path, cluster):
"""
Build a simulation JSON configuration based on profiles and execute
the simulation using wrfxpy.
:param info: the simulation info gathered from the build page
:param profiles: the job profiles available to the user
:param cluster: a cluster object that conveys information about the computing environment
:return: the simulation info object
"""
now = datetime.utcnow()
sim_id = 'from-web-%04d-%02d-%02d_%02d-%02d-%02d' % (now.year, now.month, now.day, now.hour, now.minute, now.second)
# get paths, profiles
log_path = 'logs/%s.log' % sim_id
json_path = 'jobs/%s.json' % sim_id
profile = info['profile']
# store simulation configuration
ign_lat, ign_lon = float(info['ignition_latitude']), float(info['ignition_longitude'])
# example of ignition time: Apr 10, 1975 9:45 PM
ign_time_esmf = to_esmf(datetime.strptime(info['ignition_time'], '%b %d, %Y %I:%M %p'))
sim_descr = info['description']
fc_hours = int(info['fc_hours'])
sim_info = {
'id' : sim_id,
'started_at' : to_esmf(datetime.now()),
'description' : sim_descr,
'ign_latitude' : ign_lat,
'ign_longitude' : ign_lon,
'ign_time_esmf' : ign_time_esmf,
'fc_hours' : fc_hours,
'profile' : info['profile'],
'log_file' : log_path,
'state' : make_initial_state()
}
# build a new job template
cfg = json.load(open(profile['template']))
cfg['grid_code'] = sim_id
cfg['qsys'] = cluster.qsys
cfg['num_nodes'] = 6
cfg['ppn'] = cluster.ppn
ign_time = to_utc(ign_time_esmf)
sim_start = (ign_time - timedelta(minutes=30)).replace(minute=0, second=0)
sim_end = sim_start + timedelta(hours=fc_hours)
sim_info['start_utc'] = to_esmf(sim_start)
sim_info['end_utc'] = to_esmf(sim_end)
cfg['start_utc'] = to_esmf(sim_start)
cfg['end_utc'] = to_esmf(sim_end)
# build the visualization link
wrfxpy_id = 'wfc-%s-%s-%02d' % (sim_id, to_esmf(sim_start), fc_hours)
sim_info['visualization_link'] = 'http://demo.openwfm.org/fdds/#/view1?sim_id=' + wrfxpy_id
# place top-level domain
cfg['domains']['1']['truelats'] = [ign_lat, ign_lat]
cfg['domains']['1']['stand_lon'] = ign_lon
cfg['domains']['1']['center_latlon'] = [ign_lat, ign_lon]
# all templates have exactly one ignition
domain = cfg['ignitions'].keys()[0]
cfg['ignitions'][domain][0]['time_utc'] = ign_time_esmf
# example: "latlon" : [39.894264, -103.903222]
cfg['ignitions'][domain][0]['latlon'] = [ign_lat, ign_lon]
# switch on sending results to visualization server
cfg['postproc']['shuttle'] = 'incremental'
cfg['postproc']['description'] = sim_descr
json.dump(cfg, open(json_path, 'w'))
# drop a shell script that will run the file
run_script = 'jobs/' + sim_id + '.sh'
with open(run_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
f.write('export PYTHONPATH=src\n')
f.write('cd ' + wrfxpy_path + '\n')
f.write('LOG=' + osp.abspath(log_path) + '\n')
f.write('./forecast.sh ' + osp.abspath(json_path) + ' &> $LOG \n')
# make it executable
st = os.stat(run_script)
os.chmod(run_script, st.st_mode | stat.S_IEXEC)
# execute the fire forecast and reroute into the log file provided
proc = Popen('./' + run_script, shell=True, stdin=None, stdout=None, stderr=None, close_fds=True)
return sim_info
def create_simulation(info, wrfxpy_path, cluster):
"""
Build a simulation JSON configuration based on profiles and execute
the simulation using wrfxpy.
:param info: the simulation info gathered from the build page
:param profiles: the job profiles available to the user
:param cluster: a cluster object that conveys information about the computing environment
:return: the simulation info object
"""
now = datetime.utcnow()
sim_id = 'from-web-%04d-%02d-%02d_%02d-%02d-%02d' % (
now.year, now.month, now.day, now.hour, now.minute, now.second)
# get paths, profiles
log_path = 'logs/%s.log' % sim_id
json_path = 'jobs/%s.json' % sim_id
profile = info['profile']
# store simulation configuration
ign_lat, ign_lon = float(info['ignition_latitude']), float(
info['ignition_longitude'])
# example of ignition time: Apr 10, 1975 9:45 PM
ign_time_esmf = to_esmf(
datetime.strptime(info['ignition_time'], '%b %d, %Y %I:%M %p'))
sim_descr = info['description']
fc_hours = int(info['fc_hours'])
sim_info = {
'id': sim_id,
'started_at': to_esmf(datetime.now()),
'description': sim_descr,
'ign_latitude': ign_lat,
'ign_longitude': ign_lon,
'ign_time_esmf': ign_time_esmf,
'fc_hours': fc_hours,
'profile': info['profile'],
'log_file': log_path,
'state': make_initial_state()
}
# build a new job template
cfg = json.load(open(profile['template']))
cfg['grid_code'] = sim_id
cfg['qsys'] = cluster.qsys
cfg['num_nodes'] = 6
cfg['ppn'] = cluster.ppn
ign_time = to_utc(ign_time_esmf)
sim_start = (ign_time - timedelta(minutes=30)).replace(minute=0, second=0)
sim_end = sim_start + timedelta(hours=fc_hours)
sim_info['start_utc'] = to_esmf(sim_start)
sim_info['end_utc'] = to_esmf(sim_end)
cfg['start_utc'] = to_esmf(sim_start)
cfg['end_utc'] = to_esmf(sim_end)
# build the visualization link
wrfxpy_id = 'wfc-%s-%s-%02d' % (sim_id, to_esmf(sim_start), fc_hours)
sim_info[
'visualization_link'] = 'http://demo.openwfm.org/fdds/#/view1?sim_id=' + wrfxpy_id
# place top-level domain
cfg['domains']['1']['truelats'] = [ign_lat, ign_lat]
cfg['domains']['1']['stand_lon'] = ign_lon
cfg['domains']['1']['center_latlon'] = [ign_lat, ign_lon]
# all templates have exactly one ignition
domain = cfg['ignitions'].keys()[0]
cfg['ignitions'][domain][0]['time_utc'] = ign_time_esmf
# example: "latlon" : [39.894264, -103.903222]
cfg['ignitions'][domain][0]['latlon'] = [ign_lat, ign_lon]
# switch on sending results to visualization server
cfg['postproc']['shuttle'] = 'incremental'
cfg['postproc']['description'] = sim_descr
json.dump(cfg, open(json_path, 'w'))
# drop a shell script that will run the file
run_script = 'jobs/' + sim_id + '.sh'
with open(run_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
f.write('export PYTHONPATH=src\n')
f.write('cd ' + wrfxpy_path + '\n')
f.write('LOG=' + osp.abspath(log_path) + '\n')
f.write('./forecast.sh ' + osp.abspath(json_path) + ' &> $LOG \n')
# make it executable
st = os.stat(run_script)
os.chmod(run_script, st.st_mode | stat.S_IEXEC)
# execute the fire forecast and reroute into the log file provided
proc = Popen('./' + run_script,
shell=True,
stdin=None,
stdout=None,
stderr=None,
close_fds=True)
return sim_info