def __new__(cls, year, month=1, day=1, tzinfo=None):
""
dt = to_utc(
_get_naive_datetime(year=year, month=month, day=day,
tzinfo=tzinfo))
self = datetime.date.__new__(cls, dt.year, dt.month, dt.day)
self.tzinfo = pytz.UTC
return self
def from_datetime(cls, dt):
dt = to_utc(dt)
return datetime.datetime.__new__(cls,
dt.year,
dt.month,
dt.day,
dt.hour,
dt.minute,
dt.second,
dt.microsecond,
tzinfo=pytz.UTC)
def __new__(cls,
hour=0,
minute=0,
second=0,
microsecond=0,
tzinfo=pytz.UTC):
dt = to_utc(
_get_naive_datetime(hour=hour,
minute=minute,
second=second,
microsecond=microsecond,
tzinfo=tzinfo))
return datetime.time.__new__(cls,
dt.hour,
dt.minute,
dt.second,
dt.microsecond,
tzinfo=pytz.UTC)
def create_simulation(info, conf, cluster):
"""
Build a simulation JSON configuration based on profiles and execute
the simulation using wrfxpy.
:param info: the simulation info gathered from the build page
:param conf: configuration
:param cluster: a cluster object that conveys information about the computing environment
:return: the simulation info object
"""
now = datetime.utcnow()
sim_id = 'from-web-%04d-%02d-%02d_%02d-%02d-%02d' % (
now.year, now.month, now.day, now.hour, now.minute, now.second)
# get paths of all files created
path = simulation_paths(sim_id, conf)
log_path = path['log_path']
json_path = path['json_path']
run_script = path['run_script']
# store simulation configuration
profile = info['profile']
print('profile = %s' %
json.dumps(profile, indent=1, separators=(',', ':')))
ign_lat, ign_lon = float(info['ignition_latitude']), float(
info['ignition_longitude'])
# example of ignition time: Apr 10, 1975 9:45 PM
ign_time_esmf = to_esmf(
datetime.strptime(info['ignition_time'], '%b %d, %Y %I:%M %p'))
sim_descr = info['description']
fc_hours = int(info['fc_hours'])
sim_info = {
'id': sim_id,
'started_at': to_esmf(datetime.now()),
'description': sim_descr,
'ign_latitude': ign_lat,
'ign_longitude': ign_lon,
'ign_time_esmf': ign_time_esmf,
'fc_hours': fc_hours,
'profile': info['profile'],
'log_file': log_path,
'state': make_initial_state()
}
# build a new job template
template = osp.abspath(profile['template'])
cfg = json.load(open(template))
print('Job template %s:' % template)
print json.dumps(cfg, indent=4, separators=(',', ': '))
cfg['template'] = template
cfg['profile'] = profile
cfg['grid_code'] = sim_id
# cfg['qsys'] = cluster.qsys
cfg['num_nodes'] = 6
cfg['ppn'] = cluster.ppn
ign_time = to_utc(ign_time_esmf)
sim_start = (ign_time - timedelta(minutes=30)).replace(minute=0, second=0)
sim_end = sim_start + timedelta(hours=fc_hours)
sim_info['start_utc'] = to_esmf(sim_start)
sim_info['end_utc'] = to_esmf(sim_end)
cfg['start_utc'] = to_esmf(sim_start)
cfg['end_utc'] = to_esmf(sim_end)
if not cfg.has_key('grib_source') or cfg['grib_source'] == 'auto':
cfg['grib_source'] = select_grib_source(sim_start)
print 'GRIB source not specified, selected %s from sim start time' % cfg[
'grib_source']
else:
print 'Using GRIB source %s from %s' % (cfg['grib_source'],
profile['template'])
# build wrfpy_id and the visualization link
job_id = 'wfc-%s-%s-%02d' % (sim_id, to_esmf(sim_start), fc_hours)
sim_info['job_id'] = job_id
sim_info['visualization_link'] = conf[
'wrfxweb_url'] + '/#/view1?sim_id=' + job_id
cfg['job_id'] = job_id
# place top-level domain
cfg['domains']['1']['truelats'] = [ign_lat, ign_lat]
cfg['domains']['1']['stand_lon'] = ign_lon
cfg['domains']['1']['center_latlon'] = [ign_lat, ign_lon]
# all templates have exactly one ignition
domain = cfg['ignitions'].keys()[0]
cfg['ignitions'][domain][0]['time_utc'] = ign_time_esmf
# example: "latlon" : [39.894264, -103.903222]
cfg['ignitions'][domain][0]['latlon'] = [ign_lat, ign_lon]
# switch on sending results to visualization server
cfg['postproc']['shuttle'] = 'incremental'
cfg['postproc']['description'] = sim_descr
json.dump(cfg, open(json_path, 'w'), indent=1, separators=(',', ':'))
print('Job configuration %s:' % json_path)
print json.dumps(cfg, indent=4, separators=(',', ': '))
# drop a shell script that will run the file
with open(run_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
#f.write('/usr/bin/env\n')
f.write('export PYTHONPATH=src\n')
f.write('cd ' + conf['wrfxpy_path'] + '\n')
f.write('LOG=' + osp.abspath(log_path) + '\n')
f.write('./forecast.sh ' + osp.abspath(json_path) + ' &> $LOG \n')
# make it executable
st = os.stat(run_script)
os.chmod(run_script, st.st_mode | stat.S_IEXEC)
# execute the fire forecast and reroute into the log file provided
print('Running %s' % run_script)
proc = Popen(run_script,
shell=True,
stdin=None,
stdout=None,
stderr=None,
close_fds=True)
print('Ready')
return sim_info
def __cmp(self, other):
return super(Datetime, self).__cmp(to_utc(other))
def create_simulation(info, wrfxpy_path, cluster):
"""
Build a simulation JSON configuration based on profiles and execute
the simulation using wrfxpy.
:param info: the simulation info gathered from the build page
:param profiles: the job profiles available to the user
:param cluster: a cluster object that conveys information about the computing environment
:return: the simulation info object
"""
now = datetime.utcnow()
sim_id = 'from-web-%04d-%02d-%02d_%02d-%02d-%02d' % (now.year, now.month, now.day, now.hour, now.minute, now.second)
# get paths, profiles
log_path = 'logs/%s.log' % sim_id
json_path = 'jobs/%s.json' % sim_id
profile = info['profile']
# store simulation configuration
ign_lat, ign_lon = float(info['ignition_latitude']), float(info['ignition_longitude'])
# example of ignition time: Apr 10, 1975 9:45 PM
ign_time_esmf = to_esmf(datetime.strptime(info['ignition_time'], '%b %d, %Y %I:%M %p'))
sim_descr = info['description']
fc_hours = int(info['fc_hours'])
sim_info = {
'id' : sim_id,
'started_at' : to_esmf(datetime.now()),
'description' : sim_descr,
'ign_latitude' : ign_lat,
'ign_longitude' : ign_lon,
'ign_time_esmf' : ign_time_esmf,
'fc_hours' : fc_hours,
'profile' : info['profile'],
'log_file' : log_path,
'state' : make_initial_state()
}
# build a new job template
cfg = json.load(open(profile['template']))
cfg['grid_code'] = sim_id
cfg['qsys'] = cluster.qsys
cfg['num_nodes'] = 6
cfg['ppn'] = cluster.ppn
ign_time = to_utc(ign_time_esmf)
sim_start = (ign_time - timedelta(minutes=30)).replace(minute=0, second=0)
sim_end = sim_start + timedelta(hours=fc_hours)
sim_info['start_utc'] = to_esmf(sim_start)
sim_info['end_utc'] = to_esmf(sim_end)
cfg['start_utc'] = to_esmf(sim_start)
cfg['end_utc'] = to_esmf(sim_end)
# build the visualization link
wrfxpy_id = 'wfc-%s-%s-%02d' % (sim_id, to_esmf(sim_start), fc_hours)
sim_info['visualization_link'] = 'http://demo.openwfm.org/fdds/#/view1?sim_id=' + wrfxpy_id
# place top-level domain
cfg['domains']['1']['truelats'] = [ign_lat, ign_lat]
cfg['domains']['1']['stand_lon'] = ign_lon
cfg['domains']['1']['center_latlon'] = [ign_lat, ign_lon]
# all templates have exactly one ignition
domain = cfg['ignitions'].keys()[0]
cfg['ignitions'][domain][0]['time_utc'] = ign_time_esmf
# example: "latlon" : [39.894264, -103.903222]
cfg['ignitions'][domain][0]['latlon'] = [ign_lat, ign_lon]
# switch on sending results to visualization server
cfg['postproc']['shuttle'] = 'incremental'
cfg['postproc']['description'] = sim_descr
json.dump(cfg, open(json_path, 'w'))
# drop a shell script that will run the file
run_script = 'jobs/' + sim_id + '.sh'
with open(run_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
f.write('export PYTHONPATH=src\n')
f.write('cd ' + wrfxpy_path + '\n')
f.write('LOG=' + osp.abspath(log_path) + '\n')
f.write('./forecast.sh ' + osp.abspath(json_path) + ' &> $LOG \n')
# make it executable
st = os.stat(run_script)
os.chmod(run_script, st.st_mode | stat.S_IEXEC)
# execute the fire forecast and reroute into the log file provided
proc = Popen('./' + run_script, shell=True, stdin=None, stdout=None, stderr=None, close_fds=True)
return sim_info
def create_simulation(info, conf, cluster):
"""
Build a simulation JSON configuration based on profiles and execute
the simulation using wrfxpy.
:param info: the simulation info gathered from the build page
:param conf: configuration
:param cluster: a cluster object that conveys information about the computing environment
:return: the simulation info object
"""
now = datetime.utcnow()
sim_id = 'from-web-%04d-%02d-%02d_%02d-%02d-%02d' % (now.year, now.month, now.day, now.hour, now.minute, now.second)
# get paths of all files created
path= simulation_paths(sim_id,conf)
log_path = path['log_path']
json_path = path['json_path']
run_script = path['run_script']
# store simulation configuration
profile = info['profile']
print ('profile = %s' % json.dumps(profile,indent=1, separators=(',',':')))
ign_lat, ign_lon = float(info['ignition_latitude']), float(info['ignition_longitude'])
# example of ignition time: Apr 10, 1975 9:45 PM
ign_time_esmf = to_esmf(datetime.strptime(info['ignition_time'], '%b %d, %Y %I:%M %p'))
sim_descr = info['description']
fc_hours = int(info['fc_hours'])
sim_info = {
'id' : sim_id,
'started_at' : to_esmf(datetime.now()),
'description' : sim_descr,
'ign_latitude' : ign_lat,
'ign_longitude' : ign_lon,
'ign_time_esmf' : ign_time_esmf,
'fc_hours' : fc_hours,
'profile' : info['profile'],
'log_file' : log_path,
'state' : make_initial_state()
}
# build a new job template
template = osp.abspath(profile['template'])
cfg = json.load(open(template))
print ('Job template %s:' % template)
print json.dumps(cfg, indent=4, separators=(',', ': '))
cfg['template'] = template
cfg['profile'] = profile
cfg['grid_code'] = sim_id
# cfg['qsys'] = cluster.qsys
cfg['num_nodes'] = 6
cfg['ppn'] = cluster.ppn
ign_time = to_utc(ign_time_esmf)
sim_start = (ign_time - timedelta(minutes=30)).replace(minute=0, second=0)
sim_end = sim_start + timedelta(hours=fc_hours)
sim_info['start_utc'] = to_esmf(sim_start)
sim_info['end_utc'] = to_esmf(sim_end)
cfg['start_utc'] = to_esmf(sim_start)
cfg['end_utc'] = to_esmf(sim_end)
if not cfg.has_key('grib_source') or cfg['grib_source'] == 'auto':
cfg['grib_source'] = select_grib_source(sim_start)
print 'GRIB source not specified, selected %s from sim start time' % cfg['grib_source']
else:
print 'Using GRIB source %s from %s' % (cfg['grib_source'], profile['template'])
# build wrfpy_id and the visualization link
job_id = 'wfc-%s-%s-%02d' % (sim_id, to_esmf(sim_start), fc_hours)
sim_info['job_id']=job_id
sim_info['visualization_link'] = conf['wrfxweb_url'] + '/#/view1?sim_id=' + job_id
cfg['job_id']=job_id
# place top-level domain
cfg['domains']['1']['truelats'] = [ign_lat, ign_lat]
cfg['domains']['1']['stand_lon'] = ign_lon
cfg['domains']['1']['center_latlon'] = [ign_lat, ign_lon]
# all templates have exactly one ignition
domain = cfg['ignitions'].keys()[0]
cfg['ignitions'][domain][0]['time_utc'] = ign_time_esmf
# example: "latlon" : [39.894264, -103.903222]
cfg['ignitions'][domain][0]['latlon'] = [ign_lat, ign_lon]
# switch on sending results to visualization server
cfg['postproc']['shuttle'] = 'incremental'
cfg['postproc']['description'] = sim_descr
json.dump(cfg, open(json_path, 'w'),indent=1, separators=(',',':'))
print ('Job configuration %s:' % json_path)
print json.dumps(cfg, indent=4, separators=(',', ': '))
# drop a shell script that will run the file
with open(run_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
#f.write('/usr/bin/env\n')
f.write('export PYTHONPATH=src\n')
f.write('cd ' + conf['wrfxpy_path'] + '\n')
f.write('LOG=' + osp.abspath(log_path) + '\n')
f.write('./forecast.sh ' + osp.abspath(json_path) + ' &> $LOG \n')
# make it executable
st = os.stat(run_script)
os.chmod(run_script, st.st_mode | stat.S_IEXEC)
# execute the fire forecast and reroute into the log file provided
print('Running %s' % run_script)
proc = Popen(run_script, shell=True, stdin=None, stdout=None, stderr=None, close_fds=True)
print('Ready')
return sim_info
def create_simulation(info, wrfxpy_path, cluster):
"""
Build a simulation JSON configuration based on profiles and execute
the simulation using wrfxpy.
:param info: the simulation info gathered from the build page
:param profiles: the job profiles available to the user
:param cluster: a cluster object that conveys information about the computing environment
:return: the simulation info object
"""
now = datetime.utcnow()
sim_id = 'from-web-%04d-%02d-%02d_%02d-%02d-%02d' % (
now.year, now.month, now.day, now.hour, now.minute, now.second)
# get paths, profiles
log_path = 'logs/%s.log' % sim_id
json_path = 'jobs/%s.json' % sim_id
profile = info['profile']
# store simulation configuration
ign_lat, ign_lon = float(info['ignition_latitude']), float(
info['ignition_longitude'])
# example of ignition time: Apr 10, 1975 9:45 PM
ign_time_esmf = to_esmf(
datetime.strptime(info['ignition_time'], '%b %d, %Y %I:%M %p'))
sim_descr = info['description']
fc_hours = int(info['fc_hours'])
sim_info = {
'id': sim_id,
'started_at': to_esmf(datetime.now()),
'description': sim_descr,
'ign_latitude': ign_lat,
'ign_longitude': ign_lon,
'ign_time_esmf': ign_time_esmf,
'fc_hours': fc_hours,
'profile': info['profile'],
'log_file': log_path,
'state': make_initial_state()
}
# build a new job template
cfg = json.load(open(profile['template']))
cfg['grid_code'] = sim_id
cfg['qsys'] = cluster.qsys
cfg['num_nodes'] = 6
cfg['ppn'] = cluster.ppn
ign_time = to_utc(ign_time_esmf)
sim_start = (ign_time - timedelta(minutes=30)).replace(minute=0, second=0)
sim_end = sim_start + timedelta(hours=fc_hours)
sim_info['start_utc'] = to_esmf(sim_start)
sim_info['end_utc'] = to_esmf(sim_end)
cfg['start_utc'] = to_esmf(sim_start)
cfg['end_utc'] = to_esmf(sim_end)
# build the visualization link
wrfxpy_id = 'wfc-%s-%s-%02d' % (sim_id, to_esmf(sim_start), fc_hours)
sim_info[
'visualization_link'] = 'http://demo.openwfm.org/fdds/#/view1?sim_id=' + wrfxpy_id
# place top-level domain
cfg['domains']['1']['truelats'] = [ign_lat, ign_lat]
cfg['domains']['1']['stand_lon'] = ign_lon
cfg['domains']['1']['center_latlon'] = [ign_lat, ign_lon]
# all templates have exactly one ignition
domain = cfg['ignitions'].keys()[0]
cfg['ignitions'][domain][0]['time_utc'] = ign_time_esmf
# example: "latlon" : [39.894264, -103.903222]
cfg['ignitions'][domain][0]['latlon'] = [ign_lat, ign_lon]
# switch on sending results to visualization server
cfg['postproc']['shuttle'] = 'incremental'
cfg['postproc']['description'] = sim_descr
json.dump(cfg, open(json_path, 'w'))
# drop a shell script that will run the file
run_script = 'jobs/' + sim_id + '.sh'
with open(run_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
f.write('export PYTHONPATH=src\n')
f.write('cd ' + wrfxpy_path + '\n')
f.write('LOG=' + osp.abspath(log_path) + '\n')
f.write('./forecast.sh ' + osp.abspath(json_path) + ' &> $LOG \n')
# make it executable
st = os.stat(run_script)
os.chmod(run_script, st.st_mode | stat.S_IEXEC)
# execute the fire forecast and reroute into the log file provided
proc = Popen('./' + run_script,
shell=True,
stdin=None,
stdout=None,
stderr=None,
close_fds=True)
return sim_info
def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.UTC):
dt = to_utc(_get_naive_datetime(hour=hour, minute=minute,
second=second, microsecond=microsecond,
tzinfo=tzinfo))
return datetime.time.__new__(cls, dt.hour, dt.minute, dt.second, dt.microsecond, tzinfo=pytz.UTC)
def __new__(cls, year, month=1, day=1, tzinfo=None):
""
dt = to_utc(_get_naive_datetime(year=year, month=month, day=day, tzinfo=tzinfo))
self = datetime.date.__new__(cls, dt.year, dt.month, dt.day)
self.tzinfo = pytz.UTC
return self
def __cmp(self, other):
return super(Datetime, self).__cmp(to_utc(other))
def from_datetime(cls, dt):
dt = to_utc(dt)
return datetime.datetime.__new__(cls, dt.year, dt.month, dt.day,
dt.hour, dt.minute, dt.second, dt.microsecond,
tzinfo=pytz.UTC)