def assert_write_path(self, nml, target_fname, sort=False):
tmp_fname = 'tmp.nml'
f90nml.write(nml, tmp_fname, sort=sort)
try:
self.assert_file_equal(tmp_fname, target_fname)
finally:
os.remove(tmp_fname)
def retrieve_gribs_and_run_ungrib(js, q):
"""
This function retrieves required GRIB files and runs ungrib.
It returns either 'SUCCESS' or 'FAILURE' on completion.
:param js: the JobState object containing the forecast configuration
:param q: the multiprocessing Queue into which we will send either 'SUCCESS' or 'FAILURE'
"""
try:
logging.info("retrieving GRIB files.")
# step 3: retrieve required GRIB files from the grib_source, symlink into GRIBFILE.XYZ links into wps
manifest = js.grib_source.retrieve_gribs(js.start_utc, js.end_utc)
js.grib_source.symlink_gribs(manifest, js.wps_dir)
send_email(js, 'grib2', 'Job %s - %d GRIB2 files downloaded.' % (js.job_id, len(manifest)))
logging.info("running UNGRIB")
# step 4: patch namelist for ungrib end execute ungrib
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
Ungrib(js.wps_dir).execute().check_output()
send_email(js, 'ungrib', 'Job %s - ungrib complete.' % js.job_id)
logging.info('UNGRIB complete')
q.put('SUCCESS')
except Exception as e:
logging.error('GRIB2/UNGRIB step failed with exception %s' % repr(e))
traceback.print_exc()
q.put('FAILURE')
def assert_write(self, nml, target_fname):
tmp_fname = "tmp.nml"
f90nml.write(nml, tmp_fname)
try:
self.assert_file_equal(tmp_fname, target_fname)
finally:
os.remove(tmp_fname)
def write_wxy(wxy,filename):
e = os.path.isfile(filename)
if e:
print(filename, "exist")
os.remove(filename)
print(filename,"deleted")
f90nml.write(wxy,filename);
print("write a new wxy:",filename)
def assert_write_file(self, nml, target_fname, sort=False):
tmp_fname = 'tmp.nml'
with open(tmp_fname, 'w') as tmp_file:
f90nml.write(nml, tmp_file, sort=sort)
self.assertFalse(tmp_file.closed)
try:
self.assert_file_equal(tmp_fname, target_fname)
finally:
os.remove(tmp_fname)
def init_config(self):
"""Patch input.nml as a new or restart run."""
input_fpath = os.path.join(self.work_path, 'input.nml')
input_nml = f90nml.read(input_fpath)
input_type = 'n' if self.expt.counter == 0 else 'r'
input_nml['GOLD_input_nml']['input_filename'] = input_type
f90nml.write(input_nml, input_fpath, force=True)
def init_config(self):
"""Patch input.nml as a new or restart run."""
input_fpath = os.path.join(self.work_path, 'input.nml')
input_nml = f90nml.read(input_fpath)
if self.expt.counter == 0 or self.expt.repeat_run:
input_type = 'n'
else:
input_type = 'r'
input_nml['MOM_input_nml']['input_filename'] = input_type
f90nml.write(input_nml, input_fpath, force=True)
def _write_file(self, outputfile, rawvals, do_patch=False, nml_file=None):
patch=defaultdict( dict )
for key,rawval in rawvals.items():
if rawval is None: # omit if value is None
continue
if isinstance(rawval,numpy.ndarray):
rawval=list(rawval) # necessary until f90nml supports numpy arrays
patch[key[1]][key[0]]=rawval
if do_patch:
f90nml.patch(nml_file or self._nml_file,patch,outputfile)
else:
f90nml.write(patch, outputfile, force=True)
def readpar(par_file, root):
"""Read StagYY par file.
The namelist is populated in chronological order with:
- :data:`PAR_DEFAULT`, an internal dictionary defining defaults;
- :data:`PAR_DFLT_FILE`, the global configuration par file;
- ``par_name_defaultparameters`` if it is defined in ``par_file``;
- ``par_file`` itself;
- ``parameters.dat`` if it can be found in the StagYY output directories.
Args:
par_file (:class:`pathlib.Path`): path of par file.
root (:class:`pathlib.Path`): path on which other paths are rooted.
This is usually par.parent.
Returns:
:class:`f90nml.namelist.Namelist`: case insensitive dict of dict of
values with first key being the namelist and second key the variables'
name.
"""
par_nml = deepcopy(PAR_DEFAULT)
if PAR_DFLT_FILE.is_file():
_enrich_with_par(par_nml, PAR_DFLT_FILE)
else:
PAR_DFLT_FILE.parent.mkdir(exist_ok=True)
f90nml.write(par_nml, str(PAR_DFLT_FILE))
if not par_file.is_file():
raise NoParFileError(par_file)
par_main = f90nml.read(str(par_file))
if 'default_parameters_parfile' in par_main:
par_dflt = par_main['default_parameters_parfile'].get(
'par_name_defaultparameters', 'par_defaults')
par_dflt = root / par_dflt
if not par_dflt.is_file():
raise NoParFileError(par_dflt)
_enrich_with_par(par_nml, par_dflt)
_enrich_with_par(par_nml, par_file)
par_out = root / par_nml['ioin']['output_file_stem'] / '_parameters.dat'
if par_out.is_file():
_enrich_with_par(par_nml, par_out)
par_out = root / par_nml['ioin']['hdf5_output_folder'] / 'parameters.dat'
if par_out.is_file():
_enrich_with_par(par_nml, par_out)
return par_nml
def save_InitCond(dict_initcond, year, grid=''):
InitCond = {'initialconditions': dict_initcond.copy()}
for k, v in InitCond['initialconditions'].items():
if int(v) == v:
InitCond['initialconditions'][k] = int(v)
# save RunControl to file
fn_nml = os.path.join(
'Input',
'InitialConditionstest{grid}_{year}.nml').format(
grid=grid,
year=year)
if os.path.exists(fn_nml):
os.remove(fn_nml)
f90nml.write(InitCond, fn_nml)
def save(self, filename, convkey=None):
"""パラメータをファイルに保存する.
Parameters
----------
filename : str or Path
保存するファイル名
convkey : UnitConversionKey, optional
単位変換キー, by default None
"""
convkey = convkey or self.convkey
with open(filename, 'wt', encoding='utf-8') as f:
if convkey is not None:
f.write('!!key {}\n'.format(convkey.keytext))
f90nml.write(self.nml, f, force=True)
def test_run_failure_confusing_emissions_scenarios(package, invalid_config):
f90nml.write(invalid_config,
join(package.run_dir, "MAGCFG_USER.CFG"),
force=True)
error_msg = re.escape(
"You have more than one `FILE_EMISSCEN_X` flag set. Using more than one "
"emissions scenario is hard to debug and unnecessary with Pymagicc's "
"dataframe scenario input. Please combine all your scenarios into one "
"dataframe with Pymagicc and pandas, then feed this single Dataframe into "
"Pymagicc's run API.")
with pytest.raises(ValueError, match=error_msg):
package.run()
assert package.config is None
def namelist_set(filename, setvariable, setvalue, verbose=False):
'''
Set a variable from a namelist to a value
input arguments:
filename: filename of namelist
setvariable: GROUP_NAME:VARIABLE_NAME to set from namelist
setvalue: value to set setvariable to
verbose: optional boolean argument if results should be printed to screen
'''
namelist = f90nml.read(filename)
path = setvariable.split(':')
crumb = namelist
while len(path) > 1:
crumb = crumb[path[0]]
path.pop(0)
# dealing with different types..
t = type(crumb[path[0]])
if isinstance(crumb[path[0]], int): # integer
crumb[path[0]] = int(setvalue)
elif isinstance(crumb[path[0]], float): # float
crumb[path[0]] = float(setvalue)
elif isinstance(crumb[path[0]], str): # string
crumb[path[0]] = setvalue
elif isinstance(crumb[path[0]], list): # list
t = type(crumb[path[0]][0])
# deal with trailing ',' leading to empty string, crashing int() and float()
l = setvalue.split(',')
while l[-1] == "":
l.pop()
if isinstance(crumb[path[0]][0], int): # integer
crumb[path[0]] = [int(i) for i in l]
if isinstance(crumb[path[0]][0], float): # float
crumb[path[0]] = [float(i) for i in l]
if isinstance(crumb[path[0]][0], str): # string
crumb[path[0]] = l
elif isinstance(crumb[path[0]], bool): # boolean
if setvalue == '.true.':
crumb[path[0]] = True
elif setvalue == '.false.':
crumb[path[0]] = False
else:
print "Cannot parse boolean, use .true. or .false."
else:
print "Unsupported type: ", t
# write namelist variable
f90nml.write(namelist, filename, force=True)
def namelist_set(filename, setvariable, setvalue, verbose=False):
'''
Set a variable from a namelist to a value
input arguments:
filename: filename of namelist
setvariable: GROUP_NAME:VARIABLE_NAME to set from namelist
setvalue: value to set setvariable to
verbose: optional boolean argument if results should be printed to screen
'''
namelist = f90nml.read( filename )
path = setvariable.split ( ':' )
crumb = namelist
while len(path) > 1:
crumb = crumb[ path[0] ]
path.pop(0)
# dealing with different types..
t = type(crumb[path[0]])
if isinstance(crumb[path[0]], int): # integer
crumb[ path[0] ] = int(setvalue)
elif isinstance(crumb[path[0]], float): # float
crumb[ path[0] ] = float(setvalue)
elif isinstance(crumb[path[0]], str): # string
crumb[ path[0] ] = setvalue
elif isinstance(crumb[path[0]], list): # list
t = type( crumb[path[0]][0] )
# deal with trailing ',' leading to empty string, crashing int() and float()
l = setvalue.split(',')
while l[-1] == "":
l.pop()
if isinstance(crumb[path[0]][0], int): # integer
crumb[ path[0] ] = [int(i) for i in l]
if isinstance(crumb[path[0]][0], float): # float
crumb[ path[0] ] = [float(i) for i in l]
if isinstance(crumb[path[0]][0], str): # string
crumb[ path[0] ] = l
elif isinstance(crumb[path[0]], bool): # boolean
if setvalue == '.true.':
crumb[ path[0] ] = True
elif setvalue == '.false.':
crumb[ path[0] ] = False
else:
print "Cannot parse boolean, use .true. or .false."
else:
print "Unsupported type: ", t
# write namelist variable
f90nml.write( namelist, filename, force=True )
def saveNamelist(self, filename):
'''
Write a namelist file to the requested path based on the current content of this object
:param filename: the path of the file to write
:return: None
'''
a = {'input_nml': {}}
a['input_nml']['all_qf'] = self.all_qf
a['input_nml']['sensible_qf'] = self.sensible_qf
a['input_nml']['latent_qf'] = self.latent_qf
a['input_nml']['wastewater_qf'] = self.wastewater_qf
# a['input_nml']['start_dates'] = self.dt_start.strftime('YYYY-mm-dd')
# a['input_nml']['end_dates'] = self.dt_start.strftime('YYYY-mm-dd')
a['input_nml']['start_dates'] = self.dt_start.strftime('%Y-%m-%d')
a['input_nml']['end_dates'] = self.dt_start.strftime('%Y-%m-%d')
nml.write(a, filename)
def _inject_pymagicc_compatible_magcfg_user(magicc):
"""
Overwrite ``magicc.run_dir / MAGCFG_USER.CFG`` with config that only point to ``MAGTUNE_PYMAGICC.CFG``
Parameters
----------
magicc : :obj:`pymagicc.MAGICC7`
Instance of :obj:`pymagicc.MAGICC7` to setup
"""
LOGGER.info("Writing Pymagicc compatible MAGCFG_USER.CFG in %s",
magicc.run_dir)
with open(os.path.join(magicc.run_dir, "MAGCFG_USER.CFG"),
"w") as file_handle:
f90nml.write({"nml_allcfgs": {
"file_tuningmodel_1": "PYMAGICC"
}}, file_handle)
def generate_parfiles(self):
"""
Creates separate parfiles from the main namelist container and writes
the individual parfiles to disk.
Returns
-------
parfiles : list of str
List containing the paths of the parfiles, can be passed to the legolas
runner.
"""
run_dict = {key: {} for key in self.container.keys()}
# savelist must be present
try:
run_dict["savelist"]
except KeyError:
run_dict.update({"savelist": {}})
for current_run in range(self.nb_runs):
prefix = "{:04d}".format(current_run + 1)
if self.nb_runs == 1:
prefix = ""
# generate dictionary for this specific run
for namelist, items in self.container.items():
for key, values in items.items():
run_dict[namelist].update({key: values[current_run]})
# parfile name
parfile_name = f"{prefix}{self.basename}.par"
# datfile name (no extension .dat needed)
datfile_name = "".join([
f"{prefix}",
run_dict["savelist"].get("basename_datfile", self.basename),
])
run_dict["savelist"].update({"basename_datfile": datfile_name})
# logfile name (no extension .log needed)
logfile_name = run_dict["savelist"].get("basename_logfile", None)
if logfile_name is not None:
logfile_name = f"{prefix}{logfile_name}"
run_dict["savelist"].update({"basename_logfile": logfile_name})
# set paths and write parfile
parfile_path = (self.output_dir / parfile_name).resolve()
self.parfiles.append(str(parfile_path))
f90nml.write(run_dict, parfile_path, force=True)
pylboLogger.info(f"parfiles generated and saved to {self.output_dir}")
return self.parfiles
def perform(self, args):
namelist = args.namelist["_"]
parser = f90nml.Parser()
if os.path.exists(namelist):
nml = parser.read(namelist)
else:
nml = parser.reads(namelist)
if args.outfile:
f90nml.write(nml, args.outfile["_"])
self.add_forward(data=nml.todict(complex_tuple=True))
def nmlwrite(nml, **kwargs):
output_fmt = kwargs.get("format")
_out = kwargs.get("_out", sys.stdout)
# determine output format
valid_formats = ('json', 'yaml', 'nml')
if output_fmt and output_fmt not in valid_formats:
raise ValueError(
"Error: format must be one of: {0}".format(valid_formats))
if _out != sys.stdout:
output_file = open(_out, 'w')
output_fname = _out
else:
output_file = _out
output_fname = None
# Get output format from out_file name
if not output_fmt:
if output_fname:
_, output_ext = os.path.splitext(output_fname)
if output_ext == '.json':
output_fmt = 'json'
elif output_ext in ('.yaml', '.yml'):
output_fmt = 'yaml'
else:
output_fmt = 'nml'
else:
output_fmt = 'nml'
# write nml to output_file
if output_fmt == "json":
json.dump(nml, output_file, indent=4, separators=(',', ': '))
output_file.write('\n')
elif output_fmt == "yaml":
input_data = nml.todict(complex_tuple=True)
yaml.dump(input_data, output_file, default_flow_style=False)
else:
f90nml.write(nml, output_file)
# Cleanup
if _out != sys.stdout:
output_file.close()
def set_config(self,
filename='MAGTUNE_SIMPLE.CFG',
top_level_key='nml_allcfgs',
**kwargs):
"""
Create a configuration file for MAGICC
Writes a fortran namelist in run_dir.
:param filename:
:param top_level_key:
:param kwargs: Other parameters to pass to the configuration file. No
validation on the parameters is performed.
:return: A dict containing the contents of the namelist which was
written to file
"""
fname = join(self.run_dir, filename)
data = {top_level_key: kwargs}
f90nml.write(data, fname, force=True)
return data
def write_namelists(self, run_dir):
# TODO(JLM): make the setup namelists @properties without setter (protect them)
# write hydro.namelist for the job
self.hydro_namelist_file = run_dir.joinpath(self.job_date_id +
'.hydro.namelist')
f90nml.write(self.hydro_namelist, self.hydro_namelist_file)
nlst_file = run_dir.joinpath('hydro.namelist')
if nlst_file.exists():
nlst_file.unlink()
nlst_file.symlink_to(self.hydro_namelist_file)
# write namelist.hrldas
self.namelist_hrldas_file = run_dir.joinpath(self.job_date_id +
'.namelist.hrldas')
f90nml.write(self.namelist_hrldas, self.namelist_hrldas_file)
nlst_file = run_dir.joinpath('namelist.hrldas')
if nlst_file.exists():
nlst_file.unlink()
nlst_file.symlink_to(self.namelist_hrldas_file)
def save_initcond_nml(
df_state: pd.DataFrame, site: str = "", path_dir_save: Path = Path("."),
) -> Path:
# get last time step
try:
tstep_last = df_state.index.levels[0].max()
except AttributeError:
logger_supy.exception(
(
"incorrect structure detected;"
+ " check if `df_state` is the final model state."
)
)
return
# get year for filename formatting
year_last = tstep_last.year
# generate a df with records of the last tstep
df_state_last_tstep = df_state.loc[tstep_last]
# get grid list
list_grid = df_state_last_tstep.index
# list holder for paths written out in nml
list_path_nml = []
for grid in list_grid:
# generate nml filename
filename_out_grid = f"InitialConditions{site}{grid}_{year_last}_EndofRun.nml"
# derive a save path
path_nml = path_dir_save / filename_out_grid
# retrieve initcond values from `df_state_last_tstep`
nml = {
"InitialConditions": {
key: df_state_last_tstep.loc[grid, var]
for key, var in dict_init_nml.items()
}
}
# save nml
f90nml.write(nml, path_nml, force=True)
# f90nml.write(nml, nml_file,force=True)
list_path_nml.append(path_nml)
return list_path_nml
def create_constants(constants_yaml, output_path):
"""Method to just create a constants file, bypassing
all the NetCDF setup."""
yaml = YAML(typ='safe')
# If no output_path provided, create in same place and same name as YAML
if output_path is None:
output_path = f'{os.path.splitext(constants_yaml)[0]}.nml'
# Open the YAML constants file and read contents
with open(constants_yaml, 'r') as constants_file:
constants = yaml.load(constants_file)
# Open the NML constants file and write
with open(output_path, 'w') as nml_file:
allocatable_array_sizes = {}
for grp in constants.values():
for k, v in grp.items():
if isinstance(v, list):
allocatable_array_sizes['n_{0}'.format(k)] = len(v)
f90nml.write(
{'allocatable_array_sizes': allocatable_array_sizes},
nml_file)
f90nml.write(constants, nml_file)
print('Done! Constants file saved to ' + output_path)
def update_config(self,
filename="MAGTUNE_PYMAGICC.CFG",
top_level_key="nml_allcfgs",
**kwargs):
"""Updates a configuration file for MAGICC
Updates the contents of a fortran namelist in the run directory,
creating a new namelist if none exists.
Parameters
----------
filename : str
Name of configuration file to write
top_level_key : str
Name of namelist to be written in the
configuration file
kwargs
Other parameters to pass to the configuration file. No
validation on the parameters is performed.
Returns
-------
dict
The contents of the namelist which was written to file
"""
kwargs = self._fix_any_backwards_emissions_scen_key_in_config(kwargs)
fname = join(self.run_dir, filename)
if exists(fname):
conf = f90nml.read(fname)
else:
conf = {top_level_key: {}}
conf[top_level_key].update(kwargs)
f90nml.write(conf, fname, force=True)
return conf
def write_config(p):
emis_key = "file_emissionscenario" if p.version == 6 else "FILE_EMISSCEN"
outpath = join(p.run_dir, "MAGTUNE_SIMPLE.CFG")
f90nml.write({"nml_allcfgs": {
emis_key: "RCP26.SCEN"
}},
outpath,
force=True)
# Write years config.
outpath_years = join(p.run_dir, "MAGCFG_NMLYEARS.CFG")
f90nml.write(
{
"nml_years": {
"startyear": 1765,
"endyear": 2100,
"stepsperyear": 12
}
},
outpath_years,
force=True,
)
def set_config(
self, filename="MAGTUNE_PYMAGICC.CFG", top_level_key="nml_allcfgs", **kwargs
):
"""
Create a configuration file for MAGICC
Writes a fortran namelist in run_dir.
# Parameters
filename (str): Name of configuration file to write
top_level_key (str): Name of namelist to be written in the
configuration file
kwargs: Other parameters to pass to the configuration file. No
validation on the parameters is performed.
# Returns
data (dict): The contents of the namelist which was written to file
"""
fname = join(self.run_dir, filename)
data = {top_level_key: kwargs}
f90nml.write(data, fname, force=True)
return data
def write_namelist_parameters(self,
outputfile,
do_patch=False,
nml_file=None):
patch = defaultdict(dict)
for p in self._namelist_parameters.values():
name = p["name"]
group_name = p["group_name"]
group = patch[group_name]
short = p["short"]
parameter_set_name = p.get("set_name", "parameters_" + group_name)
parameter_set = getattr(self, parameter_set_name)
if getattr(parameter_set, name) is None: # omit if value is None
continue
if is_quantity(p["default"]):
group[short] = to_quantity(getattr(
parameter_set, name)).value_in(p["default"].unit)
else:
group[short] = getattr(parameter_set, name)
if do_patch:
f90nml.patch(nml_file or self._nml_file, patch, outputfile)
else:
f90nml.write(patch, outputfile, force=True)
def skim_namelist_copy(input_path, output_path, *, date, prefix, hours=18):
old_dic = f90nml.read(os.path.join(input_path, 'namelist.wps'))
dt_object = pd.to_datetime(date)
d_init = dt_object.strftime('%Y-%m-%d_%T')
d_end = dt_object + pd.DateOffset(hours=hours)
d_end = d_end.strftime('%Y-%m-%d_%T')
old_dic['share']['start_date'] = old_dic['share']['max_dom'] * [d_init]
old_dic['share']['end_date'] = old_dic['share']['max_dom'] * [d_end]
old_dic['share']['interval_seconds'] = 6 * 3600
old_dic['ungrib']['prefix'] = prefix
sections = ['share', 'ungrib']
drops = {}
new_dic = OrderedDict()
for s in sections:
new_dic[s] = old_dic[s]
if s in drops.keys():
for d in drops[s]:
new_dic[s].pop(d)
f90nml.write(new_dic,
os.path.join(output_path, 'namelist.wps'),
force=True)
return new_dic
def retrieve_gribs_and_run_ungrib(js, q):
"""
This function retrieves required GRIB files and runs ungrib.
It returns either 'SUCCESS' or 'FAILURE' on completion.
:param js: the JobState object containing the forecast configuration
:param q: the multiprocessing Queue into which we will send either 'SUCCESS' or 'FAILURE'
"""
try:
logging.info("retrieving GRIB files.")
# step 3: retrieve required GRIB files from the grib_source, symlink into GRIBFILE.XYZ links into wps
manifest = js.grib_source.retrieve_gribs(js.start_utc, js.end_utc)
js.grib_source.symlink_gribs(manifest, js.wps_dir)
send_email(
js, 'grib2',
'Job %s - %d GRIB2 files downloaded.' % (js.job_id, len(manifest)))
logging.info("running UNGRIB")
# step 4: patch namelist for ungrib end execute ungrib
f90nml.write(js.wps_nml,
osp.join(js.wps_dir, 'namelist.wps'),
force=True)
Ungrib(js.wps_dir).execute().check_output()
send_email(js, 'ungrib', 'Job %s - ungrib complete.' % js.job_id)
logging.info('UNGRIB complete')
q.put('SUCCESS')
except Exception as e:
logging.error('GRIB2/UNGRIB step failed with exception %s' % repr(e))
traceback.print_exc()
q.put('FAILURE')
def save_prm(self, path):
f90nml.write(self.get_prm(), path)
def execute(args,job_args):
"""
Executes a weather/fire simulation.
:param args: a dictionary with all to start the simulationfollowing keys
:param job_args: a the original json given the forecast
Keys in args:
:param grid_code: the (unique) code of the grid that is used
:param sys_install_path: system installation directory
:param start_utc: start time of simulation in UTC
:param end_utc: end time of simulation in UTC
:param workspace_path: workspace directory
:param wps_install_path: installation directory of WPS that will be used
:param wrf_install_path: installation directory of WRF that will be used
:param grib_source: a string identifying a valid GRIB2 source
:param wps_namelist_path: the path to the namelist.wps file that will be used as template
:param wrf_namelist_path: the path to the namelist.input file that will be used as template
:param fire_namelist_path: the path to the namelist.fire file that will be used as template
:param wps_geog_path: the path to the geogrid data directory providing terrain/fuel data
:param email_notification: dictionary containing keys address and events indicating when a mail should be fired off
"""
# step 0 initialize the job state from the arguments
js = JobState(args)
jobdir = osp.abspath(osp.join(js.workspace_path, js.job_id))
make_clean_dir(jobdir)
json.dump(job_args, open(osp.join(jobdir,'input.json'),'w'), indent=4, separators=(',', ': '))
jsub = make_job_file(js)
json.dump(jsub, open(jsub.jobfile,'w'), indent=4, separators=(',', ': '))
logging.info("job %s starting [%d hours to forecast]." % (js.job_id, js.fc_hrs))
sys.stdout.flush()
send_email(js, 'start', 'Job %s started.' % js.job_id)
# read in all namelists
js.wps_nml = f90nml.read(js.args['wps_namelist_path'])
js.wrf_nml = f90nml.read(js.args['wrf_namelist_path'])
js.fire_nml = f90nml.read(js.args['fire_namelist_path'])
js.ems_nml = None
if 'emissions_namelist_path' in js.args:
js.ems_nml = f90nml.read(js.args['emissions_namelist_path'])
# Parse and setup the domain configuration
js.domain_conf = WPSDomainConf(js.domains)
num_doms = len(js.domain_conf)
js.wps_nml['share']['start_date'] = [utc_to_esmf(js.start_utc)] * num_doms
js.wps_nml['share']['end_date'] = [utc_to_esmf(js.end_utc)] * num_doms
js.wps_nml['share']['interval_seconds'] = 3600
logging.info("number of domains defined is %d." % num_doms)
# build directories in workspace
js.wps_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wps'))
js.wrf_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wrf'))
#check_obj(args,'args')
#check_obj(js,'Initial job state')
# step 1: clone WPS and WRF directories
logging.info("cloning WPS into %s" % js.wps_dir)
cln = WRFCloner(js.args)
cln.clone_wps(js.wps_dir, js.grib_source.vtables(), [])
# step 2: process domain information and patch namelist for geogrid
js.wps_nml['geogrid']['geog_data_path'] = js.args['wps_geog_path']
js.domain_conf.prepare_for_geogrid(js.wps_nml, js.wrf_nml, js.wrfxpy_dir, js.wps_dir)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
# do steps 2 & 3 & 4 in parallel (two execution streams)
# -> GEOGRID ->
# -> GRIB2 download -> UNGRIB ->
proc_q = Queue()
geogrid_proc = Process(target=run_geogrid, args=(js, proc_q))
grib_proc = Process(target=retrieve_gribs_and_run_ungrib, args=(js, proc_q))
logging.info('starting GEOGRID and GRIB2/UNGRIB')
geogrid_proc.start()
grib_proc.start()
# wait until both tasks are done
logging.info('waiting until both tasks are done')
grib_proc.join()
geogrid_proc.join()
if proc_q.get() != 'SUCCESS':
return
if proc_q.get() != 'SUCCESS':
return
proc_q.close()
# step 5: execute metgrid after ensuring all grids will be processed
js.domain_conf.prepare_for_metgrid(js.wps_nml)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
logging.info("running METGRID")
Metgrid(js.wps_dir).execute().check_output()
send_email(js, 'metgrid', 'Job %s - metgrid complete.' % js.job_id)
logging.info("cloning WRF into %s" % js.wrf_dir)
# step 6: clone wrf directory, symlink all met_em* files, make namelists
cln.clone_wrf(js.wrf_dir, [])
symlink_matching_files(js.wrf_dir, js.wps_dir, "met_em*")
time_ctrl = update_time_control(js.start_utc, js.end_utc, num_doms)
js.wrf_nml['time_control'].update(time_ctrl)
update_namelist(js.wrf_nml, js.grib_source.namelist_keys())
if 'ignitions' in js.args:
update_namelist(js.wrf_nml, render_ignitions(js, num_doms))
# if we have an emissions namelist, automatically turn on the tracers
if js.ems_nml is not None:
logging.debug('namelist.fire_emissions given, turning on tracers')
f90nml.write(js.ems_nml, osp.join(js.wrf_dir, 'namelist.fire_emissions'), force=True)
js.wrf_nml['dynamics']['tracer_opt'] = [2] * num_doms
f90nml.write(js.wrf_nml, osp.join(js.wrf_dir, 'namelist.input'), force=True)
f90nml.write(js.fire_nml, osp.join(js.wrf_dir, 'namelist.fire'), force=True)
# step 7: execute real.exe
logging.info("running REAL")
# try to run Real twice as it sometimes fails the first time
# it's not clear why this error happens
try:
Real(js.wrf_dir).execute().check_output()
except Exception as e:
logging.error('Real step failed with exception %s, retrying ...' % str(e))
Real(js.wrf_dir).execute().check_output()
# step 7b: if requested, do fuel moisture DA
if js.fmda is not None:
logging.info('running fuel moisture data assimilation')
for dom in js.fmda.domains:
assimilate_fm10_observations(osp.join(wrf_dir, 'wrfinput_d%02d' % dom), None, js.fmda.token)
# step 8: execute wrf.exe on parallel backend
logging.info('submitting WRF job')
send_email(js, 'wrf_submit', 'Job %s - wrf job submitted.' % js.job_id)
js.task_id = "sim-" + js.grid_code + "-" + utc_to_esmf(js.start_utc)[:10]
jsub.job_num=WRF(js.wrf_dir, js.qsys).submit(js.task_id, js.num_nodes, js.ppn, js.wall_time_hrs)
send_email(js, 'wrf_exec', 'Job %s - wrf job starting now with id %s.' % (js.job_id, js.task_id))
logging.info("WRF job %s submitted with id %s, waiting for rsl.error.0000" % (jsub.job_num, js.task_id))
jobfile = osp.abspath(osp.join(js.workspace_path, js.job_id,'job.json'))
json.dump(jsub, open(jobfile,'w'), indent=4, separators=(',', ': '))
process_output(js.job_id)
def retrieve_gribs_and_run_ungrib(js, grib_source, q):
"""
This function retrieves required GRIB files and runs ungrib.
It returns either 'SUCCESS' or 'FAILURE' on completion.
:param js: the JobState object containing the forecast configuration
:param grib_source: the GribSource object containing ungrib configuration
:param q: the multiprocessing Queue into which we will send either 'SUCCESS' or 'FAILURE'
"""
wps_dir = osp.abspath(js.wps_dir)
grib_dir = osp.join(wps_dir, grib_source.id)
make_clean_dir(grib_dir)
wps_nml = js.wps_nml
try:
logging.info("retrieving GRIB files from %s" % grib_source.id)
download_whole_cycle = js.get('download_whole_cycle', False)
manifest = grib_source.retrieve_gribs(js.start_utc, js.end_utc,
js.ref_utc, js.cycle_start_utc,
download_whole_cycle)
# logging.info('manifest: ' + str(manifest))
cache_colmet = len(manifest) > 1
have_all_colmet = False
if cache_colmet:
have_all_colmet = len(manifest.colmet_missing) == 0
colmet_dir = osp.join(grib_source.cache_dir,
manifest.colmet_prefix)
logging.info('cache colmet %s, have all colmet %s' %
(cache_colmet, have_all_colmet))
if not have_all_colmet:
# this is also if we do not cache
grib_source.symlink_gribs(manifest.grib_files, grib_dir)
send_email(
js, 'grib2', 'Job %s - %d GRIB2 files downloaded.' %
(js.job_id, len(manifest)))
logging.info("running UNGRIB for %s" % grib_source.id)
logging.info(
"step 4: patch namelist for ungrib end execute ungrib on %s files"
% grib_source.id)
update_namelist(wps_nml, grib_source.namelist_wps_keys())
if cache_colmet:
wps_nml['share']['start_date'] = [
utc_to_esmf(manifest.colmet_files_utc[0])
] * js.num_doms
wps_nml['share']['end_date'] = [
utc_to_esmf(manifest.colmet_files_utc[-1])
] * js.num_doms
# logging.info("namelist.wps for UNGRIB: %s" % json.dumps(wps_nml, indent=4, separators=(',', ': ')))
f90nml.write(wps_nml,
osp.join(grib_dir, 'namelist.wps'),
force=True)
grib_source.clone_vtables(grib_dir)
symlink_unless_exists(osp.join(wps_dir, 'ungrib.exe'),
osp.join(grib_dir, 'ungrib.exe'))
print(grib_dir + ':')
os.system('ls -l %s' % grib_dir)
Ungrib(grib_dir).execute().check_output()
print(grib_dir + ':')
os.system('ls -l %s' % grib_dir)
if cache_colmet:
# move output to cache directory
make_dir(colmet_dir)
for f in manifest.colmet_files:
move(osp.join(grib_dir, f), osp.join(colmet_dir, f))
# now all colmet files should be in the cache
if cache_colmet:
for f in manifest.colmet_files:
symlink_unless_exists(osp.join(colmet_dir, f),
osp.join(wps_dir, f))
else:
# move output
for f in glob.glob(osp.join(grib_dir, grib_source.prefix() + '*')):
move(f, wps_dir)
send_email(js, 'ungrib', 'Job %s - ungrib complete.' % js.job_id)
logging.info('UNGRIB complete for %s' % grib_source.id)
q.put('SUCCESS')
except Exception as e:
logging.error('GRIB2/UNGRIB step failed with exception %s' % repr(e))
traceback.print_exc()
q.put('FAILURE')
def execute(args, job_args):
"""
Executes a weather/fire simulation.
:param args: a dictionary with all to start the simulationfollowing keys
:param job_args: a the original json given the forecast
Keys in args:
:param grid_code: the (unique) code of the grid that is used
:param sys_install_path: system installation directory
:param start_utc: start time of simulation in UTC
:param end_utc: end time of simulation in UTC
:param workspace_path: workspace directory
:param wps_install_path: installation directory of WPS that will be used
:param wrf_install_path: installation directory of WRF that will be used
:param grib_source: a string identifying a valid GRIB2 source
:param wps_namelist_path: the path to the namelist.wps file that will be used as template
:param wrf_namelist_path: the path to the namelist.input file that will be used as template
:param fire_namelist_path: the path to the namelist.fire file that will be used as template
:param wps_geog_path: the path to the geogrid data directory providing terrain/fuel data
:param email_notification: dictionary containing keys address and events indicating when a mail should be fired off
"""
logging.info('step 0 initialize the job state from the arguments')
## logging.info('args = %s' % json.dumps(jargs, open(osp.join(jobdir,'input.json'),'w'), indent=4, separators=(',', ': ')))
js = JobState(args)
## logging.info('js = %s' % json.dumps(js, open(osp.join(jobdir,'input.json'),'w'), indent=4, separators=(',', ': ')))
jobdir = osp.abspath(osp.join(js.workspace_path, js.job_id))
make_clean_dir(jobdir)
json.dump(job_args,
open(osp.join(jobdir, 'input.json'), 'w'),
indent=4,
separators=(',', ': '))
jsub = make_job_file(js)
json.dump(jsub, open(jsub.jobfile, 'w'), indent=4, separators=(',', ': '))
logging.info("job %s starting [%d hours to forecast]." %
(js.job_id, js.fc_hrs))
sys.stdout.flush()
send_email(js, 'start', 'Job %s started.' % js.job_id)
# read in all namelists
js.wps_nml = read_namelist(js.args['wps_namelist_path'])
js.wrf_nml = read_namelist(js.args['wrf_namelist_path'])
js.fire_nml = read_namelist(js.args['fire_namelist_path'])
js.ems_nml = None
if 'emissions_namelist_path' in js.args:
js.ems_nml = read_namelist(js.args['emissions_namelist_path'])
# Parse and setup the domain configuration
js.domain_conf = WPSDomainConf(js.domains)
js.num_doms = len(js.domain_conf)
js.wps_nml['share']['interval_seconds'] = js.grib_source[
0].interval_seconds
logging.info("number of domains defined is %d." % js.num_doms)
# build directories in workspace
js.wps_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wps'))
js.wrf_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wrf'))
#check_obj(args,'args')
#check_obj(js,'Initial job state')
logging.info("step 1: clone WPS and WRF directories")
logging.info("cloning WPS into %s" % js.wps_dir)
cln = WRFCloner(js.args)
cln.clone_wps(js.wps_dir, [])
js.grib_source[0].clone_vtables(js.wps_dir)
logging.info(
"step 2: process domain information and patch namelist for geogrid")
js.wps_nml['share']['start_date'] = [utc_to_esmf(js.start_utc)
] * js.num_doms
js.wps_nml['share']['end_date'] = [utc_to_esmf(js.end_utc)] * js.num_doms
js.wps_nml['geogrid']['geog_data_path'] = js.args['wps_geog_path']
js.domain_conf.prepare_for_geogrid(js.wps_nml, js.wrf_nml, js.wrfxpy_dir,
js.wps_dir)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
# do steps 2 & 3 & 4 in parallel (two execution streams)
# -> GEOGRID ->
# -> GRIB2 download -> UNGRIB ->
proc_q = Queue()
geogrid_proc = Process(target=run_geogrid, args=(js, proc_q))
# grib_proc = Process(target=retrieve_gribs_and_run_ungrib_all, args=(js, proc_q, ref_utc))
grib_proc = {}
for grib_source in js.grib_source:
grib_proc[grib_source.id] = Process(
target=retrieve_gribs_and_run_ungrib,
args=(js, grib_source, proc_q))
logging.info('starting GEOGRID and GRIB2/UNGRIB')
if js.ungrib_only:
logging.info(
'ungrib_only set, skipping GEOGRID, will exit after UNGRIB')
else:
geogrid_proc.start()
for grib_source in js.grib_source:
grib_proc[grib_source.id].start()
# wait until all tasks are done
logging.info('waiting until all tasks are done')
for grib_source in js.grib_source:
grib_proc[grib_source.id].join()
if js.ungrib_only:
return
else:
geogrid_proc.join()
for grib_source in js.grib_source:
if proc_q.get() != 'SUCCESS':
return
if proc_q.get() != 'SUCCESS':
return
proc_q.close()
logging.info(
"step 5: execute metgrid after ensuring all grids will be processed")
update_namelist(js.wps_nml, js.grib_source[0].namelist_wps_keys())
js.domain_conf.prepare_for_metgrid(js.wps_nml)
logging.info("namelist.wps for METGRID: %s" %
json.dumps(js.wps_nml, indent=4, separators=(',', ': ')))
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
logging.info("running METGRID")
Metgrid(js.wps_dir).execute().check_output()
send_email(js, 'metgrid', 'Job %s - metgrid complete.' % js.job_id)
logging.info("METGRID complete")
logging.info("cloning WRF into %s" % js.wrf_dir)
logging.info(
"step 6: clone wrf directory, symlink all met_em* files, make namelists"
)
cln.clone_wrf(js.wrf_dir, [])
symlink_matching_files(js.wrf_dir, js.wps_dir, "met_em*")
time_ctrl = update_time_control(js.start_utc, js.end_utc, js.num_doms)
js.wrf_nml['time_control'].update(time_ctrl)
js.wrf_nml['time_control']['interval_seconds'] = js.grib_source[
0].interval_seconds
update_namelist(js.wrf_nml, js.grib_source[0].namelist_keys())
if 'ignitions' in js.args:
update_namelist(js.wrf_nml, render_ignitions(js, js.num_doms))
# if we have an emissions namelist, automatically turn on the tracers
if js.ems_nml is not None:
logging.debug('namelist.fire_emissions given, turning on tracers')
f90nml.write(js.ems_nml,
osp.join(js.wrf_dir, 'namelist.fire_emissions'),
force=True)
js.wrf_nml['dynamics']['tracer_opt'] = [2] * js.num_doms
f90nml.write(js.wrf_nml,
osp.join(js.wrf_dir, 'namelist.input'),
force=True)
f90nml.write(js.fire_nml,
osp.join(js.wrf_dir, 'namelist.fire'),
force=True)
# step 7: execute real.exe
logging.info("running REAL")
# try to run Real twice as it sometimes fails the first time
# it's not clear why this error happens
try:
Real(js.wrf_dir).execute().check_output()
except Exception as e:
logging.error('Real step failed with exception %s, retrying ...' %
str(e))
Real(js.wrf_dir).execute().check_output()
logging.info('step 7b: if requested, do fuel moisture DA')
logging.info('fmda = %s' % js.fmda)
if js.fmda is not None:
logging.info('running fuel moisture data assimilation')
for dom in js.fmda.domains:
logging.info('assimilate_fm10_observations for domain %s' % dom)
assimilate_fm10_observations(
osp.join(js.wrf_dir, 'wrfinput_d%02d' % int(dom)), None,
js.fmda.token)
# step 8: execute wrf.exe on parallel backend
logging.info('submitting WRF job')
send_email(js, 'wrf_submit', 'Job %s - wrf job submitted.' % js.job_id)
js.task_id = "sim-" + js.grid_code + "-" + utc_to_esmf(js.start_utc)[:10]
jsub.job_num = WRF(js.wrf_dir, js.qsys).submit(js.task_id, js.num_nodes,
js.ppn, js.wall_time_hrs)
send_email(
js, 'wrf_exec',
'Job %s - wrf job starting now with id %s.' % (js.job_id, js.task_id))
logging.info(
"WRF job %s submitted with id %s, waiting for rsl.error.0000" %
(jsub.job_num, js.task_id))
jobfile = osp.abspath(osp.join(js.workspace_path, js.job_id, 'job.json'))
json.dump(jsub, open(jobfile, 'w'), indent=4, separators=(',', ': '))
process_output(js.job_id)
def data_collection_hook(self):
"""Spawns data emulation using gphl simcal"""
data_collect_parameters = self.current_dc_parameters
if not HWR.beamline.gphl_workflow:
raise ValueError("Emulator requires GPhL workflow installation")
gphl_connection = HWR.beamline.gphl_connection
if not gphl_connection:
raise ValueError("Emulator requires GPhL connection installation")
# Get program locations
simcal_executive = gphl_connection.get_executable("simcal")
# Get environmental variables
envs = {
"BDG_home":
gphl_connection.software_paths["BDG_home"],
"GPHL_INSTALLATION":
gphl_connection.software_paths["GPHL_INSTALLATION"],
}
text_type = ConvertUtils.text_type
for tag, val in self["environment_variables"].getProperties().items():
envs[text_type(tag)] = text_type(val)
# get crystal data
sample_name = self.getProperty("default_sample_name")
sample = HWR.beamline.sample_changer.getLoadedSample()
if sample:
ss0 = sample.getName()
if ss0 and ss0.startswith(self.TEST_SAMPLE_PREFIX):
sample_name = ss0[len(self.TEST_SAMPLE_PREFIX):]
sample_dir = gphl_connection.software_paths.get("gphl_test_samples")
if not sample_dir:
raise ValueError(
"Emulator requires gphl_test_samples dir specified")
sample_dir = os.path.join(sample_dir, sample_name)
if not os.path.isdir(sample_dir):
raise ValueError("Sample data directory %s does not exist" %
sample_dir)
crystal_file = os.path.join(sample_dir, "crystal.nml")
if not os.path.isfile(crystal_file):
raise ValueError("Emulator crystal data file %s does not exist" %
crystal_file)
# in spite of the simcal_crystal_list name this returns an OrderdDict
crystal_data = f90nml.read(crystal_file)["simcal_crystal_list"]
if isinstance(crystal_data, list):
crystal_data = crystal_data[0]
input_data = self._get_simcal_input(data_collect_parameters,
crystal_data)
# NB outfile is the echo output of the input file;
# image files templates are set in the input file
file_info = data_collect_parameters["fileinfo"]
if not os.path.exists(file_info["directory"]):
os.makedirs(file_info["directory"])
if not os.path.exists(file_info["directory"]):
os.makedirs(file_info["directory"])
infile = os.path.join(file_info["directory"],
"simcal_in_%s.nml" % self._counter)
f90nml.write(input_data, infile, force=True)
outfile = os.path.join(file_info["directory"],
"simcal_out_%s.nml" % self._counter)
logfile = os.path.join(file_info["directory"],
"simcal_log_%s.txt" % self._counter)
self._counter += 1
hklfile = os.path.join(sample_dir, "sample.hkli")
if not os.path.isfile(hklfile):
raise ValueError("Emulator hkli file %s does not exist" % hklfile)
command_list = [
simcal_executive,
"--input",
infile,
"--output",
outfile,
"--hkl",
hklfile,
]
for tag, val in self["simcal_options"].getProperties().items():
command_list.extend(
ConvertUtils.command_option(tag, val, prefix="--"))
logging.getLogger("HWR").info("Executing command: %s", command_list)
logging.getLogger("HWR").info("Executing environment: %s" %
sorted(envs.items()))
fp1 = open(logfile, "w")
fp2 = subprocess.STDOUT
# resource.setrlimit(resource.RLIMIT_STACK, (-1,-1))
try:
running_process = subprocess.Popen(command_list,
stdout=fp1,
stderr=fp2,
env=envs)
gphl_connection.collect_emulator_process = running_process
except BaseException:
logging.getLogger("HWR").error(
"Error in spawning workflow application")
raise
finally:
fp1.close()
# This does waiting, so we want to collect the result afterwards
super(CollectEmulator, self).data_collection_hook()
logging.getLogger("HWR").info(
"Waiting for simcal collection emulation.")
# NBNB TODO put in time-out, somehow
return_code = running_process.wait()
process = gphl_connection.collect_emulator_process
gphl_connection.collect_emulator_process = None
if process == "ABORTED":
logging.getLogger("HWR").info(
"Simcal collection emulation aborted")
elif return_code:
raise RuntimeError(
"simcal process terminated with return code %s" % return_code)
else:
logging.getLogger("HWR").info(
"Simcal collection emulation successful")
return
def setup(self):
# TODO: Find a better place to generate this list
self.config_files = [f for f in os.listdir(self.control_path)
if f.startswith('data')]
self.config_files.append('eedata')
# Generic model setup
super(Mitgcm, self).setup()
# Link restart files to work directory
if self.prior_restart_path and not self.expt.repeat_run:
# Determine total number of timesteps since initialisation
core_restarts = [f for f in os.listdir(self.prior_restart_path)
if f.startswith('pickup.')]
try:
# NOTE: Use the most recent, in case of multiple restarts
n_iter0 = max([int(f.split('.')[1]) for f in core_restarts])
except ValueError:
sys.exit("payu: error: no restart files found.")
else:
n_iter0 = 0
# Update configuration file 'data'
data_path = os.path.join(self.work_path, 'data')
data_nml = f90nml.read(data_path)
# NOTE: Assumes that these are always present
dt = data_nml['parm03']['deltat']
n_timesteps = data_nml['parm03']['ntimesteps']
# NOTE: Consider permitting pchkpt_freq < dt * n_timesteps
# NOTE: May re-enable chkpt_freq in the future
data_nml['parm03']['niter0'] = n_iter0
data_nml['parm03']['pchkptfreq'] = dt * n_timesteps
data_nml['parm03']['chkptfreq'] = 0
data_nml.write(data_path, force=True)
# Patch or create data.mnc
mnc_header = os.path.join(self.work_path, 'mnc_')
data_mnc_path = os.path.join(self.work_path, 'data.mnc')
try:
data_mnc_nml = f90nml.read(data_mnc_path)
data_mnc_nml['mnc_01']['mnc_outdir_str'] = mnc_header
data_mnc_nml.write(data_mnc_path, force=True)
except IOError as exc:
if exc.errno == errno.ENOENT:
mnc_01_grp = {
'mnc_use_outdir': True,
'mnc_use_name_ni0': True,
'mnc_outdir_str': mnc_header,
'mnc_outdir_date': True,
'monitor_mnc': True
}
data_mnc_nml = {'mnc_01': mnc_01_grp}
f90nml.write(data_mnc_nml, data_mnc_path)
else:
raise
def prepare_for_metgrid(self, wps_nml):
"""
Set all domains that we use to active.
:param wps_nml: the WPS namelist
"""
wps_nml['share']['active_grid'] = [True] * len(self.domains)
if __name__ == '__main__':
import f90nml
import sys
import json
# parse a JSON domain configuration and a namelist.wps file and build the correct wps
if len(sys.argv) != 3 and len(sys.argv) != 4:
print('usage: %s <domains_json_file> <namelist.wps> [namelist.input]' % sys.argv[0])
sys.exit(1)
dcfg = WPSDomainConf(json.load(open(sys.argv[1])))
wps_nml = f90nml.read(sys.argv[2])
wrf_nml = f90nml.read(sys.argv[3]) if len(sys.argv) == 4 else None
dcfg.prepare_for_geogrid(wps_nml, wrf_nml)
f90nml.write(wps_nml, sys.argv[2], force=True)
if wrf_nml is not None:
f90nml.write(wps_nml, sys.argv[3], force=True)
def write_ppt_files(tasks):
freqgroups = cmor_utils.group(tasks, get_output_freq)
for freq1 in freqgroups:
for freq2 in freqgroups:
if freq2 > freq1 and freq2 % freq1 == 0:
freqgroups[freq2] = freqgroups[freq1] + freqgroups[freq2]
for freq in freqgroups:
mfp2df, mfpphy, mfp3dfs, mfp3dfp, mfp3dfv = [], [], [], [], []
alevs, plevs, hlevs = [], [], []
for task in freqgroups[freq]:
zaxis, levs = cmor_target.get_z_axis(task.target)
root_codes = task.source.get_root_codes()
if not zaxis:
for code in root_codes:
if code in cmor_source.ifs_source.grib_codes_3D:
log.warning(
"3D grib code %s used in 2D cmor-target %s..."
"assuming this is on model levels" %
(str(code), task.target.variable))
mfp3dfs.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
log.info(
"Adding grib code %s to MFP2DF %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
log.info(
"Adding grib code %s to MFPPHY %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfpphy.append(code)
else:
log.error("Unknown IFS grib code %s skipped" %
str(code))
else:
for code in root_codes:
if code in cmor_source.ifs_source.grib_codes_3D:
if zaxis in cmor_target.model_axes:
log.info(
"Adding grib code %s to MFP3DFS %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp3dfs.append(code)
alevs.extend(levs)
elif zaxis in cmor_target.pressure_axes:
log.info(
"Adding grib code %s to MFP3DFP %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp3dfp.append(code)
plevs.extend(levs)
elif zaxis in cmor_target.height_axes:
log.info(
"Adding grib code %s to MFP3DFV %dhr ppt file for variable "
"%s in table %s" %
(str(code), freq, task.target.variable,
task.target.table))
mfp3dfv.append(code)
hlevs.extend(levs)
else:
log.error(
"Axis type %s unknown, adding grib code %s"
"to model level variables" %
(zaxis, str(code)))
elif code in cmor_source.ifs_source.grib_codes_2D_dyn:
mfp2df.append(code)
elif code in cmor_source.ifs_source.grib_codes_2D_phy:
mfpphy.append(code)
else:
log.error("Unknown IFS grib code %s skipped" %
str(code))
# Always add the geopotential, recommended by ECMWF
if cmor_source.grib_code(129) not in mfp3dfs:
mfp2df.append(cmor_source.grib_code(129))
# Always add the surface pressure, recommended by ECMWF
mfpphy.append(cmor_source.grib_code(134))
# Always add the logarithm of surface pressure, recommended by ECMWF
mfp2df.append(cmor_source.grib_code(152))
mfp2df = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp2df))))
mfpphy = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfpphy))))
mfp3dfs = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp3dfs))))
mfp3dfp = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp3dfp))))
mfp3dfv = sorted(
list(
map(lambda c: c.var_id if c.tab_id == 128 else c.__hash__(),
set(mfp3dfv))))
plevs = sorted(list(set([float(s) for s in plevs])))[::-1]
hlevs = sorted(list(set([float(s) for s in hlevs])))
namelist = {"CFPFMT": "MODEL"}
if any(mfp2df):
namelist["NFP2DF"] = len(mfp2df)
namelist["MFP2DF"] = mfp2df
if any(mfpphy):
namelist["NFPPHY"] = len(mfpphy)
namelist["MFPPHY"] = mfpphy
if any(mfp3dfs):
namelist["NFP3DFS"] = len(mfp3dfs)
namelist["MFP3DFS"] = mfp3dfs
namelist["NRFP3S"] = -1
if any(mfp3dfp):
namelist["NFP3DFP"] = len(mfp3dfp)
namelist["MFP3DFP"] = mfp3dfp
namelist["RFP3P"] = plevs
if any(mfp3dfs):
namelist["NFP3DFV"] = len(mfp3dfv)
namelist["MFP3DFV"] = mfp3dfv
namelist["RFP3V"] = hlevs
nml = f90nml.Namelist({"NAMFPC": namelist})
nml.uppercase, nml.end_comma = True, True
f90nml.write(nml, "pptdddddd%04d" % (100 * freq, ))
def _write_default():
"""create default par file"""
if not os.path.isfile(PAR_DFLT_FILE):
f90nml.write(PAR_DEFAULT, PAR_DFLT_FILE)
def data_collection_hook(self):
"""Spawns data emulation using gphl simcal"""
data_collect_parameters = self.current_dc_parameters
logging.getLogger('HWR').debug("Emulator: nominal position "
+ ', '.join('%s=%s' % (tt)
for tt in sorted(data_collect_parameters['motors'].items())
if tt[1] is not None)
)
logging.getLogger('HWR').debug("Emulator: actual position "
+ ', '.join('%s=%s' % tt
for tt in sorted(self.diffractometer_hwobj.get_positions().items())
if tt[1] is not None)
)
# Done here as there are what-happens-first conflicts
# if you put it in init
bl_setup_hwobj = HardwareRepository().getHardwareObject(
'beamline-setup'
)
if self.gphl_workflow_hwobj is None:
self.gphl_workflow_hwobj = bl_setup_hwobj.gphl_workflow_hwobj
if not self.gphl_workflow_hwobj:
raise ValueError("Emulator requires GPhL workflow installation")
if self.gphl_connection_hwobj is None:
self.gphl_connection_hwobj = bl_setup_hwobj.gphl_connection_hwobj
if not self.gphl_connection_hwobj:
raise ValueError("Emulator requires GPhL connection installation")
# Get program locations
simcal_executive = self.gphl_connection_hwobj.get_executable('simcal')
# Get environmental variables
envs = {'BDG_home':
self.gphl_connection_hwobj.software_paths['BDG_home'],
'GPHL_INSTALLATION':
self.gphl_connection_hwobj.software_paths['GPHL_INSTALLATION']
}
for tag, val in self['environment_variables'].getProperties().items():
envs[str(tag)] = str(val)
# get crystal data
sample_name = self.getProperty('default_sample_name')
sample = self.sample_changer_hwobj.getLoadedSample()
if sample:
ss = sample.getName()
if ss and ss.startswith(self.TEST_SAMPLE_PREFIX):
sample_name = ss[len(self.TEST_SAMPLE_PREFIX):]
sample_dir = self.gphl_connection_hwobj.software_paths.get(
'gphl_test_samples'
)
if not sample_dir:
raise ValueError(
"Emulator requires gphl_test_samples dir specified"
)
sample_dir = os.path.join(sample_dir, sample_name)
if not os.path.isdir(sample_dir):
raise ValueError(
"Sample data directory %s does not exist" % sample_dir
)
crystal_file = os.path.join(sample_dir, 'crystal.nml')
if not os.path.isfile(crystal_file):
raise ValueError(
"Emulator crystal data file %s does not exist" % crystal_file
)
# in spite of the simcal_crystal_list name this returns an OrderdDict
crystal_data = f90nml.read(crystal_file)['simcal_crystal_list']
input_data = self._get_simcal_input(data_collect_parameters,
crystal_data)
# NB outfile is the echo output of the input file;
# image files templates are set in the input file
file_info = data_collect_parameters['fileinfo']
if not os.path.exists(file_info['directory']):
os.makedirs(file_info['directory'])
if not os.path.exists(file_info['directory']):
os.makedirs(file_info['directory'])
infile = os.path.join(file_info['directory'],
'simcal_in_%s.nml' % self._counter)
f90nml.write(input_data, infile, force=True)
outfile = os.path.join(file_info['directory'],
'simcal_out_%s.nml' % self._counter)
logfile = os.path.join(file_info['directory'],
'simcal_log_%s.txt' % self._counter)
self._counter += 1
hklfile = os.path.join(sample_dir, 'sample.hkli')
if not os.path.isfile(hklfile):
raise ValueError(
"Emulator hkli file %s does not exist" % hklfile
)
command_list = [simcal_executive, '--input', infile, '--output', outfile,
'--hkl', hklfile]
for tag, val in self['simcal_options'].getProperties().items():
command_list.extend(ConvertUtils.command_option(tag, val,
prefix='--'))
logging.getLogger('HWR').info("Executing command: %s" % command_list)
logging.getLogger('HWR').info("Executing environment: %s"
% sorted(envs.items()))
fp1 = open(logfile, 'w')
fp2 = subprocess.STDOUT
# resource.setrlimit(resource.RLIMIT_STACK, (-1,-1))
try:
running_process = subprocess.Popen(command_list, stdout=fp1,
stderr=fp2, env=envs)
except:
logging.getLogger('HWR').error('Error in spawning workflow application')
raise
finally:
fp1.close()
# This does waiting, so we want to collect the result afterwards
super(CollectEmulator, self).data_collection_hook()
logging.getLogger('HWR').info(
'Waiting for simcal collection emulation.'
)
# NBNB TODO put in time-out, somehow
if running_process is not None:
return_code = running_process.wait()
if return_code:
raise RuntimeError("simcal process terminated with return code %s"
% return_code)
else:
logging.getLogger('HWR').info(
'Simcal collection emulation successful'
)
return
def setup(self):
cpl_keys = {'cice': ('input_ice.nml', 'coupling_nml', 'runtime0'),
'matm': ('input_atm.nml', 'coupling', 'truntime0')}
# Keep track of this in order to set the oasis runtime.
run_runtime = 0
for model in self.expt.models:
if model.model_type == 'cice':
# Stage the supplemental input files
if model.prior_restart_path:
for f_name in model.access_restarts:
f_src = os.path.join(model.prior_restart_path, f_name)
f_dst = os.path.join(model.work_input_path, f_name)
if os.path.isfile(f_src):
make_symlink(f_src, f_dst)
if model.model_type in ('cice', 'matm'):
# Update the supplemental OASIS namelists
cpl_fname, cpl_group, runtime0_key = cpl_keys[model.model_type]
cpl_fpath = os.path.join(model.work_path, cpl_fname)
cpl_nml = f90nml.read(cpl_fpath)
# Which calendar are we using, noleap or Gregorian.
caltype = cpl_nml[cpl_group]['caltype']
init_date = cal.int_to_date(cpl_nml[cpl_group]['init_date'])
# Get time info about the beginning of this run. We're
# interested in:
# 1. start date of run
# 2. total runtime of all previous runs.
if model.prior_output_path and not self.expt.repeat_run:
prior_cpl_fpath = os.path.join(model.prior_output_path,
cpl_fname)
prior_cpl_nml = f90nml.read(prior_cpl_fpath)
cpl_nml_grp = prior_cpl_nml[cpl_group]
# The total time in seconds since the beginning of
# the experiment.
total_runtime = int(cpl_nml_grp[runtime0_key] +
cpl_nml_grp['runtime'])
run_start_date = cal.date_plus_seconds(init_date,
total_runtime,
caltype)
else:
total_runtime = 0
run_start_date = init_date
# Get new runtime for this run. We get this from either the
# 'runtime' part of the payu config, or from the namelist
if self.expt.runtime:
run_runtime = cal.runtime_from_date(
run_start_date,
self.expt.runtime['years'],
self.expt.runtime['months'],
self.expt.runtime['days'],
self.expt.runtime.get('seconds', 0),
caltype)
else:
run_runtime = cpl_nml[cpl_group]['runtime']
# Now write out new run start date and total runtime.
cpl_nml[cpl_group]['inidate'] = cal.date_to_int(run_start_date)
cpl_nml[cpl_group][runtime0_key] = total_runtime
cpl_nml[cpl_group]['runtime'] = int(run_runtime)
if model.model_type == 'cice':
if self.expt.counter and not self.expt.repeat_run:
cpl_nml[cpl_group]['jobnum'] = 1 + self.expt.counter
else:
cpl_nml[cpl_group]['jobnum'] = 1
nml_work_path = os.path.join(model.work_path, cpl_fname)
f90nml.write(cpl_nml, nml_work_path + '~')
shutil.move(nml_work_path + '~', nml_work_path)
# Now change the oasis runtime. This needs to be done after the others.
for model in self.expt.models:
if model.model_type == 'oasis':
namcouple = os.path.join(model.work_path, 'namcouple')
s = ''
with open(namcouple, 'r+') as f:
s = f.read()
m = re.search(r"^[ \t]*\$RUNTIME.*?^[ \t]*(\d+)", s,
re.MULTILINE | re.DOTALL)
assert(m is not None)
s = s[:m.start(1)] + str(run_runtime) + s[m.end(1):]
with open(namcouple, 'w') as f:
f.write(s)
def data_collection_hook(self):
"""Spawns data emulation using gphl simcal"""
data_collect_parameters = self.current_dc_parameters
if not api.gphl_workflow:
raise ValueError("Emulator requires GPhL workflow installation")
gphl_connection = api.gphl_connection
if not gphl_connection:
raise ValueError("Emulator requires GPhL connection installation")
# Get program locations
simcal_executive = gphl_connection.get_executable("simcal")
# Get environmental variables
envs = {
"BDG_home": gphl_connection.software_paths["BDG_home"],
"GPHL_INSTALLATION": gphl_connection.software_paths[
"GPHL_INSTALLATION"
],
}
for tag, val in self["environment_variables"].getProperties().items():
envs[str(tag)] = str(val)
# get crystal data
sample_name = self.getProperty("default_sample_name")
sample = self.sample_changer_hwobj.getLoadedSample()
if sample:
ss0 = sample.getName()
if ss0 and ss0.startswith(self.TEST_SAMPLE_PREFIX):
sample_name = ss0[len(self.TEST_SAMPLE_PREFIX) :]
sample_dir = gphl_connection.software_paths.get("gphl_test_samples")
if not sample_dir:
raise ValueError("Emulator requires gphl_test_samples dir specified")
sample_dir = os.path.join(sample_dir, sample_name)
if not os.path.isdir(sample_dir):
raise ValueError("Sample data directory %s does not exist" % sample_dir)
crystal_file = os.path.join(sample_dir, "crystal.nml")
if not os.path.isfile(crystal_file):
raise ValueError(
"Emulator crystal data file %s does not exist" % crystal_file
)
# in spite of the simcal_crystal_list name this returns an OrderdDict
crystal_data = f90nml.read(crystal_file)["simcal_crystal_list"]
if isinstance(crystal_data, list):
crystal_data = crystal_data[0]
input_data = self._get_simcal_input(data_collect_parameters, crystal_data)
# NB outfile is the echo output of the input file;
# image files templates are set in the input file
file_info = data_collect_parameters["fileinfo"]
if not os.path.exists(file_info["directory"]):
os.makedirs(file_info["directory"])
if not os.path.exists(file_info["directory"]):
os.makedirs(file_info["directory"])
infile = os.path.join(
file_info["directory"], "simcal_in_%s.nml" % self._counter
)
f90nml.write(input_data, infile, force=True)
outfile = os.path.join(
file_info["directory"], "simcal_out_%s.nml" % self._counter
)
logfile = os.path.join(
file_info["directory"], "simcal_log_%s.txt" % self._counter
)
self._counter += 1
hklfile = os.path.join(sample_dir, "sample.hkli")
if not os.path.isfile(hklfile):
raise ValueError("Emulator hkli file %s does not exist" % hklfile)
command_list = [
simcal_executive,
"--input",
infile,
"--output",
outfile,
"--hkl",
hklfile,
]
for tag, val in self["simcal_options"].getProperties().items():
command_list.extend(ConvertUtils.command_option(tag, val, prefix="--"))
logging.getLogger("HWR").info("Executing command: %s", command_list)
logging.getLogger("HWR").info(
"Executing environment: %s" % sorted(envs.items())
)
fp1 = open(logfile, "w")
fp2 = subprocess.STDOUT
# resource.setrlimit(resource.RLIMIT_STACK, (-1,-1))
try:
running_process = subprocess.Popen(
command_list, stdout=fp1, stderr=fp2, env=envs
)
gphl_connection.collect_emulator_process = running_process
except BaseException:
logging.getLogger("HWR").error("Error in spawning workflow application")
raise
finally:
fp1.close()
# This does waiting, so we want to collect the result afterwards
super(CollectEmulator, self).data_collection_hook()
logging.getLogger("HWR").info("Waiting for simcal collection emulation.")
# NBNB TODO put in time-out, somehow
return_code = running_process.wait()
process = gphl_connection.collect_emulator_process
gphl_connection.collect_emulator_process = None
if process == 'ABORTED':
logging.getLogger("HWR").info("Simcal collection emulation aborted")
elif return_code:
raise RuntimeError(
"simcal process terminated with return code %s" % return_code
)
else:
logging.getLogger("HWR").info("Simcal collection emulation successful")
return
def setup(self):
# TODO: Find a better place to generate this list
self.config_files = [f for f in os.listdir(self.control_path)
if f.startswith('data')]
self.config_files.append('eedata')
# Generic model setup
super(Mitgcm, self).setup()
# Link restart files to work directory
if self.prior_restart_path and not self.expt.repeat_run:
# Determine total number of timesteps since initialisation
core_restarts = [f for f in os.listdir(self.prior_restart_path)
if f.startswith('pickup.')]
try:
# NOTE: Use the most recent, in case of multiple restarts
n_iter0 = max([int(f.split('.')[1]) for f in core_restarts])
except ValueError:
sys.exit("payu: error: no restart files found.")
else:
n_iter0 = 0
# Update configuration file 'data'
data_path = os.path.join(self.work_path, 'data')
# MITgcm strips shell-style (#) comments from its namelists
nml_parser = f90nml.Parser()
nml_parser.comment_tokens += '#'
data_nml = nml_parser.read(data_path)
# Timesteps are either global (deltat) or divided into momentum
# (deltatmom) and tracer (deltat). If deltat is missing, then we just
# try deltatmom. But I am not sure how to best handle this case.
# TODO: Sort this out with an MITgcm user
try:
dt = data_nml['parm03']['deltat']
except KeyError:
dt = data_nml['parm03']['deltatmom']
# Runtime seems to be set either by timesteps (ntimesteps) or physical
# time (startTime and endTime).
# TODO: Sort this out with an MITgcm user
try:
n_timesteps = data_nml['parm03']['ntimesteps']
pchkpt_freq = dt * n_timesteps
except KeyError:
t_start = data_nml['parm03']['starttime']
t_end = data_nml['parm03']['endtime']
pchkpt_freq = t_end - t_start
# NOTE: Consider permitting pchkpt_freq < dt * n_timesteps
# NOTE: May re-enable chkpt_freq in the future
data_nml['parm03']['niter0'] = n_iter0
data_nml['parm03']['pchkptfreq'] = pchkpt_freq
data_nml['parm03']['chkptfreq'] = 0
data_nml.write(data_path, force=True)
# Patch or create data.mnc
mnc_header = os.path.join(self.work_path, 'mnc_')
data_mnc_path = os.path.join(self.work_path, 'data.mnc')
try:
data_mnc_nml = f90nml.read(data_mnc_path)
data_mnc_nml['mnc_01']['mnc_outdir_str'] = mnc_header
data_mnc_nml.write(data_mnc_path, force=True)
except IOError as exc:
if exc.errno == errno.ENOENT:
mnc_01_grp = {
'mnc_use_outdir': True,
'mnc_use_name_ni0': True,
'mnc_outdir_str': mnc_header,
'mnc_outdir_date': True,
'monitor_mnc': True
}
data_mnc_nml = {'mnc_01': mnc_01_grp}
f90nml.write(data_mnc_nml, data_mnc_path)
else:
raise
def execute(args):
"""
Executes a weather/fire simulation.
The args dictionary contains
:param args: a dictionary with the following keys
:param grid_code: the (unique) code of the grid that is used
:param sys_install_path: system installation directory
:param start_utc: start time of simulation in UTC
:param end_utc: end time of simulation in UTC
:param workspace_path: workspace directory
:param wps_install_path: installation directory of WPS that will be used
:param wrf_install_path: installation directory of WRF that will be used
:param grib_source: a string identifying a valid GRIB2 source
:param wps_namelist_path: the path to the namelist.wps file that will be used as template
:param wrf_namelist_path: the path to the namelist.input file that will be used as template
:param fire_namelist_path: the path to the namelist.fire file that will be used as template
:param wps_geog_path: the path to the geogrid data directory providing terrain/fuel data
:param email_notification: dictionary containing keys address and events indicating when a mail should be fired off
"""
logging.basicConfig(level=logging.INFO)
# initialize the job state from the arguments
js = JobState(args)
logging.info("job %s starting [%d hours to forecast]." % (js.job_id, js.fc_hrs))
send_email(js, 'start', 'Job %s started.' % js.job_id)
# read in all namelists
js.wps_nml = f90nml.read(args['wps_namelist_path'])
js.wrf_nml = f90nml.read(args['wrf_namelist_path'])
js.fire_nml = f90nml.read(args['fire_namelist_path'])
js.ems_nml = None
if 'emissions_namelist_path' in args:
js.ems_nml = f90nml.read(args['emissions_namelist_path'])
# Parse and setup the domain configuration
js.domain_conf = WPSDomainConf(js.domains)
num_doms = len(js.domain_conf)
js.wps_nml['share']['start_date'] = [utc_to_esmf(js.start_utc)] * num_doms
js.wps_nml['share']['end_date'] = [utc_to_esmf(js.end_utc)] * num_doms
js.wps_nml['share']['interval_seconds'] = 3600
logging.info("number of domains defined is %d." % num_doms)
# build directories in workspace
js.wps_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wps'))
js.wrf_dir = osp.abspath(osp.join(js.workspace_path, js.job_id, 'wrf'))
logging.info("cloning WPS into %s" % js.wps_dir)
# step 1: clone WPS and WRF directories
cln = WRFCloner(args)
cln.clone_wps(js.wps_dir, js.grib_source.vtables(), [])
# step 2: process domain information and patch namelist for geogrid
js.wps_nml['geogrid']['geog_data_path'] = args['wps_geog_path']
js.domain_conf.prepare_for_geogrid(js.wps_nml, js.wrf_nml, js.wrfxpy_dir, js.wps_dir)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
# do steps 2 & 3 & 4 in parallel (two execution streams)
# -> GEOGRID ->
# -> GRIB2 download -> UNGRIB ->
proc_q = Queue()
geogrid_proc = Process(target=run_geogrid, args=(js, proc_q))
grib_proc = Process(target=retrieve_gribs_and_run_ungrib, args=(js, proc_q))
geogrid_proc.start()
grib_proc.start()
# wait until both tasks are done
geogrid_proc.join()
grib_proc.join()
if proc_q.get() != 'SUCCESS':
return
if proc_q.get() != 'SUCCESS':
return
proc_q.close()
# step 5: execute metgrid after ensuring all grids will be processed
js.domain_conf.prepare_for_metgrid(js.wps_nml)
f90nml.write(js.wps_nml, osp.join(js.wps_dir, 'namelist.wps'), force=True)
logging.info("running METGRID")
Metgrid(js.wps_dir).execute().check_output()
send_email(js, 'metgrid', 'Job %s - metgrid complete.' % js.job_id)
logging.info("cloning WRF into %s" % js.wrf_dir)
# step 6: clone wrf directory, symlink all met_em* files
cln.clone_wrf(js.wrf_dir, [])
symlink_matching_files(js.wrf_dir, js.wps_dir, "met_em*")
logging.info("running REAL")
# step 7: patch input namelist, fire namelist, emissions namelist (if required)
# and execute real.exe
time_ctrl = update_time_control(js.start_utc, js.end_utc, num_doms)
js.wrf_nml['time_control'].update(time_ctrl)
update_namelist(js.wrf_nml, js.grib_source.namelist_keys())
if 'ignitions' in args:
update_namelist(js.wrf_nml, render_ignitions(js, num_doms))
# if we have an emissions namelist, automatically turn on the tracers
if js.ems_nml is not None:
f90nml.write(js.ems_nml, osp.join(js.wrf_dir, 'namelist.fire_emissions'), force=True)
js.wrf_nml['dynamics']['tracer_opt'] = [2] * num_doms
f90nml.write(js.wrf_nml, osp.join(js.wrf_dir, 'namelist.input'), force=True)
f90nml.write(js.fire_nml, osp.join(js.wrf_dir, 'namelist.fire'), force=True)
# try to run Real twice as it sometimes fails the first time
# it's not clear why this error happens
try:
Real(js.wrf_dir).execute().check_output()
except Exception as e:
logging.error('Real step failed with exception %s, retrying ...' % str(e))
Real(js.wrf_dir).execute().check_output()
# step 8: if requested, do fuel moisture DA
if js.fmda is not None:
logging.info('running fuel moisture data assimilation')
for dom in js.fmda.domains:
assimilate_fm10_observations(osp.join(wrf_dir, 'wrfinput_d%02d' % dom), None, js.fmda.token)
logging.info('submitting WRF job')
send_email(js, 'wrf_submit', 'Job %s - wrf job submitted.' % js.job_id)
# step 8: execute wrf.exe on parallel backend
js.task_id = "sim-" + js.grid_code + "-" + utc_to_esmf(js.start_utc)[:10]
WRF(js.wrf_dir, js.qsys).submit(js.task_id, js.num_nodes, js.ppn, js.wall_time_hrs)
send_email(js, 'wrf_exec', 'Job %s - wrf job starting now with id %s.' % (js.job_id, js.task_id))
logging.info("WRF job submitted with id %s, waiting for rsl.error.0000" % js.task_id)
# step 9: wait for appearance of rsl.error.0000 and open it
wrf_out = None
while wrf_out is None:
try:
wrf_out = open(osp.join(js.wrf_dir, 'rsl.error.0000'))
break
except IOError:
logging.info('forecast: waiting 10 seconds for rsl.error.0000 file')
time.sleep(5)
logging.info('Detected rsl.error.0000')
# step 10: track log output and check for history writes fro WRF
pp = None
already_sent_files, max_pp_dom = [], -1
if js.postproc is not None:
js.pp_dir = osp.join(js.workspace_path, js.job_id, "products")
make_dir(js.pp_dir)
pp = Postprocessor(js.pp_dir, 'wfc-' + js.grid_code)
max_pp_dom = max([int(x) for x in filter(lambda x: len(x) == 1, js.postproc)])
while True:
line = wrf_out.readline().strip()
if not line:
time.sleep(0.2)
continue
if "SUCCESS COMPLETE WRF" in line:
send_email(js, 'complete', 'Job %s - wrf job complete SUCCESS.' % js.job_id)
logging.info("WRF completion detected.")
break
if "Timing for Writing wrfout" in line:
esmf_time,domain_str = re.match(r'.*wrfout_d.._([0-9_\-:]{19}) for domain\ +(\d+):' ,line).groups()
dom_id = int(domain_str)
logging.info("Detected history write for domain %d for time %s." % (dom_id, esmf_time))
if js.postproc is not None and str(dom_id) in js.postproc:
var_list = [str(x) for x in js.postproc[str(dom_id)]]
logging.info("Executing postproc instructions for vars %s for domain %d." % (str(var_list), dom_id))
wrfout_path = find_fresh_wrfout(js.wrf_dir, dom_id)
try:
pp.process_vars(wrfout_path, dom_id, esmf_time, var_list)
except Exception as e:
logging.warning('Failed to postprocess for time %s with error %s.' % (esmf_time, str(e)))
# if this is the last processed domain for this timestamp in incremental mode, upload to server
if dom_id == max_pp_dom and js.postproc.get('shuttle', None) == 'incremental':
desc = js.postproc['description'] if 'description' in js.postproc else js.job_id
sent_files_1 = send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, desc, already_sent_files)
logging.info('sent %d files to visualization server.' % len(sent_files_1))
already_sent_files = filter(lambda x: not x.endswith('json'), already_sent_files + sent_files_1)
# if we are to send out the postprocessed files after completion, this is the time
if js.postproc.get('shuttle', None) == 'on_completion':
desc = js.postproc['description'] if 'description' in js.postproc else js.job_id
send_product_to_server(args, js.pp_dir, js.job_id, js.job_id, desc)
def parse():
parser = argparse.ArgumentParser()
parser.add_argument('--version', action='version',
version='f90nml {}'.format(f90nml.__version__))
parser.add_argument('--group', '-g', action='store')
parser.add_argument('--set', '-s', action='append')
parser.add_argument('--patch', '-p', action='store_false')
parser.add_argument('input')
parser.add_argument('output', nargs='?')
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
args = parser.parse_args()
input_fname = args.input
output_fname = args.output
# Input config
if input_fname:
input_root, input_ext = os.path.splitext(input_fname)
if input_ext == '.json':
with open(input_fname) as input_file:
input_data = json.load(input_file)
elif has_yaml and input_ext == '.yaml':
with open(input_fname) as input_file:
input_data = yaml.load(input_file)
else:
# Assume unrecognised extensions are namelists
input_data = f90nml.read(input_fname)
else:
input_data = {}
# Replace any values
if args.set:
if not args.group:
# Use the first available group
grp = list(input_data.keys())[0]
print('f90nml: warning: Assuming variables are in group \'{}\'.'
''.format(grp))
else:
grp = args.group
update_nml = '&{} {} /\n'.format(grp, ', '.join(args.set))
with io.StringIO(update_nml) as update_io:
update_data = f90nml.read(update_io)
input_data[grp].update(update_data[grp])
# Target output
if output_fname:
output_root, output_ext = os.path.splitext(output_fname)
if output_ext == '.json':
with open(output_fname, 'w') as output_file:
json.dump(input_data, output_file,
indent=4, separators=(',', ': '))
output_file.write('\n')
elif has_yaml and output_ext == '.yaml':
with open(output_fname, 'w') as output_file:
yaml.dump(input_data, output_file)
else:
# Default to namelist output
f90nml.write(input_data, output_fname)
else:
# TODO: Combine with extension output
f90nml.write(input_data, sys.stdout)
def write_input_dic(
run_type_dir, new_input_dic
):
f90nml.write(new_input_dic,
os.path.join(run_type_dir, NAMELIST_INPUT),
run_type_dir)
def setup(self):
if not self.top_level_model:
return
cpl_keys = {'cice': ('input_ice.nml', 'coupling', 'runtime0'),
'matm': ('input_atm.nml', 'coupling', 'truntime0')}
# Keep track of this in order to set the oasis runtime.
run_runtime = 0
for model in self.expt.models:
if model.model_type == 'cice' or model.model_type == 'cice5':
# Horrible hack to make a link to o2i.nc in the
# work/ice/RESTART directory
f_name = 'o2i.nc'
f_src = os.path.join(model.work_path, f_name)
f_dst = os.path.join(model.work_restart_path, f_name)
if os.path.isfile(f_src):
make_symlink(f_src, f_dst)
if model.model_type == 'cice5':
# Stage the supplemental input files
if model.prior_restart_path:
for f_name in model.access_restarts:
f_src = os.path.join(model.prior_restart_path, f_name)
f_dst = os.path.join(model.work_input_path, f_name)
if os.path.isfile(f_src):
make_symlink(f_src, f_dst)
if model.model_type in ('cice', 'matm'):
# Update the supplemental OASIS namelists
cpl_fname, cpl_group, runtime0_key = cpl_keys[model.model_type]
cpl_fpath = os.path.join(model.work_path, cpl_fname)
cpl_nml = f90nml.read(cpl_fpath)
# Which calendar are we using, noleap or Gregorian.
caltype = cpl_nml[cpl_group]['caltype']
init_date = cal.int_to_date(cpl_nml[cpl_group]['init_date'])
# Get time info about the beginning of this run. We're
# interested in:
# 1. start date of run
# 2. total runtime of all previous runs.
if model.prior_restart_path and not self.expt.repeat_run:
prior_cpl_fpath = os.path.join(model.prior_restart_path,
cpl_fname)
# With later versions this file exists in the prior restart
# path, but this was not always the case, so check, and if
# not there use prior output path
if not os.path.exists(prior_cpl_fpath):
print('payu: warning: {0} missing from prior restart '
'path; checking prior output.'.format(cpl_fname),
file=sys.stderr)
if not os.path.isdir(model.prior_output_path):
print('payu: error: No prior output path; '
'aborting run.')
sys.exit(errno.ENOENT)
prior_cpl_fpath = os.path.join(model.prior_output_path,
cpl_fname)
try:
prior_cpl_nml = f90nml.read(prior_cpl_fpath)
except IOError as exc:
if exc.errno == errno.ENOENT:
print('payu: error: {0} does not exist; aborting.'
''.format(prior_cpl_fpath), file=sys.stderr)
sys.exit(exc.errno)
else:
raise
cpl_nml_grp = prior_cpl_nml[cpl_group]
# The total time in seconds since the beginning of
# the experiment.
total_runtime = int(cpl_nml_grp[runtime0_key] +
cpl_nml_grp['runtime'])
run_start_date = cal.date_plus_seconds(init_date,
total_runtime,
caltype)
else:
total_runtime = 0
run_start_date = init_date
# Get new runtime for this run. We get this from either the
# 'runtime' part of the payu config, or from the namelist
if self.expt.runtime:
run_runtime = cal.runtime_from_date(
run_start_date,
self.expt.runtime['years'],
self.expt.runtime['months'],
self.expt.runtime['days'],
self.expt.runtime.get('seconds', 0),
caltype)
else:
run_runtime = cpl_nml[cpl_group]['runtime']
# Now write out new run start date and total runtime.
cpl_nml[cpl_group]['inidate'] = cal.date_to_int(run_start_date)
cpl_nml[cpl_group][runtime0_key] = total_runtime
cpl_nml[cpl_group]['runtime'] = int(run_runtime)
if model.model_type == 'cice':
if self.expt.counter and not self.expt.repeat_run:
cpl_nml[cpl_group]['jobnum'] = 1 + self.expt.counter
else:
cpl_nml[cpl_group]['jobnum'] = 1
nml_work_path = os.path.join(model.work_path, cpl_fname)
f90nml.write(cpl_nml, nml_work_path + '~')
shutil.move(nml_work_path + '~', nml_work_path)
# Now change the oasis runtime. This needs to be done after the others.
for model in self.expt.models:
if model.model_type == 'oasis':
namcouple = os.path.join(model.work_path, 'namcouple')
s = ''
with open(namcouple, 'r+') as f:
s = f.read()
m = re.search(r"^[ \t]*\$RUNTIME.*?^[ \t]*(\d+)", s,
re.MULTILINE | re.DOTALL)
assert(m is not None)
s = s[:m.start(1)] + str(run_runtime) + s[m.end(1):]
with open(namcouple, 'w') as f:
f.write(s)
