def simulate(**kwargs):
this_dir = os.path.dirname(os.path.realpath(__file__))
mapping_file_path = os.path.join(this_dir, "mapping.json")
sit_mdb_path = os.path.join(this_dir, "cbm3_sit.mdb")
with tempfile.TemporaryDirectory() as tempdir:
project_path = import_mdb(tempdir, sit_mdb_path, mapping_file_path)
results_path = os.path.join(tempdir, "results.mdb")
tempfiles_dir = os.path.join(tempdir, "tempfiles")
aidb_path = toolbox_defaults.get_archive_index_path()
cbm_exe_path = toolbox_defaults.get_cbm_executable_dir()
toolbox_path = toolbox_defaults.get_install_path()
run_kwargs = dict(project_path=project_path,
results_database_path=results_path,
tempfiles_output_dir=tempfiles_dir,
aidb_path=aidb_path,
cbm_exe_path=cbm_exe_path)
run_kwargs.update(kwargs)
list(
projectsimulator.run_concurrent(run_args=[run_kwargs],
toolbox_path=toolbox_path,
max_workers=1))
yield SimpleNamespace(tempdir=tempdir,
project_path=run_kwargs["project_path"],
results_path=run_kwargs["results_database_path"],
tempfiles_dir=run_kwargs["tempfiles_output_dir"],
aidb_path=run_kwargs["aidb_path"],
cbm_exe_path=run_kwargs["cbm_exe_path"],
toolbox_path=toolbox_path)
def test_keyword_args_defaults(self, loghelper, projectsimulator):
simulate.simulate_main(["my_project.mdb"])
projectsimulator.run.assert_called_with(
project_path=os.path.abspath("my_project.mdb"),
project_simulation_id=None,
n_timesteps=None,
aidb_path=toolbox_defaults.get_archive_index_path(),
toolbox_installation_dir=toolbox_defaults.get_install_path(),
cbm_exe_path=toolbox_defaults.get_cbm_executable_dir(),
results_database_path=None,
tempfiles_output_dir=None,
skip_makelist=False,
stdout_path=None,
use_existing_makelist_output=False,
copy_makelist_results=False,
dist_classes_path=None,
dist_rules_path=None,
save_svl_by_timestep=False,
loader_settings=None)
def run(project_path,
project_simulation_id=None,
n_timesteps=None,
aidb_path=None,
toolbox_installation_dir=None,
cbm_exe_path=None,
results_database_path=None,
tempfiles_output_dir=None,
skip_makelist=False,
use_existing_makelist_output=False,
copy_makelist_results=False,
stdout_path=None,
dist_classes_path=None,
dist_rules_path=None,
save_svl_by_timestep=False,
loader_settings=None):
"""runs the specified single simulation assumption project and loads the
results
Args:
project_path (str): Path to a CBM-CFS3 project to simulate.
project_simulation_id (int, optional): integer id for the simulation
scenario to run in the project (tblSimulation.SimulationID). If
not specified the highest numeric SimulationID is used.
n_timesteps (int, optional): the number of timesteps to run the
specified project. If not specified the value in tblRunTableDetails
will be used.
aidb_path (str, optional): Path to a CBM-CFS3 archive index database.
If unspecified a typical default value is used. Defaults to None.
toolbox_installation_dir (str, optional): Path of the installed
CBM-CFS3 Toolbox. If unspecified a typical default value is used.
Defaults to None.
cbm_exe_path (str, optional): Directory containing CBM.exe and
Makelist.exe. If unspecified a typical default value is used.
Defaults to None.
results_database_path (str, optional): The path to the database
produced with simulation results. If unspecified a default path
based on the simulation ID is used. The path is returned
by this function. Defaults to None.
tempfiles_output_dir (str, optional): Directory where CBM tempfiles,
which are text based cbm and makelist input and output files, will
be copied. If unspecified these files will not be copied. Defaults
to None.
skip_makelist (bool, optional): If set to true the makelist routine
will not be run. Can be used if the project contains no initially
forested stands, or if existing makelist results are stored in the
project database. Defaults to False.
use_existing_makelist_output (bool, optional): If set to True, the
existing values in the project database will be used in place of a
new set of values generated by the makelist executable. Defaults to
False.
copy_makelist_results (bool, str, optional): if set to True, makelist
*svl output in the toolbox temp dir is directly copied from the
makelist output dir to the cbm input dir rather than using the
toolbox load svl and dump svl procedures. If set to a directory
the makelist *svl output from that directory will be copied and
used by the CBM run Defaults to False.
stdout_path (str, optional): When set to a filepath all stdout from
the CBM/Makelist executables will be redirected into the specified
file. Defaults to None.
dist_classes_path (str, optional): File 1 of 2 of a pair of optional
file paths used to configure extended kf accounting. Defaults to
None.
dist_rules_path (str, optional): File 2 of 2 of a pair of optional
file paths used to configure extended kf accounting. Defaults to
None.
save_svl_by_timestep (bool, optional): if set to true the CBM
executable will be configured to write out all stand database
(SVLxxx.dat) files at the end of every time step.
loader_settings (dict, optional): If None the toolbox loader is used,
otherwise this arg specifies loader specific settings. Defaults to
None. If an empty dictionary or {"type": None} are specified then
the cbm results will not be loaded. See:
:py:func:`cbm3_python.cbm3data.cbm3_output_loader.load` for
documentation
Returns:
str: The path to the loaded CBM results database
"""
if not aidb_path:
aidb_path = toolbox_defaults.get_archive_index_path()
if not cbm_exe_path:
cbm_exe_path = toolbox_defaults.get_cbm_executable_dir()
if not toolbox_installation_dir:
toolbox_installation_dir = toolbox_defaults.get_install_path()
# don't allow relative paths here, it will cause failures later in CBM
# command line apps that have difficult to understand error messages
existing_paths = [
project_path, aidb_path, cbm_exe_path, toolbox_installation_dir,
dist_classes_path, dist_rules_path
]
output_paths = [results_database_path, tempfiles_output_dir, stdout_path]
all_paths = existing_paths + output_paths
for path in all_paths:
if path is None:
continue
if not os.path.isabs(path):
raise ValueError(
"Relative paths detected. They may cause failures in CBM "
f"model command line processes: '{path}'")
for path in existing_paths:
if path is None:
continue
if not os.path.exists(path):
raise ValueError(f"specified path does not exist '{path}'")
if tempfiles_output_dir:
_delete_old_tempfiles(tempfiles_output_dir)
with AIDB(aidb_path, False) as aidb, \
ProjectDB(project_path, False) as proj:
if use_existing_makelist_output and not skip_makelist:
raise ValueError("conflicting arguments: Cannot both use "
"existing makelist output and run makelist.")
if not use_existing_makelist_output:
clear_old_results(proj)
simId = aidb.AddProjectToAIDB(proj,
project_sim_id=project_simulation_id)
original_run_length = None
if n_timesteps:
original_run_length = proj.get_run_length(project_simulation_id)
proj.set_run_length(n_timesteps, project_simulation_id)
try:
cbm_wd = os.path.join(toolbox_installation_dir, "temp")
s = Simulator(cbm_exe_path, simId, os.path.dirname(project_path),
cbm_wd, toolbox_installation_dir, stdout_path)
aidb_path_original = s.getDefaultArchiveIndexPath()
s.setDefaultArchiveIndexPath(aidb_path)
s.removeCBMProjfile(project_path)
s.CleanupRunDirectory()
if not skip_makelist:
s.CreateMakelistFiles()
s.copyMakelist()
s.runMakelist()
else:
# one side effect of "CreateMakelistFiles" is that the
# project database is copied into the temp dir. When
# skip_makelist is enabled this step is needed to copy
# the project into the temp dir
s.CopyToWorkingDir(project_path)
s.CreateEmptyMakelistOuput()
if not use_existing_makelist_output:
s.loadMakelistSVLS()
if dist_classes_path is not None:
# support for extended "kf6" results tracking
with AccessDB(project_path, False) as proj:
cr = CreateAccountingRules(proj, dist_classes_path,
dist_rules_path)
cr.create_accounting_rules()
# copy the modified db to the working dir
s.CopyToWorkingDir(project_path)
if not copy_makelist_results:
s.DumpMakelistSVLs()
else:
makelist_src_path = None
if isinstance(copy_makelist_results, str):
makelist_src_path = copy_makelist_results
s.copyMakelistOutput(makelist_src_path)
s.CreateCBMFiles(save_svl_by_timestep=save_svl_by_timestep)
s.CopyCBMExecutable()
s.RunCBM()
if tempfiles_output_dir:
s.CopyTempFiles(output_dir=tempfiles_output_dir)
if loader_settings is None:
if results_database_path:
results_database_dir = os.path.dirname(
results_database_path)
if not os.path.exists(results_database_dir):
os.makedirs(results_database_dir)
s.LoadCBMResults(output_path=results_database_path)
elif loader_settings == {} or loader_settings["type"] is None:
return None
else:
if tempfiles_output_dir:
load_dir = tempfiles_output_dir
else:
load_dir = s.CBMTemp
cbm3_output_loader.load(
loader_settings, os.path.join(load_dir, "CBMRun",
"output"), project_path,
aidb_path)
finally:
# cleanup
if original_run_length:
proj.set_run_length(original_run_length, project_simulation_id)
s.setDefaultArchiveIndexPath(aidb_path_original)
aidb.DeleteProjectsFromAIDB(simId)
results_path = s.getDefaultResultsPath() if results_database_path \
is None else results_database_path
return results_path
def test_integration(self):
this_dir = os.path.dirname(os.path.realpath(__file__))
mapping_file_path = os.path.join(this_dir, "mapping.json")
sit_mdb_path = os.path.join(this_dir, "cbm3_sit.mdb")
sit_xls_path = os.path.join(this_dir, "cbm3_sit.xls")
with tempfile.TemporaryDirectory() as tempdir:
mdb_working_dir = os.path.join(tempdir, "mdb")
os.makedirs(mdb_working_dir)
mdb_project = import_mdb_xls(
mdb_working_dir, sit_mdb_path, mapping_file_path)
xls_working_dir = os.path.join(tempdir, "xls")
os.makedirs(xls_working_dir)
xls_project = import_mdb_xls(
xls_working_dir,
sit_xls_path,
mapping_file_path, xls=True)
csv_working_dir = os.path.join(tempdir, "csv")
os.makedirs(csv_working_dir)
mdb_to_delimited(sit_mdb_path, ".csv", csv_working_dir)
csv_project = import_delimited(
csv_working_dir, mapping_file_path, sit_helper.csv_import)
tab_working_dir = os.path.join(tempdir, "tab")
os.makedirs(tab_working_dir)
mdb_to_delimited(sit_mdb_path, ".tab", tab_working_dir)
tab_project = import_delimited(
tab_working_dir, mapping_file_path, sit_helper.tab_import)
mdb_results = os.path.join(tempdir, "mdb_results.mdb")
xls_results = os.path.join(tempdir, "xls_results.mdb")
csv_results = os.path.join(tempdir, "csv_results.mdb")
tab_results = os.path.join(tempdir, "tab_results.mdb")
run_args = [
dict(project_path=mdb_project,
results_database_path=mdb_results),
dict(project_path=xls_project,
results_database_path=xls_results),
dict(project_path=csv_project,
results_database_path=csv_results),
dict(project_path=tab_project,
results_database_path=tab_results)
]
for arg in run_args:
arg["aidb_path"] = toolbox_defaults.get_archive_index_path()
arg["cbm_exe_path"] = toolbox_defaults.get_cbm_executable_dir()
list(projectsimulator.run_concurrent(
run_args, toolbox_path=toolbox_defaults.get_install_path(),
max_workers=None))
mdb_pools = cbm3_results.load_pool_indicators(mdb_results)
xls_pools = cbm3_results.load_pool_indicators(xls_results)
csv_pools = cbm3_results.load_pool_indicators(csv_results)
tab_pools = cbm3_results.load_pool_indicators(tab_results)
# double check the results are not empty
self.assertFalse(mdb_pools.empty)
self.assertTrue(
np.allclose(mdb_pools.to_numpy(), xls_pools.to_numpy()))
self.assertTrue(
np.allclose(mdb_pools.to_numpy(), csv_pools.to_numpy()))
self.assertTrue(
np.allclose(mdb_pools.to_numpy(), tab_pools.to_numpy()))
def simulate_main(args):
try:
logpath = os.path.join("{0}_{1}.log".format(
"simulation",
datetime.datetime.now().strftime("%Y-%m-%d %H_%M_%S")))
loghelper.start_logging(logpath, 'w+')
parser = argparse.ArgumentParser(
description="CBM-CFS3 simulation script. Simulates a CBM-CFS3 "
"project access database by automating functions in "
"the Operational-Scale CBM-CFS3 toolbox")
parser.add_argument("project_path",
type=os.path.abspath,
help="path to a cbm project database file")
parser.add_argument(
"--project_simulation_id",
type=int,
help="integer id for the simulation scenario to run in the "
"project (tblSimulation.SimulationID). If not specified "
"the highest numeric SimulationID is used.")
parser.add_argument(
"--n_timesteps",
type=int,
help="the number of timesteps to run the specified project. "
"If not specified the value in tblRunTableDetails will be "
"used.")
parser.add_argument(
"--aidb_path",
nargs="?",
default=toolbox_defaults.get_archive_index_path(),
type=os.path.abspath,
help="path to a CBM-CFS3 archive index database. If unspecified a "
"typical default value is used.")
parser.add_argument(
"--toolbox_installation_dir",
nargs="?",
default=toolbox_defaults.get_install_path(),
type=os.path.abspath,
help="the Operational-Scale CBM-CFS3 toolbox installation "
"directory. If unspecified a typical default value is used.")
parser.add_argument(
"--cbm_exe_path",
nargs="?",
default=toolbox_defaults.get_cbm_executable_dir(),
type=os.path.abspath,
help="directory containing CBM.exe and Makelist.exe. If "
"unspecified a typical default value is used.")
parser.add_argument(
"--results_database_path",
type=os.path.abspath,
help="optional file path into which CBM results will be loaded."
"if unspecified a default value is used.")
parser.add_argument(
"--tempfiles_output_dir",
type=os.path.abspath,
help="optional directory where CBM tempfiles will be copied "
"after simulation. If unspecified a default directory is "
"used.")
parser.add_argument(
"--skip_makelist",
action="store_true",
help="If set then skip running makelist. Useful for "
"afforestation only projects and for re-using existing "
"makelist results.")
parser.add_argument(
"--use_existing_makelist_output",
action="store_true",
help="if set the existing values in the project "
"database will be used in place of a new set of values"
"generated by the makelist executable.")
parser.add_argument(
"--copy_makelist_results",
action="store_true",
help="If present makelist *svl output is directly copied from the "
"makelist output dir to the cbm input dir rather than using "
"the toolbox load svl and dump svl procedures. If specified "
"with a directory, the makelist output files in the "
"specified directory will be copied to the CBM input "
"directory")
parser.add_argument(
"--stdout_path",
type=os.path.abspath,
help="optionally redirect makelist and CBM standard output to "
"the specified file. If unspecified standard out is directed "
"to the console window")
parser.add_argument(
"--dist_classes_path",
type=os.path.abspath,
help="one of a pair of optional file paths used to configure "
"extended kf6 accounting")
parser.add_argument(
"--dist_rules_path",
type=os.path.abspath,
help="one of a pair of optional file paths used to configure "
"extended kf6 accounting")
parser.add_argument(
"--save_svl_by_timestep",
action="store_true",
help="if set the CBM executable will be configured to "
"write out all stand database (SVLxxx.dat) files at the end "
"of every time step.")
parser.add_argument(
"--loader_settings",
type=json.loads,
help="An optional json formatted string indicating settings for "
"loading CBM results. If omitted the CBM-Toolbox built-in "
"loader is used.")
args = parser.parse_args(args=args)
results_path = projectsimulator.run(**args.__dict__)
loghelper.get_logger().info(
f"simulation finish, results path: {results_path}")
except Exception:
loghelper.get_logger().exception("")