def CreateMakelistFiles(self):
loghelper.get_logger().info("\n\n Creating make list files...\n")
cmd = '"' + os.path.join(
self.toolboxPath, r'createMakelistFiles.exe') \
+ '" ' + str(self.simID)
loghelper.get_logger().info("Command line: " + cmd)
self.call_subprocess_cmd(cmd)
def Query(self, query, params=None):
if self.log_enabled:
loghelper.get_logger().info("{0}\n{1}".format(self.path, query))
cursor = self.connection.cursor()
params = self._floatifyIntParams(params)
cursor.execute(query, params) if params else cursor.execute(query)
return cursor
def LoadCBMResults(self, output_path=None):
loghelper.get_logger().info("\n\n Loading CBM Results...\n")
results_path = self.getDefaultResultsPath() \
if output_path is None else os.path.abspath(output_path)
cmd = '"' + os.path.join(self.toolboxPath, r'LoaderCL.exe') + '" ' \
+ str(self.simID) + ' "' + results_path + '"'
loghelper.get_logger().info("Command line: " + cmd)
self.call_subprocess_cmd(cmd)
def CleanupRunDirectory(self):
loghelper.get_logger().info("\n\n Clean up previous runs in " +
self.CBMTemp + " \n")
top = self.CBMTemp
for root, dirs, files in os.walk(self.CBMTemp, topdown=False):
for f in files:
os.remove(os.path.join(root, f))
for d in dirs:
if os.path.join(root, d) != top:
shutil.rmtree(os.path.join(root, d), ignore_errors=1)
def ExecuteMany(self, query, params):
cursor = self.connection.cursor()
try:
if not isinstance(params, Iterable):
params = self._floatifyIntParams(params)
params = [self._floatifyIntParams(p) for p in params]
cursor.executemany(query, params)
except ProgrammingError:
loghelper.get_logger().info("{}".format(query))
raise
cursor.commit()
def ExecuteQuery(self, query, params=None):
if self.log_enabled:
loghelper.get_logger().info("{0}\n{1}".format(self.path, query))
cursor = self.connection.cursor()
try:
params = self._floatifyIntParams(params)
cursor.execute(query, params) if params else cursor.execute(query)
except ProgrammingError:
loghelper.get_logger().info("{0}".format(query))
raise
cursor.commit()
def RunCBM(self):
loghelper.get_logger().info("\n\n Running CBM...\n")
cbm_path = os.path.join(self.CBMTemp, r'CBMRun\CBM.exe')
cwd = os.getcwd()
try:
# CBM is expecting the current directory to be its location
os.chdir(os.path.dirname(cbm_path))
cmd = '"' + cbm_path + '" '
loghelper.get_logger().info("Command line: " + cmd)
self.call_subprocess_cmd(cmd)
finally:
os.chdir(cwd) # change back to the original working dir
def runMakelist(self):
loghelper.get_logger().info("\n\n Running make list...\n")
makelist_path = os.path.join(self.CBMTemp, r'Makelist\Makelist.exe')
cwd = os.getcwd()
try:
# makelist is expecting the current directory to be its location
os.chdir(os.path.dirname(makelist_path))
cmd = '"' + makelist_path + '" '
loghelper.get_logger().info("Command line: " + cmd)
self.call_subprocess_cmd(cmd)
finally:
os.chdir(cwd) # change back to the original working dir
def CreateCBMFiles(self, save_svl_by_timestep=False):
loghelper.get_logger().info("\n\n Creating CBM files...\n")
cmd = '"' + os.path.join(
self.toolboxPath, r'createCBMFiles.exe') + '" ' + str(self.simID)
loghelper.get_logger().info("Command line: " + cmd)
self.call_subprocess_cmd(cmd)
cbm_run_input_dir = os.path.join(self.CBMTemp, "CBMRun", "input")
if save_svl_by_timestep:
file_replace.replace(
os.path.join(cbm_run_input_dir, "model.inf"),
self._create_model_inf_replace_func(
save_svl_every_timestep_option=1))
inf = open(os.path.join(cbm_run_input_dir, "indicate.inf"), 'w')
inf.write('0\n')
inf.flush()
inf.close()
def copyMakelistOutput(self, source_path=None):
if source_path and not os.path.exists(source_path):
raise ValueError(
f"specified makelist output dir does not exist: {source_path}")
loghelper.get_logger().info("\n\n Copying makelist outputs...\n")
CBMinpath = os.path.join(self.CBMTemp, r'CBMRun\input')
if os.path.exists(CBMinpath):
for f in glob.iglob(os.path.join(CBMinpath, '*')):
os.remove(f)
else:
os.makedirs(CBMinpath)
if source_path:
ini_glob = os.path.join(source_path, "*.ini")
else:
ini_glob = os.path.join(self.CBMTemp, r'Makelist\output\*.ini')
for f in glob.iglob(ini_glob):
shutil.copy2(f, CBMinpath)
def sit_import_main(args):
try:
parser = argparse.ArgumentParser()
parser.add_argument(
"sit_data_dir",
help="directory containing SIT formatted csv files and parameter "
"mapping json file")
parser.add_argument(
"cbm3_project_path",
help="path to the file created by imported the specified SIT "
"dataset")
parser.add_argument(
"--aidb_path",
help="path to the archive index from which default CBM parameters "
"are drawn during the SIT import process.")
parser.add_argument(
"--initialize_mapping",
help="if specified indicates the the values in the SIT dataset "
"are identical to values stored in the archive index and do "
"not require mapping.",
action="store_true",
dest="initialize_mapping")
parser.add_argument(
"--working_dir",
help="Optional working dir where logs, and intermediate files are "
"created during the sit process. If not specified a sub "
"directory in the specified 'sit_data_dir' is used.")
args = parser.parse_args(args)
aidb_path = args.aidb_path
if not aidb_path:
aidb_path = toolbox_defaults.get_archive_index_path()
sit_helper.csv_import(csv_dir=args.sit_data_dir,
imported_project_path=args.cbm3_project_path,
initialize_mapping=args.initialize_mapping,
archive_index_db_path=aidb_path,
working_dir=args.working_dir)
except Exception:
loghelper.get_logger().exception("")
def CopyCBMExecutable(self):
loghelper.get_logger().info("\n\n Copying CBM.exe to Temp dir...\n")
shutil.copy2(os.path.join(self.ExecutablePath, r'CBM.exe'),
os.path.join(self.CBMTemp, r'CBMRun'))
def loadMakelistSVLS(self):
loghelper.get_logger().info("\n\n Loading Makelist SVLs...\n")
cmd = '"' + os.path.join(self.toolboxPath, r'MakelistSVLLoader.exe') \
+ '" ' + str(self.simID)
loghelper.get_logger().info("Command line: " + cmd)
self.call_subprocess_cmd(cmd)
def copyMakelist(self):
loghelper.get_logger().info(
"\n\n Copying makelist.exe to Temp dir...\n")
shutil.copy2(os.path.join(self.ExecutablePath, r'Makelist.exe'),
os.path.join(self.CBMTemp, 'Makelist'))
def simulate_main(args):
try:
logpath = os.path.join("{0}_{1}.log".format(
"simulation",
datetime.datetime.now().strftime("%Y-%m-%d %H_%M_%S")))
loghelper.start_logging(logpath, 'w+')
parser = argparse.ArgumentParser(
description="CBM-CFS3 simulation script. Simulates a CBM-CFS3 "
"project access database by automating functions in "
"the Operational-Scale CBM-CFS3 toolbox")
parser.add_argument("project_path",
type=os.path.abspath,
help="path to a cbm project database file")
parser.add_argument(
"--project_simulation_id",
type=int,
help="integer id for the simulation scenario to run in the "
"project (tblSimulation.SimulationID). If not specified "
"the highest numeric SimulationID is used.")
parser.add_argument(
"--n_timesteps",
type=int,
help="the number of timesteps to run the specified project. "
"If not specified the value in tblRunTableDetails will be "
"used.")
parser.add_argument(
"--aidb_path",
nargs="?",
default=toolbox_defaults.get_archive_index_path(),
type=os.path.abspath,
help="path to a CBM-CFS3 archive index database. If unspecified a "
"typical default value is used.")
parser.add_argument(
"--toolbox_installation_dir",
nargs="?",
default=toolbox_defaults.get_install_path(),
type=os.path.abspath,
help="the Operational-Scale CBM-CFS3 toolbox installation "
"directory. If unspecified a typical default value is used.")
parser.add_argument(
"--cbm_exe_path",
nargs="?",
default=toolbox_defaults.get_cbm_executable_dir(),
type=os.path.abspath,
help="directory containing CBM.exe and Makelist.exe. If "
"unspecified a typical default value is used.")
parser.add_argument(
"--results_database_path",
type=os.path.abspath,
help="optional file path into which CBM results will be loaded."
"if unspecified a default value is used.")
parser.add_argument(
"--tempfiles_output_dir",
type=os.path.abspath,
help="optional directory where CBM tempfiles will be copied "
"after simulation. If unspecified a default directory is "
"used.")
parser.add_argument(
"--skip_makelist",
action="store_true",
help="If set then skip running makelist. Useful for "
"afforestation only projects and for re-using existing "
"makelist results.")
parser.add_argument(
"--use_existing_makelist_output",
action="store_true",
help="if set the existing values in the project "
"database will be used in place of a new set of values"
"generated by the makelist executable.")
parser.add_argument(
"--copy_makelist_results",
action="store_true",
help="If present makelist *svl output is directly copied from the "
"makelist output dir to the cbm input dir rather than using "
"the toolbox load svl and dump svl procedures. If specified "
"with a directory, the makelist output files in the "
"specified directory will be copied to the CBM input "
"directory")
parser.add_argument(
"--stdout_path",
type=os.path.abspath,
help="optionally redirect makelist and CBM standard output to "
"the specified file. If unspecified standard out is directed "
"to the console window")
parser.add_argument(
"--dist_classes_path",
type=os.path.abspath,
help="one of a pair of optional file paths used to configure "
"extended kf6 accounting")
parser.add_argument(
"--dist_rules_path",
type=os.path.abspath,
help="one of a pair of optional file paths used to configure "
"extended kf6 accounting")
parser.add_argument(
"--save_svl_by_timestep",
action="store_true",
help="if set the CBM executable will be configured to "
"write out all stand database (SVLxxx.dat) files at the end "
"of every time step.")
parser.add_argument(
"--loader_settings",
type=json.loads,
help="An optional json formatted string indicating settings for "
"loading CBM results. If omitted the CBM-Toolbox built-in "
"loader is used.")
args = parser.parse_args(args=args)
results_path = projectsimulator.run(**args.__dict__)
loghelper.get_logger().info(
f"simulation finish, results path: {results_path}")
except Exception:
loghelper.get_logger().exception("")
def CopyTempFiles(self, output_dir):
loghelper.get_logger().info(
"\n\n Copying Tempfiles to Project Directory...\n")
tempfilepath = os.path.abspath(output_dir)
shutil.copytree(self.CBMTemp, tempfilepath, ignore=self._ignorethese)
def DumpMakelistSVLs(self):
loghelper.get_logger().info("\n\n dumping makelist svls...\n")
cmd = '"' + os.path.join(self.toolboxPath, r'DumpMakelistSVL.exe') \
+ '" "' + str(self.simID) + ' "'
loghelper.get_logger().info("Command line: " + cmd)
self.call_subprocess_cmd(cmd)
def create_accounting_rules(self):
self.createAccountingRulesTables()
loghelper.get_logger().info(" Creating kf5 accounting rules")
self.projectAccessDb.ExecuteMany(
"INSERT INTO tblaccountingruletype (name) VALUES (?)",
[["years_since_disturbance"], ["ag_biomass"], ["total_biomass"], ["total_eco"],
["years_since_last_pass_disturbance"], ["ag_biomass_last_pass_disturbance"],
["total_biomass_last_pass_disturbance"], ["total_eco_last_pass_disturbance"]])
self.projectAccessDb.ExecuteMany(
"INSERT INTO tblAccountingRuleTrackingType (name) VALUES (?)",
[["ignore"], ["replace"], ["inherit"], ["passive"]])
with open(self.dist_classes_path) as dist_class_file:
reader = csv.DictReader(dist_class_file)
uniqueDisturbanceClasses = set([row["Category"] for row in reader])
self.projectAccessDb.ExecuteMany(
"INSERT INTO tbldisturbanceclass (name) VALUES (?)",
[[dc] for dc in uniqueDisturbanceClasses])
with open(self.dist_classes_path) as dist_class_file:
reader = csv.DictReader(dist_class_file)
self.projectAccessDb.ExecuteMany(
"""
INSERT INTO tbldisturbancetypeclassification (defaultdisttypeid, disturbance_class_id)
SELECT ? AS defaultdisttypeid, id AS disturbance_class_id
FROM tbldisturbanceclass
WHERE name LIKE ?
""",
[[row["DefaultDistTypeID"], row["Category"]] for row in reader])
with open(self.dist_rules_path) as csvfile:
add_rule_sql = \
"""
INSERT INTO tblaccountingrule (accountingrulesetid, accountingruletypeid, rule_value)
VALUES (?, ?, ?)
"""
reader = csv.DictReader(csvfile)
for row in reader:
dist_class = row["disturbance_class"]
rule_tracking_type = row["rule_tracking_type"]
rule_type = row.get("rule_type")
rule_value = row.get("rule_value")
ru = row.get("defaultSPUID")
spu = row.get("SPUID")
if spu:
rule_sets = self.get_or_add_rule_sets(dist_class, rule_tracking_type, spu=spu)
elif ru:
rule_sets = self.get_or_add_rule_sets(dist_class, rule_tracking_type, ru=ru)
else:
rule_sets = self.get_or_add_rule_sets(dist_class, rule_tracking_type)
if not rule_type:
# Rule set with no rules (i.e. to set rule_tracking_type only)
continue
rule_type_id = self.get_or_add_rule_type(rule_type)
for rule_set in rule_sets:
value = self.get_makelist_value(rule_set.spuid) if rule_value == "makelist" else rule_value
self.projectAccessDb.ExecuteQuery(
add_rule_sql,
[rule_set.accountingrulesetid, rule_type_id, value])
