def run(self) -> None:
"""
Execute a scenario run.
"""
self.cero = CERO.create_empty()
ceros = [in_conf.create_cero() for in_conf in self["input_conf"]]
if ceros:
self.cero = CERO.combine_ceros(ceros)
print("Successfully loaded scenario inputs as CERO.")
FromCERO.dataframe_out(self.cero,
(self.get_name() + "_%03d_step_%02d.xlsx" %
(self["run_no"], 0)), "xlsx")
for idx, model in enumerate(self["models"]):
m_cero = model.run(self.cero)
print(
"Completed run of model (%s) at %s." %
(model["name"], dt.datetime.now().strftime('%Y-%m-%d %H:%M')))
# If ouput_conf is not defined for a model, then None is returned...
if m_cero is None:
continue
if not CERO.is_cero(m_cero):
raise TypeError(
"Object returned from model run is *not* of CERO format.")
if model.get("export_mod_xlsx", self.get("export_mod_xlsx", True)):
# By default, export model outputs automatically to xlsx files
model_out_file = (self.get_name() + "_%03d_%s.xlsx" %
(self["run_no"], model["name"]))
print("Exporting output of %s to %s." %
(model["name"], model_out_file))
m_cero.to_excel(model_out_file)
self.cero = CERO.combine_ceros([self.cero, m_cero])
if self.get("export_int_xlsx", True):
# If true (default), export the intermediate steps to xlsx files
isfn = (self.get_name() + "_%03d_step_%02d.xlsx" %
(self["run_no"], idx + 1))
print("Exporting updated CERO to %s." % (isfn))
self.cero.to_excel(isfn)
for out_conf in self["output_conf"]:
out_conf.exec_procedures(self.cero)
else:
print("Completed generation of scenario outputs.")
def run(self, cero) -> 'CERO':
"""
Executes all data import/export operations (defined by ``input_conf`` and ``output_conf`` respectively) and the execution of any commands.
:param pandas.DataFrame cero: A CERO that contains all necessary data for conversion to input files (for \
model execution).
:return pandas.DataFrame: A CERO of relevant output data ('relevant' is defined by ``output_conf``).
"""
for input_conf in self["input_conf"]:
input_conf.exec_procedures(cero)
print(
"Completed converting CERO to model input files (%s). Now processing commands..."
% self["name"])
with _modified_environ(wd=self["wd"], **self.get("env_vars", {})):
for cmdobj in self["cmds"]:
cmd = {"type": "shell", "shell": True} # Default command
if isinstance(cmdobj, str):
# cmd is interpreted as shell command by default
# cmdobj = cmdobj.split(" ")
cmd.update({"args": cmdobj})
elif isinstance(cmdobj, dict):
cmd.update(cmdobj) # Add user updates
if "args" not in cmd:
raise ValueError(
"'args' must be provided for command of type 'dict'."
)
else:
raise TypeError(
"Invalid command object in configuration file.")
# Change to command-specific directory
cmd_run_dir = cmd.pop("wd", self["wd"])
if cmd_run_dir:
cmd_run_dir = os.path.abspath(cmd_run_dir)
cmd_type = cmd.pop("type")
# Execute commands
msg = "In directory '%s', executing command '%s'." % (
cmd_run_dir, cmd)
Model._logger.info(msg)
with _modified_environ(wd=cmd_run_dir,
**cmd.pop("env_vars", {})):
# Depending on cmd_type, execute command in different ways...
if cmd_type in ["shell"]:
args = cmd.pop("args")
Model._logger.info(
"Executing shell command: %s, with keyword args: %s."
% (args, cmd))
try:
cmd["output"] = subprocess.check_output(
args=args,
stderr=subprocess.STDOUT,
universal_newlines=True,
**cmd)
except subprocess.CalledProcessError as e:
msg = (
"Command '%s' failed with returncode: %s, and message:\n"
+ "%s\n" +
"Program logs may have more information.") % (
args, e.returncode, e.output)
Model._logger.error(msg)
print(msg)
raise e
Model._logger.info(cmd["output"])
print("Command returned: \n%s" % cmd["output"], end="")
elif cmd_type in ["python_method"]:
try:
assert ("func" in cmd)
except AssertionError:
raise ValueError(
"'func' must be defined for commands of type 'python_method'."
)
func = getattr(modfuncs, cmd.pop("func"))
cmd["output"] = func(*cmd["args"], **cmd["kwargs"])
else:
raise ValueError("Unsupported command type specified.")
if not self["output_conf"]:
return CERO.create_empty()
ceros = []
for oc in self["output_conf"]:
ceros.append(oc.create_cero())
try:
cero = CERO.combine_ceros(ceros, overwrite=False)
except CERO.CEROIndexConflict:
raise RuntimeWarning(
"Attempts to duplicate the export of data - i.e. one or more data series are being "
+
"exported more than once (which should be avoided). The last procedure will define "
+ "the intended data.")
cero = CERO.combine_ceros(ceros)
return cero
def test_create_empty(self):
empty_cero = CERO.create_empty()
self.assertTrue(CERO.is_cero(empty_cero))
