def load_scenario(scen_def: str, parent=None):
"""
:param scen_def: The file containing a single scenario definition, or a scenario definition `dict` .
:return 'Scenario': A single ``Scenario`` object.
"""
defaults = {
"search_paths": [],
"ref_dir": os.path.abspath(os.getcwd())
}
if parent is None:
parent = {}
defaults.update(parent)
if isinstance(scen_def, str):
scen_def = os.path.abspath(scen_def)
defaults["search_paths"].append(os.path.dirname(scen_def))
scen_def = read_yaml(scen_def)
if not isinstance(scen_def, dict):
raise TypeError(
"'scen_def' must be either a str to a file containing a scenario definition or a scenario definition dict."
)
return Scenario(scen_def, parent=defaults)
def load_scenarios(scen_def: str, parent=None):
"""Load one or more scenarios from a file.
:param scen_def: The file containing one or more scenario definitions.
:return "Union['Scenario',List['Scenario']]": Either a single ``Scenario`` , or a `list` of ``Scenario`` s.
"""
defaults = {"search_paths": []}
if parent is None:
parent = {}
defaults.update(parent)
if isinstance(scen_def, str):
scen_def = os.path.abspath(os.path.normpath(scen_def))
defaults["search_paths"].append(os.path.dirname(scen_def))
scen_def = read_yaml(scen_def)
msg = "defaults: %s" % (defaults)
Scenario._logger.debug(msg)
if isinstance(scen_def, dict):
# single scenario in file
return [Scenario.load_scenario(scen_def, parent=defaults)]
elif isinstance(scen_def, list):
# List of scenarios
return [
Scenario.load_scenario(scd, parent=defaults)
for scd in scen_def
]
def test_export_to_xlsx(self):
cero = pd.DataFrame.from_dict({"A": [1], "B": [2], "C": [3], "D": [4], "E": [5], "F": [6], }, orient='index',
dtype=pd.np.float32)
cero.sort_index(inplace=True)
cero.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y"))
proc = FromCERO._Procedure(read_yaml(cfg.d_td + "test_procedure_export_xlsx.yaml"))
proc.exec_ops(cero)
df1 = pd.read_excel("xlsx_export.xlsx", index_col=0)
test_list = [1, 2, 3]
df1_vals = [x[0] for x in df1.values.tolist()]
self.assertTrue(all([np.isclose(x, y) for (x, y) in zip(test_list, df1_vals)]))
test_list = ["A", "B", "C"]
self.assertTrue(all([x == y for (x, y) in zip(test_list, df1.index.tolist())]))
os.remove("xlsx_export.xlsx")
"""
Created on May 29 14:57:38 2018
.. sectionauthor:: Lyle Collins <*****@*****.**>
.. codeauthor:: Lyle Collins <*****@*****.**>
"""
import shutil
import sys
import os
from concero.format_convert_tools import read_yaml
copy_dict = read_yaml(sys.argv[1])
for k, v in copy_dict.items():
k = os.path.abspath(k)
if isinstance(type(v), str):
v = [v]
for p in v:
p = os.path.abspath(p)
try:
shutil.copy(k, p)
except FileNotFoundError:
os.mkdir(os.path.dirname(p))
shutil.copy(k, p)
def load_config(conf, parent=None):
"""
Loads configuration of FromCERO. If conf is a `str`, this is interpreted as a file (in YAML format) containing a configuration dictionary (relative to the current working directory). Otherwise conf must be a dictionary.
:param 'Union[str,dict]' conf:
:return dict:
"""
_conf = {"operations": [],
"file": "output.csv",
"sets": {},
"map": {},
"ref_dir": None, # Type: str
"procedures": [],
"libfuncs": []} # Defaults
if parent is None:
parent = {}
_conf.update(parent)
if isinstance(conf, str):
if _conf["ref_dir"] is None:
_conf["ref_dir"] = os.path.abspath(os.path.dirname(conf))
conf = read_yaml(conf) # Load configuration file
if conf is None: # Loading an empty file
raise TypeError("Attempted to load an empty file.")
_conf.update(conf)
if _conf.get("ref_dir") is None:
_conf["ref_dir"] = os.path.abspath(os.getcwd())
_conf["file"] = FromCERO.get_relpath(_conf["ref_dir"], _conf["file"])
system_libfuncs = concero.conf.find_file("libfuncs.py")
if system_libfuncs not in _conf.get("libfuncs", []):
_conf["libfuncs"].append(system_libfuncs)
# Load sets
for k in _conf.get("sets", {}).keys():
# If provided as a string, assume to be a file relative to ref_dir
if isinstance(_conf["sets"][k], str):
_conf["sets"][k] = os.path.join(_conf["ref_dir"], _conf["sets"][k])
_conf["sets"][k] = read_yaml(_conf["sets"][k])
_conf["sets"][k] = FromCERO._load_set(_conf["sets"][k])
file_ext = os.path.splitext(_conf["file"])[1][1:]
if file_ext in [".", ""]:
raise TypeError("'file' must be specified with an extension.")
for idx, procedure in enumerate(_conf["procedures"]):
if isinstance(procedure, str) and str.lower(os.path.splitext(procedure)[1][1:]) in ["yaml", "yml"]:
# If given a YAML filename, interpret as link to YAML dict
_conf["procedures"][idx] = FromCERO.get_relpath(_conf["ref_dir"], procedure)
_conf["procedures"][idx] = read_yaml(_conf["procedures"][idx])
parent = _conf.copy()
parent.pop("procedures") # Avoid recursive inheritance loop
parent.pop("file") # Procedure should not output to file unless a file is specified
if not _conf["procedures"][idx].get("name"):
_conf["procedures"][idx]["name"] = "Unnamed_proc_%d" % idx
_conf["procedures"][idx] = FromCERO._Procedure.from_obj(_conf["procedures"][idx], parent=parent)
return _conf
def load_config(proc_dict: dict, parent: 'FromCERO' = None):
# Add default options here
defaults = {"name": "Unnamed_proc",
"operations": [],
"inputs": [],
"ref_dir": None,
"sets": {},
"map": {},
"libfuncs": [],
}
if parent is None:
parent = {}
defaults.update(parent)
defaults.update(proc_dict)
if defaults.get("ref_dir") is None:
defaults["ref_dir"] = os.getcwd()
defaults["ref_dir"] = os.path.abspath(defaults["ref_dir"])
if defaults.get("file"):
defaults["file"] = os.path.join(defaults["ref_dir"], os.path.relpath(defaults["file"]))
if issubclass(type(defaults.get("libfuncs")), str):
defaults["libfuncs"] = [defaults["libfuncs"]]
lf_files = []
for lf in defaults["libfuncs"]:
if issubclass(type(lf), str) and lf in proc_dict.get("libfuncs", []):
lf = os.path.join(defaults["ref_dir"], lf)
elif issubclass(type(lf), str):
pass
elif issubclass(type(lf), ModuleType):
lf = lf.__file__
else:
raise TypeError("'libfuncs' must be provided as a list of strings and/or modules (not %s)." % type(lf))
lf_files.append(lf)
# Ensure system libfuncs is on search path...
system_libfuncs = concero.conf.find_file("libfuncs.py")
if system_libfuncs not in lf_files:
lf_files.append(system_libfuncs)
defaults["libfuncs"].append(system_libfuncs)
# Load all libfuncs modules
mods = []
for idx, (lf, mod) in enumerate(zip(lf_files, defaults["libfuncs"])):
if issubclass(type(mod), str):
spec = importlib.util.spec_from_file_location(os.path.basename(lf), lf)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
mods.append(mod)
defaults["libfuncs"] = mods
# Load sets
for k in defaults["sets"]:
if isinstance(defaults["sets"][k], str):
defaults["sets"][k] = os.path.join(defaults["ref_dir"], defaults["sets"][k])
defaults["sets"][k] = read_yaml(defaults["sets"][k])
defaults["sets"][k] = FromCERO._load_set(defaults["sets"][k])
if isinstance(defaults["inputs"], str):
defaults["inputs"] = [defaults["inputs"]]
# Determine identifiers for all inputs
defaults["inputs"] = _Identifier.get_all_idents(defaults["inputs"], sets=defaults["sets"])
if "lstrip" in defaults:
defaults["inputs"] = [_Identifier.lstrip_identifier(defaults["lstrip"], inp) for inp in defaults["inputs"]]
if "outputs" in defaults:
if isinstance(defaults["outputs"], str):
defaults["outputs"] = [defaults["outputs"]]
if issubclass(type(defaults["outputs"]), list):
defaults["outputs"] = _Identifier.get_all_idents(defaults["outputs"], sets=defaults["sets"])
elif defaults["outputs"] == True:
defaults.pop("outputs")
elif defaults["outputs"] == False:
defaults["outputs"] = None
elif defaults["outputs"] is None:
pass
else:
raise ValueError("'outputs' must be provided as a list, True or None.")
return defaults