def load_scenario(scen_def: str, parent=None): """ :param scen_def: The file containing a single scenario definition, or a scenario definition `dict` . :return 'Scenario': A single ``Scenario`` object. """ defaults = { "search_paths": [], "ref_dir": os.path.abspath(os.getcwd()) } if parent is None: parent = {} defaults.update(parent) if isinstance(scen_def, str): scen_def = os.path.abspath(scen_def) defaults["search_paths"].append(os.path.dirname(scen_def)) scen_def = read_yaml(scen_def) if not isinstance(scen_def, dict): raise TypeError( "'scen_def' must be either a str to a file containing a scenario definition or a scenario definition dict." ) return Scenario(scen_def, parent=defaults)
def load_scenarios(scen_def: str, parent=None): """Load one or more scenarios from a file. :param scen_def: The file containing one or more scenario definitions. :return "Union['Scenario',List['Scenario']]": Either a single ``Scenario`` , or a `list` of ``Scenario`` s. """ defaults = {"search_paths": []} if parent is None: parent = {} defaults.update(parent) if isinstance(scen_def, str): scen_def = os.path.abspath(os.path.normpath(scen_def)) defaults["search_paths"].append(os.path.dirname(scen_def)) scen_def = read_yaml(scen_def) msg = "defaults: %s" % (defaults) Scenario._logger.debug(msg) if isinstance(scen_def, dict): # single scenario in file return [Scenario.load_scenario(scen_def, parent=defaults)] elif isinstance(scen_def, list): # List of scenarios return [ Scenario.load_scenario(scd, parent=defaults) for scd in scen_def ]
def test_export_to_xlsx(self): cero = pd.DataFrame.from_dict({"A": [1], "B": [2], "C": [3], "D": [4], "E": [5], "F": [6], }, orient='index', dtype=pd.np.float32) cero.sort_index(inplace=True) cero.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y")) proc = FromCERO._Procedure(read_yaml(cfg.d_td + "test_procedure_export_xlsx.yaml")) proc.exec_ops(cero) df1 = pd.read_excel("xlsx_export.xlsx", index_col=0) test_list = [1, 2, 3] df1_vals = [x[0] for x in df1.values.tolist()] self.assertTrue(all([np.isclose(x, y) for (x, y) in zip(test_list, df1_vals)])) test_list = ["A", "B", "C"] self.assertTrue(all([x == y for (x, y) in zip(test_list, df1.index.tolist())])) os.remove("xlsx_export.xlsx")
""" Created on May 29 14:57:38 2018 .. sectionauthor:: Lyle Collins <*****@*****.**> .. codeauthor:: Lyle Collins <*****@*****.**> """ import shutil import sys import os from concero.format_convert_tools import read_yaml copy_dict = read_yaml(sys.argv[1]) for k, v in copy_dict.items(): k = os.path.abspath(k) if isinstance(type(v), str): v = [v] for p in v: p = os.path.abspath(p) try: shutil.copy(k, p) except FileNotFoundError: os.mkdir(os.path.dirname(p)) shutil.copy(k, p)
def load_config(conf, parent=None): """ Loads configuration of FromCERO. If conf is a `str`, this is interpreted as a file (in YAML format) containing a configuration dictionary (relative to the current working directory). Otherwise conf must be a dictionary. :param 'Union[str,dict]' conf: :return dict: """ _conf = {"operations": [], "file": "output.csv", "sets": {}, "map": {}, "ref_dir": None, # Type: str "procedures": [], "libfuncs": []} # Defaults if parent is None: parent = {} _conf.update(parent) if isinstance(conf, str): if _conf["ref_dir"] is None: _conf["ref_dir"] = os.path.abspath(os.path.dirname(conf)) conf = read_yaml(conf) # Load configuration file if conf is None: # Loading an empty file raise TypeError("Attempted to load an empty file.") _conf.update(conf) if _conf.get("ref_dir") is None: _conf["ref_dir"] = os.path.abspath(os.getcwd()) _conf["file"] = FromCERO.get_relpath(_conf["ref_dir"], _conf["file"]) system_libfuncs = concero.conf.find_file("libfuncs.py") if system_libfuncs not in _conf.get("libfuncs", []): _conf["libfuncs"].append(system_libfuncs) # Load sets for k in _conf.get("sets", {}).keys(): # If provided as a string, assume to be a file relative to ref_dir if isinstance(_conf["sets"][k], str): _conf["sets"][k] = os.path.join(_conf["ref_dir"], _conf["sets"][k]) _conf["sets"][k] = read_yaml(_conf["sets"][k]) _conf["sets"][k] = FromCERO._load_set(_conf["sets"][k]) file_ext = os.path.splitext(_conf["file"])[1][1:] if file_ext in [".", ""]: raise TypeError("'file' must be specified with an extension.") for idx, procedure in enumerate(_conf["procedures"]): if isinstance(procedure, str) and str.lower(os.path.splitext(procedure)[1][1:]) in ["yaml", "yml"]: # If given a YAML filename, interpret as link to YAML dict _conf["procedures"][idx] = FromCERO.get_relpath(_conf["ref_dir"], procedure) _conf["procedures"][idx] = read_yaml(_conf["procedures"][idx]) parent = _conf.copy() parent.pop("procedures") # Avoid recursive inheritance loop parent.pop("file") # Procedure should not output to file unless a file is specified if not _conf["procedures"][idx].get("name"): _conf["procedures"][idx]["name"] = "Unnamed_proc_%d" % idx _conf["procedures"][idx] = FromCERO._Procedure.from_obj(_conf["procedures"][idx], parent=parent) return _conf
def load_config(proc_dict: dict, parent: 'FromCERO' = None): # Add default options here defaults = {"name": "Unnamed_proc", "operations": [], "inputs": [], "ref_dir": None, "sets": {}, "map": {}, "libfuncs": [], } if parent is None: parent = {} defaults.update(parent) defaults.update(proc_dict) if defaults.get("ref_dir") is None: defaults["ref_dir"] = os.getcwd() defaults["ref_dir"] = os.path.abspath(defaults["ref_dir"]) if defaults.get("file"): defaults["file"] = os.path.join(defaults["ref_dir"], os.path.relpath(defaults["file"])) if issubclass(type(defaults.get("libfuncs")), str): defaults["libfuncs"] = [defaults["libfuncs"]] lf_files = [] for lf in defaults["libfuncs"]: if issubclass(type(lf), str) and lf in proc_dict.get("libfuncs", []): lf = os.path.join(defaults["ref_dir"], lf) elif issubclass(type(lf), str): pass elif issubclass(type(lf), ModuleType): lf = lf.__file__ else: raise TypeError("'libfuncs' must be provided as a list of strings and/or modules (not %s)." % type(lf)) lf_files.append(lf) # Ensure system libfuncs is on search path... system_libfuncs = concero.conf.find_file("libfuncs.py") if system_libfuncs not in lf_files: lf_files.append(system_libfuncs) defaults["libfuncs"].append(system_libfuncs) # Load all libfuncs modules mods = [] for idx, (lf, mod) in enumerate(zip(lf_files, defaults["libfuncs"])): if issubclass(type(mod), str): spec = importlib.util.spec_from_file_location(os.path.basename(lf), lf) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) mods.append(mod) defaults["libfuncs"] = mods # Load sets for k in defaults["sets"]: if isinstance(defaults["sets"][k], str): defaults["sets"][k] = os.path.join(defaults["ref_dir"], defaults["sets"][k]) defaults["sets"][k] = read_yaml(defaults["sets"][k]) defaults["sets"][k] = FromCERO._load_set(defaults["sets"][k]) if isinstance(defaults["inputs"], str): defaults["inputs"] = [defaults["inputs"]] # Determine identifiers for all inputs defaults["inputs"] = _Identifier.get_all_idents(defaults["inputs"], sets=defaults["sets"]) if "lstrip" in defaults: defaults["inputs"] = [_Identifier.lstrip_identifier(defaults["lstrip"], inp) for inp in defaults["inputs"]] if "outputs" in defaults: if isinstance(defaults["outputs"], str): defaults["outputs"] = [defaults["outputs"]] if issubclass(type(defaults["outputs"]), list): defaults["outputs"] = _Identifier.get_all_idents(defaults["outputs"], sets=defaults["sets"]) elif defaults["outputs"] == True: defaults.pop("outputs") elif defaults["outputs"] == False: defaults["outputs"] = None elif defaults["outputs"] is None: pass else: raise ValueError("'outputs' must be provided as a list, True or None.") return defaults