def from_config( # type: ignore
cls,
config: Dict[str, Any],
project_name: str,
config_globals=None):
"""
Construct an instance from a block of YAML config file which represents
a single Machine; loaded as a ``dict``.
Parameters
----------
config: dict
The loaded block of config which represents a 'Machine' in YAML
project_name: str
Name of the project this Machine belongs to.
config_globals:
The block of config within the YAML file within `globals`
Returns
-------
:class:`~Machine`
"""
if config_globals is None:
config_globals = dict()
name = config["name"]
model = config.get("model") or config_globals.get("model")
local_runtime = config.get("runtime", dict())
runtime = patch_dict(config_globals.get("runtime", dict()),
local_runtime)
dataset_config = patch_dict(config.get("dataset", dict()),
config_globals.get("dataset", dict()))
dataset = GordoBaseDataset.from_dict(dataset_config)
evaluation = patch_dict(config_globals.get("evaluation", dict()),
config.get("evaluation", dict()))
metadata = Metadata(
user_defined={
"global-metadata": config_globals.get("metadata", dict()),
"machine-metadata": config.get("metadata", dict()),
})
return cls(
name,
model,
dataset,
metadata=metadata,
runtime=runtime,
project_name=project_name,
evaluation=evaluation,
)
def get_default_globals(cls, gordo_version: str) -> dict:
current_version = parse(gordo_version)
unifying_version = parse(cls.UNIFYING_GORDO_VERSION)
if current_version >= unifying_version:
docker_images = cls.UNIFIED_DOCKER_IMAGES
else:
docker_images = cls.SPLITED_DOCKER_IMAGES
default_globals = cls.DEFAULT_CONFIG_GLOBALS
return patch_dict(copy(default_globals), docker_images)
def __init__(self, config: dict, project_name: str):
default_globals = self.DEFAULT_CONFIG_GLOBALS
default_globals["runtime"]["influx"][ # type: ignore
"resources"] = _calculate_influx_resources( # type: ignore
len(config["machines"]))
passed_globals = config.get("globals", dict())
patched_globals = patch_dict(default_globals, passed_globals)
if patched_globals.get("runtime"):
patched_globals["runtime"] = fix_runtime(
patched_globals.get("runtime"))
self.project_name = project_name
self.machines = [
Machine.from_config(conf,
project_name=project_name,
config_globals=patched_globals)
for conf in config["machines"]
] # type: List[Machine]
self.globals = patched_globals
def __init__(
self,
config: dict,
project_name: str,
gordo_version: Optional[str] = None,
model_builder_env: Optional[dict] = None,
):
if gordo_version is None:
gordo_version = __version__
default_globals = self.get_default_globals(gordo_version)
default_globals["runtime"]["influx"][ # type: ignore
"resources"
] = _calculate_influx_resources( # type: ignore
len(config["machines"])
)
passed_globals = config.get("globals", dict())
# keeping it for back-compatibility
if model_builder_env is not None and not (
passed_globals
and "runtime" in passed_globals
and "builder" in passed_globals["runtime"]
and "env" in passed_globals["runtime"]["builder"]
):
if "builder" not in default_globals["runtime"]:
default_globals["runtime"]["builder"] = {}
default_globals["runtime"]["builder"]["env"] = model_builder_env
patched_globals = patch_dict(default_globals, passed_globals)
patched_globals = self.prepare_patched_globals(patched_globals)
self.project_name = project_name
self.machines: List[Machine] = [
Machine.from_config(
conf, project_name=project_name, config_globals=patched_globals
)
for conf in config["machines"]
]
self.globals: dict = patched_globals
