def test_overwrites_existing():
"""Double writes to the same registry overwrites the first value"""
the_key = "akey"
with tempfile.TemporaryDirectory() as tmpdir:
first_value = "Some value"
disk_registry.write_key(tmpdir, the_key, first_value)
assert disk_registry.get_value(tmpdir, the_key) == first_value
second_value = "Some value"
disk_registry.write_key(tmpdir, the_key, second_value)
assert disk_registry.get_value(tmpdir, the_key) == second_value
def test_delete():
"""Delete removes a key"""
the_key = "akey"
with tempfile.TemporaryDirectory() as tmpdir:
first_value = "Some value"
disk_registry.write_key(tmpdir, the_key, first_value)
assert disk_registry.get_value(tmpdir, the_key) == first_value
existed_p = disk_registry.delete_value(tmpdir, the_key)
assert disk_registry.get_value(tmpdir, the_key) is None
# They key existed
assert existed_p
def test_double_delete():
"""Delete works on non-existing key, returning False"""
the_key = "akey"
with tempfile.TemporaryDirectory() as tmpdir:
existed_p = disk_registry.delete_value(tmpdir, the_key)
assert disk_registry.get_value(tmpdir, the_key) is None
assert not existed_p
def test_new_registry():
"""Tests that it works to write and read a simple value to a fresh registry with
a non-existing directory"""
with tempfile.TemporaryDirectory() as tmpdir:
registry = pathlib.Path(tmpdir).joinpath("newregistry")
disk_registry.write_key(registry, "akey", "aval")
assert disk_registry.get_value(registry, "akey") == "aval"
def check_cache(model_register_dir: Union[os.PathLike, str], cache_key: str):
"""
Checks if a model is cached, and returns its path if it exists.
Parameters
----------
model_register_dir: [os.PathLike, None]
The register dir where the model lies.
cache_key: str
A 512 byte hex value as a string based on the content of the parameters.
Returns
-------
Union[os.PathLike, None]:
The path to the cached model, or None if it does not exist.
"""
existing_model_location = disk_registry.get_value(model_register_dir,
cache_key)
# Check that the model is actually there
if existing_model_location and Path(existing_model_location).exists():
logger.debug(
f"Found existing model at path {existing_model_location}, returning it"
)
return existing_model_location
elif existing_model_location:
logger.warning(
f"Found that the model-path {existing_model_location} stored in the "
f"registry did not exist.")
return None
else:
logger.info(
f"Did not find the model with key {cache_key} in the register at "
f"{model_register_dir}.")
return None
def test_complicated_happy_path():
"""Tests that it works to write and read a 'complicated' value"""
with tempfile.TemporaryDirectory() as tmpdir:
value = """
A long
value with many weird character lie åøæ
and some linebreaks"""
disk_registry.write_key(tmpdir, "akey", value)
assert disk_registry.get_value(tmpdir, "akey") == value
def test_simple_happy_path():
"""Tests that it works to write and read a simple value to a fresh registry"""
with tempfile.TemporaryDirectory() as tmpdir:
disk_registry.write_key(tmpdir, "akey", "aval")
assert disk_registry.get_value(tmpdir, "akey") == "aval"
def test_get_value_without_registry_dir():
"""Test that it works to have registry_dir as None, and that it returns None as expected"""
assert disk_registry.get_value(None, "akey") is None
def provide_saved_model(
name: str,
model_config: dict,
data_config: dict,
metadata: dict,
output_dir: Union[os.PathLike, str],
model_register_dir: Union[os.PathLike, str] = None,
replace_cache=False,
) -> Union[os.PathLike, str]:
"""
Ensures that the desired model exists on disk, and returns the path to it.
Builds the model if needed, or finds it among already existing models if
``model_register_dir`` is non-None, and we find the model there. If
`model_register_dir` is set we will also store the model-location of the generated
model there for future use. Think about it as a cache that is never emptied.
Parameters
----------
name: str
Name of the model to be built
model_config: dict
Config for the model. See
:func:`gordo_components.builder.build_model.build_model`.
data_config: dict
Config for the data-configuration. See
:func:`gordo_components.builder.build_model.build_model`.
metadata: dict
Extra metadata to be added to the built models if it is built. See
:func:`gordo_components.builder.build_model.build_model`.
output_dir: Union[os.PathLike, str]
A path to where the model will be deposited if it is built.
model_register_dir:
A path to a register, see `gordo_components.util.disk_registry`. If this is None
then always build the model, otherwise try to resolve the model from the
registry.
replace_cache: bool
Forces a rebuild of the model, and replaces the entry in the cache with the new
model.
Returns
-------
Union[os.PathLike, str]:
Path to the model
"""
cache_key = calculate_model_key(name,
model_config,
data_config,
metadata=metadata)
if model_register_dir:
logger.info(
f"Model caching activated, attempting to read model-location with key "
f"{cache_key} from register {model_register_dir}")
if replace_cache:
logger.info(
"replace_cache activated, deleting any existing cache entry")
cache_key = calculate_model_key(name,
model_config,
data_config,
metadata=metadata)
disk_registry.delete_value(model_register_dir, cache_key)
existing_model_location = disk_registry.get_value(
model_register_dir, cache_key)
# Check that the model is actually there
if existing_model_location and Path(existing_model_location).exists():
logger.debug(
f"Found existing model at path {existing_model_location}, returning it"
)
return existing_model_location
elif existing_model_location:
logger.warning(
f"Found that the model-path {existing_model_location} stored in the "
f"registry did not exist.")
else:
logger.info(
f"Did not find the model with key {cache_key} in the register at "
f"{model_register_dir}.")
model, metadata = build_model(name=name,
model_config=model_config,
data_config=data_config,
metadata=metadata)
model_location = _save_model_for_workflow(model=model,
metadata=metadata,
output_dir=output_dir)
logger.info(f"Successfully built model, and deposited at {model_location}")
if model_register_dir:
logger.info(f"Writing model-location to model registry")
disk_registry.write_key(model_register_dir, cache_key, model_location)
return model_location
