def test_overwrites_existing(): """Double writes to the same registry overwrites the first value""" the_key = "akey" with tempfile.TemporaryDirectory() as tmpdir: first_value = "Some value" disk_registry.write_key(tmpdir, the_key, first_value) assert disk_registry.get_value(tmpdir, the_key) == first_value second_value = "Some value" disk_registry.write_key(tmpdir, the_key, second_value) assert disk_registry.get_value(tmpdir, the_key) == second_value
def test_delete(): """Delete removes a key""" the_key = "akey" with tempfile.TemporaryDirectory() as tmpdir: first_value = "Some value" disk_registry.write_key(tmpdir, the_key, first_value) assert disk_registry.get_value(tmpdir, the_key) == first_value existed_p = disk_registry.delete_value(tmpdir, the_key) assert disk_registry.get_value(tmpdir, the_key) is None # They key existed assert existed_p
def test_double_delete(): """Delete works on non-existing key, returning False""" the_key = "akey" with tempfile.TemporaryDirectory() as tmpdir: existed_p = disk_registry.delete_value(tmpdir, the_key) assert disk_registry.get_value(tmpdir, the_key) is None assert not existed_p
def test_new_registry(): """Tests that it works to write and read a simple value to a fresh registry with a non-existing directory""" with tempfile.TemporaryDirectory() as tmpdir: registry = pathlib.Path(tmpdir).joinpath("newregistry") disk_registry.write_key(registry, "akey", "aval") assert disk_registry.get_value(registry, "akey") == "aval"
def check_cache(model_register_dir: Union[os.PathLike, str], cache_key: str): """ Checks if a model is cached, and returns its path if it exists. Parameters ---------- model_register_dir: [os.PathLike, None] The register dir where the model lies. cache_key: str A 512 byte hex value as a string based on the content of the parameters. Returns ------- Union[os.PathLike, None]: The path to the cached model, or None if it does not exist. """ existing_model_location = disk_registry.get_value(model_register_dir, cache_key) # Check that the model is actually there if existing_model_location and Path(existing_model_location).exists(): logger.debug( f"Found existing model at path {existing_model_location}, returning it" ) return existing_model_location elif existing_model_location: logger.warning( f"Found that the model-path {existing_model_location} stored in the " f"registry did not exist.") return None else: logger.info( f"Did not find the model with key {cache_key} in the register at " f"{model_register_dir}.") return None
def test_complicated_happy_path(): """Tests that it works to write and read a 'complicated' value""" with tempfile.TemporaryDirectory() as tmpdir: value = """ A long value with many weird character lie åøæ and some linebreaks""" disk_registry.write_key(tmpdir, "akey", value) assert disk_registry.get_value(tmpdir, "akey") == value
def test_simple_happy_path(): """Tests that it works to write and read a simple value to a fresh registry""" with tempfile.TemporaryDirectory() as tmpdir: disk_registry.write_key(tmpdir, "akey", "aval") assert disk_registry.get_value(tmpdir, "akey") == "aval"
def test_get_value_without_registry_dir(): """Test that it works to have registry_dir as None, and that it returns None as expected""" assert disk_registry.get_value(None, "akey") is None
def provide_saved_model( name: str, model_config: dict, data_config: dict, metadata: dict, output_dir: Union[os.PathLike, str], model_register_dir: Union[os.PathLike, str] = None, replace_cache=False, ) -> Union[os.PathLike, str]: """ Ensures that the desired model exists on disk, and returns the path to it. Builds the model if needed, or finds it among already existing models if ``model_register_dir`` is non-None, and we find the model there. If `model_register_dir` is set we will also store the model-location of the generated model there for future use. Think about it as a cache that is never emptied. Parameters ---------- name: str Name of the model to be built model_config: dict Config for the model. See :func:`gordo_components.builder.build_model.build_model`. data_config: dict Config for the data-configuration. See :func:`gordo_components.builder.build_model.build_model`. metadata: dict Extra metadata to be added to the built models if it is built. See :func:`gordo_components.builder.build_model.build_model`. output_dir: Union[os.PathLike, str] A path to where the model will be deposited if it is built. model_register_dir: A path to a register, see `gordo_components.util.disk_registry`. If this is None then always build the model, otherwise try to resolve the model from the registry. replace_cache: bool Forces a rebuild of the model, and replaces the entry in the cache with the new model. Returns ------- Union[os.PathLike, str]: Path to the model """ cache_key = calculate_model_key(name, model_config, data_config, metadata=metadata) if model_register_dir: logger.info( f"Model caching activated, attempting to read model-location with key " f"{cache_key} from register {model_register_dir}") if replace_cache: logger.info( "replace_cache activated, deleting any existing cache entry") cache_key = calculate_model_key(name, model_config, data_config, metadata=metadata) disk_registry.delete_value(model_register_dir, cache_key) existing_model_location = disk_registry.get_value( model_register_dir, cache_key) # Check that the model is actually there if existing_model_location and Path(existing_model_location).exists(): logger.debug( f"Found existing model at path {existing_model_location}, returning it" ) return existing_model_location elif existing_model_location: logger.warning( f"Found that the model-path {existing_model_location} stored in the " f"registry did not exist.") else: logger.info( f"Did not find the model with key {cache_key} in the register at " f"{model_register_dir}.") model, metadata = build_model(name=name, model_config=model_config, data_config=data_config, metadata=metadata) model_location = _save_model_for_workflow(model=model, metadata=metadata, output_dir=output_dir) logger.info(f"Successfully built model, and deposited at {model_location}") if model_register_dir: logger.info(f"Writing model-location to model registry") disk_registry.write_key(model_register_dir, cache_key, model_location) return model_location