def test_create_asset(spec_dict, valid): if valid: AssetSpec(**spec_dict) # major_minor is default system AssetSpec(**spec_dict, versioning="major_minor") else: with pytest.raises(errors.InvalidAssetSpecError): AssetSpec(**spec_dict)
def test_asset_spec_is_version_complete(): spec = AssetSpec(name="name", version="1.1", versioning="major_minor") assert spec.is_version_complete() spec = AssetSpec(name="name", version="1", versioning="major_minor") assert not spec.is_version_complete() spec = AssetSpec(name="name", versioning="major_minor") assert not spec.is_version_complete()
def test_create_asset(): spec = AssetSpec(name="name", version="2020-11-15T17-30-56Z", versioning="simple_date") assert isinstance(spec.versioning, SimpleDateAssetsVersioningSystem) with pytest.raises(errors.InvalidVersionError): AssetSpec(name="name", version="2020-11-15T17-30-56", versioning="simple_date")
def test_asset_spec_set_latest_version(): spec = AssetSpec(name="a", versioning="simple_date") spec.set_latest_version(["2021-11-15T17-31-06Z", "2021-11-14T18-00-00Z"]) assert spec.version == "2021-11-15T17-31-06Z" spec = AssetSpec(name="a", version="2021-11-14T18-00-00Z", versioning="simple_date") spec.set_latest_version(["2021-11-15T17-31-06Z", "2021-11-14T18-00-00Z"]) assert spec.version == "2021-11-15T17-31-06Z"
def test_asset_spec_set_latest_version(): spec = AssetSpec(name="a", versioning="major_minor") spec.set_latest_version(["3", "2.1", "1.3"]) assert spec.version == "3" spec = AssetSpec(name="a", version="2", versioning="major_minor") spec.set_latest_version(["3", "2.1", "2.0", "1.3"]) assert spec.version == "2.1" spec = AssetSpec(name="a", version="1.1", versioning="major_minor") spec.set_latest_version(["3", "2.1", "2.0", "1.3"]) assert spec.version == "1.3"
def update(self, asset_path: str, name: str, version: str, dry_run=False): """ Update an existing asset version """ spec = AssetSpec(name=name, version=version) versions_object_name = self.get_versions_object_name(spec.name) if not self.driver.exists(versions_object_name): raise errors.AssetDoesNotExistError(spec.name) logger.info( "Updating asset", name=spec.name, version=spec.version, asset_path=asset_path, ) versions_list = self.get_versions_info(spec.name) self.push(asset_path, spec.name, spec.version, dry_run=dry_run) with tempfile.TemporaryDirectory() as tmp_dir: versions_fn = os.path.join(tmp_dir, "versions.json") versions = spec.sort_versions([spec.version] + versions_list) with open(versions_fn, "w") as f: json.dump({"versions": versions}, f) logger.debug( "Pushing updated versions file", name=spec.name, versions=versions, ) if not dry_run: self.driver.upload_object(versions_fn, versions_object_name)
def test_fetch_asset_version_no_storage_provider(version_asset_name, version, versioning): manager = AssetsManager(assets_dir=os.path.join( TEST_DIR, "testdata", "test-bucket", "assets-prefix")) asset_name = os.path.join("category", version_asset_name) spec = AssetSpec(name=asset_name, version=version, versioning=versioning) asset_dict = manager._fetch_asset_version( spec=spec, _force_download=False, ) assert asset_dict == { "from_cache": True, "version": version, "path": os.path.join(manager.assets_dir, asset_name, version), } with pytest.raises(errors.StorageDriverError): manager._fetch_asset_version( spec=spec, _force_download=True, ) spec.name = os.path.join("not-existing-asset", version_asset_name) with pytest.raises(errors.LocalAssetDoesNotExistError): manager._fetch_asset_version( spec=spec, _force_download=False, )
def test_asset_spec_get_local_versions(): spec = AssetSpec(name="name", versioning="major_minor") assert spec.get_local_versions("not_a_dir") == [] asset_dir = [ "testdata", "test-bucket", "assets-prefix", "category", "asset" ] local_path = os.path.join(tests.TEST_DIR, *asset_dir) assert spec.get_local_versions(local_path) == ["1.0", "0.1", "0.0"]
def test_asset_spec_sort_versions(): spec = AssetSpec(name="name", versioning="simple_date") version_list = [ "2021-11-15T17-30-56Z", "2020-11-15T17-30-56Z", "2021-10-15T17-30-56Z", ] result = [ "2021-11-15T17-30-56Z", "2021-10-15T17-30-56Z", "2020-11-15T17-30-56Z", ] assert spec.sort_versions(version_list) == result
def test_asset_spec_get_local_versions(): spec = AssetSpec(name="name", versioning="simple_date") assert spec.get_local_versions("not_a_dir") == [] asset_dir = [ "testdata", "test-bucket", "assets-prefix", "category", "simple_date_asset", ] local_path = os.path.join(tests.TEST_DIR, *asset_dir) assert spec.get_local_versions(local_path) == [ "2021-11-15T17-31-06Z", "2021-11-14T18-00-00Z", ]
def test_fetch_asset_version_with_sub_parts(version_asset_name, version, versioning, working_dir): manager = AssetsManager(assets_dir=os.path.join( TEST_DIR, "testdata", "test-bucket", "assets-prefix")) asset_name = os.path.join("category", version_asset_name) sub_part = "sub_part" spec = AssetSpec(name=asset_name, version=version, sub_part=sub_part, versioning=versioning) # no _has_succeeded cache => fetch asset_dict = manager._fetch_asset_version( spec=spec, _force_download=False, ) assert asset_dict == { "from_cache": True, "version": version, "path": os.path.join(manager.assets_dir, asset_name, version, sub_part), }
def test_fetch_asset_version_with_storage_provider(version_asset_name, version, versioning, working_dir): manager = AssetsManager( assets_dir=working_dir, storage_provider=StorageProvider( provider="local", bucket=os.path.join(TEST_DIR, "testdata", "test-bucket"), prefix="assets-prefix", ), ) asset_name = os.path.join("category", version_asset_name) spec = AssetSpec(name=asset_name, version=version, versioning=versioning) # no _has_succeeded cache => fetch asset_dict = manager._fetch_asset_version( spec=spec, _force_download=False, ) del asset_dict["meta"] # fetch meta data assert asset_dict == { "from_cache": False, "version": version, "path": os.path.join(working_dir, asset_name, version), } # cache asset_dict = manager._fetch_asset_version( spec=spec, _force_download=False, ) assert asset_dict == { "from_cache": True, "version": version, "path": os.path.join(working_dir, asset_name, version), } # cache but force download asset_dict = manager._fetch_asset_version( spec=spec, _force_download=True, ) del asset_dict["meta"] # fetch meta data assert asset_dict == { "from_cache": False, "version": version, "path": os.path.join(working_dir, asset_name, version), } # Re-Download asset when missing version os.remove(os.path.join(working_dir, asset_name, version)) asset_dict = manager._fetch_asset_version( spec=spec, _force_download=False, ) del asset_dict["meta"] # fetch meta data assert asset_dict == { "from_cache": False, "version": version, "path": os.path.join(working_dir, asset_name, version), }
def test_versioning_values(): AssetSpec(name="a") AssetSpec(name="a", versioning="major_minor") AssetSpec(name="a", versioning="simple_date") with pytest.raises(errors.UnknownAssetsVersioningSystemError): AssetSpec(name="a", versioning="unk_versioning")
def test_string_asset_spec(s, spec): assert AssetSpec.from_string(s) == AssetSpec(**spec) assert AssetSpec.from_string(s, versioning="major_minor") == AssetSpec( versioning="major_minor", **spec)
def test_asset_spec_sort_versions(version_list, result): spec = AssetSpec(name="name", versioning="major_minor") assert spec.sort_versions(version_list) == result
def _resolve_assets(self, model_name): """ This function fetches assets for the current model and its dependent models and populates the assets_info dictionary with the paths. """ logger.debug("Resolving asset for Model", model_name=model_name) configuration = self.configuration[model_name] # First, resolve assets from dependent models for dep_name in configuration.model_dependencies.values(): self._resolve_assets(dep_name) if not configuration.asset: # If the model has no asset to load return model_settings = { **configuration.model_settings, **self.required_models.get(model_name, {}), } # If the asset is overriden in the model_settings if "asset_path" in model_settings: asset_path = model_settings.pop("asset_path") logger.debug( "Overriding asset from Model settings", model_name=model_name, asset_path=asset_path, ) self.assets_info[configuration.asset] = AssetInfo(path=asset_path) asset_spec = AssetSpec.from_string(configuration.asset) # If the model's asset is overriden with environment variables venv = "MODELKIT_{}_FILE".format( re.sub(r"[\/\-\.]+", "_", asset_spec.name).upper()) local_file = os.environ.get(venv) if local_file: logger.debug( "Overriding asset from environment variable", asset_name=asset_spec.name, path=local_file, ) self.assets_info[configuration.asset] = AssetInfo(path=local_file) # The assets should be retrieved # possibly override version venv = "MODELKIT_{}_VERSION".format( re.sub(r"[\/\-\.]+", "_", asset_spec.name).upper()) version = os.environ.get(venv) if version: logger.debug( "Overriding asset version from environment variable", asset_name=asset_spec.name, path=local_file, ) asset_spec = AssetSpec.from_string(asset_spec.name + ":" + version) if self.override_assets_manager: try: self.assets_info[configuration.asset] = AssetInfo( **self.override_assets_manager.fetch_asset( spec=AssetSpec(name=asset_spec.name, sub_part=asset_spec.sub_part), return_info=True, )) logger.debug( "Asset has been overriden", name=asset_spec.name, ) except modelkit.assets.errors.AssetDoesNotExistError: logger.debug( "Asset not found in overriden prefix", name=asset_spec.name, ) if configuration.asset not in self.assets_info: self.assets_info[configuration.asset] = AssetInfo( **self.assets_manager.fetch_asset(asset_spec, return_info=True))
def test_string_asset_spec(s, spec): assert AssetSpec.from_string(s, versioning="simple_date") == AssetSpec( versioning="simple_date", **spec)