def test_create_asset(spec_dict, valid):
if valid:
AssetSpec(**spec_dict) # major_minor is default system
AssetSpec(**spec_dict, versioning="major_minor")
else:
with pytest.raises(errors.InvalidAssetSpecError):
AssetSpec(**spec_dict)
def test_asset_spec_is_version_complete():
spec = AssetSpec(name="name", version="1.1", versioning="major_minor")
assert spec.is_version_complete()
spec = AssetSpec(name="name", version="1", versioning="major_minor")
assert not spec.is_version_complete()
spec = AssetSpec(name="name", versioning="major_minor")
assert not spec.is_version_complete()
def test_create_asset():
spec = AssetSpec(name="name",
version="2020-11-15T17-30-56Z",
versioning="simple_date")
assert isinstance(spec.versioning, SimpleDateAssetsVersioningSystem)
with pytest.raises(errors.InvalidVersionError):
AssetSpec(name="name",
version="2020-11-15T17-30-56",
versioning="simple_date")
def test_asset_spec_set_latest_version():
spec = AssetSpec(name="a", versioning="simple_date")
spec.set_latest_version(["2021-11-15T17-31-06Z", "2021-11-14T18-00-00Z"])
assert spec.version == "2021-11-15T17-31-06Z"
spec = AssetSpec(name="a",
version="2021-11-14T18-00-00Z",
versioning="simple_date")
spec.set_latest_version(["2021-11-15T17-31-06Z", "2021-11-14T18-00-00Z"])
assert spec.version == "2021-11-15T17-31-06Z"
def test_asset_spec_set_latest_version():
spec = AssetSpec(name="a", versioning="major_minor")
spec.set_latest_version(["3", "2.1", "1.3"])
assert spec.version == "3"
spec = AssetSpec(name="a", version="2", versioning="major_minor")
spec.set_latest_version(["3", "2.1", "2.0", "1.3"])
assert spec.version == "2.1"
spec = AssetSpec(name="a", version="1.1", versioning="major_minor")
spec.set_latest_version(["3", "2.1", "2.0", "1.3"])
assert spec.version == "1.3"
def update(self, asset_path: str, name: str, version: str, dry_run=False):
"""
Update an existing asset version
"""
spec = AssetSpec(name=name, version=version)
versions_object_name = self.get_versions_object_name(spec.name)
if not self.driver.exists(versions_object_name):
raise errors.AssetDoesNotExistError(spec.name)
logger.info(
"Updating asset",
name=spec.name,
version=spec.version,
asset_path=asset_path,
)
versions_list = self.get_versions_info(spec.name)
self.push(asset_path, spec.name, spec.version, dry_run=dry_run)
with tempfile.TemporaryDirectory() as tmp_dir:
versions_fn = os.path.join(tmp_dir, "versions.json")
versions = spec.sort_versions([spec.version] + versions_list)
with open(versions_fn, "w") as f:
json.dump({"versions": versions}, f)
logger.debug(
"Pushing updated versions file",
name=spec.name,
versions=versions,
)
if not dry_run:
self.driver.upload_object(versions_fn, versions_object_name)
def test_fetch_asset_version_no_storage_provider(version_asset_name, version,
versioning):
manager = AssetsManager(assets_dir=os.path.join(
TEST_DIR, "testdata", "test-bucket", "assets-prefix"))
asset_name = os.path.join("category", version_asset_name)
spec = AssetSpec(name=asset_name, version=version, versioning=versioning)
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
assert asset_dict == {
"from_cache": True,
"version": version,
"path": os.path.join(manager.assets_dir, asset_name, version),
}
with pytest.raises(errors.StorageDriverError):
manager._fetch_asset_version(
spec=spec,
_force_download=True,
)
spec.name = os.path.join("not-existing-asset", version_asset_name)
with pytest.raises(errors.LocalAssetDoesNotExistError):
manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
def test_asset_spec_get_local_versions():
spec = AssetSpec(name="name", versioning="major_minor")
assert spec.get_local_versions("not_a_dir") == []
asset_dir = [
"testdata", "test-bucket", "assets-prefix", "category", "asset"
]
local_path = os.path.join(tests.TEST_DIR, *asset_dir)
assert spec.get_local_versions(local_path) == ["1.0", "0.1", "0.0"]
def test_asset_spec_sort_versions():
spec = AssetSpec(name="name", versioning="simple_date")
version_list = [
"2021-11-15T17-30-56Z",
"2020-11-15T17-30-56Z",
"2021-10-15T17-30-56Z",
]
result = [
"2021-11-15T17-30-56Z",
"2021-10-15T17-30-56Z",
"2020-11-15T17-30-56Z",
]
assert spec.sort_versions(version_list) == result
def test_asset_spec_get_local_versions():
spec = AssetSpec(name="name", versioning="simple_date")
assert spec.get_local_versions("not_a_dir") == []
asset_dir = [
"testdata",
"test-bucket",
"assets-prefix",
"category",
"simple_date_asset",
]
local_path = os.path.join(tests.TEST_DIR, *asset_dir)
assert spec.get_local_versions(local_path) == [
"2021-11-15T17-31-06Z",
"2021-11-14T18-00-00Z",
]
def test_fetch_asset_version_with_sub_parts(version_asset_name, version,
versioning, working_dir):
manager = AssetsManager(assets_dir=os.path.join(
TEST_DIR, "testdata", "test-bucket", "assets-prefix"))
asset_name = os.path.join("category", version_asset_name)
sub_part = "sub_part"
spec = AssetSpec(name=asset_name,
version=version,
sub_part=sub_part,
versioning=versioning)
# no _has_succeeded cache => fetch
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
assert asset_dict == {
"from_cache": True,
"version": version,
"path": os.path.join(manager.assets_dir, asset_name, version,
sub_part),
}
def test_fetch_asset_version_with_storage_provider(version_asset_name, version,
versioning, working_dir):
manager = AssetsManager(
assets_dir=working_dir,
storage_provider=StorageProvider(
provider="local",
bucket=os.path.join(TEST_DIR, "testdata", "test-bucket"),
prefix="assets-prefix",
),
)
asset_name = os.path.join("category", version_asset_name)
spec = AssetSpec(name=asset_name, version=version, versioning=versioning)
# no _has_succeeded cache => fetch
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
del asset_dict["meta"] # fetch meta data
assert asset_dict == {
"from_cache": False,
"version": version,
"path": os.path.join(working_dir, asset_name, version),
}
# cache
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
assert asset_dict == {
"from_cache": True,
"version": version,
"path": os.path.join(working_dir, asset_name, version),
}
# cache but force download
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=True,
)
del asset_dict["meta"] # fetch meta data
assert asset_dict == {
"from_cache": False,
"version": version,
"path": os.path.join(working_dir, asset_name, version),
}
# Re-Download asset when missing version
os.remove(os.path.join(working_dir, asset_name, version))
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
del asset_dict["meta"] # fetch meta data
assert asset_dict == {
"from_cache": False,
"version": version,
"path": os.path.join(working_dir, asset_name, version),
}
def test_versioning_values():
AssetSpec(name="a")
AssetSpec(name="a", versioning="major_minor")
AssetSpec(name="a", versioning="simple_date")
with pytest.raises(errors.UnknownAssetsVersioningSystemError):
AssetSpec(name="a", versioning="unk_versioning")
def test_string_asset_spec(s, spec):
assert AssetSpec.from_string(s) == AssetSpec(**spec)
assert AssetSpec.from_string(s, versioning="major_minor") == AssetSpec(
versioning="major_minor", **spec)
def test_asset_spec_sort_versions(version_list, result):
spec = AssetSpec(name="name", versioning="major_minor")
assert spec.sort_versions(version_list) == result
def _resolve_assets(self, model_name):
"""
This function fetches assets for the current model and its dependent models
and populates the assets_info dictionary with the paths.
"""
logger.debug("Resolving asset for Model", model_name=model_name)
configuration = self.configuration[model_name]
# First, resolve assets from dependent models
for dep_name in configuration.model_dependencies.values():
self._resolve_assets(dep_name)
if not configuration.asset:
# If the model has no asset to load
return
model_settings = {
**configuration.model_settings,
**self.required_models.get(model_name, {}),
}
# If the asset is overriden in the model_settings
if "asset_path" in model_settings:
asset_path = model_settings.pop("asset_path")
logger.debug(
"Overriding asset from Model settings",
model_name=model_name,
asset_path=asset_path,
)
self.assets_info[configuration.asset] = AssetInfo(path=asset_path)
asset_spec = AssetSpec.from_string(configuration.asset)
# If the model's asset is overriden with environment variables
venv = "MODELKIT_{}_FILE".format(
re.sub(r"[\/\-\.]+", "_", asset_spec.name).upper())
local_file = os.environ.get(venv)
if local_file:
logger.debug(
"Overriding asset from environment variable",
asset_name=asset_spec.name,
path=local_file,
)
self.assets_info[configuration.asset] = AssetInfo(path=local_file)
# The assets should be retrieved
# possibly override version
venv = "MODELKIT_{}_VERSION".format(
re.sub(r"[\/\-\.]+", "_", asset_spec.name).upper())
version = os.environ.get(venv)
if version:
logger.debug(
"Overriding asset version from environment variable",
asset_name=asset_spec.name,
path=local_file,
)
asset_spec = AssetSpec.from_string(asset_spec.name + ":" + version)
if self.override_assets_manager:
try:
self.assets_info[configuration.asset] = AssetInfo(
**self.override_assets_manager.fetch_asset(
spec=AssetSpec(name=asset_spec.name,
sub_part=asset_spec.sub_part),
return_info=True,
))
logger.debug(
"Asset has been overriden",
name=asset_spec.name,
)
except modelkit.assets.errors.AssetDoesNotExistError:
logger.debug(
"Asset not found in overriden prefix",
name=asset_spec.name,
)
if configuration.asset not in self.assets_info:
self.assets_info[configuration.asset] = AssetInfo(
**self.assets_manager.fetch_asset(asset_spec,
return_info=True))
def test_string_asset_spec(s, spec):
assert AssetSpec.from_string(s, versioning="simple_date") == AssetSpec(
versioning="simple_date", **spec)