def test_fetch_asset_version_no_storage_provider(version_asset_name, version,
versioning):
manager = AssetsManager(assets_dir=os.path.join(
TEST_DIR, "testdata", "test-bucket", "assets-prefix"))
asset_name = os.path.join("category", version_asset_name)
spec = AssetSpec(name=asset_name, version=version, versioning=versioning)
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
assert asset_dict == {
"from_cache": True,
"version": version,
"path": os.path.join(manager.assets_dir, asset_name, version),
}
with pytest.raises(errors.StorageDriverError):
manager._fetch_asset_version(
spec=spec,
_force_download=True,
)
spec.name = os.path.join("not-existing-asset", version_asset_name)
with pytest.raises(errors.LocalAssetDoesNotExistError):
manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
def test_fetch_asset_version_with_sub_parts(version_asset_name, version,
versioning, working_dir):
manager = AssetsManager(assets_dir=os.path.join(
TEST_DIR, "testdata", "test-bucket", "assets-prefix"))
asset_name = os.path.join("category", version_asset_name)
sub_part = "sub_part"
spec = AssetSpec(name=asset_name,
version=version,
sub_part=sub_part,
versioning=versioning)
# no _has_succeeded cache => fetch
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
assert asset_dict == {
"from_cache": True,
"version": version,
"path": os.path.join(manager.assets_dir, asset_name, version,
sub_part),
}
def test_fetch_asset_version_with_storage_provider(version_asset_name, version,
versioning, working_dir):
manager = AssetsManager(
assets_dir=working_dir,
storage_provider=StorageProvider(
provider="local",
bucket=os.path.join(TEST_DIR, "testdata", "test-bucket"),
prefix="assets-prefix",
),
)
asset_name = os.path.join("category", version_asset_name)
spec = AssetSpec(name=asset_name, version=version, versioning=versioning)
# no _has_succeeded cache => fetch
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
del asset_dict["meta"] # fetch meta data
assert asset_dict == {
"from_cache": False,
"version": version,
"path": os.path.join(working_dir, asset_name, version),
}
# cache
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
assert asset_dict == {
"from_cache": True,
"version": version,
"path": os.path.join(working_dir, asset_name, version),
}
# cache but force download
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=True,
)
del asset_dict["meta"] # fetch meta data
assert asset_dict == {
"from_cache": False,
"version": version,
"path": os.path.join(working_dir, asset_name, version),
}
# Re-Download asset when missing version
os.remove(os.path.join(working_dir, asset_name, version))
asset_dict = manager._fetch_asset_version(
spec=spec,
_force_download=False,
)
del asset_dict["meta"] # fetch meta data
assert asset_dict == {
"from_cache": False,
"version": version,
"path": os.path.join(working_dir, asset_name, version),
}