def test_delete_artifacts(gcs_mock):
experiment_root_path = "/experiment_id/"
repo = GCSArtifactRepository("gs://test_bucket" + experiment_root_path,
gcs_mock)
def delete_file():
del obj_mock.name
del obj_mock.size
return obj_mock
obj_mock = mock.Mock()
run_id_path = experiment_root_path + "run_id/"
file_path = "file"
attrs = {
"name": run_id_path + file_path,
"size": 1,
"delete.side_effect": delete_file
}
obj_mock.configure_mock(**attrs)
def get_mock_listing(prefix, **kwargs):
"""
Produces a mock listing that only contains content if the
specified prefix is the artifact root. This allows us to mock
`list_artifacts` during the `_download_artifacts_into` subroutine
without recursively listing the same artifacts at every level of the
directory traversal.
"""
# pylint: disable=unused-argument
if hasattr(obj_mock, "name") and hasattr(obj_mock, "size"):
mock_results = mock.MagicMock()
mock_results.__iter__.return_value = [obj_mock]
return mock_results
else:
mock_empty_results = mock.MagicMock()
mock_empty_results.__iter__.return_value = []
return mock_empty_results
mock_method_chain(
gcs_mock,
[
"Client",
"bucket",
"list_blobs",
],
side_effect=get_mock_listing,
)
artifact_file_names = [obj.path for obj in repo.list_artifacts()]
assert "run_id/file" in artifact_file_names
repo.delete_artifacts()
artifact_file_names = [obj.path for obj in repo.list_artifacts()]
assert not artifact_file_names
def test_list_artifacts_with_subdir(gcs_mock):
artifact_root_path = "/experiment_id/run_id/"
repo = GCSArtifactRepository("gs://test_bucket" + artifact_root_path, gcs_mock)
# mocked bucket/blob structure
# gs://test_bucket/experiment_id/run_id/
# |- model
# |- model.pb
# |- variables
# list artifacts at sub directory level
dir_name = "model"
obj_mock = mock.Mock()
file_path = dir_name + "/" + 'model.pb'
obj_mock.configure_mock(name=artifact_root_path + file_path, size=1)
subdir_mock = mock.Mock()
subdir_name = dir_name + "/" + 'variables'
subdir_mock.configure_mock(prefixes=(artifact_root_path + subdir_name + "/",))
mock_results = mock.MagicMock()
mock_results.configure_mock(pages=[subdir_mock])
mock_results.__iter__.return_value = [obj_mock]
gcs_mock.Client.return_value.bucket.return_value\
.list_blobs.return_value = mock_results
artifacts = repo.list_artifacts(path=dir_name)
assert len(artifacts) == 2
assert artifacts[0].path == file_path
assert artifacts[0].is_dir is False
assert artifacts[0].file_size == obj_mock.size
assert artifacts[1].path == subdir_name
assert artifacts[1].is_dir is True
assert artifacts[1].file_size is None
def test_list_artifacts(gcs_mock):
artifact_root_path = "/experiment_id/run_id/"
repo = GCSArtifactRepository("gs://test_bucket" + artifact_root_path,
gcs_mock)
# mocked bucket/blob structure
# gs://test_bucket/experiment_id/run_id/
# |- file
# |- model
# |- model.pb
# mocking a single blob returned by bucket.list_blobs iterator
# https://googlecloudplatform.github.io/google-cloud-python/latest/storage/buckets.html#google.cloud.storage.bucket.Bucket.list_blobs
# list artifacts at artifact root level
obj_mock = mock.Mock()
file_path = 'file'
obj_mock.configure_mock(name=artifact_root_path + file_path, size=1)
dir_mock = mock.Mock()
dir_name = "model"
dir_mock.configure_mock(prefixes=(artifact_root_path + dir_name + "/", ))
mock_results = mock.MagicMock()
mock_results.configure_mock(pages=[dir_mock])
mock_results.__iter__.return_value = [obj_mock]
gcs_mock.Client.return_value.get_bucket.return_value\
.list_blobs.return_value = mock_results
artifacts = repo.list_artifacts(path=None)
assert len(artifacts) == 2
assert artifacts[0].path == file_path
assert artifacts[0].is_dir is False
assert artifacts[0].file_size == obj_mock.size
assert artifacts[1].path == dir_name
assert artifacts[1].is_dir is True
assert artifacts[1].file_size is None
def test_list_artifacts_empty(gcs_mock):
repo = GCSArtifactRepository("gs://test_bucket/some/path", gcs_mock)
gcs_mock.Client.return_value.bucket.return_value.list_blobs.return_value = mock.MagicMock(
)
assert repo.list_artifacts() == []