def test_download_dir_with_basename(self, client):
dirname1 = tempfile.mkdtemp()
dirname2 = tempfile.mkdtemp(prefix=dirname1 + '/')
def mkfile(container, cloud_path, fname):
return open(fname, 'w')
client.return_value.get_blob_to_path.side_effect = mkfile
store = AzureStore()
blob_path = '/path/to/'
azure_url = self.wasbs_base + blob_path
rel_path2 = dirname2.split('/')[-1]
# Mock return list
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock1 = Blob(blob_path + 'foo/test1.txt', props=blob_props)
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock2 = Blob(blob_path + 'foo/test2.txt', props=blob_props)
subdir_mock = BlobPrefix()
subdir_mock.name = blob_path + 'foo/' + rel_path2
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock3 = Blob(blob_path + 'foo/' + rel_path2 + '/' + 'test3.txt',
props=blob_props)
# Create some files to return
def list_side_effect(container_name,
prefix,
delimiter='/',
marker=None):
if prefix == blob_path + 'foo/':
return MockBlobList([subdir_mock, obj_mock1, obj_mock2])
return MockBlobList([obj_mock3])
client.return_value.list_blobs.side_effect = list_side_effect
dirname3 = tempfile.mkdtemp()
# Test without basename
store.download_dir(blob=azure_url + 'foo',
local_path=dirname3,
use_basename=True)
client.return_value.get_blob_to_path.assert_has_calls([
mock.call('container', '{}foo/test1.txt'.format(blob_path),
'{}/foo/test1.txt'.format(dirname3)),
mock.call('container', '{}foo/test2.txt'.format(blob_path),
'{}/foo/test2.txt'.format(dirname3)),
mock.call('container', '{}foo/{}/test3.txt'.format(
blob_path, rel_path2), '{}/foo/{}/test3.txt'.format(
dirname3, rel_path2)),
],
any_order=True)
def test_download_directory_artifact_succeeds_when_artifact_root_is_blob_container_root(
mock_client, tmpdir):
repo = AzureBlobArtifactRepository(TEST_BLOB_CONTAINER_ROOT, mock_client)
subdir_path = "my_directory"
dir_prefix = BlobPrefix()
dir_prefix.name = subdir_path
file_path_1 = "file_1"
file_path_2 = "file_2"
blob_props_1 = BlobProperties()
blob_props_1.content_length = 42
blob_1 = Blob(os.path.join(subdir_path, file_path_1), props=blob_props_1)
blob_props_2 = BlobProperties()
blob_props_2.content_length = 42
blob_2 = Blob(os.path.join(subdir_path, file_path_2), props=blob_props_2)
def get_mock_listing(*args, **kwargs):
"""
Produces a mock listing that only contains content if the specified prefix is the artifact
root or a relevant subdirectory. This allows us to mock `list_artifacts` during the
`_download_artifacts_into` subroutine without recursively listing the same artifacts at
every level of the directory traversal.
"""
# pylint: disable=unused-argument
if posixpath.abspath(kwargs["prefix"]) == "/":
return MockBlobList([dir_prefix])
if posixpath.abspath(
kwargs["prefix"]) == posixpath.abspath(subdir_path):
return MockBlobList([blob_1, blob_2])
else:
return MockBlobList([])
def create_file(container, cloud_path, local_path):
# pylint: disable=unused-argument
fname = os.path.basename(local_path)
f = tmpdir.join(fname)
f.write("hello world!")
mock_client.list_blobs.side_effect = get_mock_listing
mock_client.get_blob_to_path.side_effect = create_file
# Ensure that the root directory can be downloaded successfully
repo.download_artifacts("")
# Ensure that the `mkfile` side effect copied all of the download artifacts into `tmpdir`
dir_contents = os.listdir(tmpdir.strpath)
assert file_path_1 in dir_contents
assert file_path_2 in dir_contents
def test_download_artifact_throws_value_error_when_listed_blobs_do_not_contain_artifact_root_prefix(
mock_client):
repo = AzureBlobArtifactRepository(TEST_URI, mock_client)
# Create a "bad blob" with a name that is not prefixed by the root path of the artifact store
bad_blob_props = BlobProperties()
bad_blob_props.content_length = 42
bad_blob = Blob("file_path", props=bad_blob_props)
def get_mock_listing(*args, **kwargs):
"""
Produces a mock listing that only contains content if the
specified prefix is the artifact root. This allows us to mock
`list_artifacts` during the `_download_artifacts_into` subroutine
without recursively listing the same artifacts at every level of the
directory traversal.
"""
# pylint: disable=unused-argument
if os.path.abspath(
kwargs["prefix"]) == os.path.abspath(TEST_ROOT_PATH):
# Return a blob that is not prefixed by the root path of the artifact store. This
# should result in an exception being raised
return MockBlobList([bad_blob])
else:
return MockBlobList([])
mock_client.list_blobs.side_effect = get_mock_listing
with pytest.raises(ValueError) as exc:
repo.download_artifacts("")
assert "Azure blob does not begin with the specified artifact path" in str(
exc)
def test_list_non_empty(self, client):
base_path = '/path/'
# Create some files to return
dir_prefix = BlobPrefix()
dir_prefix.name = base_path + 'dir'
blob_props = BlobProperties()
blob_props.content_length = 42
blob = Blob(base_path + 'file', props=blob_props)
client.return_value.list_blobs.return_value = MockBlobList([dir_prefix, blob])
store = AzureStore()
key_path = self.wasbs_base + base_path
results = store.list(key=key_path)
assert len(results['blobs']) == 1
assert len(results['prefixes']) == 1
assert results['prefixes'][0] == 'dir'
assert results['blobs'][0][0] == 'file'
assert results['blobs'][0][1] == 42
def test_list_artifacts(mock_client):
repo = AzureBlobArtifactRepository(TEST_URI, mock_client)
# Create some files to return
dir_prefix = BlobPrefix()
dir_prefix.name = TEST_ROOT_PATH + "/dir"
blob_props = BlobProperties()
blob_props.content_length = 42
blob = Blob(TEST_ROOT_PATH + "/file", props=blob_props)
mock_client.list_blobs.return_value = MockBlobList([dir_prefix, blob])
artifacts = repo.list_artifacts()
assert artifacts[0].path == "dir"
assert artifacts[0].is_dir is True
assert artifacts[0].file_size is None
assert artifacts[1].path == "file"
assert artifacts[1].is_dir is False
assert artifacts[1].file_size == 42
def test_list_non_empty(self, client):
base_path = "/path/"
# Create some files to return
dir_prefix = BlobPrefix()
dir_prefix.name = base_path + "dir"
blob_props = BlobProperties()
blob_props.content_length = 42
blob = Blob(base_path + "file", props=blob_props)
client.return_value.list_blobs.return_value = MockBlobList([dir_prefix, blob])
store = AzureBlobStoreService()
key_path = self.wasbs_base + base_path
results = store.list(key=key_path)
assert len(results["blobs"]) == 1
assert len(results["prefixes"]) == 1
assert results["prefixes"][0] == "dir"
assert results["blobs"][0][0] == "file"
assert results["blobs"][0][1] == 42
def test_size_of_file(self):
props = BlobProperties()
props.content_length = 12
self.storage._service.get_blob_properties.return_value = Blob(
props=props)
self.assertEqual(12, self.storage.size("name"))
def test_size_of_file(self):
props = BlobProperties()
props.content_length = 12
self.storage._service.get_blob_properties.return_value = Blob(props=props)
self.assertEqual(12, self.storage.size("name"))
def test_download_dir(self, client):
dirname1 = tempfile.mkdtemp()
dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/")
def mkfile(container, cloud_path, fname):
return open(fname, "w")
client.return_value.get_blob_to_path.side_effect = mkfile
store = AzureStore()
blob_path = "/path/to/"
azure_url = self.wasbs_base + blob_path
rel_path2 = dirname2.split("/")[-1]
# Mock return list
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock1 = Blob(blob_path + "test1.txt", props=blob_props)
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock2 = Blob(blob_path + "test2.txt", props=blob_props)
subdir_mock = BlobPrefix()
subdir_mock.name = blob_path + rel_path2
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock3 = Blob(blob_path + rel_path2 + "/" + "test3.txt",
props=blob_props)
# Create some files to return
def list_side_effect(container_name,
prefix,
delimiter="/",
marker=None):
if prefix == blob_path:
return MockBlobList([subdir_mock, obj_mock1, obj_mock2])
return MockBlobList([obj_mock3])
client.return_value.list_blobs.side_effect = list_side_effect
dirname3 = tempfile.mkdtemp()
# Test without basename
store.download_dir(blob=azure_url,
local_path=dirname3,
use_basename=False)
client.return_value.get_blob_to_path.assert_has_calls(
[
mock.call(
"container",
"{}test1.txt".format(blob_path),
"{}/test1.txt".format(dirname3),
),
mock.call(
"container",
"{}test2.txt".format(blob_path),
"{}/test2.txt".format(dirname3),
),
mock.call(
"container",
"{}{}/test3.txt".format(blob_path, rel_path2),
"{}/{}/test3.txt".format(dirname3, rel_path2),
),
],
any_order=True,
)