def test_last_modified_of_file(self):
props = BlobProperties()
accepted_time = datetime.datetime(2017, 5, 11, 8, 52, 4)
props.last_modified = accepted_time
self.storage._service.get_blob_properties.return_value = Blob(props=props)
time = self.storage.modified_time("name")
self.assertEqual(accepted_time, time)
def test_download_artifact_throws_value_error_when_listed_blobs_do_not_contain_artifact_root_prefix(
mock_client):
repo = AzureBlobArtifactRepository(TEST_URI, mock_client)
# Create a "bad blob" with a name that is not prefixed by the root path of the artifact store
bad_blob_props = BlobProperties()
bad_blob_props.content_length = 42
bad_blob = Blob("file_path", props=bad_blob_props)
def get_mock_listing(*args, **kwargs):
"""
Produces a mock listing that only contains content if the
specified prefix is the artifact root. This allows us to mock
`list_artifacts` during the `_download_artifacts_into` subroutine
without recursively listing the same artifacts at every level of the
directory traversal.
"""
# pylint: disable=unused-argument
if os.path.abspath(
kwargs["prefix"]) == os.path.abspath(TEST_ROOT_PATH):
# Return a blob that is not prefixed by the root path of the artifact store. This
# should result in an exception being raised
return MockBlobList([bad_blob])
else:
return MockBlobList([])
mock_client.list_blobs.side_effect = get_mock_listing
with pytest.raises(ValueError) as exc:
repo.download_artifacts("")
assert "Azure blob does not begin with the specified artifact path" in str(
exc)
def test_size_of_file(self):
props = BlobProperties()
props.size = 12
client_mock = mock.MagicMock()
client_mock.get_blob_properties.return_value = props
self.storage._client.get_blob_client.return_value = client_mock
self.assertEqual(12, self.storage.size("name"))
def test_last_modified_of_file(self):
props = BlobProperties()
accepted_time = datetime.datetime(2017, 5, 11, 8, 52, 4)
props.last_modified = accepted_time
self.storage._client.get_blob_properties.return_value = props
time = self.storage.modified_time("name")
self.assertEqual(accepted_time, time)
def test_list_artifacts(mock_client, root_path):
repo = AzureBlobArtifactRepository(
posixpath.join(TEST_BLOB_CONTAINER_ROOT, root_path), mock_client
)
# Create some files to return
dir_prefix = BlobPrefix()
dir_prefix.name = posixpath.join(TEST_ROOT_PATH, "dir")
blob_props = BlobProperties()
blob_props.size = 42
blob_props.name = posixpath.join(TEST_ROOT_PATH, "file")
mock_client.get_container_client().walk_blobs.return_value = MockBlobList(
[dir_prefix, blob_props]
)
artifacts = repo.list_artifacts()
mock_client.get_container_client().walk_blobs.assert_called_with(name_starts_with="some/path/")
assert artifacts[0].path == "dir"
assert artifacts[0].is_dir is True
assert artifacts[0].file_size is None
assert artifacts[1].path == "file"
assert artifacts[1].is_dir is False
assert artifacts[1].file_size == 42
def test_list_artifacts_single_file(mock_client):
repo = AzureBlobArtifactRepository(TEST_URI, mock_client)
# Evaluate single file
blob_props = BlobProperties()
blob_props.name = posixpath.join(TEST_ROOT_PATH, "file")
mock_client.get_container_client().walk_blobs.return_value = MockBlobList([blob_props])
assert repo.list_artifacts("file") == []
def test_download_dir_with_basename(self, client):
dirname1 = tempfile.mkdtemp()
dirname2 = tempfile.mkdtemp(prefix=dirname1 + '/')
def mkfile(container, cloud_path, fname):
return open(fname, 'w')
client.return_value.get_blob_to_path.side_effect = mkfile
store = AzureStore()
blob_path = '/path/to/'
azure_url = self.wasbs_base + blob_path
rel_path2 = dirname2.split('/')[-1]
# Mock return list
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock1 = Blob(blob_path + 'foo/test1.txt', props=blob_props)
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock2 = Blob(blob_path + 'foo/test2.txt', props=blob_props)
subdir_mock = BlobPrefix()
subdir_mock.name = blob_path + 'foo/' + rel_path2
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock3 = Blob(blob_path + 'foo/' + rel_path2 + '/' + 'test3.txt',
props=blob_props)
# Create some files to return
def list_side_effect(container_name,
prefix,
delimiter='/',
marker=None):
if prefix == blob_path + 'foo/':
return MockBlobList([subdir_mock, obj_mock1, obj_mock2])
return MockBlobList([obj_mock3])
client.return_value.list_blobs.side_effect = list_side_effect
dirname3 = tempfile.mkdtemp()
# Test without basename
store.download_dir(blob=azure_url + 'foo',
local_path=dirname3,
use_basename=True)
client.return_value.get_blob_to_path.assert_has_calls([
mock.call('container', '{}foo/test1.txt'.format(blob_path),
'{}/foo/test1.txt'.format(dirname3)),
mock.call('container', '{}foo/test2.txt'.format(blob_path),
'{}/foo/test2.txt'.format(dirname3)),
mock.call('container', '{}foo/{}/test3.txt'.format(
blob_path, rel_path2), '{}/foo/{}/test3.txt'.format(
dirname3, rel_path2)),
],
any_order=True)
def test_list_non_empty(self, client):
base_path = "/path/"
# Create some files to return
dir_prefix = BlobPrefix()
dir_prefix.name = base_path + "dir"
blob1_props = BlobProperties()
blob1_props.size = 42
blob1_props.name = base_path + "file1"
blob2_props = BlobProperties()
blob2_props.size = 0
blob2_props.name = base_path + "file2"
client.get_container_client().walk_blobs.return_value = MockBlobList(
[dir_prefix, blob1_props, blob2_props]
)
store = AzureBlobStoreService()
store.set_connection(connection=client)
key_path = self.wasbs_base + base_path
results = store.list(key=key_path)
assert len(results["blobs"]) == 2
assert len(results["prefixes"]) == 1
assert results["prefixes"][0] == "dir"
assert results["blobs"][0][0] == "file1"
assert results["blobs"][0][1] == 42
assert results["blobs"][1][0] == "file2"
assert results["blobs"][1][1] == 0
def get_blob_properties(filename):
logger = logging.getLogger(__name__)
props = BlobProperties()
content = ContentSettings()
logger.debug('Settings props for file %s', filename)
props.name = filename
if '.json' in filename:
content.content_type = 'application/json'
elif '.html' in filename:
content.content_type = 'text/html'
else:
content.content_type = 'text/plain'
logger.debug('Content-type set to %s', content.content_type)
content.content_disposition = f'inline; filename={filename}'
props.content_settings = content
return props
def test_list_non_empty(self, client):
base_path = "/path/"
# Create some files to return
dir_prefix = BlobPrefix()
dir_prefix.name = base_path + "dir"
blob_props = BlobProperties()
blob_props.content_length = 42
blob = Blob(base_path + "file", props=blob_props)
client.return_value.list_blobs.return_value = MockBlobList([dir_prefix, blob])
store = AzureBlobStoreService()
key_path = self.wasbs_base + base_path
results = store.list(key=key_path)
assert len(results["blobs"]) == 1
assert len(results["prefixes"]) == 1
assert results["prefixes"][0] == "dir"
assert results["blobs"][0][0] == "file"
assert results["blobs"][0][1] == 42
def test_list_artifacts(mock_client):
repo = AzureBlobArtifactRepository(TEST_URI, mock_client)
# Create some files to return
dir_prefix = BlobPrefix()
dir_prefix.name = TEST_ROOT_PATH + "/dir"
blob_props = BlobProperties()
blob_props.content_length = 42
blob = Blob(TEST_ROOT_PATH + "/file", props=blob_props)
mock_client.list_blobs.return_value = MockBlobList([dir_prefix, blob])
artifacts = repo.list_artifacts()
assert artifacts[0].path == "dir"
assert artifacts[0].is_dir is True
assert artifacts[0].file_size is None
assert artifacts[1].path == "file"
assert artifacts[1].is_dir is False
assert artifacts[1].file_size == 42
def test_list_non_empty(self, client):
base_path = '/path/'
# Create some files to return
dir_prefix = BlobPrefix()
dir_prefix.name = base_path + 'dir'
blob_props = BlobProperties()
blob_props.content_length = 42
blob = Blob(base_path + 'file', props=blob_props)
client.return_value.list_blobs.return_value = MockBlobList([dir_prefix, blob])
store = AzureStore()
key_path = self.wasbs_base + base_path
results = store.list(key=key_path)
assert len(results['blobs']) == 1
assert len(results['prefixes']) == 1
assert results['prefixes'][0] == 'dir'
assert results['blobs'][0][0] == 'file'
assert results['blobs'][0][1] == 42
def test_download_directory_artifact_succeeds_when_artifact_root_is_blob_container_root(
mock_client, tmpdir):
repo = AzureBlobArtifactRepository(TEST_BLOB_CONTAINER_ROOT, mock_client)
subdir_path = "my_directory"
dir_prefix = BlobPrefix()
dir_prefix.name = subdir_path
file_path_1 = "file_1"
file_path_2 = "file_2"
blob_props_1 = BlobProperties()
blob_props_1.size = 42
blob_props_1.name = posixpath.join(subdir_path, file_path_1)
blob_props_2 = BlobProperties()
blob_props_2.size = 42
blob_props_2.name = posixpath.join(subdir_path, file_path_2)
def get_mock_listing(*args, **kwargs):
"""
Produces a mock listing that only contains content if the specified prefix is the artifact
root or a relevant subdirectory. This allows us to mock `list_artifacts` during the
`_download_artifacts_into` subroutine without recursively listing the same artifacts at
every level of the directory traversal.
"""
# pylint: disable=unused-argument
if posixpath.abspath(kwargs["name_starts_with"]) == "/":
return MockBlobList([dir_prefix])
if posixpath.abspath(
kwargs["name_starts_with"]) == posixpath.abspath(subdir_path):
return MockBlobList([blob_props_1, blob_props_2])
else:
return MockBlobList([])
def create_file(buffer):
fname = os.path.basename(buffer.name)
f = tmpdir.join(fname)
f.write("hello world!")
mock_client.get_container_client(
).walk_blobs.side_effect = get_mock_listing
mock_client.get_container_client().download_blob(
).readinto.side_effect = create_file
# Ensure that the root directory can be downloaded successfully
repo.download_artifacts("")
# Ensure that the `mkfile` side effect copied all of the download artifacts into `tmpdir`
dir_contents = os.listdir(tmpdir.strpath)
assert file_path_1 in dir_contents
assert file_path_2 in dir_contents
def upload_blob(self, file_name):
content_settings = ContentSettings(content_type='application/json')
file_temp_folder = self.temp_folder + file_name
blob_client = self.service_client.get_blob_client(
container=self.container, blob=BlobProperties(name=file_name))
with open(file_temp_folder, 'rb') as data:
blob_client.upload_blob(data)
# Set type to application/json, access tier to hot and delete temp files
blob_client.set_http_headers(content_settings=content_settings)
blob_client.set_standard_blob_tier(standard_blob_tier='Hot')
def get_blob_properties(self):
path = self.root / self.key
if path.exists() and path.is_file():
return BlobProperties(
**{
"name": self.key,
"Last-Modified": datetime.fromtimestamp(path.stat().st_mtime),
"ETag": "etag",
}
)
else:
raise ResourceNotFoundError
def upload_blob_to_archive(self, jenkins_name, file_name):
file_dir = jenkins_name + '/' + file_name
content_settings = ContentSettings(content_type='application/json')
blob_client = self.service_client.get_blob_client(
container=self.container,
blob=BlobProperties(name='archive/' + file_dir))
with open(self.temp_folder + '/' + file_dir, 'rb') as data:
blob_client.upload_blob(data)
# Set type to application/json, access tier to COLD and delete temp files
blob_client.set_http_headers(content_settings=content_settings)
blob_client.set_standard_blob_tier(standard_blob_tier='Cool')
def download_blobs_to_local_temp_folder(self, project):
blobs_list = self.list_blobs_for_project(project)
# Remove last blob to keep one on Azure Blob Storage
blobs_list.pop()
print("BLOBS FOUND: ", project, blobs_list)
for file in blobs_list:
print("DOWNLOADING BLOB: ", file)
blob_client = self.service_client.get_blob_client(
container=self.container, blob=BlobProperties(name=file))
os.makedirs(self.temp_folder + '/' + project, exist_ok=True)
with open(self.temp_folder + '/' + file, "wb") as my_blob:
download_stream = blob_client.download_blob()
my_blob.write(download_stream.readall())
def mock_item_paged(root, name_starts_with=None):
items = []
if not name_starts_with:
name_starts_with = ""
for f in root.glob("**/*"):
if ((not f.name.startswith(".")) and f.is_file()
and (root / name_starts_with) in [f, *f.parents]):
items.append(f)
for ix in items:
# BlobProperties
# https://github.com/Azure/azure-sdk-for-python/blob/b83018de46d4ecb6554ab33ecc22d4c7e7b77129/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py#L517
yield BlobProperties(
**{
"name": str(ix.relative_to(root)),
"Last-Modified": datetime.fromtimestamp(ix.stat().st_mtime),
"ETag": "etag",
})
def test_download_directory_artifact_succeeds_when_artifact_root_is_not_blob_container_root(
mock_client, tmpdir):
assert TEST_URI is not TEST_BLOB_CONTAINER_ROOT
repo = AzureBlobArtifactRepository(TEST_URI, mock_client)
file_path_1 = "file_1"
file_path_2 = "file_2"
blob_props_1 = BlobProperties()
blob_props_1.content_length = 42
blob_1 = Blob(os.path.join(TEST_ROOT_PATH, file_path_1),
props=blob_props_1)
blob_props_2 = BlobProperties()
blob_props_2.content_length = 42
blob_2 = Blob(os.path.join(TEST_ROOT_PATH, file_path_2),
props=blob_props_2)
def get_mock_listing(*args, **kwargs):
"""
Produces a mock listing that only contains content if the
specified prefix is the artifact root. This allows us to mock
`list_artifacts` during the `_download_artifacts_into` subroutine
without recursively listing the same artifacts at every level of the
directory traversal.
"""
# pylint: disable=unused-argument
if os.path.abspath(
kwargs["prefix"]) == os.path.abspath(TEST_ROOT_PATH):
return MockBlobList([blob_1, blob_2])
else:
return MockBlobList([])
def create_file(container, cloud_path, local_path):
# pylint: disable=unused-argument
fname = os.path.basename(local_path)
f = tmpdir.join(fname)
f.write("hello world!")
mock_client.list_blobs.side_effect = get_mock_listing
mock_client.get_blob_to_path.side_effect = create_file
# Ensure that the root directory can be downloaded successfully
repo.download_artifacts("")
# Ensure that the `mkfile` side effect copied all of the download artifacts into `tmpdir`
dir_contents = os.listdir(tmpdir.strpath)
assert file_path_1 in dir_contents
assert file_path_2 in dir_contents
def test_size_of_file(self):
props = BlobProperties()
props.content_length = 12
self.storage._service.get_blob_properties.return_value = Blob(
props=props)
self.assertEqual(12, self.storage.size("name"))
def test_download_dir_with_basename(self, client):
dirname1 = tempfile.mkdtemp()
dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/")
def mkfile(fname):
return open(fname, "w")
client.return_value.get_blob_to_path.side_effect = mkfile
store = AzureBlobStoreService()
store.set_connection(connection=client)
blob_path = "/path/to/"
azure_url = self.wasbs_base + blob_path
rel_path2 = dirname2.split("/")[-1]
# Mock return list
obj_mock1 = BlobProperties()
obj_mock1.size = 42
obj_mock1.name = blob_path + "foo/test1.txt"
obj_mock2 = BlobProperties()
obj_mock2.size = 42
obj_mock2.name = blob_path + "foo/test2.txt"
subdir_mock = BlobPrefix()
subdir_mock.name = blob_path + "foo/" + rel_path2
obj_mock3 = BlobProperties()
obj_mock3.size = 42
obj_mock3.name = blob_path + "foo/" + rel_path2 + "/" + "test3.txt"
# Create some files to return
def list_side_effect(container_name, prefix, delimiter="/", marker=None):
if prefix == blob_path + "foo/":
return MockBlobList([subdir_mock, obj_mock1, obj_mock2])
return MockBlobList([obj_mock3])
client.return_value.walk_blobs.side_effect = list_side_effect
dirname3 = tempfile.mkdtemp()
# Test without basename
store.download_dir(
blob=azure_url + "foo", local_path=dirname3, use_basename=True
)
call_args_list = client.get_container_client().download_blob.call_args_list
for call_args in call_args_list:
call_arg1 = call_args[0]
if call_arg1 == "{}foo/test1.txt".format(blob_path):
assert os.path.exists("{}/foo/test1.txt".format(dirname3))
elif call_arg1 == "{}foo/test2.txt".format(blob_path):
assert os.path.exists("{}/foo/test2.txt".format(dirname3))
elif call_arg1 == "{}foo/{}/test3.txt".format(blob_path, rel_path2):
assert os.path.exists("{}/foo/{}/test3.txt".format(dirname3, rel_path2))
else:
assert False
def delete_blob_from_main_storage(self, jenkins_name, file_name):
file_dir = jenkins_name + '/' + file_name
blob_client = self.service_client.get_blob_client(
container=self.container, blob=BlobProperties(name=file_dir))
# Delete blob from main 'Hot' folder - later upload it to Archive
blob_client.delete_blob(delete_snapshots='include')
def test_size_of_file(self):
props = BlobProperties()
props.content_length = 12
self.storage._service.get_blob_properties.return_value = Blob(props=props)
self.assertEqual(12, self.storage.size("name"))
config.read(the_cwd + '\\episode_vii.ini')
ws = Workspace.get(name=config['azure.settings']['workspace_name'],
subscription_id=config['azure.settings']['subscription_id'],
resource_group=config['azure.settings']['resource_group'])
print('Workspace loaded.')
storage_connection_string = config['azure.settings'][
'storage_connection_string']
blob_service_client = BlobServiceClient.from_connection_string(
storage_connection_string)
credential = blob_service_client.credential
container_client = None
for container in blob_service_client.list_containers():
if container.name == '154fd9d2-2831-4ad7-a2e7-61ee7e620cab':
container_client = blob_service_client.get_container_client(container)
print(type(container_client))
#csv_file_name = the_cwd + '\\data\\regression.csv'
#csv_file = open(csv_file_name, 'rb')
#container_client.upload_blob(name='regression.csv', data=csv_file)
#print('Upload complete.')
from azure.storage.blob import BlobProperties
download_file = the_cwd + '\\data\\regression_download.csv'
with open(download_file, "wb") as my_blob:
download_stream = container_client.download_blob(
BlobProperties(name='regression.csv'))
my_blob.write(download_stream.readall())
def test_download_dir(self, client):
dirname1 = tempfile.mkdtemp()
dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/")
def mkfile(container, cloud_path, fname):
return open(fname, "w")
client.return_value.get_blob_to_path.side_effect = mkfile
store = AzureStore()
blob_path = "/path/to/"
azure_url = self.wasbs_base + blob_path
rel_path2 = dirname2.split("/")[-1]
# Mock return list
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock1 = Blob(blob_path + "test1.txt", props=blob_props)
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock2 = Blob(blob_path + "test2.txt", props=blob_props)
subdir_mock = BlobPrefix()
subdir_mock.name = blob_path + rel_path2
blob_props = BlobProperties()
blob_props.content_length = 42
obj_mock3 = Blob(blob_path + rel_path2 + "/" + "test3.txt",
props=blob_props)
# Create some files to return
def list_side_effect(container_name,
prefix,
delimiter="/",
marker=None):
if prefix == blob_path:
return MockBlobList([subdir_mock, obj_mock1, obj_mock2])
return MockBlobList([obj_mock3])
client.return_value.list_blobs.side_effect = list_side_effect
dirname3 = tempfile.mkdtemp()
# Test without basename
store.download_dir(blob=azure_url,
local_path=dirname3,
use_basename=False)
client.return_value.get_blob_to_path.assert_has_calls(
[
mock.call(
"container",
"{}test1.txt".format(blob_path),
"{}/test1.txt".format(dirname3),
),
mock.call(
"container",
"{}test2.txt".format(blob_path),
"{}/test2.txt".format(dirname3),
),
mock.call(
"container",
"{}{}/test3.txt".format(blob_path, rel_path2),
"{}/{}/test3.txt".format(dirname3, rel_path2),
),
],
any_order=True,
)