def test_last_modified_of_file(self): props = BlobProperties() accepted_time = datetime.datetime(2017, 5, 11, 8, 52, 4) props.last_modified = accepted_time self.storage._service.get_blob_properties.return_value = Blob(props=props) time = self.storage.modified_time("name") self.assertEqual(accepted_time, time)
def test_download_artifact_throws_value_error_when_listed_blobs_do_not_contain_artifact_root_prefix( mock_client): repo = AzureBlobArtifactRepository(TEST_URI, mock_client) # Create a "bad blob" with a name that is not prefixed by the root path of the artifact store bad_blob_props = BlobProperties() bad_blob_props.content_length = 42 bad_blob = Blob("file_path", props=bad_blob_props) def get_mock_listing(*args, **kwargs): """ Produces a mock listing that only contains content if the specified prefix is the artifact root. This allows us to mock `list_artifacts` during the `_download_artifacts_into` subroutine without recursively listing the same artifacts at every level of the directory traversal. """ # pylint: disable=unused-argument if os.path.abspath( kwargs["prefix"]) == os.path.abspath(TEST_ROOT_PATH): # Return a blob that is not prefixed by the root path of the artifact store. This # should result in an exception being raised return MockBlobList([bad_blob]) else: return MockBlobList([]) mock_client.list_blobs.side_effect = get_mock_listing with pytest.raises(ValueError) as exc: repo.download_artifacts("") assert "Azure blob does not begin with the specified artifact path" in str( exc)
def test_size_of_file(self): props = BlobProperties() props.size = 12 client_mock = mock.MagicMock() client_mock.get_blob_properties.return_value = props self.storage._client.get_blob_client.return_value = client_mock self.assertEqual(12, self.storage.size("name"))
def test_last_modified_of_file(self): props = BlobProperties() accepted_time = datetime.datetime(2017, 5, 11, 8, 52, 4) props.last_modified = accepted_time self.storage._client.get_blob_properties.return_value = props time = self.storage.modified_time("name") self.assertEqual(accepted_time, time)
def test_list_artifacts(mock_client, root_path): repo = AzureBlobArtifactRepository( posixpath.join(TEST_BLOB_CONTAINER_ROOT, root_path), mock_client ) # Create some files to return dir_prefix = BlobPrefix() dir_prefix.name = posixpath.join(TEST_ROOT_PATH, "dir") blob_props = BlobProperties() blob_props.size = 42 blob_props.name = posixpath.join(TEST_ROOT_PATH, "file") mock_client.get_container_client().walk_blobs.return_value = MockBlobList( [dir_prefix, blob_props] ) artifacts = repo.list_artifacts() mock_client.get_container_client().walk_blobs.assert_called_with(name_starts_with="some/path/") assert artifacts[0].path == "dir" assert artifacts[0].is_dir is True assert artifacts[0].file_size is None assert artifacts[1].path == "file" assert artifacts[1].is_dir is False assert artifacts[1].file_size == 42
def test_list_artifacts_single_file(mock_client): repo = AzureBlobArtifactRepository(TEST_URI, mock_client) # Evaluate single file blob_props = BlobProperties() blob_props.name = posixpath.join(TEST_ROOT_PATH, "file") mock_client.get_container_client().walk_blobs.return_value = MockBlobList([blob_props]) assert repo.list_artifacts("file") == []
def test_download_dir_with_basename(self, client): dirname1 = tempfile.mkdtemp() dirname2 = tempfile.mkdtemp(prefix=dirname1 + '/') def mkfile(container, cloud_path, fname): return open(fname, 'w') client.return_value.get_blob_to_path.side_effect = mkfile store = AzureStore() blob_path = '/path/to/' azure_url = self.wasbs_base + blob_path rel_path2 = dirname2.split('/')[-1] # Mock return list blob_props = BlobProperties() blob_props.content_length = 42 obj_mock1 = Blob(blob_path + 'foo/test1.txt', props=blob_props) blob_props = BlobProperties() blob_props.content_length = 42 obj_mock2 = Blob(blob_path + 'foo/test2.txt', props=blob_props) subdir_mock = BlobPrefix() subdir_mock.name = blob_path + 'foo/' + rel_path2 blob_props = BlobProperties() blob_props.content_length = 42 obj_mock3 = Blob(blob_path + 'foo/' + rel_path2 + '/' + 'test3.txt', props=blob_props) # Create some files to return def list_side_effect(container_name, prefix, delimiter='/', marker=None): if prefix == blob_path + 'foo/': return MockBlobList([subdir_mock, obj_mock1, obj_mock2]) return MockBlobList([obj_mock3]) client.return_value.list_blobs.side_effect = list_side_effect dirname3 = tempfile.mkdtemp() # Test without basename store.download_dir(blob=azure_url + 'foo', local_path=dirname3, use_basename=True) client.return_value.get_blob_to_path.assert_has_calls([ mock.call('container', '{}foo/test1.txt'.format(blob_path), '{}/foo/test1.txt'.format(dirname3)), mock.call('container', '{}foo/test2.txt'.format(blob_path), '{}/foo/test2.txt'.format(dirname3)), mock.call('container', '{}foo/{}/test3.txt'.format( blob_path, rel_path2), '{}/foo/{}/test3.txt'.format( dirname3, rel_path2)), ], any_order=True)
def test_list_non_empty(self, client): base_path = "/path/" # Create some files to return dir_prefix = BlobPrefix() dir_prefix.name = base_path + "dir" blob1_props = BlobProperties() blob1_props.size = 42 blob1_props.name = base_path + "file1" blob2_props = BlobProperties() blob2_props.size = 0 blob2_props.name = base_path + "file2" client.get_container_client().walk_blobs.return_value = MockBlobList( [dir_prefix, blob1_props, blob2_props] ) store = AzureBlobStoreService() store.set_connection(connection=client) key_path = self.wasbs_base + base_path results = store.list(key=key_path) assert len(results["blobs"]) == 2 assert len(results["prefixes"]) == 1 assert results["prefixes"][0] == "dir" assert results["blobs"][0][0] == "file1" assert results["blobs"][0][1] == 42 assert results["blobs"][1][0] == "file2" assert results["blobs"][1][1] == 0
def get_blob_properties(filename): logger = logging.getLogger(__name__) props = BlobProperties() content = ContentSettings() logger.debug('Settings props for file %s', filename) props.name = filename if '.json' in filename: content.content_type = 'application/json' elif '.html' in filename: content.content_type = 'text/html' else: content.content_type = 'text/plain' logger.debug('Content-type set to %s', content.content_type) content.content_disposition = f'inline; filename={filename}' props.content_settings = content return props
def test_list_non_empty(self, client): base_path = "/path/" # Create some files to return dir_prefix = BlobPrefix() dir_prefix.name = base_path + "dir" blob_props = BlobProperties() blob_props.content_length = 42 blob = Blob(base_path + "file", props=blob_props) client.return_value.list_blobs.return_value = MockBlobList([dir_prefix, blob]) store = AzureBlobStoreService() key_path = self.wasbs_base + base_path results = store.list(key=key_path) assert len(results["blobs"]) == 1 assert len(results["prefixes"]) == 1 assert results["prefixes"][0] == "dir" assert results["blobs"][0][0] == "file" assert results["blobs"][0][1] == 42
def test_list_artifacts(mock_client): repo = AzureBlobArtifactRepository(TEST_URI, mock_client) # Create some files to return dir_prefix = BlobPrefix() dir_prefix.name = TEST_ROOT_PATH + "/dir" blob_props = BlobProperties() blob_props.content_length = 42 blob = Blob(TEST_ROOT_PATH + "/file", props=blob_props) mock_client.list_blobs.return_value = MockBlobList([dir_prefix, blob]) artifacts = repo.list_artifacts() assert artifacts[0].path == "dir" assert artifacts[0].is_dir is True assert artifacts[0].file_size is None assert artifacts[1].path == "file" assert artifacts[1].is_dir is False assert artifacts[1].file_size == 42
def test_list_non_empty(self, client): base_path = '/path/' # Create some files to return dir_prefix = BlobPrefix() dir_prefix.name = base_path + 'dir' blob_props = BlobProperties() blob_props.content_length = 42 blob = Blob(base_path + 'file', props=blob_props) client.return_value.list_blobs.return_value = MockBlobList([dir_prefix, blob]) store = AzureStore() key_path = self.wasbs_base + base_path results = store.list(key=key_path) assert len(results['blobs']) == 1 assert len(results['prefixes']) == 1 assert results['prefixes'][0] == 'dir' assert results['blobs'][0][0] == 'file' assert results['blobs'][0][1] == 42
def test_download_directory_artifact_succeeds_when_artifact_root_is_blob_container_root( mock_client, tmpdir): repo = AzureBlobArtifactRepository(TEST_BLOB_CONTAINER_ROOT, mock_client) subdir_path = "my_directory" dir_prefix = BlobPrefix() dir_prefix.name = subdir_path file_path_1 = "file_1" file_path_2 = "file_2" blob_props_1 = BlobProperties() blob_props_1.size = 42 blob_props_1.name = posixpath.join(subdir_path, file_path_1) blob_props_2 = BlobProperties() blob_props_2.size = 42 blob_props_2.name = posixpath.join(subdir_path, file_path_2) def get_mock_listing(*args, **kwargs): """ Produces a mock listing that only contains content if the specified prefix is the artifact root or a relevant subdirectory. This allows us to mock `list_artifacts` during the `_download_artifacts_into` subroutine without recursively listing the same artifacts at every level of the directory traversal. """ # pylint: disable=unused-argument if posixpath.abspath(kwargs["name_starts_with"]) == "/": return MockBlobList([dir_prefix]) if posixpath.abspath( kwargs["name_starts_with"]) == posixpath.abspath(subdir_path): return MockBlobList([blob_props_1, blob_props_2]) else: return MockBlobList([]) def create_file(buffer): fname = os.path.basename(buffer.name) f = tmpdir.join(fname) f.write("hello world!") mock_client.get_container_client( ).walk_blobs.side_effect = get_mock_listing mock_client.get_container_client().download_blob( ).readinto.side_effect = create_file # Ensure that the root directory can be downloaded successfully repo.download_artifacts("") # Ensure that the `mkfile` side effect copied all of the download artifacts into `tmpdir` dir_contents = os.listdir(tmpdir.strpath) assert file_path_1 in dir_contents assert file_path_2 in dir_contents
def upload_blob(self, file_name): content_settings = ContentSettings(content_type='application/json') file_temp_folder = self.temp_folder + file_name blob_client = self.service_client.get_blob_client( container=self.container, blob=BlobProperties(name=file_name)) with open(file_temp_folder, 'rb') as data: blob_client.upload_blob(data) # Set type to application/json, access tier to hot and delete temp files blob_client.set_http_headers(content_settings=content_settings) blob_client.set_standard_blob_tier(standard_blob_tier='Hot')
def get_blob_properties(self): path = self.root / self.key if path.exists() and path.is_file(): return BlobProperties( **{ "name": self.key, "Last-Modified": datetime.fromtimestamp(path.stat().st_mtime), "ETag": "etag", } ) else: raise ResourceNotFoundError
def upload_blob_to_archive(self, jenkins_name, file_name): file_dir = jenkins_name + '/' + file_name content_settings = ContentSettings(content_type='application/json') blob_client = self.service_client.get_blob_client( container=self.container, blob=BlobProperties(name='archive/' + file_dir)) with open(self.temp_folder + '/' + file_dir, 'rb') as data: blob_client.upload_blob(data) # Set type to application/json, access tier to COLD and delete temp files blob_client.set_http_headers(content_settings=content_settings) blob_client.set_standard_blob_tier(standard_blob_tier='Cool')
def download_blobs_to_local_temp_folder(self, project): blobs_list = self.list_blobs_for_project(project) # Remove last blob to keep one on Azure Blob Storage blobs_list.pop() print("BLOBS FOUND: ", project, blobs_list) for file in blobs_list: print("DOWNLOADING BLOB: ", file) blob_client = self.service_client.get_blob_client( container=self.container, blob=BlobProperties(name=file)) os.makedirs(self.temp_folder + '/' + project, exist_ok=True) with open(self.temp_folder + '/' + file, "wb") as my_blob: download_stream = blob_client.download_blob() my_blob.write(download_stream.readall())
def mock_item_paged(root, name_starts_with=None): items = [] if not name_starts_with: name_starts_with = "" for f in root.glob("**/*"): if ((not f.name.startswith(".")) and f.is_file() and (root / name_starts_with) in [f, *f.parents]): items.append(f) for ix in items: # BlobProperties # https://github.com/Azure/azure-sdk-for-python/blob/b83018de46d4ecb6554ab33ecc22d4c7e7b77129/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py#L517 yield BlobProperties( **{ "name": str(ix.relative_to(root)), "Last-Modified": datetime.fromtimestamp(ix.stat().st_mtime), "ETag": "etag", })
def test_download_directory_artifact_succeeds_when_artifact_root_is_not_blob_container_root( mock_client, tmpdir): assert TEST_URI is not TEST_BLOB_CONTAINER_ROOT repo = AzureBlobArtifactRepository(TEST_URI, mock_client) file_path_1 = "file_1" file_path_2 = "file_2" blob_props_1 = BlobProperties() blob_props_1.content_length = 42 blob_1 = Blob(os.path.join(TEST_ROOT_PATH, file_path_1), props=blob_props_1) blob_props_2 = BlobProperties() blob_props_2.content_length = 42 blob_2 = Blob(os.path.join(TEST_ROOT_PATH, file_path_2), props=blob_props_2) def get_mock_listing(*args, **kwargs): """ Produces a mock listing that only contains content if the specified prefix is the artifact root. This allows us to mock `list_artifacts` during the `_download_artifacts_into` subroutine without recursively listing the same artifacts at every level of the directory traversal. """ # pylint: disable=unused-argument if os.path.abspath( kwargs["prefix"]) == os.path.abspath(TEST_ROOT_PATH): return MockBlobList([blob_1, blob_2]) else: return MockBlobList([]) def create_file(container, cloud_path, local_path): # pylint: disable=unused-argument fname = os.path.basename(local_path) f = tmpdir.join(fname) f.write("hello world!") mock_client.list_blobs.side_effect = get_mock_listing mock_client.get_blob_to_path.side_effect = create_file # Ensure that the root directory can be downloaded successfully repo.download_artifacts("") # Ensure that the `mkfile` side effect copied all of the download artifacts into `tmpdir` dir_contents = os.listdir(tmpdir.strpath) assert file_path_1 in dir_contents assert file_path_2 in dir_contents
def test_size_of_file(self): props = BlobProperties() props.content_length = 12 self.storage._service.get_blob_properties.return_value = Blob( props=props) self.assertEqual(12, self.storage.size("name"))
def test_download_dir_with_basename(self, client): dirname1 = tempfile.mkdtemp() dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/") def mkfile(fname): return open(fname, "w") client.return_value.get_blob_to_path.side_effect = mkfile store = AzureBlobStoreService() store.set_connection(connection=client) blob_path = "/path/to/" azure_url = self.wasbs_base + blob_path rel_path2 = dirname2.split("/")[-1] # Mock return list obj_mock1 = BlobProperties() obj_mock1.size = 42 obj_mock1.name = blob_path + "foo/test1.txt" obj_mock2 = BlobProperties() obj_mock2.size = 42 obj_mock2.name = blob_path + "foo/test2.txt" subdir_mock = BlobPrefix() subdir_mock.name = blob_path + "foo/" + rel_path2 obj_mock3 = BlobProperties() obj_mock3.size = 42 obj_mock3.name = blob_path + "foo/" + rel_path2 + "/" + "test3.txt" # Create some files to return def list_side_effect(container_name, prefix, delimiter="/", marker=None): if prefix == blob_path + "foo/": return MockBlobList([subdir_mock, obj_mock1, obj_mock2]) return MockBlobList([obj_mock3]) client.return_value.walk_blobs.side_effect = list_side_effect dirname3 = tempfile.mkdtemp() # Test without basename store.download_dir( blob=azure_url + "foo", local_path=dirname3, use_basename=True ) call_args_list = client.get_container_client().download_blob.call_args_list for call_args in call_args_list: call_arg1 = call_args[0] if call_arg1 == "{}foo/test1.txt".format(blob_path): assert os.path.exists("{}/foo/test1.txt".format(dirname3)) elif call_arg1 == "{}foo/test2.txt".format(blob_path): assert os.path.exists("{}/foo/test2.txt".format(dirname3)) elif call_arg1 == "{}foo/{}/test3.txt".format(blob_path, rel_path2): assert os.path.exists("{}/foo/{}/test3.txt".format(dirname3, rel_path2)) else: assert False
def delete_blob_from_main_storage(self, jenkins_name, file_name): file_dir = jenkins_name + '/' + file_name blob_client = self.service_client.get_blob_client( container=self.container, blob=BlobProperties(name=file_dir)) # Delete blob from main 'Hot' folder - later upload it to Archive blob_client.delete_blob(delete_snapshots='include')
def test_size_of_file(self): props = BlobProperties() props.content_length = 12 self.storage._service.get_blob_properties.return_value = Blob(props=props) self.assertEqual(12, self.storage.size("name"))
config.read(the_cwd + '\\episode_vii.ini') ws = Workspace.get(name=config['azure.settings']['workspace_name'], subscription_id=config['azure.settings']['subscription_id'], resource_group=config['azure.settings']['resource_group']) print('Workspace loaded.') storage_connection_string = config['azure.settings'][ 'storage_connection_string'] blob_service_client = BlobServiceClient.from_connection_string( storage_connection_string) credential = blob_service_client.credential container_client = None for container in blob_service_client.list_containers(): if container.name == '154fd9d2-2831-4ad7-a2e7-61ee7e620cab': container_client = blob_service_client.get_container_client(container) print(type(container_client)) #csv_file_name = the_cwd + '\\data\\regression.csv' #csv_file = open(csv_file_name, 'rb') #container_client.upload_blob(name='regression.csv', data=csv_file) #print('Upload complete.') from azure.storage.blob import BlobProperties download_file = the_cwd + '\\data\\regression_download.csv' with open(download_file, "wb") as my_blob: download_stream = container_client.download_blob( BlobProperties(name='regression.csv')) my_blob.write(download_stream.readall())
def test_download_dir(self, client): dirname1 = tempfile.mkdtemp() dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/") def mkfile(container, cloud_path, fname): return open(fname, "w") client.return_value.get_blob_to_path.side_effect = mkfile store = AzureStore() blob_path = "/path/to/" azure_url = self.wasbs_base + blob_path rel_path2 = dirname2.split("/")[-1] # Mock return list blob_props = BlobProperties() blob_props.content_length = 42 obj_mock1 = Blob(blob_path + "test1.txt", props=blob_props) blob_props = BlobProperties() blob_props.content_length = 42 obj_mock2 = Blob(blob_path + "test2.txt", props=blob_props) subdir_mock = BlobPrefix() subdir_mock.name = blob_path + rel_path2 blob_props = BlobProperties() blob_props.content_length = 42 obj_mock3 = Blob(blob_path + rel_path2 + "/" + "test3.txt", props=blob_props) # Create some files to return def list_side_effect(container_name, prefix, delimiter="/", marker=None): if prefix == blob_path: return MockBlobList([subdir_mock, obj_mock1, obj_mock2]) return MockBlobList([obj_mock3]) client.return_value.list_blobs.side_effect = list_side_effect dirname3 = tempfile.mkdtemp() # Test without basename store.download_dir(blob=azure_url, local_path=dirname3, use_basename=False) client.return_value.get_blob_to_path.assert_has_calls( [ mock.call( "container", "{}test1.txt".format(blob_path), "{}/test1.txt".format(dirname3), ), mock.call( "container", "{}test2.txt".format(blob_path), "{}/test2.txt".format(dirname3), ), mock.call( "container", "{}{}/test3.txt".format(blob_path, rel_path2), "{}/{}/test3.txt".format(dirname3, rel_path2), ), ], any_order=True, )