def delete_data_from_blob(prefix):
from azure.storage.blob.blockblobservice import BlockBlobService
ws = get_workspace()
def_blob_store = ws.get_default_datastore()
print("Deleting blobs from folder:", prefix)
blob_service = BlockBlobService(
def_blob_store.account_name, def_blob_store.account_key
)
generator = blob_service.list_blobs(
def_blob_store.container_name, prefix=prefix
)
for blob in generator:
if blob.name.endswith("mp4"):
print("Deleting: " + blob.name)
blob_service.delete_blob(def_blob_store.container_name, blob.name)
generator = blob_service.list_blobs(
def_blob_store.container_name, prefix=prefix
)
for blob in generator:
print("Deleting: " + blob.name)
blob_service.delete_blob(def_blob_store.container_name, blob.name)
class AzureStorageContainer(Common.Contracts.StorageContainer):
def __init__(self, container_name, config: AzureStorageConfig):
self._container_name = container_name
self._blob_service = BlockBlobService(
account_name=config.account_name,
account_key=config.account_key
)
self._blob_service.create_container(self._container_name)
def upload_text(self, blob_name, text):
self._blob_service.create_blob_from_text(self._container_name, blob_name, text)
def list_blobs(self):
return self._blob_service.list_blobs(self._container_name)
def get_blob_to_text(self, file_name):
return self._blob_service.get_blob_to_text(self._container_name, file_name)
class AzureBlobFileSystem(object):
def __init__(self,
account_name=None,
account_key=None,
sas_token=None,
connection_string=None,
**storage_options):
account_name = account_name or os.environ.get(
"AZURE_BLOB_ACCOUNT_NAME")
account_key = account_key or os.environ.get("AZURE_BLOB_ACCOUNT_KEY")
sas_token = sas_token or os.environ.get("AZURE_BLOB_SAS_TOKEN")
connection_string = connection_string or os.environ.get(
"AZURE_BLOB_CONNECTION_STRING")
print(account_name, account_key)
self.connection = BlockBlobService(
account_name=account_name,
account_key=account_key,
sas_token=sas_token,
connection_string=connection_string,
protocol=storage_options.get("protocol") or "https",
endpoint_suffix=storage_options.get("endpoint_suffix"),
custom_domain=storage_options.get("custom_domain"))
self.sep = "/"
def ls(self, container, pattern=None):
return list(
set(
filter(
lambda x: fnmatch.fnmatch(x, pattern) if pattern else x,
map(lambda x: x.name,
self.connection.list_blobs(container)))))
def mkdir(self, container, dir_name):
self.touch(container, "{dir_name}/".format(dir_name=dir_name))
def rm(self, container, full_path):
if self.connection.exists(container, full_path):
path_delete_lease = None
try:
path_delete_lease = self.connection.acquire_blob_lease(
container, full_path)
self.connection.delete_blob(container,
full_path,
lease_id=path_delete_lease)
except:
if path_delete_lease is not None:
self.connection.release_blob_lease(container, full_path,
path_delete_lease)
else:
raise IOError(
"File '{file}' does not exist under container '{container}'".
format(file=full_path, container=container))
def touch(self, container, full_path):
container_lease = None
try:
container_lease = self.connection.acquire_container_lease(
container)
self.connection.create_blob_from_text(container, full_path, "")
finally:
if container_lease is not None:
self.connection.release_container_lease(
container, container_lease)
return full_path
def mv(self, container, src_path, dst_path):
try:
self.cp(container, src_path, dst_path)
self.rm(container, src_path)
return True
except:
self.rm(container, dst_path)
return False
def cp(self, container, full_src_path, full_dst_path):
copy_container_lease = None
try:
copy_container_lease = self.connection.acquire_container_lease(
container)
self.connection.copy_blob(
container, full_dst_path,
self.connection.make_blob_url(container, full_src_path))
finally:
if copy_container_lease is not None:
self.connection.release_container_lease(
container, copy_container_lease)
def du(self, container):
return {
blob.name: blob.properties.content_length
for blob in self.connection.list_blobs(container)
}
def last_modified(self, container, full_path):
return self.connection.get_blob_properties(
container, full_path).properties.last_modified
def head(self, container, full_path, bytes_count):
return self.connection.get_blob_to_bytes(container,
full_path,
start_range=0,
end_range=bytes_count -
1).content
def tail(self, container, full_path, bytes_count):
size = self.connection.get_blob_properties(
container, full_path).properties.content_length
return self.connection.get_blob_to_bytes(container,
full_path,
start_range=size -
bytes_count,
end_range=size - 1).content
def exists(self, container, full_path):
return self.connection.exists(container, full_path)
class FileLayer(object):
def __init__(self, storage_account_name="", storage_account_access_key=""):
self.storage_account_name = storage_account_name
self.storage_account_access_key = storage_account_access_key
self.file_service = BlockBlobService(
account_name=self.storage_account_name,
account_key=self.storage_account_access_key)
def download_blob(self,
blobpath="",
local_file_path="",
container="testing-environment"):
"""
Downloads file in the Azure Blob to current local directory
Args:
blobpath: blob name oz azure
local_file_path:File path on local system
container: Azure Blob Container name
Returns: Downloaded local file path
"""
if len(local_file_path) == 0:
local_path = "/" + blobpath
else:
local_path = local_file_path
os.makedirs("/".join(local_path.split("/")[:-1]), exist_ok=True)
self.file_service.get_blob_to_path(container_name=container,
blob_name=blobpath,
file_path=local_path)
print("downloaded to = {}".format(local_path))
return local_path
def upload_to_blob(self,
local_file_path="",
blobpath="",
container="testing-environment"):
"""
Uploads local file to the blob
Args:
blobpath: blob name oz azure
local_file_path:File path on local system
container: Azure Blob Container name
Returns: Blob name on azure
"""
if len(bloppath) == 0:
blob_file = "/".join(local_file_path.split("/")[1:])
else:
blob_file = blobpath
self.file_service.create_blob_from_path(container_name=container,
blob_name=blob_file,
file_path=local_file_path)
return blob_file
def read_pickle(self, path_on_blob, container="testing-environment"):
"""
Method reads serialized object stored in azure containers
Args:
path_on_blob: pickle file Path on the blob
container: Azure Blob Container name
Returns:Serialized object
"""
model_obj = pickle.loads(
self.file_service.get_blob_to_bytes(
container_name="testing-environment",
blob_name=path_on_blob).content)
return model_obj
def read_config(self, path_on_blob="", container="testing-environment"):
"""
Reads the json config file present on the blob
Args:
path_on_blob: Json file Path on the blob
container: Azure Blob Container name
Returns:
"""
config_file = self.file_service.get_blob_to_text(
container_name=container, blob_name=path_on_blob)
return json.loads(config_file.content)
def list_folders_in_blob_path(self,
blob_path="",
container="testing-environment"):
"""
Lists the Azure blob folder contents
Args:
blob_path: Folder path pn the blob
container: Azure Blob Container name
Returns:
"""
list_generator = self.file_service.list_blobs(container_name=container,
prefix=blob_path)
folders_under_blob = list(set([pth.name for pth in list_generator]))
return folders_under_blob
def copy_blob_same_storage(self,
sourceblobpath="",
destinationblobpath="",
sourcecontainer="",
destinationcontainer=""):
"""
This method copies blob across containers in same storage account.
Args:
sourceblobpath:source blob name
destinationblobpath:destionation blob name
sourcecontainer:source container name
destinationcontainer:destination container
"""
if len(destinationblobpath) == 0:
destinationblobpath = sourceblobpath
source_blob_url = self.file_service.make_blob_url(
sourcecontainer, sourceblobpath)
self.file_service.copy_blob(destinationcontainer, destinationblobpath,
source_blob_url)
def copy_blob_across_storage(self,
sourceblobpath="",
destinationblobpath="",
sourcecontainer="",
destinationcontainer="",
destination_source_account_name="",
destination_source_account_key=""):
"""
This method copies blob across different storage accounts.
Args:
sourceblobpath:source blob name
destinationblobpath:destionation blob name
sourcecontainer:source container name
destinationcontainer:destination container
destination_source_account_name: storage account name for destination storage account
destination_source_account_key:storage account key for destination storage account
"""
destinationfileservice = BlockBlobService(
account_name=destination_source_account_name,
account_key=destination_source_account_key)
local_path = self.download_blob(blobpath=sourceblobpath,
container=sourcecontainer)
if len(destinationblobpath) == 0:
destinationblobpath = "/".join(local_path.split("/")[1:])
destinationfileservice.create_blob_from_path(
container_name=destinationcontainer,
blob_name=destinationblobpath,
file_path=local_path)
os.remove(local_path)