class RemoteStorage():
def __init__(self, user, key, container):
self.bs = BlockBlobService(account_name=user, account_key=key)
self.container = container
def get_obj(self, blob_name):
try:
return self.bs.get_blob_properties(self.container, blob_name)
except AzureMissingResourceHttpError:
return None
def download_blob(self, blob_name, localf):
self.bs.get_blob_to_path(self.container, blob_name, localf)
def del_blob(self, blob_name):
try:
self.bs.delete_blob(self.container, blob_name)
except:
pass
def upload(self, blob_name, localf):
self.bs.create_blob_from_path(
self.container,
blob_name,
localf,
content_settings=ContentSettings(content_type='text/xml'))
def new_docs(self):
res = []
for blob in self.bs.list_blobs(self.container):
res.append(blob.name)
return res
class AzureStorageInterface:
def __init__(self, AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY,\
AZURE_DIRECTORY_NAME, AZURE_CONTAINER_NAME):
self.AZURE_ACCOUNT_KEY = AZURE_ACCOUNT_KEY
self.AZURE_ACCOUNT_NAME = AZURE_ACCOUNT_NAME
self.AZURE_DIRECTORY_NAME = AZURE_DIRECTORY_NAME
self.AZURE_CONTAINER_NAME = AZURE_CONTAINER_NAME
self.block_blob_service = BlockBlobService(AZURE_ACCOUNT_NAME,
AZURE_ACCOUNT_KEY)
def updateSRTFiles(self, videoid, srtFilePath):
#.2017-05-22T15:18:29.573
self.createBlob(videoid, srtFilePath)
print("Updated SRT file for videoid ", videoid, srtFilePath)
timedSRTFILEPATH = srtFilePath + '.' + str(
datetime.utcnow().isoformat()[:-3])
shutil.copy(srtFilePath, timedSRTFILEPATH)
self.createBlob(videoid, timedSRTFILEPATH)
print("Updated SRT file for videoid ", videoid, timedSRTFILEPATH)
def getVideoFilesFromAzureStorage(self, videoid, videodirpath):
videoblobname = self.AZURE_DIRECTORY_NAME + videoid + '/' + videoid + '.mp4'
if not os.path.exists(videodirpath):
os.makedirs(videodirpath)
videopath = videodirpath + "/" + videoid + '.mp4'
if self.block_blob_service.exists(self.AZURE_CONTAINER_NAME,
videoblobname):
self.getBlob(videoblobname, videopath)
if os.path.exists(videopath):
return True, videopath
else:
return False, ''
def isSRTExist(self, videoid):
srtblobname = self.AZURE_DIRECTORY_NAME + videoid + '/' + videoid + '.en.srt'
issrtexist = self.block_blob_service.exists(self.AZURE_CONTAINER_NAME,
srtblobname)
return issrtexist
def isVideoExist(self, videoid):
videoblobname = self.AZURE_DIRECTORY_NAME + videoid + '/' + videoid + ".mp4"
isvideoexist = self.block_blob_service.exists(
self.AZURE_CONTAINER_NAME, videoblobname)
return isvideoexist
def deleteBlob(self, blobname):
self.block_blob_service.delete_blob(self.AZURE_CONTAINER_NAME,
blobname)
def getBlob(self, blobname, filepath):
self.block_blob_service.get_blob_to_path(self.AZURE_CONTAINER_NAME,
blobname, filepath)
def createBlob(self, videoid, filepath):
path, filename = os.path.split(filepath)
blobname = self.AZURE_DIRECTORY_NAME + videoid + '/' + filename
self.block_blob_service.create_blob_from_path(
self.AZURE_CONTAINER_NAME, blobname, filepath)
def delete_blob(blob_name):
block_blob_service = BlockBlobService(account_name, account_key)
# Set the permission so the blobs are public.
block_blob_service.set_container_acl(corpus_seg_container_name,
public_access=PublicAccess.Container)
block_blob_service.delete_blob(corpus_seg_container_name, blob_name)
print(blob_name + " was deleted from corpus-segments-container")
def remove_unused_blobs(self):
"""
Cleanup script to remove unused blobs: Managed vhds and unmanaged disks
Returns list of removed disks
"""
# removing unmanaged disks
removed_blobs = []
container_client = BlockBlobService(self.storage_account, self.storage_key)
for container in container_client.list_containers():
for blob in container_client.list_blobs(container_name=container.name, prefix='test'):
if blob.properties.lease.status == 'unlocked':
self.logger.info("Removing Blob {b} "
"from containter {c}".format(b=blob.name, c=container.name))
container_client.delete_blob(container_name=container.name,
blob_name=blob.name)
removed_blobs.append({'container': container.name, 'blob': blob.name})
# removing managed disks
removed_disks = []
for disk in self.compute_client.disks.list():
if disk.name.startswith('test') and disk.owner_id is None:
self.logger.info("Removing disk {d}".format(d=disk.name))
self.compute_client.disks.delete(resource_group_name=self.resource_group,
disk_name=disk.name)
removed_disks.append({'resource_group': self.resource_group,
'disk': disk.name})
return {'Managed': removed_disks, 'Unmanaged': removed_blobs}
class AzureStorage:
def __init__(self, connectionString, container):
self.BlobService = BlockBlobService(connection_string=connectionString)
nameValue = UtilityHelper.connectStringToDictionary(connectionString)
self.AccountName = nameValue['AccountName']
self.container = container
def getBaseURL(self):
return 'https://' + self.AccountName + '.blob.core.windows.net/'
def uploadByLocalFile(self, localFullFileName, remoteBlobName):
self.BlobService.create_blob_from_path(self.container, remoteBlobName,
localFullFileName)
blobURL = 'https://' + self.AccountName + '.blob.core.windows.net/' + self.container + '/' + remoteBlobName
return blobURL
def uploadByStream(self, streamData, remoteBlobName):
self.BlobService.create_blob_from_stream(self.container,
remoteBlobName, streamData)
blobURL = 'https://' + self.AccountName + '.blob.core.windows.net/' + self.container + '/' + remoteBlobName
return blobURL
def uploadByBytes(self, bytesData, remoteBlobName):
self.BlobService.create_blob_from_bytes(self.container, remoteBlobName,
bytesData)
blobURL = 'https://' + self.AccountName + '.blob.core.windows.net/' + self.container + '/' + remoteBlobName
return blobURL
def delete(self, blobName):
self.BlobService.delete_blob(self.container, blobName)
def copy(self, sourceBlobURL, targetBlobName):
self.BlobService.copy_blob(self.container, targetBlobName,
sourceBlobURL)
def delete_data_from_blob(prefix):
from azure.storage.blob import BlockBlobService
from azureml.core.authentication import ServicePrincipalAuthentication
from azureml.core import Workspace
import json
ws = get_workspace()
def_blob_store = ws.get_default_datastore()
print("Deleting blobs from folder:", prefix)
blob_service = BlockBlobService(def_blob_store.account_name,
def_blob_store.account_key)
generator = blob_service.list_blobs(def_blob_store.container_name,
prefix=prefix)
for blob in generator:
if blob.name.endswith("mp4"):
print("Deleting: " + blob.name)
blob_service.delete_blob(def_blob_store.container_name, blob.name)
generator = blob_service.list_blobs(def_blob_store.container_name,
prefix=prefix)
for blob in generator:
print("Deleting: " + blob.name)
blob_service.delete_blob(def_blob_store.container_name, blob.name)
def deleteBlob(uri):
conn_string = os.environ.get('BlockStorageAccount')
uri = unquote(uri)
container_name, file_name = extractBlobInfoFromUri(uri)
account = BlockBlobService(connection_string=conn_string)
logging.info(f'{container_name}, {file_name}')
account.delete_blob(container_name, file_name)
def basic_blockblob_operations(account):
# # Create a Block Blob Service object
blockblob_service = account.create_block_blob_service()
blockblob_service = BlockBlobService(account_name, account_key)
# Check if blobs exist in Azure container, and
# List and download all the blobs in the container
dpContainer = 'jsoncontdp'
print('List Blobs in Container')
generator = blockblob_service.list_blobs(dpContainer)
if sum(1 for _ in generator) == 0:
print('No blobs to process -- program ending')
exit(0)
for blob in generator:
print('\tBlob Name: ' + blob.name, blob.properties)
file = blob.name
# Download the blob
downloadDir = 'c:/DownloadsDP/'
print('Download the blob', file, 'path', downloadDir)
blockblob_service.get_blob_to_path(dpContainer, file, downloadDir+file)
# Delete .h264 blob from Azure container. Keep .jpeg - log of crop locations
if ".h264" in file:
print('Delete Blob ', downloadDir+file)
blockblob_service.delete_blob(dpContainer, file)
def delete_blob(blob_name, container_name):
block_blob_service = BlockBlobService(account_name=storage_acc_name,
account_key=storage_acc_key)
# Set the permission so the blobs are public.
block_blob_service.set_container_acl(container_name,
public_access=PublicAccess.Container)
block_blob_service.delete_blob(container_name=container_name,
blob_name=blob_name)
def delete_image(sender, instance, **kwargs):
image_name = instance.img.split("/")[-1]
if not settings.DEBUG:
block_blob_service = BlockBlobService(account_name=os.environ['AZURE_IMAGE_CONTAINER_NAME'], account_key=os.environ['AZURE_IMAGE_CONTAINER_KEY'])
block_blob_service.delete_blob('images', image_name)
logger.debug(f"Image: {image_name} deleted from Azure")
else:
logger.debug(f"Image: {image_name} would of been deleted if not in DEBUG mode")
def delete_blob(blob_name, container_name, bbs=None):
if not bbs:
bbs = BlockBlobService(account_name=config["account_name"],
account_key=config["account_key"])
blob_exists = check_blob_exists(blob_name, container_name, bbs)
if not blob_exists:
return
bbs.delete_blob(container_name, blob_name)
def wipe_blob_container(container_name):
blob_service = BlockBlobService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY)
generator = blob_service.list_blobs(container_name)
for blob in generator:
if blob.name.startswith("results"):
print("Deleting blob: {}".format(blob.name))
blob_service.delete_blob(CONTAINER_NAME, blob.name)
def delete_picture(request, pk):
pic = Picture.objects.get(pk=pk)
block_blob_service = BlockBlobService(account_name='djangodemo', account_key='wHisO/rq7ZD5FgKETqdPFWDraDdyVIveAFRW/TpZnHJ0t1KjNJcP9mOvMS9cIDX+QHBOMLVgXlUXft+QHrp1eA==')
block_blob_service.delete_blob('media', '{}'.format(pic.picture))
Picture.objects.filter(pk=pk).delete()
return redirect('display_gallery_toapprove')
def put_file_in_azure(azure_emulator_coords: azure_utils.StorageCoordinates,
azure_service: blob.BlockBlobService,
sample_stream_content: str) -> typing.Generator:
filename = 'test_put_file.txt'
azure_service.create_blob_from_text(azure_emulator_coords.container,
filename, sample_stream_content)
yield filename
azure_service.delete_blob(azure_emulator_coords.container, filename)
def clearStorageOldData(self):
blobService = BlockBlobService(
account_name=self.storageAccountName,
account_key=self.storageAccountKey,
endpoint_suffix=self.storageEndpointSuffix)
blobs = blobService.list_blobs(self.storageContainer)
for blob in blobs:
blobService.delete_blob(self.storageContainer, blob.name)
print('delete blob : ' + blob.name)
def remove_blob(self, blob):
try:
block_blob_service = BlockBlobService(
account_name=self.config['blob_account_name'],
account_key=self.config['blob_account_key'])
block_blob_service.delete_blob(
container_name=self.config['blob_image_container'],
blob_name=blob)
except Exception as e:
print(e)
class azure_operations(operations):
"""Defines ops for Azure"""
block_blob_service = None
bucket_name = None
def __init__(self, profiles, index, path):
operations.__init__(self, profiles, "Azure", path)
account_name = profiles[index]['accesskey']
access_key = profiles[index]['secretkey']
self.block_blob_service = BlockBlobService(account_name=account_name,
account_key=access_key)
self.bucket_name = profiles[index]['bucketname']
try:
self.block_blob_service.create_container(
self.bucket_name, public_access=PublicAccess.Container)
except Exception:
pass
def get(self):
ret = ''
if (self.path is None) or (self.path == ''):
generator = self.block_blob_service.list_blobs(self.bucket_name)
for blob in generator:
ret += blob.name + "\n"
else:
path = getsKeyNameFromPath(self.path)
self.block_blob_service.get_blob_to_path(self.bucket_name, path,
path)
ret += 'Download Done'
return ret
def put(self):
path = getsKeyNameFromPath(self.path)
self.block_blob_service.create_blob_from_path(
self.bucket_name, # container should user provide the name
path, # to be uploaded object
path, # real file path
)
return "Put Done"
def delete(self):
path = getsKeyNameFromPath(self.path)
self.block_blob_service.delete_blob(self.bucket_name, path)
return "Delete Done"
def create(self):
# create container
return "Done"
def checkExists(self):
print("Azure check exists")
return "Done"
def create_blob_sas_defintion(self, storage_account_name, vault_url):
"""
Creates a service SAS definition with access to a blob container.
"""
from azure.storage.blob import BlockBlobService, ContainerPermissions
from azure.keyvault.models import SasTokenType, SasDefinitionAttributes
from azure.keyvault import SecretId
# create the blob sas definition template
# the sas template uri for service sas definitions contains the storage entity url with the template token
# this sample demonstrates constructing the template uri for a blob container, but a similar approach can
# be used for all other storage service, i.e. File, Queue, Table
# create a template sas token for the container
service = BlockBlobService(account_name=storage_account_name,
# don't sign the template with the storage account key use key 00000000
account_key='00000000')
permissions = ContainerPermissions(read=True, write=True, delete=True, list=True)
temp_token = service.generate_container_shared_access_signature(container_name='blobcontainer',
permission=permissions,
expiry='2020-01-01')
# use the BlockBlobService to construct the template uri for the container sas definition
blob_sas_template_uri = service.make_container_url(container_name='blobcontainer',
protocol='https',
sas_token=temp_token)
# create the sas definition in the vault
attributes = SasDefinitionAttributes(enabled=True)
blob_sas_def = self.client.set_sas_definition(vault_base_url=vault_url,
storage_account_name=storage_account_name,
sas_definition_name='blobcontall',
template_uri=blob_sas_template_uri,
sas_type=SasTokenType.service,
validity_period='PT2H',
sas_definition_attributes=attributes)
# use the sas definition to provision a sas token and use it to create a BlockBlobClient
# which can interact with blobs in the container
# get the secret_id of the container sas definition and get the token from the vault as a secret
sas_secret_id = SecretId(uri=blob_sas_def.secret_id)
blob_sas_token = self.client.get_secret(vault_base_url=sas_secret_id.vault,
secret_name=sas_secret_id.name,
secret_version=sas_secret_id.version).value
service = BlockBlobService(account_name=storage_account_name,
sas_token=blob_sas_token)
service.create_blob_from_text(container_name='blobcontainer',
blob_name='blob2',
text=u'test blob2 data')
blobs = list(service.list_blobs(container_name='blobcontainer'))
for blob in blobs:
service.delete_blob(container_name='blobcontainer',
blob_name=blob.name)
def azure_blob_storage():
try:
# To create a blob service or connection to azure blob storage
block_blob_service = BlockBlobService(account_name='account_name', account_key='account_key')
# To create a container
container_name ='container_name'
block_blob_service.create_container(container_name)
# Set access to a container such as public ,readonly,private
block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container)
# Create a file to upload to azure blob storage
local_file_name ="Test.txt"
path_to_file =os.path.join('local_path', local_file_name)
# To write to the file
local_file = open(full_path_to_file, 'w')
local_file.write("hi peoplee")
local_file.close()
# Upload the created file, use local_file_name for the blob name
block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file)
# To List all the blobs in the container
container_List = block_blob_service.list_blobs(container_name)
for file in container_List:
print("Blob name: "+{file.name})
# Download the blob(s).
download_file_path = os.path.join('local_path', 'local_file_name')
block_blob_service.get_blob_to_path(container_name, local_file_name, download_file_path)
#delete a blob
block_blob_service.delete_blob('container_name', 'blob_name')
#Append to a blob service
append_blob_service = AppendBlobService(account_name='myaccount', account_key='mykey')
# The same containers can hold all types of blobs
append_blob_service.create_container('container_name')
#To append file must exists
append_blob_service.create_blob('container_name', 'append_blob')
#Append to a blob service
append_blob_service.append_blob_from_text('container_name', 'append_blob', 'Hello, world!')
append_blob = append_blob_service.get_blob_to_text('container_name', 'append_blob')
# Clean up resources. This includes the container and the temp files
block_blob_service.delete_container(container_name)
except Exception as e:
def upload_timestamp_file(
azure_emulator_coords: azure_utils.StorageCoordinates,
azure_service: blob.BlockBlobService,
sample_timestamp: str) -> typing.Generator:
azure_service.create_blob_from_text(azure_emulator_coords.container,
endpoint.TIMESTAMP_FILENAME,
sample_timestamp)
yield
azure_service.delete_blob(azure_emulator_coords.container,
endpoint.TIMESTAMP_FILENAME)
def startProcessing(accountName, key, ccontainer):
print 'Processor started using path: ' + os.getcwd()
block_blob_service = BlockBlobService(account_name=accountName, account_key=key)
generator = block_blob_service.list_blobs(ccontainer)
for blob in generator:
if blob.properties.content_length != 0:
print('Downloaded a non empty blob: ' + blob.name)
cleanName = string.replace(blob.name, '/', '_')
block_blob_service.get_blob_to_path(ccontainer, blob.name, cleanName)
processBlob(cleanName)
os.remove(cleanName)
block_blob_service.delete_blob(ccontainer, blob.name)
class AzureStore(DataStoreInterface):
service_cache = _ClientCache()
def __init__(self):
service = self.service_cache.get('azure')
if service is None:
self.service = BlockBlobService(**env.AZURE_STORAGE_CREDENTIALS)
else:
self.service: BlockBlobService = service
self.service_cache.get('azure')
self.service_cache.put('azure', self.service)
def get(self, sample_url: str) -> BinaryIO:
container_name, blob_name = self._parser_url(sample_url)
stream = io.BytesIO()
self.service.get_blob_to_stream(container_name, blob_name, stream)
stream.seek(0)
return stream
def put(self, sample_url: str, stream: BinaryIO):
container_name, blob_name = self._parser_url(sample_url)
self.service.create_container(container_name)
self.service.create_blob_from_stream(container_name, blob_name, stream)
def exists(self, sample_url) -> bool:
container_name, blob_name = self._parser_url(sample_url)
return self.service.exists(container_name, blob_name)
def delete(self, sample_url):
container_name, blob_name = self._parser_url(sample_url)
self.service.delete_blob(container_name, blob_name)
def get_signed_url(self, sample_url) -> str:
container_name, blob_name = self._parser_url(sample_url)
signature = self.service.generate_blob_shared_access_signature(
container_name,
blob_name,
permission=BlobPermissions(read=True),
expiry=datetime.datetime.utcnow() + datetime.timedelta(minutes=15),
)
url = self.service.make_blob_url(container_name,
blob_name,
protocol='https',
sas_token=signature)
return url
@staticmethod
def _parser_url(url) -> Tuple[str, str]:
if not url.startswith(AZURE_PREFIX):
raise ValueError(f"URL should start with {AZURE_PREFIX}")
container_name, blob_name = url[len(AZURE_PREFIX):].split('/',
maxsplit=1)
return container_name, blob_name
def delete_blob(blob_name, container_name):
block_blob_service = BlockBlobService(account_name=storage_account,
account_key=storage_key)
# Set the permission so the blobs are public.
block_blob_service.set_container_acl(container_name,
public_access=PublicAccess.Container)
try:
block_blob_service.delete_blob(container_name=container_name,
blob_name=blob_name)
except:
print("The blob not exist int the container")
print("%s deleted from container: %s" % (blob_name, container_name))
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# Check for a DELETE request.
if (req.method == "DELETE"):
try:
# check url parameter for machineID
machineID = req.params.get('MachineID')
logging.info("Getting machine id from url.")
logging.debug(machineID)
# storage connection string
connect_str = os.getenv('StorageConnStr')
accuntName = os.getenv('StorageName')
accuntKey = os.getenv('StorageConnectionKey')
block_blob_service = BlockBlobService(account_name=accuntName,
account_key=accuntKey)
logging.info("Successful connection to blob storage.")
# blob name
blobName = 'machineimages'
BlobPermissions(read=False,
add=False,
create=False,
write=False,
delete=True,
_str=None)
# delete an image from storage
block_blob_service.delete_blob(blobName, machineID)
logging.info('Image deleted successfully')
# clean up
connect_str.close()
logging.info("Machine successfully deleted")
return func.HttpResponse(f"Image successfully deleted")
except ValueError as e:
logging.error("Invalid json format " + str(e))
logging.info("Storage account connection failed")
pass
finally:
logging.info("Image successfuly deleted")
logging.info("Closing connection to the storage...")
# returns a Http 400 status bad request.
return func.HttpResponse(
"Please pass a DELETE request on the query string or in the request body",
status_code=400)
def process(data):
user=os.environ['STORAGE_USERNAME']
password=os.environ['STORAGE_PASSWORD']
container_landing ='landingzone'
container_staging = 'staging'
block_blob_service = BlockBlobService(account_name=user, account_key=password)
blobname = data["data"]["url"].split("/")[-1]
content = block_blob_service.get_blob_to_text(container_landing, blobname).content
block_blob_service.create_blob_from_text(container_staging,blobname,preprocessing(content))
block_blob_service.delete_blob(container_landing,blobname,delete_snapshots='include')
def delete_resources_for_node(self, node):
logger.info('deleting node {}'.format(node.name))
resource_management_client = get_mgmt_service_client(
ResourceManagementClient)
compute_management_client = get_mgmt_service_client(
ComputeManagementClient)
# save disk location
vm_details = compute_management_client.virtual_machines.get(
self.resource_group_name, node.name, None)
storage_infos = vm_details.storage_profile.os_disk.vhd.uri.split('/')
account_name = storage_infos[2].split('.')[0]
container_name = storage_infos[3]
blob_name = storage_infos[4]
# delete vm
logger.info('Deleting VM for {}'.format(node.name))
delete_vm_op = resource_management_client.resources.delete(
self.resource_group_name, 'Microsoft.Compute', '',
'virtualMachines', node.name, '2016-03-30')
delete_vm_op.wait()
# delete nic
logger.info('Deleting NIC for {}'.format(node.name))
name_parts = node.name.split('-')
nic_name = '{}-{}-{}-nic-{}'.format(name_parts[0], name_parts[1],
name_parts[2], name_parts[3])
delete_nic_op = resource_management_client.resources.delete(
self.resource_group_name, 'Microsoft.Network', '',
'networkInterfaces', nic_name, '2016-03-30')
delete_nic_op.wait()
# delete os blob
logger.info('Deleting OS disk for {}'.format(node.name))
storage_management_client = get_mgmt_service_client(
StorageManagementClient)
keys = storage_management_client.storage_accounts.list_keys(
self.resource_group_name, account_name)
key = keys.keys[0].value
for i in range(5):
try:
block_blob_service = BlockBlobService(
account_name=account_name, account_key=key)
block_blob_service.delete_blob(container_name, blob_name)
except AzureHttpError as err:
print(err.message)
continue
break
def clearStorageOldData(self):
blobService = BlockBlobService(
account_name=self.storageAccountName,
account_key=self.storageAccountKey,
endpoint_suffix=self.storageEndpointSuffix)
try:
blobs = blobService.list_blobs(self.storageContainer)
for blob in blobs:
blobService.delete_blob(self.storageContainer, blob.name)
print('delete blob : ' + blob.name)
except:
print('blob was locked. Re-try after 30 seconds.')
time.sleep(30)
self.clearStorageOldData()
def save_model_metadata(self, blob_name, text_data):
block_blob_service = BlockBlobService(
account_name=self.config.storage_account,
account_key=self.config.storage_account_key)
block_blob_service.create_container(self.config.models_container_name,
fail_on_exist=False,
public_access=None)
if block_blob_service.exists(self.config.models_container_name,
blob_name):
block_blob_service.delete_blob(self.config.models_container_name,
blob_name)
block_blob_service.create_blob_from_text(
self.config.models_container_name, blob_name, text_data)
return
class AzureStorage(ObjectStorage):
def __init__(self, config):
self.account_name = config.get("ACCOUNT_NAME", None)
self.account_key = config.get("ACCOUNT_KEY", None)
self.container_name = config.get("CONTAINER_NAME", None)
self.endpoint_suffix = config.get("ENDPOINT_SUFFIX",
'core.chinacloudapi.cn')
if self.account_name and self.account_key:
self.client = BlockBlobService(
account_name=self.account_name,
account_key=self.account_key,
endpoint_suffix=self.endpoint_suffix)
else:
self.client = None
def upload(self, src, target):
try:
self.client.create_blob_from_path(self.container_name, target, src)
return True, None
except Exception as e:
return False, e
def download(self, src, target):
try:
os.makedirs(os.path.dirname(target), 0o755, exist_ok=True)
self.client.get_blob_to_path(self.container_name, src, target)
return True, None
except Exception as e:
return False, e
def delete(self, path):
try:
self.client.delete_blob(self.container_name, path)
return True, False
except Exception as e:
return False, e
def exists(self, path):
return self.client.exists(self.container_name, path)
def list_buckets(self):
response = self.client.list_containers()
return ([c.name for c in response.items])
@property
def type(self):
return 'azure'
class AzureStorageService():
def __init__(self):
self.account_name = config.get(
'ckanext.dataextractor.azure_storage_account_name', None)
self.account_key = config.get(
'ckanext.dataextractor.azure_storage_account_key', None)
self.container_name = config.get(
'ckanext.dataextractor.azure_storage_container_name', None)
self.blob_expiration_days = config.get(
'ckanext.dataextractor.blob_expiration_days', 10)
if not self.account_name or not self.account_key or not self.container_name:
raise ValueError(
'azure_storage_account_name, azure_storage_account_key, '
'azure_storage_container_name')
self.service = BlockBlobService(account_name=self.account_name,
account_key=self.account_key)
def _get_blob_reference(self, prefix, format):
prefix = prefix.replace('/', '')
return '{}{}.{}'.format(prefix + '-',
str(uuid.uuid4()).replace('-', ''),
format.lower())
def blob_create(self, stream, format, resource_name):
blob_name = self._get_blob_reference(resource_name, format)
self.service.create_blob_from_stream(self.container_name, blob_name,
stream)
blob_url = self.service.make_blob_url(self.container_name, blob_name)
return blob_url
def blobs_delete(self):
blobs = self.service.list_blobs(self.container_name)
blobs_deleted = 0
for blob in blobs:
time_diff = datetime.now() - blob.properties.last_modified.replace(
tzinfo=None)
if time_diff.days >= int(self.blob_expiration_days):
blobs_deleted += 1
self.service.delete_blob(self.container_name, blob.name)
return blobs_deleted
def delete_blob():
try:
block_blob_service = BlockBlobService(account_name=name,
account_key=key)
container_name = input("Enter Container Name to List the blobs: ")
generator = block_blob_service.list_blobs(container_name)
for blob in generator:
print(" Blob name: ", blob.name)
blob_name = input("Enter blob name you want to delete: ")
print("Deleting blob: ", blob_name)
block_blob_service.delete_blob(container_name,
blob_name,
snapshot=None)
print("Successfully deleted : ", blob_name)
except Exception as e:
print(e)
class _BlobStorageTestCase(_TestCase):
def _get_container_name(self, handler_name):
container = _get_handler_config_value(handler_name, 'container')
if container:
container = container.replace('_', '-').lower()
return container
def setUp(self):
self.service = BlockBlobService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY,
is_emulated=_EMULATED)
# ensure that there's no log file in the container before each test
containers = [c.name for c in self.service.list_containers()]
for handler in LOGGING['handlers']:
container = self._get_container_name(handler)
if container in containers:
filename = _get_handler_config_value(handler, 'filename')
basename = os.path.basename(filename)
for blob in self.service.list_blobs(container, prefix=basename):
self.service.delete_blob(container, blob.name)
class AzureTransfer(BaseTransfer):
def __init__(self, account_name, account_key, container_name, prefix=None):
# NOTE: Azure wants all paths to start with a slash
prefix = "/{}".format(prefix.lstrip("/") if prefix else "")
super().__init__(prefix=prefix)
self.account_name = account_name
self.account_key = account_key
self.container_name = container_name
self.conn = BlockBlobService(account_name=self.account_name, account_key=self.account_key)
self.container = self.get_or_create_container(self.container_name)
self.log.debug("AzureTransfer initialized")
# XXX: AzureTransfer isn't actively tested and hasn't its error handling is probably lacking
#self.log.warning("AzureTransfer is experimental and has not been thoroughly tested")
def get_metadata_for_key(self, key):
key = self.format_key_for_backend(key)
return self._list_blobs(key)[0]["metadata"]
def _metadata_for_key(self, key):
return self._list_blobs(key)[0]["metadata"]
def list_path(self, key):
path = self.format_key_for_backend(key, trailing_slash=True)
return self._list_blobs(path)
def _list_blobs(self, path):
self.log.debug("Listing path %r", path)
items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata")
result = []
for item in items:
result.append({
"last_modified": item.properties.last_modified.isoformat(),
"metadata": item.metadata,
"name": self.format_key_from_backend(item.name),
"size": item.properties.content_length,
})
return result
def delete_key(self, key):
key = self.format_key_for_backend(key)
self.log.debug("Deleting key: %r", key)
return self.conn.delete_blob(self.container_name, key)
def get_contents_to_file(self, key, filepath_to_store_to, *, progress_callback=None):
key = self.format_key_for_backend(key)
self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to)
meta = self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to)
if progress_callback:
progress_callback(1, 1)
return meta
def get_contents_to_fileobj(self, key, fileobj_to_store_to, *, progress_callback=None):
key = self.format_key_for_backend(key)
self.log.debug("Starting to fetch the contents of: %r", key)
meta = self.conn.get_blob_to_file(self.container_name, key, fileobj_to_store_to)
if progress_callback:
progress_callback(1, 1)
return meta
def get_contents_to_string(self, key):
key = self.format_key_for_backend(key)
self.log.debug("Starting to fetch the contents of: %r", key)
return self.conn.get_blob_to_bytes(self.container_name, key), self._metadata_for_key(key)
def store_file_from_memory(self, key, memstring, metadata=None):
key = self.format_key_for_backend(key)
self.conn.create_blob_from_bytes(self.container_name, key, memstring,
x_ms_meta_name_values=self.sanitize_metadata(metadata))
def store_file_from_disk(self, key, filepath, metadata=None, multipart=None):
key = self.format_key_for_backend(key)
self.conn.create_blob_from_path(self.container_name, key, filepath,None)
# x_ms_meta_name_values=self.sanitize_metadata(metadata))
def get_or_create_container(self, container_name):
start_time = time.time()
self.conn.create_container(container_name)
self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time)
return container_name
class BlobStorage(object):
# void
def __init__(self, wasbs_path, dryrun=False):
parsed = urlparse(wasbs_path)
if parsed.scheme not in ('wasbs', 'wasb'):
raise InvalidBlobStorePath('Remote path is not supported! Expected format: `wasb[s]://container/blob-path`')
self.dryrun = dryrun
self.schema, self.container, self.blob_path = parsed.scheme, parsed.netloc, parsed.path
if self.blob_path and self.blob_path[0] == u'/':
self.blob_path = self.blob_path[1:]
self.blob_path = self.blob_path or None
self.pbar = None
self.service = BlockBlobService(
account_name=os.environ['AZURE_STORAGE_ACCOUNT'].strip(),
account_key=os.environ['AZURE_STORAGE_ACCESS_KEY'].strip())
@property
def url(self):
return u'{}://{}'.format(self.schema, self.container)
@property
def path(self):
return os.path.join(self.url, self.blob_path)
# Blob
def get_blob(self):
for blob in self.list_blobs():
if blob.path == self.blob_path:
return blob
# genexp<list<Blob>>
def list_blobs(self):
marker = None
while True:
batch = self.service.list_blobs(self.container, prefix=self.blob_path, marker=marker)
for blob in batch:
yield Blob(self, blob)
if not batch.next_marker:
break
marker = batch.next_marker
# void
def execute(self, executable_fn, message, end=None, **kwargs):
# Print the original message
print(message % kwargs, end=end)
# If dryrun, write the message and exit
if self.dryrun:
print('IGNORE (--dryrun)')
return
try:
executable_fn(**kwargs)
print('OK')
except Exception as e:
print('FAIL\n{}'.format(e))
# void
def remove_fn(self, path, url=None):
self.service.delete_blob(self.container, path)
# void
def remove_blobs(self, prefix=False):
if not self.blob_path:
print(u'Have to specify the path of the blob.')
sys.exit(1)
if not prefix:
return self.execute(self.remove_fn, 'Remove blob from `%(url)s` ... ', path=self.blob_path, url=self.path, end='')
for blob in self.list_blobs():
self.execute(self.remove_fn, 'Remove blob from `%(url)s` ... ', path=blob.path, url=blob.url, end='')
# void
def upload_fn(self, blob_path, file_path, rel_file_path=None, url=None):
self.service.create_blob_from_path(self.container, blob_path, file_path, \
max_connections=int(os.environ.get('AZURE_STORAGE_MAX_CONNECTIONS',1)), \
progress_callback=self.show_progress)
self.pbar.finish()
self.pbar = None
# tuple<str,str>
def get_upload_path_pair(self, file_path, common_prefix=None):
is_directory_ending = self.blob_path and self.blob_path.endswith('/')
is_container_path = self.blob_path is None
blob_path = os.path.join(self.blob_path or u'', os.path.split(file_path)[-1]) \
if any([is_container_path, is_directory_ending]) and common_prefix is None \
else self.blob_path
if common_prefix and blob_path:
blob_path = os.path.join(blob_path, file_path.split(common_prefix)[-1].strip('/'))
elif common_prefix and not blob_path:
blob_path = file_path.split(common_prefix)[-1].strip('/')
elif common_prefix == u'' and blob_path:
blob_path = os.path.join(blob_path, file_path.strip('/'))
elif common_prefix == u'' and not blob_path:
blob_path = file_path.strip('/')
return (file_path, blob_path)
# genexp<tuple<str,str>>
def get_upload_path_pairs(self, file_paths):
if len(file_paths) == 1:
yield self.get_upload_path_pair(file_paths[0])
return
common_prefix = os.path.split(os.path.commonprefix(file_paths))[0]
if self.blob_path and not self.blob_path.endswith('/'): self.blob_path += '/'
for file_path in file_paths:
yield self.get_upload_path_pair(file_path, common_prefix=common_prefix)
# void
def upload_blobs(self, file_paths):
for file_path, blob_path in self.get_upload_path_pairs(file_paths):
self.execute(self.upload_fn, 'Upload `%(rel_file_path)s` into `%(url)s`', \
file_path=file_path, rel_file_path=os.path.relpath(file_path), blob_path=blob_path, \
url=u'{}/{}'.format(self.url, blob_path))
# void
def show_progress(self, current, total):
def filesize(n,pow=0,b=1024,u='B',pre=['']+[p+'i'for p in'KMGTPEZY']):
pow,n=min(int(log(max(n*b**pow,1),b)),len(pre)-1),n*b**pow
return "%%.%if %%s%%s"%abs(pow%(-pow-1))%(n/b**float(pow),pre[pow],u)
if self.pbar is None:
self.pbar = ProgressBar(widgets=[
' '*5,
'Size: {}'.format(filesize(total)),
' ',
Percentage(),
' ',
Bar(),
' ',
ETA(),
' ',
FileTransferSpeed(),
' '*5
],
maxval=total
).start()
self.pbar.update(current)
# void
def download_fn(self, blob_path, file_path, **kwargs):
self.service.get_blob_to_path(self.container, blob_path, file_path, \
max_connections=int(os.environ.get('AZURE_STORAGE_MAX_CONNECTIONS',1)), \
progress_callback=self.show_progress)
self.pbar.finish()
self.pbar = None
# tuple<str,str>
def get_download_path_pair(self, blob_path, file_path, common_prefix=None):
file_path = os.path.join(file_path, os.path.split(blob_path)[-1]) \
if os.path.exists(file_path) and os.path.isdir(file_path) and common_prefix is None \
else file_path
if common_prefix:
file_path = os.path.join(file_path, blob_path.split(common_prefix)[-1].strip('/'))
elif common_prefix == u'':
file_path = os.path.join(file_path, blob_path.strip('/'))
dir_path = os.path.split(file_path)[0]
if dir_path and not os.path.exists(dir_path):
os.makedirs(dir_path)
return blob_path, file_path
# genexp<tuple<str,str>>
def get_download_path_pairs(self, file_path, prefix=False, skip_existing=False, sync=False):
# Ignore if no blob path is defined.
if not self.blob_path:
raise BlobPathRequired(u'Blob path is required for `get` command.')
# Single file download scenario.
if not prefix:
# Skip if the file is already existing.
if skip_existing and os.path.exists(file_path):
return
# Only downloads the not existing or the updated files (based on file size).
if sync and os.path.exists(file_path):
blob = self.get_blob()
if blob and get_fresher(blob, file_path) != blob:
return
# Return the caluclated path of the file.
yield self.get_download_path_pair(self.blob_path, file_path)
return
# List the blobs with the given prefix in the ABS.
blob_paths, blob_paths_dict = [], {}
for blob in self.list_blobs():
blob_paths.append(blob.path)
blob_paths_dict[blob.path] = blob
# Determine the common prefix between the blobs.
common_prefix = os.path.dirname(self.blob_path) \
if not self.blob_path.endswith('/') \
else self.blob_path
resolved_file_paths = []
for blob_path in blob_paths:
# Determine the input, output path pairs.
bp, fp = self.get_download_path_pair(blob_path, file_path, common_prefix=common_prefix)
# If any of the files want to write to the same file, raise an error.
if fp in resolved_file_paths:
raise DirectoryRequired('Can not use the same path (`{}`) for multiple blob!' \
.format(fp))
# Ignore the files that already exists.
if skip_existing and os.path.exists(fp):
continue
# Only downloads the not existing or the updated files (based on file size).
if sync and os.path.exists(fp) and get_fresher(blob_paths_dict[blob_path], fp) != blob_paths_dict[blob_path]:
continue
resolved_file_paths.append(fp)
yield bp, fp
# void
def download_blobs(self, file_path, prefix=False, skip_existing=False, sync=False):
# Iterates over the final input, output paths and download them.
for blob_path, file_path in self.get_download_path_pairs(file_path, prefix=prefix, skip_existing=skip_existing, sync=sync):
self.execute(self.download_fn, 'Download `%(url)s` into `%(rel_file_path)s`', \
blob_path=blob_path, file_path=file_path, rel_file_path=os.path.relpath(file_path), \
url=u'{}/{}'.format(self.url, blob_path))
class AzureTransfer(BaseTransfer):
def __init__(self, account_name, account_key, bucket_name, prefix=None):
prefix = "{}".format(prefix.lstrip("/") if prefix else "")
super().__init__(prefix=prefix)
self.account_name = account_name
self.account_key = account_key
self.container_name = bucket_name
self.conn = BlockBlobService(account_name=self.account_name, account_key=self.account_key)
self.container = self.get_or_create_container(self.container_name)
self.log.debug("AzureTransfer initialized, %r", self.container_name)
def get_metadata_for_key(self, key):
key = self.format_key_for_backend(key, remove_slash_prefix=True, trailing_slash=False)
results = self._list_blobs(key)
if not results:
raise FileNotFoundFromStorageError(key)
return results[0]["metadata"]
def _metadata_for_key(self, key):
return self._list_blobs(key)[0]["metadata"]
def list_path(self, key, trailing_slash=True): # pylint: disable=arguments-differ
# Trailing slash needed when listing directories, without when listing individual files
path = self.format_key_for_backend(key, remove_slash_prefix=True, trailing_slash=trailing_slash)
return self._list_blobs(path)
def _list_blobs(self, path):
self.log.debug("Listing path %r", path)
if path:
items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata")
else: # If you give Azure an empty path, it gives you an authentication error
items = self.conn.list_blobs(self.container_name, delimiter="/", include="metadata")
results = []
for item in items:
if not isinstance(item, BlobPrefix):
results.append({
"last_modified": item.properties.last_modified,
# Azure Storage cannot handle '-' so we turn them into underscores and back again
"metadata": dict((k.replace("_", "-"), v) for k, v in item.metadata.items()),
"name": self.format_key_from_backend(item.name),
"size": item.properties.content_length,
})
return results
def delete_key(self, key):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.log.debug("Deleting key: %r", key)
try:
return self.conn.delete_blob(self.container_name, key)
except azure.common.AzureMissingResourceHttpError as ex:
raise FileNotFoundFromStorageError(key) from ex
def get_contents_to_file(self, key, filepath_to_store_to, *, progress_callback=None):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to)
try:
self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to)
except azure.common.AzureMissingResourceHttpError as ex:
raise FileNotFoundFromStorageError(key) from ex
if progress_callback:
progress_callback(1, 1)
return self._metadata_for_key(key)
def get_contents_to_fileobj(self, key, fileobj_to_store_to, *, progress_callback=None):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.log.debug("Starting to fetch the contents of: %r", key)
try:
self.conn.get_blob_to_stream(self.container_name, key, fileobj_to_store_to)
except azure.common.AzureMissingResourceHttpError as ex:
raise FileNotFoundFromStorageError(key) from ex
if progress_callback:
progress_callback(1, 1)
return self._metadata_for_key(key)
def get_contents_to_string(self, key):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.log.debug("Starting to fetch the contents of: %r", key)
try:
blob = self.conn.get_blob_to_bytes(self.container_name, key)
return blob.content, self._metadata_for_key(key)
except azure.common.AzureMissingResourceHttpError as ex:
raise FileNotFoundFromStorageError(key) from ex
def store_file_from_memory(self, key, memstring, metadata=None):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.conn.create_blob_from_bytes(self.container_name, key, memstring,
metadata=self.sanitize_metadata(metadata, replace_hyphen_with="_"))
def store_file_from_disk(self, key, filepath, metadata=None, multipart=None):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.conn.create_blob_from_path(self.container_name, key, filepath,
metadata=self.sanitize_metadata(metadata, replace_hyphen_with="_"))
def get_or_create_container(self, container_name):
start_time = time.monotonic()
self.conn.create_container(container_name)
self.log.debug("Got/Created container: %r successfully, took: %.3fs",
container_name, time.monotonic() - start_time)
return container_name
class AzureStorage(plugins.IStoragePlugin):
def configure(self, config):
self.storage = BlockBlobService(account_name=config['account_name'], account_key=config['account_key'])
self.container = config['container']
try:
container = self.storage.get_container_properties(self.container)
log.info("Configuring Azure blob storage %s/%s", self.storage.account_name, self.container)
except AzureMissingResourceHttpError as e:
log.warning("Container '%s' is missing in account '%s', trying to create new", self.container, self.storage.account_name)
try:
self.storage.create_container(self.container)
self.storage.set_container_acl(self.container, public_access=PublicAccess.Container)
except Exception as e:
log.critical("Cannot create new container: %s", e)
raise plugins.PluginInitException("Cannot create new container")
except AzureHttpError as e:
log.critical("Cannot access container '%s' in account '%s': %s", self.container, self.storage.account_name, e)
raise plugins.PluginInitException("Cannot access container")
except Exception as e:
log.critical("Cannot access container '%s' in account '%s': %s", self.container, self.storage.account_name, e)
raise plugins.PluginInitException("Cannot access container")
def delete(self, key):
log.info("Deleting file '%s' from %s/%s", key, self.storage.account_name, self.container)
try:
self.storage.delete_blob(self.container, key)
except AzureMissingResourceHttpError:
log.error("File '%s' was not found in %s/%s", key, self.storage.account_name, self.container)
raise common.NotFound('File not found')
except Exception as e:
log.error("Cannot delete '%s' from %s/%s: %s", key, self.storage.account_name, self.container, e)
raise common.FatalError(e)
def put(self, key, filename=None, file=None):
storage_key = key
try:
if filename:
log.debug("Uploading %s to %s", filename, self.storage.make_blob_url(self.container, storage_key))
self.storage.create_blob_from_path(self.container, storage_key, filename, content_settings=ContentSettings(content_type='application/octet-stream'))
elif file:
old_pos = file.tell()
file.seek(0)
log.debug("Uploading from stream to %s", self.storage.make_blob_url(self.container, storage_key))
self.storage.create_blob_from_stream(self.container, storage_key, file, content_settings=ContentSettings(content_type='application/octet-stream'))
file.seek(old_pos)
except Exception as e:
# TODO: more detailed error inspection
log.critical("Error uploading to %s/%s: %s", self.storage.account_name, self.container, e)
raise common.FatalError(e)
return storage_key
def get(self, key, stream):
# current azure python sdk barely can work with non-seekable streams,
# so we have to implement chunking by our own
# TODO: proper ranging? RFC says server SHOULD return 406 once range is unsatisfiable,
# but Azure is OK with end pos > blob length unless blob is not empty
chunk_size = 4*1024*1024
chunk_start = 0
chunk_end = chunk_size - 1
while True:
try:
chunk = self.storage._get_blob(self.container, key, start_range=chunk_start, end_range=chunk_end)
log.debug("Writing %s bytes from %s", len(chunk.content), chunk_start)
stream.write(chunk.content)
except IOError:
# remote side closed connection
return
except AzureMissingResourceHttpError as e:
raise common.NotFound(e)
except (AzureHttpError, AzureException) as e:
raise common.TemporaryError('Error while downloading {}: {}'.format(key, e))
blob_length = int(chunk.properties.content_range.split('/')[1])
chunk_start, chunk_end, blob_size = map(int, re.match(r'^bytes\s+(\d+)-(\d+)/(\d+)$', chunk.properties.content_range).groups())
if chunk_end == blob_size - 1:
# no more data to stream
break
else:
chunk_start = chunk_end + 1
chunk_end += chunk_size
return 0
#from azure.storage.blob import ContentSettings
#block_blob_service.create_blob_from_path(
# 'myseccontainer',
# 'remotesecfiles3.txt',
# 'localfiles3.txt',
# content_settings=ContentSettings(content_type='text/html')
# )
#### To list the blobs in a container, use the list_blobs method. This method returns a generator.
#### The following code outputs the name of each blob in a container to the console.
#generator = block_blob_service.list_blobs('myseccontainer')
#for blob in generator:
# print(blob.name)
#### The following example demonstrates using get_blob_to_path to download the contents of the myblob blob and store it to the out-sunset.png file.
#block_blob_service.get_blob_to_path('myseccontainer', 'remotesecf.txt', 'fromazure-out.txt')
#### Finally, to delete a blob, call delete_blob.
block_blob_service.delete_blob('myseccontainer', 'remotesecf.txt')
#### The example below creates a new append blob and appends some data to it, simulating a simple logging operation.
from azure.storage.blob import AppendBlobService
append_blob_service = AppendBlobService(myaccount, mykey)
#The same containers can hold all types of blobs
append_blob_service.create_container('mycontainer')
#Append blobs must be created before they are appended to
append_blob_service.create_blob('mycontainer', 'myappendblob')
append_blob_service.append_blob_from_text('mycontainer', 'myappendblob', u'Sinaq, cumle!')
append_blob = append_blob_service.get_blob_to_text('mycontainer', 'myappendblob')
print(append_blob)