def get_cloud_storage_account(_ctx=ctx):
'''Gets the Azure Blob storage service'''
# Get the storage account
storage_account = utils.get_parent(
_ctx.instance,
rel_type=constants.REL_CONTAINED_IN_SA)
storage_account_name = utils.get_resource_name(_ctx=storage_account)
# Get the storage account keys
keys = StorageAccount(_ctx=storage_account).list_keys()
if not isinstance(keys, list) or len(keys) < 1:
raise RecoverableError(
'StorageAccount reported no usable authentication keys')
# Get an interface to the Storage Account
storage_account_key = keys[0].get('key')
return CloudStorageAccount(
account_name=storage_account_name,
account_key=storage_account_key)
def get_cloud_storage_account(_ctx=ctx):
"""Gets the Azure Blob storage service"""
# Get the storage account
storage_account = utils.get_parent(_ctx.instance,
rel_type=constants.REL_CONTAINED_IN_SA)
resource_group_name = utils.get_resource_group(_ctx)
storage_account_name = utils.get_resource_name(_ctx=storage_account)
azure_config = _ctx.node.properties.get("azure_config")
if not azure_config.get("subscription_id"):
azure_config = ctx.node.properties.get('client_config')
else:
ctx.logger.warn("azure_config is deprecated please use client_config, "
"in later version it will be removed")
# Get the storage account keys
keys = StorageAccount(azure_config,
_ctx.logger).list_keys(resource_group_name,
storage_account_name)
if not keys or not keys.get("key1"):
raise RecoverableError(
'StorageAccount reported no usable authentication keys')
# Get an interface to the Storage Account
storage_account_key = keys.get("key1")
return CloudStorageAccount(account_name=storage_account_name,
account_key=storage_account_key)
def connection(self):
if self._connection is None:
account = CloudStorageAccount(self.account_name, self.account_key)
self._connection = account.create_block_blob_service()
return self._connection
def __init__(self, *args, **kwargs):
if "stay_on_remote" in kwargs:
del kwargs["stay_on_remote"]
self.azure = AzureStorageAccount(**kwargs).create_block_blob_service()
class AzureStorageHelper(object):
def __init__(self, *args, **kwargs):
if "stay_on_remote" in kwargs:
del kwargs["stay_on_remote"]
self.azure = AzureStorageAccount(**kwargs).create_block_blob_service()
def container_exists(self, container_name):
try:
self.azure.exists(container_name=container_name)
return True
except:
return False
def upload_to_azure_storage(
self,
container_name,
file_path,
blob_name=None,
use_relative_path_for_blob_name=True,
relative_start_dir=None,
extra_args=None,
):
""" Upload a file to Azure Storage
This function uploads a file to an Azure Storage Container as a blob.
Args:
container_name: the name of the Azure container to use
file_path: The path to the file to upload.
blob_name: The name to set for the blob on Azure. If not specified, this will default to the
name of the file.
Returns: The blob_name of the file on Azure if written, None otherwise
"""
file_path = os.path.realpath(os.path.expanduser(file_path))
assert container_name, "container_name must be specified"
assert os.path.exists(file_path), (
"The file path specified does not exist: %s" % file_path)
assert os.path.isfile(file_path), (
"The file path specified does not appear to be a file: %s" %
file_path)
if not self.azure.exists(container_name):
self.azure.create_container(container_name=container_name)
if not blob_name:
if use_relative_path_for_blob_name:
if relative_start_dir:
path_blob_name = os.path.relpath(file_path,
relative_start_dir)
else:
path_blob_name = os.path.relpath(file_path)
else:
path_blob_name = os.path.basename(file_path)
blob_name = path_blob_name
b = self.azure
try:
b.create_blob_from_path(container_name,
file_path=file_path,
blob_name=blob_name)
return b.get_blob_properties(container_name,
blob_name=blob_name).name
except:
raise WorkflowError("Error in creating blob. %s" % e.msg)
# return None
def download_from_azure_storage(
self,
container_name,
blob_name,
destination_path=None,
expandBlobNameIntoDirs=True,
make_dest_dirs=True,
create_stub_only=False,
):
""" Download a file from Azure Storage
This function downloads an object from a specified Azure Storage container.
Args:
container_name: the name of the Azure Storage container to use (container name only)
destination_path: If specified, the file will be saved to this path, otherwise cwd.
expandBlobNameIntoDirs: Since Azure blob names can include slashes, if this is True (defult)
then Azure blob names with slashes are expanded into directories on the receiving end.
If it is False, the blob name is passed to os.path.basename() to get the substring
following the last slash.
make_dest_dirs: If this is True (default) and the destination path includes directories
that do not exist, they will be created.
Returns:
The destination path of the downloaded file on the receiving end, or None if the destination_path
could not be downloaded
"""
assert container_name, "container_name must be specified"
assert blob_name, "blob_name must be specified"
if destination_path:
destination_path = os.path.realpath(
os.path.expanduser(destination_path))
else:
if expandBlobNameIntoDirs:
destination_path = os.path.join(os.getcwd(), blob_name)
else:
destination_path = os.path.join(os.getcwd(),
os.path.basename(blob_name))
# if the destination path does not exist
if make_dest_dirs:
os.makedirs(os.path.dirname(destination_path), exist_ok=True)
b = self.azure
try:
if not create_stub_only:
b.get_blob_to_path(
container_name=container_name,
blob_name=blob_name,
file_path=destination_path,
)
else:
# just create an empty file with the right timestamps
with open(destination_path, "wb") as fp:
os.utime(
fp.name,
(b.last_modified.timestamp(),
b.last_modified.timestamp()),
)
return destination_path
except:
return None
def delete_from_container(self, container_name, blob_name):
""" Delete a file from Azure Storage container
This function deletes an object from a specified Azure Storage container.
Args:
container_name: the name of the Azure Storage container to use (container name only, not endpoint)
blob_name: the name of the blob to delete from the container
Returns:
nothing
"""
assert container_name, "container_name must be specified"
assert blob_name, "blob_name must be specified"
b = self.azure
b.delete_blob(container_name, blob_name)
def exists_in_container(self, container_name, blob_name):
""" Returns whether the blob exists in the container
Args:
container_name: the name of the Azure Storage container (container name only, not endpoint)
blob_name: the blob_name of the object to delete from the container
Returns:
True | False
"""
assert container_name, "container_name must be specified"
assert blob_name, "blob_name must be specified"
try:
return self.azure.exists(container_name, blob_name)
except:
return None
def blob_size(self, container_name, blob_name):
""" Returns the size of a blob
Args:
container_name: the name of the Azure Storage container (container name only, not endpoint)
blob_name: the blob_name of the object to delete from the container
Returns:
Size in kb
"""
assert container_name, "container_name must be specified"
assert blob_name, "blob_name must be specified"
try:
b = self.azure.get_blob_properties(container_name, blob_name)
return b.properties.content_length // 1024
except:
print("blob or container do not exist")
return None
def blob_last_modified(self, container_name, blob_name):
""" Returns a timestamp of a blob
Args:
container_name: the name of the Azure Storage container (container name only, not endpoint)
blob_name: the blob_name of the object to delete from the container
Returns:
timestamp
"""
assert container_name, "container_name must be specified"
assert blob_name, "blob_name must be specified"
try:
b = self.azure.get_blob_properties(container_name, blob_name)
return b.properties.last_modified.timestamp()
except:
print("blob or container do not exist")
return None
def list_blobs(self, container_name):
""" Returns a list of blobs from the container
Args:
container_name: the name of the Azure Storage container (container name only, not endpoint)
Returns:
list of blobs
"""
assert container_name, "container_name must be specified"
try:
b = self.azure.list_blobs(container_name)
return [o.name for o in b]
except:
print("Did you provide a valid container_name?")
return None
def create_file_share(**_):
'''Creates an Azure File Share'''
# Get resource config values
res_cfg = utils.get_resource_config() or dict()
share_name = ctx.node.properties.get('name')
metadata = res_cfg.get('metadata')
quota = res_cfg.get('quota')
fail_on_exist = res_cfg.get('fail_on_exist', False)
# Check if invalid external resource
if ctx.node.properties.get('use_external_resource', False) and \
not share_name:
raise NonRecoverableError(
'"use_external_resource" specified without a resource "name"')
# Get the storage account
storage_account = utils.get_parent(ctx.instance,
rel_type=constants.REL_CONTAINED_IN_SA)
storage_account_name = utils.get_resource_name(_ctx=storage_account)
# Get the storage account keys
keys = StorageAccount(_ctx=storage_account).list_keys()
if not isinstance(keys, list) or len(keys) < 1:
raise RecoverableError(
'StorageAccount reported no usable authentication keys')
# Get an interface to the Storage Account
storage_account_key = keys[0].get('key')
storageacct = CloudStorageAccount(account_name=storage_account_name,
account_key=storage_account_key)
# Get an interface to the File Service
filesvc = storageacct.create_file_service()
if ctx.node.properties.get('use_external_resource', False):
# Attempt to use an existing File Share (if specified)
ctx.logger.debug(
'Checking for existing File Share "{0}"'.format(share_name))
try:
share = filesvc.get_share_properties(share_name)
metadata = share.get('metadata', dict())
quota = share.get('properties', dict()).get('quota')
created = False
except Exception as ex:
ctx.logger.error(
'File Share "{0}" does not exist and '
'"use_external_resource" is set to true'.format(share_name))
raise NonRecoverableError(ex)
else:
# Generate a new File Share name if needed
if not share_name:
ctx.logger.info('Generating a new File Share name')
for _ in xrange(0, 10):
tmpname = file_share_name_generator()
if not file_share_exists(filesvc, tmpname):
share_name = tmpname
break
# Handle name error
if not share_name:
raise NonRecoverableError(
'Error generating a new File Share name. Failed '
'after 10 tries.')
# Attempt to create the File Share
ctx.logger.debug('Creating File Share "{0}"'.format(share_name))
created = filesvc.create_share(share_name=share_name,
metadata=metadata,
quota=quota,
fail_on_exist=False)
if not created:
ctx.logger.warn('File Share already exists')
if fail_on_exist:
raise NonRecoverableError(
'File Share already exists in the storage account and '
'"fail_on_exist" set to True')
# Set run-time properties
ctx.instance.runtime_properties['name'] = share_name
ctx.instance.runtime_properties['quota'] = quota
ctx.instance.runtime_properties['metadata'] = metadata
ctx.instance.runtime_properties['created'] = created
ctx.instance.runtime_properties['storage_account'] = storage_account_name
ctx.instance.runtime_properties['username'] = storage_account_name
ctx.instance.runtime_properties['password'] = storage_account_key
ctx.instance.runtime_properties['uri'] = '{0}.file.{1}/{2}'.format(
storage_account_name, constants.CONN_STORAGE_FILE_ENDPOINT, share_name)
def create_file_share(**_):
'''Creates an Azure File Share'''
# Get resource config values
res_cfg = utils.get_resource_config() or dict()
share_name = ctx.node.properties.get('name')
metadata = res_cfg.get('metadata')
quota = res_cfg.get('quota')
fail_on_exist = res_cfg.get('fail_on_exist', False)
# Check if invalid external resource
if ctx.node.properties.get('use_external_resource', False) and \
not share_name:
raise NonRecoverableError(
'"use_external_resource" specified without a resource "name"')
# Get the storage account
storage_account = utils.get_parent(
ctx.instance,
rel_type=constants.REL_CONTAINED_IN_SA)
storage_account_name = utils.get_resource_name(_ctx=storage_account)
# Get the storage account keys
keys = StorageAccount(_ctx=storage_account).list_keys()
if not isinstance(keys, list) or len(keys) < 1:
raise RecoverableError(
'StorageAccount reported no usable authentication keys')
# Get an interface to the Storage Account
storage_account_key = keys[0].get('key')
storageacct = CloudStorageAccount(
account_name=storage_account_name,
account_key=storage_account_key)
# Get an interface to the File Service
filesvc = storageacct.create_file_service()
if ctx.node.properties.get('use_external_resource', False):
# Attempt to use an existing File Share (if specified)
ctx.logger.debug('Checking for existing File Share "{0}"'
.format(share_name))
try:
share = filesvc.get_share_properties(share_name)
metadata = share.get('metadata', dict())
quota = share.get('properties', dict()).get('quota')
created = False
except Exception as ex:
ctx.logger.error('File Share "{0}" does not exist and '
'"use_external_resource" is set to true'
.format(share_name))
raise NonRecoverableError(ex)
else:
# Generate a new File Share name if needed
if not share_name:
ctx.logger.info('Generating a new File Share name')
for _ in xrange(0, 10):
tmpname = file_share_name_generator()
if not file_share_exists(filesvc, tmpname):
share_name = tmpname
break
# Handle name error
if not share_name:
raise NonRecoverableError(
'Error generating a new File Share name. Failed '
'after 10 tries.')
# Attempt to create the File Share
ctx.logger.debug('Creating File Share "{0}"'.format(share_name))
created = filesvc.create_share(
share_name=share_name,
metadata=metadata,
quota=quota,
fail_on_exist=False)
if not created:
ctx.logger.warn('File Share already exists')
if fail_on_exist:
raise NonRecoverableError(
'File Share already exists in the storage account and '
'"fail_on_exist" set to True')
# Set run-time properties
ctx.instance.runtime_properties['name'] = share_name
ctx.instance.runtime_properties['quota'] = quota
ctx.instance.runtime_properties['metadata'] = metadata
ctx.instance.runtime_properties['created'] = created
ctx.instance.runtime_properties['storage_account'] = storage_account_name
ctx.instance.runtime_properties['username'] = storage_account_name
ctx.instance.runtime_properties['password'] = storage_account_key
ctx.instance.runtime_properties['uri'] = '{0}.file.{1}/{2}'.format(
storage_account_name, constants.CONN_STORAGE_FILE_ENDPOINT, share_name
)