def set_storage_service_cors_properties(account_name, account_key, cors_rules):
"""
Assigns the specified CORS rules to the specified Blob service.
blob_service: Target BlobService object.
cors_rules: A Cors instance specifying the rules to apply.
"""
blob_svc_props = StorageServiceProperties()
blob_svc_props.metrics = None
blob_svc_props.logging = None
setattr(blob_svc_props, 'cors', cors_rules)
def request_filter(request, next_filter):
""" Intercepts request to modify headers."""
request.headers = [(k, v) for (k, v) in request.headers
if k not in ('x-ms-version', 'Authorization')]
request.headers.append(('x-ms-version', '2013-08-15'))
request.headers.append(
('Authorization',
_sign_storage_blob_request(request, account_name, account_key)))
response = next_filter(request)
return response
blob_service = BlobService(account_name,
account_key).with_filter(request_filter)
blob_service.set_blob_service_properties(blob_svc_props)
def get_options2(url, account_key, path, filename, extension, type, ls):
options.url = url
options.account_key = account_key
options.path = path
options.filename = filename
options.extension = extension
options.type = 4
options.ls = ls
options.extension and options.filename and exit(print_warning())
tmp = urlparse(options.url)
options.account_name = tmp.netloc.split('.')[0]
options.container = tmp.path.split('/')[1]
options.vhd = tmp.path.split('/')[2]
options.host_base = tmp.netloc[tmp.netloc.find('.'):]
if options.account_key:
options.blob_service = BlobService(options.account_name,
options.account_key,
host_base=options.host_base)
else:
options.blob_service = None
options.path_list = split_path(options.path)
def test_no_retry_on_keyboadinterrupt(collect):
"""Ensure that KeyboardInterrupts are forwarded."""
key_name = 'test-key-name'
b = B(name=key_name)
# If vanilla KeyboardInterrupt is used, then sending SIGINT to the
# test can cause it to pass improperly, so use a subtype instead.
class MarkedKeyboardInterrupt(KeyboardInterrupt):
pass
collect.inject(MarkedKeyboardInterrupt('SIGINT, probably'))
d = wabs_deleter.Deleter(BlobService('test', 'ing'), 'test-container')
with pytest.raises(MarkedKeyboardInterrupt):
d.delete(b)
# Exactly when coroutines are scheduled is non-deterministic,
# so spin while yielding to provoke the
# MarkedKeyboardInterrupt being processed within the
# pytest.raises context manager.
while True:
gevent.sleep(0.1)
# Only one key should have been aborted, since the purpose is to
# *not* retry when processing KeyboardInterrupt.
assert collect.aborted_keys == [key_name]
# Turn off fault injection and flush/synchronize with close().
collect.inject(None)
d.close()
# Since there is no retrying, no keys should be deleted.
assert not collect.deleted_keys
def main():
"""
Run the main loop for the magical bus detection sensor
:return:
"""
blob_service = BlobService(account_name=azureAccount,
account_key=accountKey)
subprocess.call(
'adb shell "am start -a android.media.action.STILL_IMAGE_CAMERA"',
shell=True)
while 1:
distance = readAio()
trigger = trigDistance.read()
print(distance, ' : ', trigger)
if distance < trigger and distance: # probs need to have a better deciding logic
led.write(1)
images = takePhoto()
print images
for image in images:
print("uploading {0}".format(image))
putImage(image, blob_service)
putToApi(images)
deleteImages(images)
led.write(0)
time.sleep(.1)
def make_blob_sas_url(account_name,
account_key,
container_name,
blob_name,
permission='w',
duration=16):
"""
Generate a Blob SAS URL to allow a client to upload a file.
account_name: Storage account name.
account_key: Storage account key.
container_name: Storage container.
blob_name: Blob name.
duration: A timedelta representing duration until SAS expiration.
SAS start date will be utcnow() minus one minute. Expiry date
is start date plus duration.
Returns the SAS URL.
"""
sas = SharedAccessSignature(account_name, account_key)
resource_path = '%s/%s' % (container_name, blob_name)
date_format = "%Y-%m-%dT%H:%M:%SZ"
start = datetime.datetime.utcnow() - datetime.timedelta(minutes=5)
expiry = start + datetime.timedelta(minutes=duration)
sap = SharedAccessPolicy(AccessPolicy(
start.strftime(date_format),
expiry.strftime(date_format),
permission))
sas_token = sas.generate_signed_query_string(resource_path, 'b', sap)
blob_url = BlobService(account_name, account_key)
url = blob_url.make_blob_url(container_name=container_name, blob_name=blob_name, sas_token=sas_token)
return url
def download_azure_blob(account_name, account_key, file_uri, download_dir):
(blob_name, container_name) = parse_blob_uri(file_uri)
host_base = get_host_base_from_uri(file_uri)
download_path = os.path.join(download_dir, blob_name)
blob_service = BlobService(account_name, account_key, host_base=host_base)
max_retry = 3
for retry in range(1, max_retry + 1):
try:
blob_service.get_blob_to_path(container_name, blob_name, download_path)
except Exception:
hutil.error('Failed to download Azure blob, retry = ' + str(retry) + ', max_retry = ' + str(max_retry))
if retry != max_retry:
hutil.log('Sleep 10 seconds')
time.sleep(10)
else:
waagent.AddExtensionEvent(name=ExtensionShortName,
op=Operation.Download,
isSuccess=False,
message="(03303)Failed to download file from Azure Storage")
raise Exception('Failed to download azure blob: ' + blob_name)
waagent.AddExtensionEvent(name=ExtensionShortName,
op=Operation.Download,
isSuccess=True,
message="(03301)Succeeded to download file from Azure Storage")
return download_path
def _get_service(self):
if not hasattr(self, '_blob_service'):
self._blob_service = BlobService(account_name=self.account_name,
account_key=self.account_key,
protocol=self._get_protocol())
return self._blob_service
def test_close_error():
"""Ensure that attempts to use a closed Deleter results in an error."""
d = wabs_deleter.Deleter(BlobService('test', 'ing'), 'test-container')
d.close()
with pytest.raises(exception.UserCritical):
d.delete('no value should work')
def generate_blob_service(self):
if self.blob_service is None:
# if storage info doesn't exist in config.py upload file function stop working
self.blob_service = BlobService(
account_name=self.util.get_config(
"storage.azure.account_name"),
account_key=self.util.get_config("storage.azure.account_key"),
host_base=self.util.get_config(
"storage.azure.blob_service_host_base"))
def connect_stroage(self,
account_name,
account_key,
protocol="http",
host_base=".blob.core.windows.net"):
self.account_info = AccountInfo(None, account_name, account_key,
protocol, host_base)
self.service = BlobService(account_name, account_key, protocol,
host_base)
def download_azure_blob(account_name, account_key, file_uri, download_dir):
waagent.AddExtensionEvent(name=ExtensionShortName,
op="EnableInProgress",
isSuccess=True,
message="Downloading from azure blob")
try:
(blob_name, container_name) = parse_blob_uri(file_uri)
host_base = get_host_base_from_uri(file_uri)
blob_parent_path = os.path.join(download_dir,
os.path.dirname(blob_name))
if not os.path.exists(blob_parent_path):
os.makedirs(blob_parent_path)
download_path = os.path.join(download_dir, blob_name)
blob_service = BlobService(account_name,
account_key,
host_base=host_base)
except Exception as e:
waagent.AddExtensionEvent(
name=ExtensionShortName,
op='DownloadInProgress',
isSuccess=True,
message=
'Enable failed with the azure storage error : {0}, stack trace: {1}'
.format(str(e), traceback.format_exc()))
hutil.error(
'Failed to enable the extension with error: %s, stack trace: %s' %
(str(e), traceback.format_exc()))
hutil.do_exit(1, 'Enable', 'error', '1',
'Enable failed: {0}'.format(e))
max_retry = 3
for retry in range(1, max_retry + 1):
try:
blob_service.get_blob_to_path(container_name, blob_name,
download_path)
except Exception:
hutil.error('Failed to download Azure blob, retry = ' +
str(retry) + ', max_retry = ' + str(max_retry))
if retry != max_retry:
hutil.log('Sleep 10 seconds')
time.sleep(10)
else:
waagent.AddExtensionEvent(
name=ExtensionShortName,
op=Operation.Download,
isSuccess=False,
message="(03303)Failed to download file from Azure Storage"
)
raise Exception('Failed to download azure blob: ' + blob_name)
waagent.AddExtensionEvent(
name=ExtensionShortName,
op=Operation.Download,
isSuccess=True,
message="(03301)Succeeded to download file from Azure Storage")
return download_path
def test_processes_one_deletion(collect):
key_name = 'test-key-name'
b = B(name=key_name)
d = wabs_deleter.Deleter(BlobService('test', 'ing'), 'test-container')
d.delete(b)
d.close()
assert collect.deleted_keys == [key_name]
def setUp(self):
self.workspace = Workspace(
settings.workspace.id,
settings.workspace.token,
settings.workspace.endpoint
)
self.blob = BlobService(
settings.storage.account_name,
settings.storage.account_key
)
def upload_log():
blob_service = BlobService(account_name=os.getenv('ACC_NAME'),
account_key=os.getenv('ACCESS_KEY'))
fpath = os.path.join(os.getenv('LOGS_DIR'), "log.log")
blob_service.put_block_blob_from_path('log',
"log.log",
fpath,
x_ms_blob_content_type="text/plain")
def __init__(self, path=None, config=None):
self._config = config
self._container = self._config.azure_storage_container
protocol = 'https' if self._config.azure_use_https else 'http'
acct_name = self._config.azure_storage_account_name
acct_key = self._config.azure_storage_account_key
self._blob = BlobService(account_name=acct_name,
account_key=acct_key,
protocol=protocol)
self._init_container()
logger.debug("Initialized azureblob storage driver")
def download_blob(storage_account_name, storage_account_key, blob_uri, seqNo,
command, hutil):
container_name = get_container_name_from_uri(blob_uri)
blob_name = get_blob_name_from_uri(blob_uri)
download_dir = get_download_directory(seqNo)
download_path = os.path.join(download_dir, blob_name)
# Guest agent already ensure the plugin is enabled one after another. The blob download will not conflict.
blob_service = BlobService(storage_account_name, storage_account_key)
try:
blob_service.get_blob_to_path(container_name, blob_name, download_path)
except Exception, e:
hutil.error("Failed to download blob with uri:" + blob_uri +
"with error:" + str(e))
raise
def run(itk_source_dir, externaldata_object_store, account_name, account_key):
blob_service = BlobService(account_name=account_name,
account_key=account_key)
blobs = blob_service.list_blobs("md5")
current_blobs = [blob.name for blob in blobs]
md5files = []
for root, dirnames, filenames in os.walk(itk_source_dir):
for filename in fnmatch.filter(filenames, "*.md5"):
md5files.append(os.path.join(root, filename))
for content_link in md5files:
upload_to_azure(content_link, externaldata_object_store, blob_service,
current_blobs)
def __init__(self, account_name, account_key, container_name, prefix=None):
# NOTE: Azure wants all paths to start with a slash
prefix = "/{}".format(prefix.lstrip("/") if prefix else "")
super().__init__(prefix=prefix)
self.account_name = account_name
self.account_key = account_key
self.container_name = container_name
self.conn = BlobService(account_name=self.account_name,
account_key=self.account_key)
self.container = self.get_or_create_container(self.container_name)
self.log.debug("AzureTransfer initialized")
# XXX: AzureTransfer isn't actively tested and hasn't its error handling is probably lacking
self.log.warning(
"AzureTransfer is experimental and has not been thoroughly tested")
def uri_get_file(creds, uri, conn=None):
assert uri.startswith('wabs://')
url_tup = urlparse(uri)
if conn is None:
conn = BlobService(creds.account_name,
creds.account_key,
protocol='https')
# Determin the size of the target blob
props = conn.get_blob_properties(url_tup.netloc, url_tup.path)
blob_size = int(props['content-length'])
ret_size = 0
data = ''
# WABS requires large files to be downloaded in 4MB chunks
while ret_size < blob_size:
ms_range = 'bytes={}-{}'.format(ret_size,
ret_size + WABS_CHUNK_SIZE - 1)
while True:
# Because we're downloading in chunks, catch rate limiting and
# connection errors here instead of letting them bubble up to the
# @retry decorator so that we don't have to start downloading the
# whole file over again.
try:
part = conn.get_blob(url_tup.netloc,
url_tup.path,
x_ms_range=ms_range)
except EnvironmentError as e:
if e.errno in (errno.EBUSY, errno.ECONNRESET):
logger.warning(
msg="retrying after encountering exception",
detail=("Exception traceback:\n{0}".format(
traceback.format_exception(*sys.exc_info()))),
hint="")
gevent.sleep(30)
else:
raise
else:
break
length = len(part)
ret_size += length
data += part
if length > 0 and length < WABS_CHUNK_SIZE:
break
elif length == 0:
break
return data
def _deleteBlob(self, storageAccount, mediaLink):
primary = self._getStorageAccountKey(storageAccount)
# BlobServiceオブジェクトを作成
blobService = BlobService(storageAccount, primary)
(container, blob) = self._getBlobFromMediaLink(blobService, mediaLink)
rs = blobService.delete_blob(container_name=container.name,
blob_name=blob.name)
try:
updatedBlob = blobService.get_blob_properties(
container_name=container.name, blob_name=blob.name)
except WindowsAzureMissingResourceError as e:
return True
return False
def _ensureStorageContainersExist(self):
"""
Creates Blob storage containers required by the service.
"""
logger.info("Checking for existence of Blob containers.")
account_name = self.config.getServiceStorageAccountName()
account_key = self._getStorageAccountKey(account_name)
blob_service = BlobService(account_name, account_key)
name_and_access_list = [(self.config.getServicePublicStorageContainer(), 'blob'),
(self.config.getServiceBundleStorageContainer(), None)]
for name, access in name_and_access_list:
logger.info("Checking for existence of Blob container %s.", name)
blob_service.create_container(name, x_ms_blob_public_access=access, fail_on_exist=False)
access_info = 'private' if access is None else 'public {0}'.format(access)
logger.info("Blob container %s is ready (access: %s).", name, access_info)
def connect(config=False):
# Connect to the cloud service.
if not config: config = misc.config['_private']
from azure.storage import BlobService
container = 'streams'
if not 'azure' in config:
return None, None
blob_service = BlobService(config['azure']['storage_account_name'],
config['azure']['primary_access_key'])
blob_service.create_container(container,
x_ms_blob_public_access='container')
return blob_service, container
def __init__(self,
accountName,
namePrefix,
config=None,
jobChunkSize=maxAzureTablePropertySize):
self.jobChunkSize = jobChunkSize
self.keyPath = None
self.account_key = _fetchAzureAccountKey(accountName)
# Table names have strict requirements in Azure
self.namePrefix = self._sanitizeTableName(namePrefix)
logger.debug("Creating job store with name prefix '%s'" %
self.namePrefix)
# These are the main API entrypoints.
self.tableService = TableService(account_key=self.account_key,
account_name=accountName)
self.blobService = BlobService(account_key=self.account_key,
account_name=accountName)
# Register our job-store in the global table for this storage account
self.registryTable = self._getOrCreateTable('toilRegistry')
exists = self.registryTable.get_entity(row_key=self.namePrefix)
self._checkJobStoreCreation(config is not None, exists,
accountName + ":" + self.namePrefix)
self.registryTable.insert_or_replace_entity(row_key=self.namePrefix,
entity={'exists': True})
# Serialized jobs table
self.jobItems = self._getOrCreateTable(self.qualify('jobs'))
# Job<->file mapping table
self.jobFileIDs = self._getOrCreateTable(self.qualify('jobFileIDs'))
# Container for all shared and unshared files
self.files = self._getOrCreateBlobContainer(self.qualify('files'))
# Stats and logging strings
self.statsFiles = self._getOrCreateBlobContainer(
self.qualify('statsfiles'))
# File IDs that contain stats and logging strings
self.statsFileIDs = self._getOrCreateTable(
self.qualify('statsFileIDs'))
super(AzureJobStore, self).__init__(config=config)
if self.config.cseKey is not None:
self.keyPath = self.config.cseKey
def main():
service = BlobService(
credentials.getStorageServicesName(),
credentials.getStorageServicesKey(),
)
service.create_container(CONTAINER_NAME)
process(service,
LOCAL_BLOCK_BLOB_FILES,
CONNECTION_COUNTS,
is_page_blob=False)
process(service,
LOCAL_PAGE_BLOB_FILES,
CONNECTION_COUNTS,
is_page_blob=True)
def download_blob(storage_account_name, storage_account_key,
blob_uri, dst, hutil):
seqNo = hutil.get_seq_no()
container_name = get_container_name_from_uri(blob_uri)
blob_name = get_blob_name_from_uri(blob_uri)
download_dir = prepare_download_dir(seqNo)
download_path = os.path.join(download_dir, dst)
#Guest agent already ensure the plugin is enabled one after another.
#The blob download will not conflict.
blob_service = BlobService(storage_account_name, storage_account_key)
try:
blob_service.get_blob_to_path(container_name, blob_name, download_path)
except Exception, e:
hutil.log_and_syslog(logging.ERROR, ("Failed to download blob with uri:{0} "
"with error {1}").format(blob_uri,e))
raise
def do_step(context):
settings = context.meta['settings']
# Prepare the containers
storage_account_name = settings["STORAGE-ACCOUNT-NAME"]
storage_access_key = settings["STORAGE-ACCESS-KEY"]
blob_service = BlobService(storage_account_name, storage_access_key)
blob_service.create_container('bosh')
blob_service.create_container(container_name='stemcell', x_ms_blob_public_access='blob')
# Prepare the table for storing meta datas of storage account and stemcells
table_service = TableService(storage_account_name, storage_access_key)
table_service.create_table('stemcells')
context.meta['settings'] = settings
return context
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# Import Microsoft Azure Storage SDK for Python library.
try:
import azure
import azure.storage
if hasattr(azure.storage, 'BlobService'):
# v0.11.1 and below
from azure.storage import BlobService
self.AzureMissingResourceError = azure.WindowsAzureMissingResourceError
self.AzureConflictError = azure.WindowsAzureConflictError
else:
# v1.0.0 and above
from azure.storage.blob import BlobService
self.AzureMissingResourceError = azure.common.AzureMissingResourceHttpError
self.AzureConflictError = azure.common.AzureConflictHttpError
except ImportError:
raise BackendException(
'Azure backend requires Microsoft Azure Storage SDK for Python '
'(https://pypi.python.org/pypi/azure-storage/).')
if 'AZURE_ACCOUNT_NAME' not in os.environ:
raise BackendException(
'AZURE_ACCOUNT_NAME environment variable not set.')
if 'AZURE_ACCOUNT_KEY' not in os.environ:
raise BackendException(
'AZURE_ACCOUNT_KEY environment variable not set.')
self.blob_service = BlobService(
account_name=os.environ['AZURE_ACCOUNT_NAME'],
account_key=os.environ['AZURE_ACCOUNT_KEY'])
# TODO: validate container name
self.container = parsed_url.path.lstrip('/')
try:
self.blob_service.create_container(self.container,
fail_on_exist=True)
except self.AzureConflictError:
# Indicates that the resource could not be created because it already exists.
pass
except Exception as e:
log.FatalError(
"Could not create Azure container: %s" %
unicode(e.message).split('\n', 1)[0],
log.ErrorCode.connection_failed)
def _leaseBlob(self, storageAccount, mediaLink):
primary = self._getStorageAccountKey(storageAccount)
# BlobServiceオブジェクトを作成
blobService = BlobService(storageAccount, primary)
(container, blob) = self._getBlobFromMediaLink(blobService, mediaLink)
prop = blob.properties
# Lease StatusがlockedだったらBlobのリース解放を試みる
if prop.lease_status == 'locked':
# unlockedの時に実行すると、 azure.WindowsAzureConflictError
res = blobService.lease_blob(container_name=container.name,
blob_name=blob.name,
x_ms_lease_action='break')
# (成功すると？){}が返ってくる
updatedBlob = blobService.get_blob_properties(
container_name=container.name, blob_name=blob.name)
def test_processes_many_deletions(collect):
# Generate a target list of keys in a stable order
target = sorted(['test-key-' + str(x) for x in range(20001)])
# Construct boto S3 Keys from the generated names and delete them
# all.
blobs = [B(name=key_name) for key_name in target]
d = wabs_deleter.Deleter(BlobService('test', 'ing'), 'test-container')
for b in blobs:
d.delete(b)
d.close()
# Sort the deleted key names to obtain another stable order and
# then ensure that everything was passed for deletion
# successfully.
assert sorted(collect.deleted_keys) == target
def create_container(storage_account_name, container_name, storage_keys):
"""
Creates a file share in the specified Microsoft Azure Storage account.
A container is like a folder within a storage account
:param storage_account_name:
:param container_name:
:param storage_keys:
:return:
"""
logging.info('Creating Container \'{0}\' in Storage account {1}'.format(
container_name, storage_account_name))
blob_svc = BlobService(
account_name=storage_account_name,
account_key=storage_keys.storage_service_keys.primary)
blob_svc.create_container(container_name)
logging.info(
'Creating Container \'{0}\' in Storage account {1} complete'.format(
container_name, storage_account_name))
