def test_azure_call(request): import os try: from azure.storage import BlobService bs = BlobService(os.environ["AZURE_STORAGE_ACCOUNT"], os.environ["AZURE_STORAGE_ACCESS_KEY"]) import random container_name = hex(int(random.random() * 1000000000)) bs.create_container(container_name) bs.put_blob(container_name, 'testblob', 'hello world\n', 'BlockBlob') blob = bs.get_blob(container_name, 'testblob') if blob != 'hello world\n': return HttpResponse("Failed!", status = '404') bs.delete_blob(container_name, 'testblob') bs.delete_container(container_name) return HttpResponse("Succeeded!") except: try: import traceback return HttpResponse(traceback.format_exc() + str(os.environ.keys())) except: import traceback return HttpResponse(traceback.format_exc())
class FileService(Component): def __init__(self): self.blob_service = None def generate_blob_service(self): if self.blob_service is None: # if storage info doesn't exist in config.py upload file function stop working self.blob_service = BlobService(account_name=self.util.get_config("storage.azure.account_name"), account_key=self.util.get_config("storage.azure.account_key"), host_base=self.util.get_config("storage.azure.blob_service_host_base")) def create_container_in_storage(self, container_name, access): """ create a container if doesn't exist :param container_name: :param access: :return: """ self.generate_blob_service() try: names = map(lambda x: x.name, self.blob_service.list_containers()) if container_name not in names: self.blob_service.create_container(container_name, x_ms_blob_public_access=access) else: self.log.debug("container already exsit in storage") return True except Exception as e: self.log.error(e) return False def upload_file_to_azure(self, stream, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_file(container_name, blob_name, stream) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_path(self, path, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_path(container_name, blob_name, path) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def delete_file_from_azure(self, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.delete_blob(container_name, blob_name) except Exception as e: self.log.error(e) return None
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, container_name): BaseTransfer.__init__(self) self.account_name = account_name self.account_key = account_key self.container_name = container_name self.conn = BlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized") def get_metadata_for_key(self, key): key = fix_path(key) return self.list_path(key)[0]['metadata'] def list_path(self, path): return_list = [] path = fix_path(path) self.log.info("Asking for listing of: %r", path) for r in self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata"): entry = {"name": r.name, "size": r.properties.content_length, "last_modified": dateutil.parser.parse(r.properties.last_modified), "metadata": r.metadata} return_list.append(entry) return return_list def delete_key(self, key_name): key_name = fix_path(key_name) self.log.debug("Deleting key: %r", key_name) return self.conn.delete_blob(self.container_name, key_name) def get_contents_to_file(self, obj_key, filepath_to_store_to): obj_key = fix_path(obj_key) self.log.debug("Starting to fetch the contents of: %r to: %r", obj_key, filepath_to_store_to) return self.conn.get_blob_to_path(self.container_name, obj_key, filepath_to_store_to) def get_contents_to_string(self, obj_key): obj_key = fix_path(obj_key) self.log.debug("Starting to fetch the contents of: %r", obj_key) return self.conn.get_blob_to_bytes(self.container_name, obj_key), self.get_metadata_for_key(obj_key) def store_file_from_memory(self, key, memstring, metadata=None): # For whatever reason Azure requires all values to be strings at the point of sending metadata_to_send = dict((str(k), str(v)) for k, v in metadata.items()) self.conn.put_block_blob_from_bytes(self.container_name, key, memstring, x_ms_meta_name_values=metadata_to_send) def store_file_from_disk(self, key, filepath, metadata=None): # For whatever reason Azure requires all values to be strings at the point of sending metadata_to_send = dict((str(k), str(v)) for k, v in metadata.items()) self.conn.put_block_blob_from_path(self.container_name, key, filepath, x_ms_meta_name_values=metadata_to_send) def get_or_create_container(self, container_name): start_time = time.time() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time) return container_name
def main(): service = BlobService( credentials.getStorageServicesName(), credentials.getStorageServicesKey(), ) service.create_container(CONTAINER_NAME) process(service, LOCAL_BLOCK_BLOB_FILES, CONNECTION_COUNTS, is_page_blob=False) process(service, LOCAL_PAGE_BLOB_FILES, CONNECTION_COUNTS, is_page_blob=True)
def main(argv): options = _parse_options(argv) blob_service = BlobService(options[_opt_azure_acc_name], options[_opt_azure_acc_key]) _print_container_names(blob_service) blob_service.create_container(options[_opt_azure_container_name]) _print_blobs(blob_service, options[_opt_azure_container_name]) export_dir = _export_repo(options) export_zip = _zip_export(export_dir) _upload_zip(blob_service, options[_opt_azure_container_name], export_zip)
def connect(config=False): # Connect to the cloud service. if not config: config = misc.config['_private'] from azure.storage import BlobService container = 'streams' if not 'azure' in config: return None, None blob_service = BlobService(config['azure']['storage_account_name'], config['azure']['primary_access_key']) blob_service.create_container(container, x_ms_blob_public_access='container') return blob_service, container
def create_container(storage_account_name, container_name, storage_keys): """ Creates a file share in the specified Microsoft Azure Storage account. A container is like a folder within a storage account :param storage_account_name: :param container_name: :param storage_keys: :return: """ logging.info('Creating Container \'{0}\' in Storage account {1}'.format(container_name, storage_account_name)) blob_svc = BlobService(account_name=storage_account_name, account_key=storage_keys.storage_service_keys.primary) blob_svc.create_container(container_name) logging.info('Creating Container \'{0}\' in Storage account {1} complete'.format(container_name, storage_account_name))
def connect(config=False): # Connect to the cloud service. if not config: config = misc.config['_private'] from azure.storage import BlobService container = 'streams' if not 'azure' in config: return None, None blob_service = BlobService(config['azure']['storage_account_name'], config['azure']['primary_access_key']) blob_service.create_container(container, x_ms_blob_public_access='container') return blob_service, container
def _ensureStorageContainersExist(self): """ Creates Blob storage containers required by the service. """ logger.info("Checking for existence of Blob containers.") account_name = self.config.getServiceStorageAccountName() account_key = self._getStorageAccountKey(account_name) blob_service = BlobService(account_name, account_key) name_and_access_list = [(self.config.getServicePublicStorageContainer(), 'blob'), (self.config.getServiceBundleStorageContainer(), None)] for name, access in name_and_access_list: logger.info("Checking for existence of Blob container %s.", name) blob_service.create_container(name, x_ms_blob_public_access=access, fail_on_exist=False) access_info = 'private' if access is None else 'public {0}'.format(access) logger.info("Blob container %s is ready (access: %s).", name, access_info)
def _ensureStorageContainersExist(self): """ Creates Blob storage containers required by the service. """ logger.info("Checking for existence of Blob containers.") account_name = self.config.getServiceStorageAccountName() account_key = self._getStorageAccountKey(account_name) blob_service = BlobService(account_name, account_key) name_and_access_list = [(self.config.getServicePublicStorageContainer(), 'blob'), (self.config.getServiceBundleStorageContainer(), None)] for name, access in name_and_access_list: logger.info("Checking for existence of Blob container %s.", name) blob_service.create_container(name, x_ms_blob_public_access=access, fail_on_exist=False) access_info = 'private' if access is None else 'public {0}'.format(access) logger.info("Blob container %s is ready (access: %s).", name, access_info)
def main(): service = BlobService( credentials.getStorageServicesName(), credentials.getStorageServicesKey(), ) service.create_container(CONTAINER_NAME) process(service, LOCAL_BLOCK_BLOB_FILES, CONNECTION_COUNTS, is_page_blob=False) process(service, LOCAL_PAGE_BLOB_FILES, CONNECTION_COUNTS, is_page_blob=True)
def do_step(context): settings = context.meta['settings'] # Prepare the containers storage_account_name = settings["STORAGE-ACCOUNT-NAME"] storage_access_key = settings["STORAGE-ACCESS-KEY"] blob_service = BlobService(storage_account_name, storage_access_key) blob_service.create_container('bosh') blob_service.create_container(container_name='stemcell', x_ms_blob_public_access='blob') # Prepare the table for storing meta datas of storage account and stemcells table_service = TableService(storage_account_name, storage_access_key) table_service.create_table('stemcells') context.meta['settings'] = settings return context
def do_step(context): settings = context.meta['settings'] # Prepare the containers storage_account_name = settings["STORAGE-ACCOUNT-NAME"] storage_access_key = settings["STORAGE-ACCESS-KEY"] blob_service = BlobService(storage_account_name, storage_access_key) blob_service.create_container('bosh') blob_service.create_container(container_name='stemcell', x_ms_blob_public_access='blob') # Prepare the table for storing meta datas of storage account and stemcells table_service = TableService(storage_account_name, storage_access_key) table_service.create_table('stemcells') context.meta['settings'] = settings return context
class AzureBlobStorage(StorageBase): def __init__(self, account_name, account_key, container_name): self.__container_name = container_name self.__blob_service = BlobService(account_name=account_name, account_key=account_key) self.__blob_service.create_container(container_name) def get(self, key, default=None): logger.info('get: key = %s' % key) return pickle.loads(self.__blob_service .get_blob_to_text(self.__container_name, key)) def set(self, key, value): logger.info('get: key = %s, value = %s' % (key, value)) self.__blob_service.put_block_blob_from_text(self.__container_name, key, pickle.dumps(value))
def create_container(storage_account_name, container_name, storage_keys): """ Creates a file share in the specified Microsoft Azure Storage account. A container is like a folder within a storage account :param storage_account_name: :param container_name: :param storage_keys: :return: """ logging.info('Creating Container \'{0}\' in Storage account {1}'.format( container_name, storage_account_name)) blob_svc = BlobService( account_name=storage_account_name, account_key=storage_keys.storage_service_keys.primary) blob_svc.create_container(container_name) logging.info( 'Creating Container \'{0}\' in Storage account {1} complete'.format( container_name, storage_account_name))
class BlobCache: ''' Simplistic cache toolkit targetting an Azure Blob Service. name: the name of a storage account. key: the access key for the storage account. container: the name of the container to use. ''' def __init__(self, name, key, container): self.container = container self.blobstore = BlobService(name, key) self.blobstore.create_container(self.container) def getresponse(self, cachekey): ''' Get a value from the cache. cachekey: The key. Kilroy notes that this throws an exception rather than returning a value on failure. ''' return self.blobstore.get_blob_to_text(self.container, str2blobname(cachekey)) def putresponse(self, cachekey, value): ''' Put a value in the cache with the given key. cachekey: The key. value: The value to associate with the key. ''' return self.blobstore.put_block_blob_from_text(self.container, str2blobname(cachekey), value) def invalidate(self, cachekey): ''' Invalidate a value in the cache. Immediate. Permanent. cachekey: The key. ''' self.blobstore.delete_blob(self.container, str2blobname(cachekey))
class BlobCache: ''' Simplistic cache toolkit targetting an Azure Blob Service. name: the name of a storage account. key: the access key for the storage account. container: the name of the container to use. ''' def __init__(self, name, key, container): self.container = container self.blobstore = BlobService(name, key) self.blobstore.create_container(self.container) def getresponse(self, cachekey): ''' Get a value from the cache. cachekey: The key. Kilroy notes that this throws an exception rather than returning a value on failure. ''' return self.blobstore.get_blob_to_text(self.container,str2blobname(cachekey)) def putresponse(self, cachekey, value): ''' Put a value in the cache with the given key. cachekey: The key. value: The value to associate with the key. ''' return self.blobstore.put_block_blob_from_text(self.container, str2blobname(cachekey), value) def invalidate(self, cachekey): ''' Invalidate a value in the cache. Immediate. Permanent. cachekey: The key. ''' self.blobstore.delete_blob(self.container, str2blobname(cachekey))
class AzureBackend(BakthatBackend): """Backend to handle Azure upload/download.""" def __init__(self, conf={}, profile="default"): BakthatBackend.__init__(self, conf, profile) self.blob_service = BlobService(self.conf["account_name"], self.conf["account_key"]) try: self.containe_properties = self.blob_service.get_container_properties(self.conf["azure_container"]) except WindowsAzureError, e: if type(e)==WindowsAzureMissingResourceError: self.containe_properties =self.blob_service.create_container(self.conf["azure_container"]) else: raise e self.container = self.conf["azure_container"] self.container_key = "azure_container"
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, container_name, prefix=None): # NOTE: Azure wants all paths to start with a slash prefix = "/{}".format(prefix.lstrip("/") if prefix else "") super().__init__(prefix=prefix) self.account_name = account_name self.account_key = account_key self.container_name = container_name self.conn = BlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized") # XXX: AzureTransfer isn't actively tested and hasn't its error handling is probably lacking self.log.warning("AzureTransfer is experimental and has not been thoroughly tested") def get_metadata_for_key(self, key): key = self.format_key_for_backend(key) return self._list_blobs(key)[0]["metadata"] def _metadata_for_key(self, key): return self._list_blobs(key)[0]["metadata"] def list_path(self, key): path = self.format_key_for_backend(key, trailing_slash=True) return self._list_blobs(path) def _list_blobs(self, path): self.log.debug("Listing path %r", path) items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata") result = [] for item in items: result.append({ "last_modified": dateutil.parser.parse(item.properties.last_modified), "metadata": item.metadata, "name": self.format_key_from_backend(item.name), "size": item.properties.content_length, }) return result def delete_key(self, key): key = self.format_key_for_backend(key) self.log.debug("Deleting key: %r", key) return self.conn.delete_blob(self.container_name, key) def get_contents_to_file(self, key, filepath_to_store_to): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to) return self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to) def get_contents_to_fileobj(self, key, fileobj_to_store_to): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_file(self.container_name, key, fileobj_to_store_to) def get_contents_to_string(self, key): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_bytes(self.container_name, key), self._metadata_for_key(key) def store_file_from_memory(self, key, memstring, metadata=None): key = self.format_key_for_backend(key) # Azure requires all metadata keys and values to be strings metadata_to_send = {str(k): str(v) for k, v in metadata.items()} self.conn.put_block_blob_from_bytes(self.container_name, key, memstring, x_ms_meta_name_values=metadata_to_send) def store_file_from_disk(self, key, filepath, metadata=None, multipart=None): key = self.format_key_for_backend(key) # Azure requires all metadata keys and values to be strings metadata_to_send = {str(k): str(v) for k, v in metadata.items()} self.conn.put_block_blob_from_path(self.container_name, key, filepath, x_ms_meta_name_values=metadata_to_send) def get_or_create_container(self, container_name): start_time = time.time() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time) return container_name
class Storage(driver.Base): supports_bytes_range = True def __init__(self, path=None, config=None): self._config = config self._container = self._config.azure_storage_container protocol = 'https' if self._config.azure_use_https else 'http' acct_name = self._config.azure_storage_account_name acct_key = self._config.azure_storage_account_key self._blob = BlobService(account_name=acct_name, account_key=acct_key, protocol=protocol) self._init_container() logger.debug("Initialized azureblob storage driver") def _init_container(self): '''Initializes image container on Azure blob storage if the container does not exist. ''' created = self._blob.create_container(self._container, x_ms_blob_public_access='blob', fail_on_exist=False) if created: logger.info('Created blob container for image registry.') else: logger.debug('Registry container already exists.') return created @lru.get def get_content(self, path): try: return self._blob.get_blob(self._container, path) except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path) @lru.set def put_content(self, path, content): self._blob.put_blob(self._container, path, content, 'BlockBlob') return path def stream_read(self, path, bytes_range=None): try: f = io.BytesIO() self._blob.get_blob_to_file(self._container, path, f) if bytes_range: f.seek(bytes_range[0]) total_size = bytes_range[1] - bytes_range[0] + 1 else: f.seek(0) while True: buf = None if bytes_range: # Bytes Range is enabled buf_size = self.buffer_size if nb_bytes + buf_size > total_size: # We make sure we don't read out of the range buf_size = total_size - nb_bytes if buf_size > 0: buf = f.read(buf_size) nb_bytes += len(buf) else: # We're at the end of the range buf = '' else: buf = f.read(self.buffer_size) if not buf: break yield buf except IOError: raise exceptions.FileNotFoundError('%s is not there' % path) def stream_write(self, path, fp): self._blob.put_block_blob_from_file(self._container, path, fp) def list_directory(self, path=None): if not path.endswith('/'): path += '/' # path=a would list a/b.txt as well as 'abc.txt' blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) return [b.name for b in blobs] def exists(self, path): try: self._blob.get_blob_properties(self._container, path) return True except azure.WindowsAzureMissingResourceError: return False @lru.remove def remove(self, path): is_blob = self.exists(path) if is_blob: self._blob.delete_blob(self._container, path) return exists = False blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) for b in blobs: self._blob.delete_blob(self._container, b.name) def get_size(self, path): try: properties = self._blob.get_blob_properties(self._container, path) return int(properties['content-length']) # auto-converted to long except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path)
class BlobServiceAdapter(Component): """The :class:`BlobServiceAdapter` class is a thin wrapper over azure.storage.BlobService. All the attributes of the wrapper stream are proxied by the adapter so it's possible to do ``adapter.create_container()`` instead of the long form ``adapter.blob_service.adapter()``. """ def __init__(self): self.blob_service = BlobService(account_name=self.util.get_config("storage.azure.account_name"), account_key=self.util.get_config("storage.azure.account_key"), host_base=self.util.get_config("storage.azure.blob_service_host_base")) def __getattr__(self, name): return getattr(self.blob_service, name) def create_container_in_storage(self, container_name, access="container"): """create a container if doesn't exist :type container_name: str|unicode :param container_name: Name of container to create. :type access: str|unicode :param access: Optional. Possible values include: container, blob :return: """ try: names = [x.name for x in self.blob_service.list_containers()] if container_name not in names: return self.blob_service.create_container(container_name, x_ms_blob_public_access=access) else: self.log.debug("container already exists in storage") return True except Exception as e: self.log.error(e) return False def upload_file_to_azure(self, container_name, blob_name, stream): """ Creates a new block blob from a file/stream, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str | unicode :param blob_name: Name of blob to create or update. :type stream: file :param stream: Opened file/stream to upload as the blob content. """ try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_file(container_name, blob_name, stream) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_bytes(self, container_name, blob_name, blob): """ Creates a new block blob from an array of bytes, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type blob: bytes :param blob: Content of blob as an array of bytes. """ try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_bytes(container_name, blob_name, blob) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_text(self, container_name, blob_name, text): """ Creates a new block blob from str/unicode, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type text: str|unicode :param text: Text to upload to the blob. """ try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_text(container_name, blob_name, text) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_path(self, container_name, blob_name, path): """ Creates a new page blob from a file path, or updates the content of an existing page blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type path: str|unicode :param path: Path of the file to upload as the blob content. """ try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_path(container_name, blob_name, path) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def delete_file_from_azure(self, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.delete_blob(container_name, blob_name) except Exception as e: self.log.error(e) return None
def upload(): file = request.files['fileInput'] print "File is" + file.filename if file: data = file.read() blob_service = BlobService(account_name='squadshots', account_key='UgxaWKAKv2ZvhHrPt0IHi4EQedPpZw35r+RXkAYB2eICPrG3TjSwk2G8gUzG/PNDDTV+4CVCYWCvZSiad5xMQQ==') blob_service.create_container('album') blob_service.put_block_blob_from_bytes( 'album', file.filename + "_blob", data, x_ms_blob_content_type='image/png' ) if 'username' in session: un = session['username'] else: print "not in session" blob_service.set_blob_metadata(container_name="album", blob_name=file.filename + "_blob", x_ms_meta_name_values={'metaun': un}) blob_service.get_blob_to_path('album',file.filename + "_blob",'static/output.png') f = open('input_person.png','w+') f.write(data) f.close() [X,y] = read_images(OUTPUT_DIRECTORY, (256,256)) # Convert labels to 32bit integers. This is a workaround for 64bit machines, y = np.asarray(y, dtype=np.int32) # Create the Eigenfaces model. model = cv2.createEigenFaceRecognizer() # Learn the model. Remember our function returns Python lists, # so we use np.asarray to turn them into NumPy lists to make # the OpenCV wrapper happy: model.train(np.asarray(X), np.asarray(y)) # Save the model for later use model.save("eigenModel.xml") # Create an Eign Face recogniser t = float(100000) model = cv2.createEigenFaceRecognizer(threshold=t) # Load the model model.load("eigenModel.xml") # Read the image we're looking for try: sampleImage = cv2.imread('static/output.png', cv2.IMREAD_GRAYSCALE) if sampleImage != None: sampleImage = cv2.resize(sampleImage, (256,256)) else: print "sample image is null" except IOError: print "IO error" # Look through the model and find the face it matches [p_label, p_confidence] = model.predict(sampleImage) # Print the confidence levels print "Predicted label = %d (confidence=%.2f)" % (p_label, p_confidence) # If the model found something, print the file path if (p_label > -1): count = 0 for dirname, dirnames, filenames in os.walk(OUTPUT_DIRECTORY): for subdirname in dirnames: subject_path = os.path.join(dirname, subdirname) if (count == p_label): for filename in os.listdir(subject_path): print "subject path = " + subject_path count = count+1 return "uploaded"
class AzureFS(LoggingMixIn, Operations): """Azure Blob Storage filesystem""" blobs = None containers = dict() # <cname, dict(stat:dict, #files:None|dict<fname, stat>) fds = dict() # <fd, (path, bytes, dirty)> fd = 0 def __init__(self, account, key): self.blobs = BlobService(account, key) self.rebuild_container_list() def convert_to_epoch(self, date): """Converts Tue, 31 Jul 2012 07:17:34 GMT format to epoch""" return int(time.mktime(time.strptime(date, TIME_FORMAT))) def rebuild_container_list(self): cmap = dict() cnames = set() for c in self.blobs.list_containers(): date = c.properties.last_modified cstat = dict(st_mode=(S_IFDIR | 0755), st_uid=getuid(), st_size=0, st_mtime=self.convert_to_epoch(date)) cname = c.name cmap['/' + cname] = dict(stat=cstat, files=None) cnames.add(cname) cmap['/'] = dict(files={}, stat=dict(st_mode=(S_IFDIR | 0755), st_uid=getuid(), st_size=0, st_mtime=int(time.time()))) self.containers = cmap # destroys fs tree cache resistant to misses def _parse_path(self, path): # returns </dir, file(=None)> if path.count('/') > 1: # file return str(path[:path.rfind('/')]), str(path[path.rfind('/') + 1:]) else: # dir pos = path.rfind('/', 1) if pos == -1: return path, None else: return str(path[:pos]), None def parse_container(self, path): base_container = path[1:] # /abc/def/g --> abc if base_container.find('/') > -1: base_container = base_container[:base_container.find('/')] return str(base_container) def _get_dir(self, path, contents_required=False): if not self.containers: self.rebuild_container_list() if path in self.containers and not (contents_required and \ self.containers[path]['files'] is None): return self.containers[path] cname = self.parse_container(path) if '/' + cname not in self.containers: raise FuseOSError(ENOENT) else: if self.containers['/' + cname]['files'] is None: # fetch contents of container log.info("------> CONTENTS NOT FOUND: %s" % cname) blobs = self.blobs.list_blobs(cname) dirstat = dict(st_mode=(S_IFDIR | 0755), st_size=0, st_uid=getuid(), st_mtime=time.time()) if self.containers['/' + cname]['files'] is None: self.containers['/' + cname]['files'] = dict() for f in blobs: blob_name = f.name blob_date = f.properties.last_modified blob_size = long(f.properties.content_length) node = dict(st_mode=(S_IFREG | 0644), st_size=blob_size, st_mtime=self.convert_to_epoch(blob_date), st_uid=getuid()) if blob_name.find('/') == -1: # file just under container self.containers['/' + cname]['files'][blob_name] = node return self.containers['/' + cname] return None def _get_file(self, path): d, f = self._parse_path(path) dir = self._get_dir(d, True) if dir is not None and f in dir['files']: return dir['files'][f] def getattr(self, path, fh=None): d, f = self._parse_path(path) if f is None: dir = self._get_dir(d) return dir['stat'] else: file = self._get_file(path) if file: return file raise FuseOSError(ENOENT) # FUSE def mkdir(self, path, mode): if path.count('/') <= 1: # create on root name = path[1:] if not 3 <= len(name) <= 63: log.error("Container names can be 3 through 63 chars long.") raise FuseOSError(ENAMETOOLONG) if name is not name.lower(): log.error("Container names cannot contain uppercase \ characters.") raise FuseOSError(EACCES) if name.count('--') > 0: log.error('Container names cannot contain consecutive \ dashes (-).') raise FuseOSError(EAGAIN) #TODO handle all "-"s must be preceded by letter or numbers #TODO starts with only letter or number, can contain letter, nr,'-' resp = self.blobs.create_container(name) if resp: self.rebuild_container_list() log.info("CONTAINER %s CREATED" % name) else: raise FuseOSError(EACCES) log.error("Invalid container name or container already \ exists.") else: raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs def rmdir(self, path): if path.count('/') == 1: c_name = path[1:] resp = self.blobs.delete_container(c_name) if resp: if path in self.containers: del self.containers[path] else: raise FuseOSError(EACCES) else: raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs def create(self, path, mode): node = dict(st_mode=(S_IFREG | mode), st_size=0, st_nlink=1, st_uid=getuid(), st_mtime=time.time()) d, f = self._parse_path(path) if not f: log.error("Cannot create files on root level: /") raise FuseOSError(ENOSYS) dir = self._get_dir(d, True) if not dir: raise FuseOSError(EIO) dir['files'][f] = node return self.open(path, data='') # reusing handler provider def open(self, path, flags=0, data=None): if data == None: # download contents c_name = self.parse_container(path) f_name = path[path.find('/', 1) + 1:] try: data = self.blobs.get_blob(c_name, f_name) except WindowsAzureMissingResourceError: dir = self._get_dir('/' + c_name, True) if f_name in dir['files']: del dir['files'][f_name] raise FuseOSError(ENOENT) except WindowsAzureError as e: log.error("Read blob failed HTTP %d" % e.code) raise FuseOSError(EAGAIN) self.fd += 1 self.fds[self.fd] = (path, data, False) return self.fd def flush(self, path, fh=None): if not fh: raise FuseOSError(EIO) else: if fh not in self.fds: raise FuseOSError(EIO) path = self.fds[fh][0] data = self.fds[fh][1] dirty = self.fds[fh][2] if not dirty: return 0 # avoid redundant write d, f = self._parse_path(path) c_name = self.parse_container(path) if data is None: data = '' try: if len(data) < 64 * 1024 * 1024: # 64 mb self.blobs.put_blob(c_name, f, data, 'BlockBlob') else: # divide file by blocks and upload block_size = 8 * 1024 * 1024 num_blocks = int(math.ceil(len(data) * 1.0 / block_size)) rd = str(random.randint(1, 1e8)) block_ids = list() for i in range(num_blocks): part = data[i * block_size:min((i + 1) * block_size, len(data))] block_id = base64.encodestring('%s_%s' % (rd, (8 - len(str(i))) * '0' + str(i))) self.blobs.put_block(c_name, f, part, block_id) block_ids.append(block_id) self.blobs.put_block_list(c_name, f, block_ids) except WindowsAzureError: raise FuseOSError(EAGAIN) dir = self._get_dir(d, True) if not dir or f not in dir['files']: raise FuseOSError(EIO) # update local data dir['files'][f]['st_size'] = len(data) dir['files'][f]['st_mtime'] = time.time() self.fds[fh] = (path, data, False) # mark as not dirty return 0 def release(self, path, fh=None): if fh is not None and fh in self.fds: del self.fds[fh] def truncate(self, path, length, fh=None): return 0 # assume done, no need def write(self, path, data, offset, fh=None): if not fh or fh not in self.fds: raise FuseOSError(ENOENT) else: d = self.fds[fh][1] if d is None: d = "" self.fds[fh] = (self.fds[fh][0], d[:offset] + data, True) return len(data) def unlink(self, path): c_name = self.parse_container(path) d, f = self._parse_path(path) try: self.blobs.delete_blob(c_name, f) _dir = self._get_dir(path, True) if _dir and f in _dir['files']: del _dir['files'][f] return 0 except WindowsAzureMissingResourceError: raise FuseOSError(ENOENT) except Exception as e: raise FuseOSError(EAGAIN) def readdir(self, path, fh): if path == '/': return ['.', '..'] + [x[1:] for x in self.containers.keys() \ if x is not '/'] dir = self._get_dir(path, True) if not dir: raise FuseOSError(ENOENT) return ['.', '..'] + dir['files'].keys() def read(self, path, size, offset, fh): if not fh or fh not in self.fds: raise FuseOSError(ENOENT) f_name = path[path.find('/', 1) + 1:] c_name = path[1:path.find('/', 1)] try: data = self.blobs.get_blob(c_name, f_name) self.fds[fh] = (self.fds[fh][0], data, False) return data[offset:offset + size] except URLError, e: if e.code == 404: raise FuseOSError(ENOENT) elif e.code == 403: raise FUSEOSError(EPERM) else: log.error("Read blob failed HTTP %d" % e.code) raise FuseOSError(EAGAIN) data = self.fds[fh][1] if data is None: data = "" return data[offset:offset + size]
imagesQueue = 'imagesqueue' tableName = 'photos' tablePartitionKey = 'allphotos' # Get queue credentials # accountName = environ["AZURE_STORAGE_ACCOUNT"] with open("ASA.key", "r") as myfile: accountName = myfile.read().replace('\n', '') # accountKey = environ["AZURE_STORAGE_ACCESS_KEY"] with open("ASK.key", "r") as myfile: accountKey = myfile.read().replace('\n', '') # Create blob service blob_service = BlobService(account_name=accountName, account_key=accountKey) blob_service.create_container(blob_container) blob_service.create_container(blob_analysis) # Open queue with given credentials queue_service = QueueService(account_name=accountName, account_key=accountKey) # Open table service table_service = TableService(account_name=accountName, account_key=accountKey) # Analysis results results = None # Regions for analysis region = 4 # Repeat while (True):
imagesQueue = 'imagesqueue' tableName = 'photos' tablePartitionKey = 'allphotos' # Get queue credentials # accountName = environ["AZURE_STORAGE_ACCOUNT"] with open ("ASA.key", "r") as myfile: accountName=myfile.read().replace('\n', '') # accountKey = environ["AZURE_STORAGE_ACCESS_KEY"] with open ("ASK.key", "r") as myfile: accountKey=myfile.read().replace('\n', '') # Create blob service blob_service = BlobService( account_name=accountName, account_key=accountKey ) blob_service.create_container( blob_container ) blob_service.create_container( blob_analysis ) # Open queue with given credentials queue_service = QueueService( account_name=accountName, account_key=accountKey ) # Open table service table_service = TableService( account_name=accountName, account_key=accountKey ) # Analysis results results = None # Regions for analysis region = 4 # Repeat while(True):
timeStamp = datetime.today().strftime("%Y-%m-%dT%H%M%SZ") pathToZip = '' sevenZip = '' fileName = 'Backup_'+timeStamp+'.7z' activeContainer = '' if platform.system() == 'Windows' : pathToZip = 'C:\\SQLBackups\\' + fileName sevenZip = 'C:\\SQLBackups\\7Zip\\7za.exe' activeContainer = regVPS else: # sudo apt-get install p7zip-full pathToZip = '/var/tmp/'+fileName sevenZip = '7z' activeContainer = DOVPS #Create archive.7z containing Backups Directory with max compression zipArgs = "a "+ pathToZip +" Backups -mx9" subprocess.call(sevenZip + " " + zipArgs) blob_service = BlobService('storageAccountName', 'storageKey') blob_service.create_container(regVPS) blob_service.create_container(DOVPS) blob_service.put_block_blob_from_path(activeContainer, fileName, pathToZip) os.remove(fileName)
class FileService(Component): def __init__(self): self.blob_service = None def generate_blob_service(self): if self.blob_service is None: # if storage info doesn't exist in config.py upload file function stop working self.blob_service = BlobService( account_name=self.util.get_config( "storage.azure.account_name"), account_key=self.util.get_config("storage.azure.account_key"), host_base=self.util.get_config( "storage.azure.blob_service_host_base")) def create_container_in_storage(self, container_name, access): """ create a container if doesn't exist :param container_name: :param access: :return: """ self.generate_blob_service() try: names = map(lambda x: x.name, self.blob_service.list_containers()) if container_name not in names: self.blob_service.create_container( container_name, x_ms_blob_public_access=access) else: self.log.debug("container already exsit in storage") return True except Exception as e: self.log.error(e) return False def upload_file_to_azure(self, stream, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_file( container_name, blob_name, stream) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_path(self, path, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_path( container_name, blob_name, path) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def delete_file_from_azure(self, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.delete_blob(container_name, blob_name) except Exception as e: self.log.error(e) return None
# Get settings from CustomScriptForLinux extension configurations waagent.LoggerInit('/var/log/waagent.log', '/dev/stdout') hutil = Util.HandlerUtility(waagent.Log, waagent.Error, "bosh-deploy-script") hutil.do_parse_context("enable") settings = hutil.get_public_settings() with open(os.path.join('bosh', 'settings'), "w") as tmpfile: tmpfile.write(json.dumps(settings, indent=4, sort_keys=True)) username = settings["username"] home_dir = os.path.join("/home", username) install_log = os.path.join(home_dir, "install.log") # Prepare the containers storage_account_name = settings["STORAGE-ACCOUNT-NAME"] storage_access_key = settings["STORAGE-ACCESS-KEY"] blob_service = BlobService(storage_account_name, storage_access_key) blob_service.create_container('bosh') blob_service.create_container('stemcell') # Generate the private key and certificate call("sh create_cert.sh", shell=True) call("cp bosh.key ./bosh/bosh", shell=True) with open('bosh_cert.pem', 'r') as tmpfile: ssh_cert = tmpfile.read() indentation = " " * 8 ssh_cert = ("\n" + indentation).join([line for line in ssh_cert.split('\n')]) # Render the yml template for bosh-init bosh_template = 'bosh.yml' if os.path.exists(bosh_template): with open(bosh_template, 'r') as tmpfile: contents = tmpfile.read()
def do_step(context): settings = context.meta['settings'] index_file = context.meta['index-file'] pivnetAPIToken = settings["pivnet-api-token"] f = open("manifests/{0}".format(index_file)) manifests = yaml.safe_load(f) f.close() eula_urls = [ "https://network.pivotal.io/api/v2/products/{0}/releases/{1}/eula_acceptance" .format(m['release-name'], m['release-number']) for m in manifests['manifests'] ] release_urls = [ "https://network.pivotal.io/api/v2/products/{0}/releases/{1}/product_files/{2}/download" .format(m['release-name'], m['release-number'], m['file-number']) for m in manifests['manifests'] ] stemcell_urls = [m['stemcell'] for m in manifests['manifests']] # accept eula for each product for url in eula_urls: print url if not "concourse" in url: res = authorizedPost(url, pivnetAPIToken) code = res.getcode() # releases is_release_file = re.compile("^releases\/.+") if not os.path.exists("/tmp/releases"): os.makedirs("/tmp/releases") client = bosh_client.BoshClient("https://10.0.0.4:25555", "admin", "admin") storage_account_name = settings["STORAGE-ACCOUNT-NAME"] storage_access_key = settings["STORAGE-ACCESS-KEY"] blob_service = BlobService(storage_account_name, storage_access_key) blob_service.create_container(container_name='tempreleases', x_ms_blob_public_access='container') print "Processing releases." for url in release_urls: print "Downloading {0}.".format(url) if "concourse" in url: release_url = "https://s3-us-west-2.amazonaws.com/bosh-azure-releases/concourse.zip" res = urllib2.urlopen(release_url) else: res = authorizedPost(url, pivnetAPIToken) code = res.getcode() length = int(res.headers["Content-Length"]) # content-length if code is 200: total = 0 pcent = 0.0 CHUNK = 16 * 1024 with tempfile.TemporaryFile() as temp: while True: chunk = res.read(CHUNK) total += CHUNK pcent = (float(total) / float(length)) * 100 sys.stdout.write("Download progress: %.2f%% (%.2fM)\r" % (pcent, total / 1000000.0)) sys.stdout.flush() if not chunk: break temp.write(chunk) print "Download complete." z = zipfile.ZipFile(temp) for name in z.namelist(): # is this a release? if is_release_file.match(name): release_filename = "/tmp/{0}".format(name) print "Unpacking {0}.".format(name) z.extract(name, "/tmp") print "Uploading {0} to Azure blob store".format(name) blob_service.put_block_blob_from_path( 'tempreleases', name, "/tmp/{0}".format(name), x_ms_blob_content_type='application/x-compressed') os.unlink(release_filename) blob_url = "http://{0}.blob.core.windows.net/{1}/{2}".format( storage_account_name, 'tempreleases', name) print "Uploading release {0} to BOSH director.".format( name) task_id = client.upload_release(blob_url) client.wait_for_task(task_id) z.close() temp.close() blob_service.delete_container("tempreleases") # stemcells print "Processing stemcells." for url in stemcell_urls: print "Processing stemcell {0}".format(url) task_id = client.upload_stemcell(url) client.wait_for_task(task_id) return context
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, container_name, prefix=None): # NOTE: Azure wants all paths to start with a slash prefix = "/{}".format(prefix.lstrip("/") if prefix else "") super().__init__(prefix=prefix) self.account_name = account_name self.account_key = account_key self.container_name = container_name self.conn = BlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized") # XXX: AzureTransfer isn't actively tested and hasn't its error handling is probably lacking self.log.warning( "AzureTransfer is experimental and has not been thoroughly tested") def get_metadata_for_key(self, key): key = self.format_key_for_backend(key) return self._list_blobs(key)[0]["metadata"] def _metadata_for_key(self, key): return self._list_blobs(key)[0]["metadata"] def list_path(self, key): path = self.format_key_for_backend(key, trailing_slash=True) return self._list_blobs(path) def _list_blobs(self, path): self.log.debug("Listing path %r", path) items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata") result = [] for item in items: result.append({ "last_modified": dateutil.parser.parse(item.properties.last_modified), "metadata": item.metadata, "name": self.format_key_from_backend(item.name), "size": item.properties.content_length, }) return result def delete_key(self, key): key = self.format_key_for_backend(key) self.log.debug("Deleting key: %r", key) return self.conn.delete_blob(self.container_name, key) def get_contents_to_file(self, key, filepath_to_store_to): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to) return self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to) def get_contents_to_fileobj(self, key, fileobj_to_store_to): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_file(self.container_name, key, fileobj_to_store_to) def get_contents_to_string(self, key): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_bytes(self.container_name, key), self._metadata_for_key(key) def store_file_from_memory(self, key, memstring, metadata=None): key = self.format_key_for_backend(key) # Azure requires all metadata keys and values to be strings metadata_to_send = {str(k): str(v) for k, v in metadata.items()} self.conn.put_block_blob_from_bytes( self.container_name, key, memstring, x_ms_meta_name_values=metadata_to_send) def store_file_from_disk(self, key, filepath, metadata=None, multipart=None): key = self.format_key_for_backend(key) # Azure requires all metadata keys and values to be strings metadata_to_send = {str(k): str(v) for k, v in metadata.items()} self.conn.put_block_blob_from_path( self.container_name, key, filepath, x_ms_meta_name_values=metadata_to_send) def get_or_create_container(self, container_name): start_time = time.time() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time) return container_name
def generate_and_upload(gauge_factory, config, logger): start = datetime.datetime.now() twitter_followers = gauge_factory('twitter.followers') twitter_tweets = gauge_factory('twitter.tweets') fb_friends = gauge_factory('facebook.friends') foursq_checkins = gauge_factory('foursquare.checkins') klout_score = gauge_factory('klout.score') runkeeper_activities = gauge_factory('runkeeper.activities') runkeeper_calories = gauge_factory('runkeeper.calories_burned') runkeeper_weight = gauge_factory('runkeeper.weight') tmp102_celsius = gauge_factory('tmp102.temperature', gauge_type='hourly') lastfm_listened = gauge_factory('lastfm.listened') jawbone_sleeps = gauge_factory('jawbone.sleeps') jawbone_heartrate = gauge_factory('jawbone.resting_heartrate') jawbone_steps = gauge_factory('jawbone.steps') jawbone_caffeine = gauge_factory('jawbone.caffeine') data = {} data_sources = [ # (output key, gauge, days back, aggregator, postprocessors) ('twitter.followers', twitter_followers, 30, None, [zero_fill_daily, interpolators.linear]), ('twitter.tweets', twitter_tweets, 20, None, [zero_fill_daily]), ('facebook.friends', fb_friends, 180, monthly_max, None), ('foursquare.checkins', foursq_checkins, 14, None, [zero_fill_daily]), ('lastfm.listened', lastfm_listened, 14, None, [zero_fill_daily]), ('klout.score', klout_score, 30, weekly_max, [zero_fill_weekly, interpolators.linear]), ('runkeeper.calories', runkeeper_calories, 60, weekly_sum, [zero_fill_weekly]), ('runkeeper.activities', runkeeper_activities, 60, weekly_sum, [zero_fill_weekly]), ('runkeeper.weight', runkeeper_weight, 180, weekly_min, [zero_fill_weekly, interpolators.linear]), ('sleeps', jawbone_sleeps, 14, None, [zero_fill_daily, interpolators.linear]), ('heartrate', jawbone_heartrate, 21, None, [zero_fill_daily, interpolators.linear]), ('steps', jawbone_steps, 14, None, [zero_fill_daily, interpolators.linear]), ('caffeine', jawbone_caffeine, 30, None, [zero_fill_daily]), ('tmp102.temperature', tmp102_celsius, 2.5, None, None) ] for ds in data_sources: data[ds[0]] = ds[1].aggregate(today_utc() - timedelta(days=ds[2]), aggregator=ds[3], post_processors=ds[4]) report = { 'generated': str(now_utc()), 'data': data, 'took': (datetime.datetime.now() - start).seconds } report_json = json.dumps(report, indent=4, default=json_date_serializer) report_content = '{0}({1})'.format(JSONP_CALLBACK_NAME, report_json) blob_service = BlobService(config['azure.account'], config['azure.key']) blob_service.create_container(config['azure.blob.container']) blob_service.set_container_acl(config['azure.blob.container'], x_ms_blob_public_access='container') blob_service.put_blob(config['azure.blob.container'], config['azure.blob.name'], report_content, 'BlockBlob') took = (datetime.datetime.now() - start).seconds logger.info('Report generated and uploaded. Took {0} s.'.format(took))
from azure import WindowsAzureMissingResourceError import glob import os import engaged.data.azure_connect as AC ACCOUNT_NAME = 'sounds' ACCOUNT_KEY = AC.getAccountKey() # primary access key HOST_BASE = '.blob.core.windows.net' blob_service = BlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY, host_base=HOST_BASE) CONTAINER = 'bat-detective' # or whatever else you like created = blob_service.create_container(CONTAINER, x_ms_blob_public_access='container') print "Created" if created else "Not created (probably already existing)" audio_dir = '../../data/wav/' SOUND_FILES = glob.glob(audio_dir + '*.wav') for f in SOUND_FILES: print "uploading", os.path.basename(f) blob_service.put_block_blob_from_path( CONTAINER, # container os.path.basename(f), # blob f, # path x_ms_blob_content_type='audio/wav' )
class Command(BaseCommand): help = "Synchronizes static media to cloud files." option_list = BaseCommand.option_list + ( optparse.make_option( '-w', '--wipe', action='store_true', dest='wipe', default=False, help="Wipes out entire contents of container first."), optparse.make_option('-t', '--test-run', action='store_true', dest='test_run', default=False, help="Performs a test run of the sync."), optparse.make_option('-c', '--container', dest='container', help="Override STATIC_CONTAINER."), ) # settings from azurite.settings ACCOUNT_NAME = AZURITE['ACCOUNT_NAME'] ACCOUNT_KEY = AZURITE['ACCOUNT_KEY'] STATIC_CONTAINER = AZURITE['STATIC_CONTAINER'] # paths DIRECTORY = os.path.abspath(settings.STATIC_ROOT) STATIC_URL = settings.STATIC_URL if not DIRECTORY.endswith('/'): DIRECTORY = DIRECTORY + '/' if STATIC_URL.startswith('/'): STATIC_URL = STATIC_URL[1:] local_object_names = [] create_count = 0 upload_count = 0 update_count = 0 skip_count = 0 delete_count = 0 service = None def handle(self, *args, **options): self.wipe = options.get('wipe') self.test_run = options.get('test_run') self.verbosity = int(options.get('verbosity')) if hasattr(options, 'container'): self.STATIC_CONTAINER = options.get('container') self.sync_files() def sync_files(self): self.service = BlobService(account_name=self.ACCOUNT_NAME, account_key=self.ACCOUNT_KEY) try: self.service.get_container_properties(self.STATIC_CONTAINER) except WindowsAzureMissingResourceError: self.service.create_container(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') self.service.set_container_acl(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') # if -w option is provided, wipe out the contents of the container if self.wipe: blob_count = len(self.service.list_blobs(self.STATIC_CONTAINER)) if self.test_run: print "Wipe would delete %d objects." % blob_count else: print "Deleting %d objects..." % blob_count for blob in self.service.list_blobs(self.STATIC_CONTAINER): self.service.delete_blob(self.STATIC_CONTAINER, blob.name) # walk through the directory, creating or updating files on the cloud os.path.walk(self.DIRECTORY, self.upload_files, "foo") # remove any files on remote that don't exist locally self.delete_files() # print out the final tally to the cmd line self.update_count = self.upload_count - self.create_count print if self.test_run: print "Test run complete with the following results:" print "Skipped %d. Created %d. Updated %d. Deleted %d." % ( self.skip_count, self.create_count, self.update_count, self.delete_count) def upload_files(self, arg, dirname, names): # upload or skip items for item in names: file_path = os.path.join(dirname, item) if os.path.isdir(file_path): continue # Don't try to upload directories object_name = self.STATIC_URL + file_path.split(self.DIRECTORY)[1] self.local_object_names.append(object_name) try: properties = self.service.get_blob_properties( self.STATIC_CONTAINER, object_name) except WindowsAzureMissingResourceError: properties = {} self.create_count += 1 cloud_datetime = None if 'last-modified' in properties: cloud_datetime = ( properties['last-modified'] and datetime.datetime.strptime(properties['last-modified'], "%a, %d %b %Y %H:%M:%S %Z") or None) local_datetime = datetime.datetime.utcfromtimestamp( os.stat(file_path).st_mtime) if cloud_datetime and local_datetime < cloud_datetime: self.skip_count += 1 if self.verbosity > 1: print "Skipped %s: not modified." % object_name continue if not self.test_run: file_contents = open(file_path, 'r').read() content_type, encoding = mimetypes.guess_type(file_path) self.service.put_blob(self.STATIC_CONTAINER, object_name, file_contents, x_ms_blob_type='BlockBlob', x_ms_blob_content_type=content_type, content_encoding=encoding) # sync_headers(cloud_obj) self.upload_count += 1 if self.verbosity > 1: print "Uploaded", object_name def delete_files(self): # remove any objects in the container that don't exist locally for blob in self.service.list_blobs(self.STATIC_CONTAINER): if blob.name not in self.local_object_names: self.delete_count += 1 if self.verbosity > 1: print "Deleted %s" % blob.name if not self.test_run: self.service.delete_blob(self.STATIC_CONTAINER, blob.name)
class AzureBackend(duplicity.backend.Backend): """ Backend for Azure Blob Storage Service """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) # Import Microsoft Azure Storage SDK for Python library. try: import azure import azure.storage if hasattr(azure.storage, 'BlobService'): # v0.11.1 and below from azure.storage import BlobService self.AzureMissingResourceError = azure.WindowsAzureMissingResourceError self.AzureConflictError = azure.WindowsAzureConflictError else: # v1.0.0 and above import azure.storage.blob if hasattr(azure.storage.blob, 'BlobService'): from azure.storage.blob import BlobService else: from azure.storage.blob.blockblobservice import BlockBlobService as BlobService self.AzureMissingResourceError = azure.common.AzureMissingResourceHttpError self.AzureConflictError = azure.common.AzureConflictHttpError except ImportError as e: raise BackendException("""\ Azure backend requires Microsoft Azure Storage SDK for Python (https://pypi.python.org/pypi/azure-storage/). Exception: %s""" % str(e)) # TODO: validate container name self.container = parsed_url.path.lstrip('/') if 'AZURE_ACCOUNT_NAME' not in os.environ: raise BackendException('AZURE_ACCOUNT_NAME environment variable not set.') if 'AZURE_ACCOUNT_KEY' in os.environ: if 'AZURE_ENDPOINT_SUFFIX' in os.environ: self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], account_key=os.environ['AZURE_ACCOUNT_KEY'], endpoint_suffix=os.environ['AZURE_ENDPOINT_SUFFIX']) else: self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], account_key=os.environ['AZURE_ACCOUNT_KEY']) self._create_container() elif 'AZURE_SHARED_ACCESS_SIGNATURE' in os.environ: if 'AZURE_ENDPOINT_SUFFIX' in os.environ: self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], sas_token=os.environ['AZURE_SHARED_ACCESS_SIGNATURE'], endpoint_suffix=os.environ['AZURE_ENDPOINT_SUFFIX']) else: self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], sas_token=os.environ['AZURE_SHARED_ACCESS_SIGNATURE']) else: raise BackendException( 'Neither AZURE_ACCOUNT_KEY nor AZURE_SHARED_ACCESS_SIGNATURE environment variable not set.') if globals.azure_max_single_put_size: # check if we use azure-storage>=0.30.0 try: _ = self.blob_service.MAX_SINGLE_PUT_SIZE self.blob_service.MAX_SINGLE_PUT_SIZE = globals.azure_max_single_put_size # fallback for azure-storage<0.30.0 except AttributeError: self.blob_service._BLOB_MAX_DATA_SIZE = globals.azure_max_single_put_size if globals.azure_max_block_size: # check if we use azure-storage>=0.30.0 try: _ = self.blob_service.MAX_BLOCK_SIZE self.blob_service.MAX_BLOCK_SIZE = globals.azure_max_block_size # fallback for azure-storage<0.30.0 except AttributeError: self.blob_service._BLOB_MAX_CHUNK_DATA_SIZE = globals.azure_max_block_size def _create_container(self): try: self.blob_service.create_container(self.container, fail_on_exist=True) except self.AzureConflictError: # Indicates that the resource could not be created because it already exists. pass except Exception as e: log.FatalError("Could not create Azure container: %s" % unicode(e.message).split('\n', 1)[0], log.ErrorCode.connection_failed) def _put(self, source_path, remote_filename): kwargs = {} if globals.azure_max_connections: kwargs['max_connections'] = globals.azure_max_connections # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#upload-a-blob-into-a-container try: self.blob_service.create_blob_from_path(self.container, remote_filename, source_path.name, **kwargs) except AttributeError: # Old versions use a different method name self.blob_service.put_block_blob_from_path(self.container, remote_filename, source_path.name, **kwargs) def _get(self, remote_filename, local_path): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#download-blobs self.blob_service.get_blob_to_path(self.container, remote_filename, local_path.name) def _list(self): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#list-the-blobs-in-a-container blobs = [] marker = None while True: batch = self.blob_service.list_blobs(self.container, marker=marker) blobs.extend(batch) if not batch.next_marker: break marker = batch.next_marker return [blob.name for blob in blobs] def _delete(self, filename): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#delete-blobs self.blob_service.delete_blob(self.container, filename) def _query(self, filename): prop = self.blob_service.get_blob_properties(self.container, filename) try: info = {'size': int(prop.properties.content_length)} except AttributeError: # old versions directly returned the properties info = {'size': int(prop['content-length'])} return info def _error_code(self, operation, e): if isinstance(e, self.AzureMissingResourceError): return log.ErrorCode.backend_not_found
imagesQueue = 'mosaicqueue' tableName = 'photos' tablePartitionKey = 'allphotos' # Get queue credentials # accountName = environ["AZURE_STORAGE_ACCOUNT"] with open("ASA.key", "r") as myfile: accountName = myfile.read().replace('\n', '') # accountKey = environ["AZURE_STORAGE_ACCESS_KEY"] with open("ASK.key", "r") as myfile: accountKey = myfile.read().replace('\n', '') # Create blob service blob_service = BlobService(account_name=accountName, account_key=accountKey) blob_service.create_container(blob_small) blob_service.create_container(blob_big) blob_service.create_container(blob_analysis) # Open queue with given credentials queue_service = QueueService(account_name=accountName, account_key=accountKey) # Open table service table_service = TableService(account_name=accountName, account_key=accountKey) # Repeat while (True): # get images form *imagesQueue* - it is invoked by CRON messages = queue_service.get_messages(imagesQueue) if len(messages) == 0:
# Get settings from CustomScriptForLinux extension configurations waagent.LoggerInit('/var/log/waagent.log', '/dev/stdout') hutil = Util.HandlerUtility(waagent.Log, waagent.Error, "bosh-deploy-script") hutil.do_parse_context("enable") settings = hutil.get_public_settings() with open (os.path.join('bosh','settings'), "w") as tmpfile: tmpfile.write(json.dumps(settings, indent=4, sort_keys=True)) username = settings["username"] home_dir = os.path.join("/home", username) install_log = os.path.join(home_dir, "install.log") # Prepare the containers storage_account_name = settings["STORAGE-ACCOUNT-NAME"] storage_access_key = settings["STORAGE-ACCESS-KEY"] blob_service = BlobService(storage_account_name, storage_access_key) blob_service.create_container('bosh') blob_service.create_container('stemcell') # Generate the private key and certificate call("sh create_cert.sh", shell=True) call("cp bosh.key ./bosh/bosh", shell=True) with open ('bosh_cert.pem', 'r') as tmpfile: ssh_cert = tmpfile.read() indentation = " " * 8 ssh_cert=("\n"+indentation).join([line for line in ssh_cert.split('\n')]) # Render the yml template for bosh-init bosh_template = 'bosh.yml' if os.path.exists(bosh_template): with open (bosh_template, 'r') as tmpfile: contents = tmpfile.read()
def generate_and_upload(gauge_factory, config, logger): start = datetime.datetime.now() twitter_followers = gauge_factory('twitter.followers') twitter_tweets = gauge_factory('twitter.tweets') fb_friends = gauge_factory('facebook.friends') foursq_checkins = gauge_factory('foursquare.checkins') klout_score = gauge_factory('klout.score') runkeeper_activities = gauge_factory('runkeeper.activities') runkeeper_calories = gauge_factory('runkeeper.calories_burned') runkeeper_weight = gauge_factory('runkeeper.weight') tmp102_celsius = gauge_factory('tmp102.te mperature', gauge_type='hourly') lastfm_listened = gauge_factory('lastfm.listened') jawbone_sleeps = gauge_factory('jawbone.sleeps') jawbone_steps = gauge_factory('jawbone.steps') jawbone_caffeine = gauge_factory('jawbone.caffeine') data = {} data_sources = [ # (output key, gauge, days back, aggregator, postprocessors) ('twitter.followers', twitter_followers, 30, None, [zero_fill_daily, interpolators.linear]), ('twitter.tweets', twitter_tweets, 20, None, [zero_fill_daily]), ('facebook.friends', fb_friends, 180, monthly_max, None), ('foursquare.checkins', foursq_checkins, 14, None, [zero_fill_daily]), ('lastfm.listened', lastfm_listened, 14, None, [zero_fill_daily]), ('klout.score', klout_score, 30, weekly_max, [zero_fill_weekly, interpolators.linear]), ('runkeeper.calories', runkeeper_calories, 60, weekly_sum, [zero_fill_weekly]), ('runkeeper.activities', runkeeper_activities, 60,weekly_sum, [zero_fill_weekly]), ('runkeeper.weight', runkeeper_weight, 180,weekly_min, [zero_fill_weekly, interpolators.linear]), ('sleeps', jawbone_sleeps, 14, None, [zero_fill_daily, interpolators.linear]), ('steps', jawbone_steps, 14, None, [zero_fill_daily, interpolators.linear]), ('caffeine', jawbone_caffeine, 30, None, [zero_fill_daily]), ('tmp102.temperature', tmp102_celsius, 2.5, None, None) ] for ds in data_sources: data[ds[0]] = ds[1].aggregate(today_utc() - timedelta(days=ds[2]), aggregator=ds[3], post_processors=ds[4]) report = { 'generated': str(now_utc()), 'data': data, 'took': (datetime.datetime.now() - start).seconds } report_json = json.dumps(report, indent=4, default=json_date_serializer) report_content = '{0}({1})'.format(JSONP_CALLBACK_NAME, report_json) blob_service = BlobService(config['azure.account'], config['azure.key']) blob_service.create_container(config['azure.blob.container']) blob_service.set_container_acl(config['azure.blob.container'], x_ms_blob_public_access='container') blob_service.put_blob(config['azure.blob.container'], config['azure.blob.name'], report_content, 'BlockBlob') took = (datetime.datetime.now() - start).seconds logger.info('Report generated and uploaded. Took {0} s.'.format(took))
import glob import os import time logtime = time.strftime("%Y%m%d-%H%M%S") logmessage = "uploadmp3.py started" command = "sed -i '1s/^/" + logtime + " " + logmessage + "\\n/' /home/pi/selcuk/log.txt" os.system(command) from azure.storage import BlobService blob_service = BlobService(account_name='account_name', account_key='account_key') blob_service.create_container('record') blob_service.create_container('record', x_ms_blob_public_access='container') blob_service.set_container_acl('record', x_ms_blob_public_access='container') directory = "/home/pi/selcuk/mp3" os.chdir(directory) for file in glob.glob("*.mp3"): full_path = directory + "/" + file blob_service.put_block_blob_from_path( 'record', file, full_path, x_ms_blob_content_type='audio/mpeg3' ) delete_command = "rm " + file os.system(delete_command) logtime = time.strftime("%Y%m%d-%H%M%S") logmessage = file + " uploaded to cloud and deleted from device"
class AzureBackend(duplicity.backend.Backend): """ Backend for Azure Blob Storage Service """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) # Import Microsoft Azure Storage SDK for Python library. try: import azure import azure.storage if hasattr(azure.storage, 'BlobService'): # v0.11.1 and below from azure.storage import BlobService self.AzureMissingResourceError = azure.WindowsAzureMissingResourceError self.AzureConflictError = azure.WindowsAzureConflictError else: # v1.0.0 and above from azure.storage.blob import BlobService self.AzureMissingResourceError = azure.common.AzureMissingResourceHttpError self.AzureConflictError = azure.common.AzureConflictHttpError except ImportError as e: raise BackendException("""\ Azure backend requires Microsoft Azure Storage SDK for Python (https://pypi.python.org/pypi/azure-storage/). Exception: %s""" % str(e)) if 'AZURE_ACCOUNT_NAME' not in os.environ: raise BackendException('AZURE_ACCOUNT_NAME environment variable not set.') if 'AZURE_ACCOUNT_KEY' not in os.environ: raise BackendException('AZURE_ACCOUNT_KEY environment variable not set.') self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], account_key=os.environ['AZURE_ACCOUNT_KEY']) # TODO: validate container name self.container = parsed_url.path.lstrip('/') try: self.blob_service.create_container(self.container, fail_on_exist=True) except self.AzureConflictError: # Indicates that the resource could not be created because it already exists. pass except Exception as e: log.FatalError("Could not create Azure container: %s" % unicode(e.message).split('\n', 1)[0], log.ErrorCode.connection_failed) def _put(self, source_path, remote_filename): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#upload-a-blob-into-a-container self.blob_service.put_block_blob_from_path(self.container, remote_filename, source_path.name) def _get(self, remote_filename, local_path): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#download-blobs self.blob_service.get_blob_to_path(self.container, remote_filename, local_path.name) def _list(self): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#list-the-blobs-in-a-container blobs = [] marker = None while True: batch = self.blob_service.list_blobs(self.container, marker=marker) blobs.extend(batch) if not batch.next_marker: break marker = batch.next_marker return [blob.name for blob in blobs] def _delete(self, filename): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#delete-blobs self.blob_service.delete_blob(self.container, filename) def _query(self, filename): prop = self.blob_service.get_blob_properties(self.container, filename) return {'size': int(prop['content-length'])} def _error_code(self, operation, e): if isinstance(e, self.AzureMissingResourceError): return log.ErrorCode.backend_not_found
from azure.storage import BlobService import sys key = raw_input("Please enter azure vidoepath blob storage key: ") blob_service = BlobService(account_name='videopathmobilefiles', account_key=key) source = sys.argv[1] target = sys.argv[2] print source + " -> " + target blob_service.create_container(target, x_ms_blob_public_access='container') # blob_service.copy_blob('test2', 'copiedkey', '/videopathmobilefiles/test/key') blobs = blob_service.list_blobs(source) for b in blobs: name = b.name source_path = '/videopathmobilefiles/' + source + '/' + name blob_service.copy_blob(target, name, source_path) print name
class AzureBlobStorage(Storage): def __init__(self, account='nyxstorage', container='pxo'): self.base_storage_uri = 'http://%s.blob.core.windows.net/%s/' % ( account, container) self.blob_service = BlobService( account, get_env_variable('AZURE_BLOB_STORAGE_KEY')) self.container = container def _open(self, name, mode='rb'): data = self.blob_service.get_blob(self.container, name) return ContentFile(data) def _save(self, name, content): _file = content.read() file_name = content.name[-35:] self.blob_service.put_blob( self.container, file_name, _file, x_ms_blob_type='BlockBlob') return self.base_storage_uri + file_name def create_container(self, container_name): result = self.blob_service.create_container( container_name, x_ms_blob_public_access='container') return result def delete(self, name): self.blob_service.delete_blob(self.container, name) def exists(self, name): try: self.blob_service.get_blob_properties(self.container, name) except: return False else: return True def get_available_name(self, name): return name def get_blobs(self): blobs = self.blob_service.list_blobs(self.container) return blobs def get_valid_name(self, name): return name def modified_time(self, name): metadata = self.blob_service.get_blob_metadata(self.container, name) modified_time = float(metadata.get('x-ms-meta-modified_time')) return datetime.fromtimestamp(modified_time) def set_public_container(self, container_name): result = self.blob_service.set_container_acl( container_name, x_ms_blob_public_access='container') return result def size(self, name): properties = self.blob_service.get_blob_properties( self.container, name) return properties.get('content-length') def url(self, name): blob = self.blob_service.list_blobs(self.container, prefix=name) return blob.blobs[0].url
try: blob_service.put_block_blob_from_path( container_name, key, source, x_ms_blob_content_type=content_type, x_ms_blob_cache_control='public, max-age=600') break except: print "retrying..." time.sleep(1) # walk all files in dir and push to bucket codepath = os.path.dirname(os.path.abspath(__file__)) + "/upload/" + videoname container_name = videoname.lower() blob_service.create_container(container_name, x_ms_blob_public_access='container') for path, subdirs, files in os.walk(codepath): for name in files: # don't upload hidden files if name[0] == ".": continue pathname = os.path.join(path, name) keyname = pathname.replace(codepath, "")[1:] upload_file(pathname, container_name, name)
class BlobServiceAdapter(Component): """The :class:`BlobServiceAdapter` class is a thin wrapper over azure.storage.BlobService. All the attributes of the wrapper stream are proxied by the adapter so it's possible to do ``adapter.create_container()`` instead of the long form ``adapter.blob_service.adapter()``. """ def __init__(self): self.blob_service = BlobService( account_name=self.util.get_config("storage.azure.account_name"), account_key=self.util.get_config("storage.azure.account_key"), host_base=self.util.get_config("storage.azure.blob_service_host_base"), ) def __getattr__(self, name): return getattr(self.blob_service, name) def create_container_in_storage(self, container_name, access="container"): """create a container if doesn't exist :type container_name: str|unicode :param container_name: Name of container to create. :type access: str|unicode :param access: Optional. Possible values include: container, blob :return: """ try: names = [x.name for x in self.blob_service.list_containers()] if container_name not in names: return self.blob_service.create_container(container_name, x_ms_blob_public_access=access) else: self.log.debug("container already exists in storage") return True except Exception as e: self.log.error(e) return False def upload_file_to_azure(self, container_name, blob_name, stream): """ Creates a new block blob from a file/stream, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str | unicode :param blob_name: Name of blob to create or update. :type stream: file :param stream: Opened file/stream to upload as the blob content. """ try: if self.create_container_in_storage(container_name, "container"): self.blob_service.put_block_blob_from_file(container_name, blob_name, stream) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_bytes(self, container_name, blob_name, blob): """ Creates a new block blob from an array of bytes, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type blob: bytes :param blob: Content of blob as an array of bytes. """ try: if self.create_container_in_storage(container_name, "container"): self.blob_service.put_block_blob_from_bytes(container_name, blob_name, blob) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_text(self, container_name, blob_name, text): """ Creates a new block blob from str/unicode, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type text: str|unicode :param text: Text to upload to the blob. """ try: if self.create_container_in_storage(container_name, "container"): self.blob_service.put_block_blob_from_text(container_name, blob_name, text) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_path(self, container_name, blob_name, path): """ Creates a new page blob from a file path, or updates the content of an existing page blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type path: str|unicode :param path: Path of the file to upload as the blob content. """ try: if self.create_container_in_storage(container_name, "container"): self.blob_service.put_block_blob_from_path(container_name, blob_name, path) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def delete_file_from_azure(self, container_name, blob_name): try: if self.create_container_in_storage(container_name, "container"): self.blob_service.delete_blob(container_name, blob_name) except Exception as e: self.log.error(e) return None
from azure.storage import BlobService from azure import WindowsAzureError, WindowsAzureMissingResourceError import logging from logging.handlers import RotatingFileHandler app = Flask(__name__) service_name = 'azure-storage' vcap_services = json.loads(os.environ['VCAP_SERVICES']) account_name = vcap_services[service_name][0]['credentials']['storage_account_name'] account_key = vcap_services[service_name][0]['credentials']['primary_access_key'] blob_service = BlobService(account_name, account_key) container_name = '{0}{1}'.format('cloud-foundry-', str(uuid.uuid4()).replace('-', '')) blob_service.create_container(container_name) @app.route('/') def index(): blobs = blob_service.list_blobs(container_name) return render_template("index.html", filenames=[blob.name for blob in blobs]) @app.route('/upload', methods= ['POST']) def upload(): file = request.files['file'] if not file or not file.filename: msg = 'The file to upload is invalid' blobs = blob_service.list_blobs(container_name) return render_template("index.html", msg=msg, filenames=[blob.name for blob in blobs]) try: app.logger.info('uploading file {0}'.format(file.filename))
def upload_to_blob(filename): #uploads to azure blob storage blob_service = BlobService(account_name=storage_account_name, account_key=storage_account_key) blob_service.create_container('pubsubcat-pics') blob_service.put_block_blob_from_path("pubsubcat-pics", hostname + "/" + filename, 'temp/' + filename)
class AzureJobStore(AbstractJobStore): """ A job store that uses Azure's blob store for file storage and Table Service to store job info with strong consistency.""" def __init__(self, accountName, namePrefix, config=None, jobChunkSize=maxAzureTablePropertySize): self.jobChunkSize = jobChunkSize self.keyPath = None self.account_key = _fetchAzureAccountKey(accountName) # Table names have strict requirements in Azure self.namePrefix = self._sanitizeTableName(namePrefix) log.debug("Creating job store with name prefix '%s'" % self.namePrefix) # These are the main API entrypoints. self.tableService = TableService(account_key=self.account_key, account_name=accountName) self.blobService = BlobService(account_key=self.account_key, account_name=accountName) # Register our job-store in the global table for this storage account self.registryTable = self._getOrCreateTable('toilRegistry') exists = self.registryTable.get_entity(row_key=self.namePrefix) self._checkJobStoreCreation(config is not None, exists, accountName + ":" + self.namePrefix) self.registryTable.insert_or_replace_entity(row_key=self.namePrefix, entity={'exists': True}) # Serialized jobs table self.jobItems = self._getOrCreateTable(self.qualify('jobs')) # Job<->file mapping table self.jobFileIDs = self._getOrCreateTable(self.qualify('jobFileIDs')) # Container for all shared and unshared files self.files = self._getOrCreateBlobContainer(self.qualify('files')) # Stats and logging strings self.statsFiles = self._getOrCreateBlobContainer(self.qualify('statsfiles')) # File IDs that contain stats and logging strings self.statsFileIDs = self._getOrCreateTable(self.qualify('statsFileIDs')) super(AzureJobStore, self).__init__(config=config) if self.config.cseKey is not None: self.keyPath = self.config.cseKey # Table names must be alphanumeric nameSeparator = 'xx' # Length of a jobID - used to test if a stats file has been read already or not jobIDLength = len(str(uuid.uuid4())) def qualify(self, name): return self.namePrefix + self.nameSeparator + name def jobs(self): for jobEntity in self.jobItems.query_entities(): yield AzureJob.fromEntity(jobEntity) def create(self, command, memory, cores, disk, predecessorNumber=0): jobStoreID = self._newJobID() job = AzureJob(jobStoreID=jobStoreID, command=command, memory=memory, cores=cores, disk=disk, remainingRetryCount=self._defaultTryCount(), logJobStoreFileID=None, predecessorNumber=predecessorNumber) entity = job.toItem(chunkSize=self.jobChunkSize) entity['RowKey'] = jobStoreID self.jobItems.insert_entity(entity=entity) return job def exists(self, jobStoreID): if self.jobItems.get_entity(row_key=jobStoreID) is None: return False return True def load(self, jobStoreID): jobEntity = self.jobItems.get_entity(row_key=jobStoreID) if jobEntity is None: raise NoSuchJobException(jobStoreID) return AzureJob.fromEntity(jobEntity) def update(self, job): self.jobItems.update_entity(row_key=job.jobStoreID, entity=job.toItem(chunkSize=self.jobChunkSize)) def delete(self, jobStoreID): try: self.jobItems.delete_entity(row_key=jobStoreID) except WindowsAzureMissingResourceError: # Job deletion is idempotent, and this job has been deleted already return filterString = "PartitionKey eq '%s'" % jobStoreID for fileEntity in self.jobFileIDs.query_entities(filter=filterString): jobStoreFileID = fileEntity.RowKey self.deleteFile(jobStoreFileID) def deleteJobStore(self): self.registryTable.delete_entity(row_key=self.namePrefix) self.jobItems.delete_table() self.jobFileIDs.delete_table() self.files.delete_container() self.statsFiles.delete_container() self.statsFileIDs.delete_table() def getEnv(self): return dict(AZURE_ACCOUNT_KEY=self.account_key) def writeFile(self, localFilePath, jobStoreID=None): jobStoreFileID = self._newFileID() self.updateFile(jobStoreFileID, localFilePath) self._associateFileWithJob(jobStoreFileID, jobStoreID) return jobStoreFileID def updateFile(self, jobStoreFileID, localFilePath): with open(localFilePath) as read_fd: with self._uploadStream(jobStoreFileID, self.files) as write_fd: while True: buf = read_fd.read(self._maxAzureBlockBytes) write_fd.write(buf) if len(buf) == 0: break def readFile(self, jobStoreFileID, localFilePath): try: with self._downloadStream(jobStoreFileID, self.files) as read_fd: with open(localFilePath, 'w') as write_fd: while True: buf = read_fd.read(self._maxAzureBlockBytes) write_fd.write(buf) if not buf: break except WindowsAzureMissingResourceError: raise NoSuchFileException(jobStoreFileID) def deleteFile(self, jobStoreFileID): try: self.files.delete_blob(blob_name=jobStoreFileID) self._dissociateFileFromJob(jobStoreFileID) except WindowsAzureMissingResourceError: pass def fileExists(self, jobStoreFileID): # As Azure doesn't have a blob_exists method (at least in the # python API) we just try to download the metadata, and hope # the metadata is small so the call will be fast. try: self.files.get_blob_metadata(blob_name=jobStoreFileID) return True except WindowsAzureMissingResourceError: return False @contextmanager def writeFileStream(self, jobStoreID=None): # TODO: this (and all stream methods) should probably use the # Append Blob type, but that is not currently supported by the # Azure Python API. jobStoreFileID = self._newFileID() with self._uploadStream(jobStoreFileID, self.files) as fd: yield fd, jobStoreFileID self._associateFileWithJob(jobStoreFileID, jobStoreID) @contextmanager def updateFileStream(self, jobStoreFileID): with self._uploadStream(jobStoreFileID, self.files, checkForModification=True) as fd: yield fd def getEmptyFileStoreID(self, jobStoreID=None): jobStoreFileID = self._newFileID() self.files.put_blob(blob_name=jobStoreFileID, blob='', x_ms_blob_type='BlockBlob') self._associateFileWithJob(jobStoreFileID, jobStoreID) return jobStoreFileID @contextmanager def readFileStream(self, jobStoreFileID): if not self.fileExists(jobStoreFileID): raise NoSuchFileException(jobStoreFileID) with self._downloadStream(jobStoreFileID, self.files) as fd: yield fd @contextmanager def writeSharedFileStream(self, sharedFileName, isProtected=None): sharedFileID = self._newFileID(sharedFileName) with self._uploadStream(sharedFileID, self.files, encrypted=isProtected) as fd: yield fd @contextmanager def readSharedFileStream(self, sharedFileName): sharedFileID = self._newFileID(sharedFileName) if not self.fileExists(sharedFileID): raise NoSuchFileException(sharedFileID) with self._downloadStream(sharedFileID, self.files) as fd: yield fd def writeStatsAndLogging(self, statsAndLoggingString): # TODO: would be a great use case for the append blobs, once implemented in the Azure SDK jobStoreFileID = self._newFileID() encrypted = self.keyPath is not None if encrypted: statsAndLoggingString = encryption.encrypt(statsAndLoggingString, self.keyPath) self.statsFiles.put_block_blob_from_text(blob_name=jobStoreFileID, text=statsAndLoggingString, x_ms_meta_name_values=dict( encrypted=str(encrypted))) self.statsFileIDs.insert_entity(entity={'RowKey': jobStoreFileID}) def readStatsAndLogging(self, callback, readAll=False): suffix = '_old' numStatsFiles = 0 for entity in self.statsFileIDs.query_entities(): jobStoreFileID = entity.RowKey hasBeenRead = len(jobStoreFileID) > self.jobIDLength if not hasBeenRead: with self._downloadStream(jobStoreFileID, self.statsFiles) as fd: callback(fd) # Mark this entity as read by appending the suffix self.statsFileIDs.insert_entity(entity={'RowKey': jobStoreFileID + suffix}) self.statsFileIDs.delete_entity(row_key=jobStoreFileID) numStatsFiles += 1 elif readAll: # Strip the suffix to get the original ID jobStoreFileID = jobStoreFileID[:-len(suffix)] with self._downloadStream(jobStoreFileID, self.statsFiles) as fd: callback(fd) numStatsFiles += 1 return numStatsFiles _azureTimeFormat = "%Y-%m-%dT%H:%M:%SZ" def getPublicUrl(self, jobStoreFileID): try: self.files.get_blob_properties(blob_name=jobStoreFileID) except WindowsAzureMissingResourceError: raise NoSuchFileException(jobStoreFileID) # Compensate of a little bit of clock skew startTimeStr = (datetime.utcnow() - timedelta(minutes=5)).strftime(self._azureTimeFormat) endTime = datetime.utcnow() + self.publicUrlExpiration endTimeStr = endTime.strftime(self._azureTimeFormat) sap = SharedAccessPolicy(AccessPolicy(startTimeStr, endTimeStr, BlobSharedAccessPermissions.READ)) sas_token = self.files.generate_shared_access_signature(blob_name=jobStoreFileID, shared_access_policy=sap) return self.files.make_blob_url(blob_name=jobStoreFileID) + '?' + sas_token def getSharedPublicUrl(self, sharedFileName): jobStoreFileID = self._newFileID(sharedFileName) return self.getPublicUrl(jobStoreFileID) def _newJobID(self): # raw UUIDs don't work for Azure property names because the '-' character is disallowed. return str(uuid.uuid4()).replace('-', '_') # A dummy job ID under which all shared files are stored. sharedFileJobID = uuid.UUID('891f7db6-e4d9-4221-a58e-ab6cc4395f94') def _newFileID(self, sharedFileName=None): if sharedFileName is None: ret = str(uuid.uuid4()) else: ret = str(uuid.uuid5(self.sharedFileJobID, str(sharedFileName))) return ret.replace('-', '_') def _associateFileWithJob(self, jobStoreFileID, jobStoreID=None): if jobStoreID is not None: self.jobFileIDs.insert_entity(entity={'PartitionKey': jobStoreID, 'RowKey': jobStoreFileID}) def _dissociateFileFromJob(self, jobStoreFileID): entities = self.jobFileIDs.query_entities(filter="RowKey eq '%s'" % jobStoreFileID) if entities: assert len(entities) == 1 jobStoreID = entities[0].PartitionKey self.jobFileIDs.delete_entity(partition_key=jobStoreID, row_key=jobStoreFileID) def _getOrCreateTable(self, tableName): # This will not fail if the table already exists. for attempt in retry_on_error(): with attempt: self.tableService.create_table(tableName) return AzureTable(self.tableService, tableName) def _getOrCreateBlobContainer(self, containerName): for attempt in retry_on_error(): with attempt: self.blobService.create_container(containerName) return AzureBlobContainer(self.blobService, containerName) def _sanitizeTableName(self, tableName): """ Azure table names must start with a letter and be alphanumeric. This will never cause a collision if uuids are used, but otherwise may not be safe. """ return 'a' + filter(lambda x: x.isalnum(), tableName) # Maximum bytes that can be in any block of an Azure block blob # https://github.com/Azure/azure-storage-python/blob/4c7666e05a9556c10154508335738ee44d7cb104/azure/storage/blob/blobservice.py#L106 _maxAzureBlockBytes = 4 * 1024 * 1024 @contextmanager def _uploadStream(self, jobStoreFileID, container, checkForModification=False, encrypted=None): """ :param encrypted: True to enforce encryption (will raise exception unless key is set), False to prevent encryption or None to encrypt if key is set. """ if checkForModification: try: expectedVersion = container.get_blob_properties(blob_name=jobStoreFileID)['etag'] except WindowsAzureMissingResourceError: expectedVersion = None if encrypted is None: encrypted = self.keyPath is not None elif encrypted: if self.keyPath is None: raise RuntimeError('Encryption requested but no key was provided') maxBlockSize = self._maxAzureBlockBytes if encrypted: # There is a small overhead for encrypted data. maxBlockSize -= encryption.overhead readable_fh, writable_fh = os.pipe() with os.fdopen(readable_fh, 'r') as readable: with os.fdopen(writable_fh, 'w') as writable: def reader(): blockIDs = [] try: while True: buf = readable.read(maxBlockSize) if len(buf) == 0: # We're safe to break here even if we never read anything, since # putting an empty block list creates an empty blob. break if encrypted: buf = encryption.encrypt(buf, self.keyPath) blockID = self._newFileID() container.put_block(blob_name=jobStoreFileID, block=buf, blockid=blockID) blockIDs.append(blockID) except: # This is guaranteed to delete any uncommitted # blocks. container.delete_blob(blob_name=jobStoreFileID) raise if checkForModification and expectedVersion is not None: # Acquire a (60-second) write lock, leaseID = container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action='acquire')['x-ms-lease-id'] # check for modification, blobProperties = container.get_blob_properties(blob_name=jobStoreFileID) if blobProperties['etag'] != expectedVersion: container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action='release', x_ms_lease_id=leaseID) raise ConcurrentFileModificationException(jobStoreFileID) # commit the file, container.put_block_list(blob_name=jobStoreFileID, block_list=blockIDs, x_ms_lease_id=leaseID, x_ms_meta_name_values=dict( encrypted=str(encrypted))) # then release the lock. container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action='release', x_ms_lease_id=leaseID) else: # No need to check for modification, just blindly write over whatever # was there. container.put_block_list(blob_name=jobStoreFileID, block_list=blockIDs, x_ms_meta_name_values=dict( encrypted=str(encrypted))) thread = ExceptionalThread(target=reader) thread.start() yield writable # The writable is now closed. This will send EOF to the readable and cause that # thread to finish. thread.join() @contextmanager def _downloadStream(self, jobStoreFileID, container): # The reason this is not in the writer is so we catch non-existant blobs early blobProps = container.get_blob_properties(blob_name=jobStoreFileID) encrypted = strict_bool(blobProps['x-ms-meta-encrypted']) if encrypted and self.keyPath is None: raise AssertionError('Content is encrypted but no key was provided.') readable_fh, writable_fh = os.pipe() with os.fdopen(readable_fh, 'r') as readable: with os.fdopen(writable_fh, 'w') as writable: def writer(): try: chunkStartPos = 0 fileSize = int(blobProps['Content-Length']) while chunkStartPos < fileSize: chunkEndPos = chunkStartPos + self._maxAzureBlockBytes - 1 buf = container.get_blob(blob_name=jobStoreFileID, x_ms_range="bytes=%d-%d" % (chunkStartPos, chunkEndPos)) if encrypted: buf = encryption.decrypt(buf, self.keyPath) writable.write(buf) chunkStartPos = chunkEndPos + 1 finally: # Ensure readers aren't left blocking if this thread crashes. # This close() will send EOF to the reading end and ultimately cause the # yield to return. It also makes the implict .close() done by the enclosing # "with" context redundant but that should be ok since .close() on file # objects are idempotent. writable.close() thread = ExceptionalThread(target=writer) thread.start() yield readable thread.join()
class SAzure(SyncStorage): def __init__(self): super().__init__() self.msg_key_na = _('Key not available') try: import alxlib.key key = alxlib.key.Key() if os.path.isfile(key.get_path()): sys.path.insert(0, key.get_dir()) import alxkey self.key = alxkey.alxkey_azure """self.blob = BlobService(account_name=self.key['AZURE_STORAGE_ACCOUNT_NAME'], account_key=self.key['AZURE_ACCESS_KEY'])""" else: # raise (self.msg_key_na) self.key = None except: pass # raise (self.msg_key_na) def connect(self): try: self.blob = BlobService(account_name=self.key['AZURE_STORAGE_ACCOUNT_NAME'], account_key=self.key['AZURE_ACCESS_KEY']) return self.blob.list_containers(maxresults=1) except: return None def connect_blob(self, az_account_name=None, az_account_key=None): try: if az_account_name != None: self.key['AZURE_STORAGE_ACCOUNT_NAME'] = az_account_name self.key['AZURE_ACCESS_KEY'] = az_account_key return self.connect() except: return None def path_clean(self, path: str): try: i = path.index("//") + 2 self.container = path[0:i] if path[len(path) - 1] != "/": path += "/" return path[i:] except: print(_("Bad Path")) exit(1) def spath(self, container, root, b): spath = SyncPath() spath.BasePath = container if b.name[len(b.name)-1]=="/": spath.IsDir= True else: spath.IsFile= True spath.AbsPath = b.name if len(root)>0: spath.SPath = b.name[len(root) - 1:] else: spath.SPath=b.name spath.Size = b.properties.content_length import alxlib.time_help spath.ModifiedTS = alxlib.time_help.to_timestamp(b.properties.last_modified) spath.MD5 = b.properties.content_md5 spath.sys="azure" return spath def path_split(self, path: str): try: list = path.split("/") container = list[0] uri = "" if len(list) > 1: uri = "/".join(map(str, list[1:])) return container, uri except: print(_("Bad path")) exit(1) def path_list_blobs(self, container, uri): try: if len(uri)>0: blobs = self.blob.list_blobs(container, prefix=uri) else: blobs = self.blob.list_blobs(container) """for blob in blobs: print(blob.properties.__dict__) print(blob.name) print(blob.url)""" return blobs except Exception as e: print(_("Bad connection")) logging.warning("container {0}, path {1}".format(container, uri)) exit(1) def path_list(self, path): try: logging.debug("path_list {0}".format(path)) container, uri = self.path_split(path) logging.debug("Container: {0}, Uri: {1}".format(container, uri)) self.connect() self.blob.create_container(container) blobs = self.path_list_blobs(container, uri) d = {} for b in blobs: spath = self.spath(container, uri, b) # print(b.__dict__) #print(str(b.properties.last_modified.__dict__)) #print(str(spath.ModifiedTS)) d[spath.SPath] = spath # print(d) return d except Exception as e: print(e) def remove(self, src: SyncPath): try: logging.debug("Removing {0}".format(src.AbsPath)) self.connect() self.blob.create_container(src.BasePath) self.blob.delete_blob(src.BasePath, src.AbsPath) except: pass def copy_local2azure(self, src, base_dir): try: container, uri = self.path_split(base_dir) if len(src.SPath)>0 and src.SPath[0]=="/": path= uri+ src.SPath[1:] else: path= uri+src.SPath logging.debug("copy_local2azure Spath {0}. path:{1}".format(src.SPath, path)) self.connect() if not src.IsDir: self.blob.put_block_blob_from_path (container, path, src.AbsPath) else: self.blob.put_block_blob_from_text(container, path+"/", "") except Exception as e: print("Error Copying") print(e) def copy_azure2local(self, src, base_dir): try: if len(src.SPath)>0 and (src.SPath[0] == "/" or src.SPath[0] == "\\") : path = src.SPath[1:] else: path = src.SPath path= os.path.normpath(os.path.join(base_dir, path)) logging.debug("copy_azure2local basedir:{0} Spath {1}, path {2}, abs: {3}".format( base_dir, src.SPath, path, src.AbsPath)) if not os.path.isdir(path): os.makedirs(os.path.dirname(path), exist_ok=True) #print( os.path.dirname(path)+"***************") if not (len(src.AbsPath)>0 and src.AbsPath[len(src.AbsPath)-1]=="/"): self.blob.get_blob_to_path(src.BasePath, src.AbsPath, path) """container, uri = self.path_split(base_dir) if len(src.SPath)>0 and src.SPath[0]=="/": path= uri+ src.SPath[1:] else: path= uri+src.SPath self.connect() if not src.IsDir: self.blob.get_blob_to_path(src.BasePath, path, src.AbsPath) else: self.blob.put_block_blob_from_text(container, path, "")""" except Exception as e: print("Error copying") print(e)
def do_step(context): settings = context.meta['settings'] index_file = context.meta['index-file'] pivnetAPIToken = settings["pivnet-api-token"] f = open("manifests/{0}".format(index_file)) manifests = yaml.safe_load(f) f.close() eula_urls = [ "https://network.pivotal.io/api/v2/products/{0}/releases/{1}/eula_acceptance".format( m['release-name'], m['release-number']) for m in manifests['manifests']] release_urls = [ "https://network.pivotal.io/api/v2/products/{0}/releases/{1}/product_files/{2}/download".format( m['release-name'], m['release-number'], m['file-number']) for m in manifests['manifests']] stemcell_urls = [m['stemcell'] for m in manifests['manifests']] # accept eula for each product for url in eula_urls: print url if not "concourse" in url: res = authorizedPost(url, pivnetAPIToken) code = res.getcode() # releases is_release_file = re.compile("^releases\/.+") if not os.path.exists("/tmp/releases"): os.makedirs("/tmp/releases") client = bosh_client.BoshClient("https://10.0.0.4:25555", "admin", "admin") storage_account_name = settings["STORAGE-ACCOUNT-NAME"] storage_access_key = settings["STORAGE-ACCESS-KEY"] blob_service = BlobService(storage_account_name, storage_access_key) blob_service.create_container( container_name='tempreleases', x_ms_blob_public_access='container') print "Processing releases." for url in release_urls: print "Downloading {0}.".format(url) if "concourse" in url: release_url = "https://s3-us-west-2.amazonaws.com/bosh-azure-releases/concourse.zip" res = urllib2.urlopen(release_url) else: res = authorizedPost(url, pivnetAPIToken) code = res.getcode() length = int(res.headers["Content-Length"]) # content-length if code is 200: total = 0 pcent = 0.0 CHUNK = 16 * 1024 with tempfile.TemporaryFile() as temp: while True: chunk = res.read(CHUNK) total += CHUNK pcent = (float(total) / float(length)) * 100 sys.stdout.write( "Download progress: %.2f%% (%.2fM)\r" % (pcent, total / 1000000.0)) sys.stdout.flush() if not chunk: break temp.write(chunk) print "Download complete." z = zipfile.ZipFile(temp) for name in z.namelist(): # is this a release? if is_release_file.match(name): release_filename = "/tmp/{0}".format(name) print "Unpacking {0}.".format(name) z.extract(name, "/tmp") print "Uploading {0} to Azure blob store".format(name) blob_service.put_block_blob_from_path( 'tempreleases', name, "/tmp/{0}".format(name), x_ms_blob_content_type='application/x-compressed' ) os.unlink(release_filename) blob_url = "http://{0}.blob.core.windows.net/{1}/{2}".format( storage_account_name, 'tempreleases', name) print "Uploading release {0} to BOSH director.".format(name) task_id = client.upload_release(blob_url) client.wait_for_task(task_id) z.close() temp.close() blob_service.delete_container("tempreleases") # stemcells print "Processing stemcells." for url in stemcell_urls: print "Processing stemcell {0}".format(url) task_id = client.upload_stemcell(url) client.wait_for_task(task_id) return context
import logging from logging.handlers import RotatingFileHandler app = Flask(__name__) service_name = 'azure-storage' vcap_services = json.loads(os.environ['VCAP_SERVICES']) account_name = vcap_services[service_name][0]['credentials'][ 'storage_account_name'] account_key = vcap_services[service_name][0]['credentials'][ 'primary_access_key'] blob_service = BlobService(account_name, account_key) container_name = '{0}{1}'.format('cloud-foundry-', str(uuid.uuid4()).replace('-', '')) blob_service.create_container(container_name) @app.route('/') def index(): blobs = blob_service.list_blobs(container_name) return render_template("index.html", filenames=[blob.name for blob in blobs]) @app.route('/upload', methods=['POST']) def upload(): file = request.files['file'] if not file or not file.filename: msg = 'The file to upload is invalid' blobs = blob_service.list_blobs(container_name)
# Get settings from CustomScriptForLinux extension configurations waagent.LoggerInit('/var/log/waagent.log', '/dev/stdout') hutil = Util.HandlerUtility(waagent.Log, waagent.Error, "bosh-deploy-script") hutil.do_parse_context("enable") settings = hutil.get_public_settings() with open(os.path.join('bosh', 'settings'), "w") as tmpfile: tmpfile.write(json.dumps(settings, indent=4, sort_keys=True)) username = settings["username"] home_dir = os.path.join("/home", username) install_log = os.path.join(home_dir, "install.log") # Prepare the containers storage_account_name = settings["STORAGE-ACCOUNT-NAME"] storage_access_key = settings["STORAGE-ACCESS-KEY"] blob_service = BlobService(storage_account_name, storage_access_key) blob_service.create_container('bosh') blob_service.create_container(container_name='stemcell', x_ms_blob_public_access='blob') # Prepare the table for storing meta datas of storage account and stemcells table_service = TableService(storage_account_name, storage_access_key) table_service.create_table('stemcells') # Generate the private key and certificate call("sh create_cert.sh", shell=True) call("cp bosh.key ./bosh/bosh", shell=True) with open('bosh_cert.pem', 'r') as tmpfile: ssh_cert = tmpfile.read() ssh_cert = "|\n" + ssh_cert ssh_cert = "\n ".join([line for line in ssh_cert.split('\n')])
class AzureJobStore(AbstractJobStore): """ A job store that uses Azure's blob store for file storage and Table Service to store job info with strong consistency.""" def __init__(self, accountName, namePrefix, config=None, jobChunkSize=maxAzureTablePropertySize): self.jobChunkSize = jobChunkSize self.keyPath = None self.account_key = _fetchAzureAccountKey(accountName) # Table names have strict requirements in Azure self.namePrefix = self._sanitizeTableName(namePrefix) logger.debug("Creating job store with name prefix '%s'" % self.namePrefix) # These are the main API entrypoints. self.tableService = TableService(account_key=self.account_key, account_name=accountName) self.blobService = BlobService(account_key=self.account_key, account_name=accountName) # Register our job-store in the global table for this storage account self.registryTable = self._getOrCreateTable('toilRegistry') exists = self.registryTable.get_entity(row_key=self.namePrefix) self._checkJobStoreCreation(config is not None, exists, accountName + ":" + self.namePrefix) self.registryTable.insert_or_replace_entity(row_key=self.namePrefix, entity={'exists': True}) # Serialized jobs table self.jobItems = self._getOrCreateTable(self.qualify('jobs')) # Job<->file mapping table self.jobFileIDs = self._getOrCreateTable(self.qualify('jobFileIDs')) # Container for all shared and unshared files self.files = self._getOrCreateBlobContainer(self.qualify('files')) # Stats and logging strings self.statsFiles = self._getOrCreateBlobContainer( self.qualify('statsfiles')) # File IDs that contain stats and logging strings self.statsFileIDs = self._getOrCreateTable( self.qualify('statsFileIDs')) super(AzureJobStore, self).__init__(config=config) if self.config.cseKey is not None: self.keyPath = self.config.cseKey # Table names must be alphanumeric nameSeparator = 'xx' # Length of a jobID - used to test if a stats file has been read already or not jobIDLength = len(str(uuid.uuid4())) def qualify(self, name): return self.namePrefix + self.nameSeparator + name def jobs(self): # How many jobs have we done? total_processed = 0 for jobEntity in self.jobItems.query_entities_auto(): # Process the items in the page yield AzureJob.fromEntity(jobEntity) total_processed += 1 if total_processed % 1000 == 0: # Produce some feedback for the user, because this can take # a long time on, for example, Azure logger.info("Processed %d total jobs" % total_processed) logger.info("Processed %d total jobs" % total_processed) def create(self, command, memory, cores, disk, predecessorNumber=0): jobStoreID = self._newJobID() job = AzureJob(jobStoreID=jobStoreID, command=command, memory=memory, cores=cores, disk=disk, remainingRetryCount=self._defaultTryCount(), logJobStoreFileID=None, predecessorNumber=predecessorNumber) entity = job.toItem(chunkSize=self.jobChunkSize) entity['RowKey'] = jobStoreID self.jobItems.insert_entity(entity=entity) return job def exists(self, jobStoreID): if self.jobItems.get_entity(row_key=jobStoreID) is None: return False return True def load(self, jobStoreID): jobEntity = self.jobItems.get_entity(row_key=jobStoreID) if jobEntity is None: raise NoSuchJobException(jobStoreID) return AzureJob.fromEntity(jobEntity) def update(self, job): self.jobItems.update_entity( row_key=job.jobStoreID, entity=job.toItem(chunkSize=self.jobChunkSize)) def delete(self, jobStoreID): try: self.jobItems.delete_entity(row_key=jobStoreID) except WindowsAzureMissingResourceError: # Job deletion is idempotent, and this job has been deleted already return filterString = "PartitionKey eq '%s'" % jobStoreID for fileEntity in self.jobFileIDs.query_entities(filter=filterString): jobStoreFileID = fileEntity.RowKey self.deleteFile(jobStoreFileID) def deleteJobStore(self): self.registryTable.delete_entity(row_key=self.namePrefix) self.jobItems.delete_table() self.jobFileIDs.delete_table() self.files.delete_container() self.statsFiles.delete_container() self.statsFileIDs.delete_table() def getEnv(self): return dict(AZURE_ACCOUNT_KEY=self.account_key) def writeFile(self, localFilePath, jobStoreID=None): jobStoreFileID = self._newFileID() self.updateFile(jobStoreFileID, localFilePath) self._associateFileWithJob(jobStoreFileID, jobStoreID) return jobStoreFileID def updateFile(self, jobStoreFileID, localFilePath): with open(localFilePath) as read_fd: with self._uploadStream(jobStoreFileID, self.files) as write_fd: while True: buf = read_fd.read(self._maxAzureBlockBytes) write_fd.write(buf) if len(buf) == 0: break def readFile(self, jobStoreFileID, localFilePath): try: with self._downloadStream(jobStoreFileID, self.files) as read_fd: with open(localFilePath, 'w') as write_fd: while True: buf = read_fd.read(self._maxAzureBlockBytes) write_fd.write(buf) if not buf: break except WindowsAzureMissingResourceError: raise NoSuchFileException(jobStoreFileID) def deleteFile(self, jobStoreFileID): try: self.files.delete_blob(blob_name=jobStoreFileID) self._dissociateFileFromJob(jobStoreFileID) except WindowsAzureMissingResourceError: pass def fileExists(self, jobStoreFileID): # As Azure doesn't have a blob_exists method (at least in the # python API) we just try to download the metadata, and hope # the metadata is small so the call will be fast. try: self.files.get_blob_metadata(blob_name=jobStoreFileID) return True except WindowsAzureMissingResourceError: return False @contextmanager def writeFileStream(self, jobStoreID=None): # TODO: this (and all stream methods) should probably use the # Append Blob type, but that is not currently supported by the # Azure Python API. jobStoreFileID = self._newFileID() with self._uploadStream(jobStoreFileID, self.files) as fd: yield fd, jobStoreFileID self._associateFileWithJob(jobStoreFileID, jobStoreID) @contextmanager def updateFileStream(self, jobStoreFileID): with self._uploadStream(jobStoreFileID, self.files, checkForModification=True) as fd: yield fd def getEmptyFileStoreID(self, jobStoreID=None): jobStoreFileID = self._newFileID() self.files.put_blob(blob_name=jobStoreFileID, blob='', x_ms_blob_type='BlockBlob') self._associateFileWithJob(jobStoreFileID, jobStoreID) return jobStoreFileID @contextmanager def readFileStream(self, jobStoreFileID): if not self.fileExists(jobStoreFileID): raise NoSuchFileException(jobStoreFileID) with self._downloadStream(jobStoreFileID, self.files) as fd: yield fd @contextmanager def writeSharedFileStream(self, sharedFileName, isProtected=None): sharedFileID = self._newFileID(sharedFileName) with self._uploadStream(sharedFileID, self.files, encrypted=isProtected) as fd: yield fd @contextmanager def readSharedFileStream(self, sharedFileName): sharedFileID = self._newFileID(sharedFileName) if not self.fileExists(sharedFileID): raise NoSuchFileException(sharedFileID) with self._downloadStream(sharedFileID, self.files) as fd: yield fd def writeStatsAndLogging(self, statsAndLoggingString): # TODO: would be a great use case for the append blobs, once implemented in the Azure SDK jobStoreFileID = self._newFileID() encrypted = self.keyPath is not None if encrypted: statsAndLoggingString = encryption.encrypt(statsAndLoggingString, self.keyPath) self.statsFiles.put_block_blob_from_text( blob_name=jobStoreFileID, text=statsAndLoggingString, x_ms_meta_name_values=dict(encrypted=str(encrypted))) self.statsFileIDs.insert_entity(entity={'RowKey': jobStoreFileID}) def readStatsAndLogging(self, callback, readAll=False): suffix = '_old' numStatsFiles = 0 for entity in self.statsFileIDs.query_entities(): jobStoreFileID = entity.RowKey hasBeenRead = len(jobStoreFileID) > self.jobIDLength if not hasBeenRead: with self._downloadStream(jobStoreFileID, self.statsFiles) as fd: callback(fd) # Mark this entity as read by appending the suffix self.statsFileIDs.insert_entity( entity={'RowKey': jobStoreFileID + suffix}) self.statsFileIDs.delete_entity(row_key=jobStoreFileID) numStatsFiles += 1 elif readAll: # Strip the suffix to get the original ID jobStoreFileID = jobStoreFileID[:-len(suffix)] with self._downloadStream(jobStoreFileID, self.statsFiles) as fd: callback(fd) numStatsFiles += 1 return numStatsFiles _azureTimeFormat = "%Y-%m-%dT%H:%M:%SZ" def getPublicUrl(self, jobStoreFileID): try: self.files.get_blob_properties(blob_name=jobStoreFileID) except WindowsAzureMissingResourceError: raise NoSuchFileException(jobStoreFileID) # Compensate of a little bit of clock skew startTimeStr = (datetime.utcnow() - timedelta(minutes=5)).strftime( self._azureTimeFormat) endTime = datetime.utcnow() + self.publicUrlExpiration endTimeStr = endTime.strftime(self._azureTimeFormat) sap = SharedAccessPolicy( AccessPolicy(startTimeStr, endTimeStr, BlobSharedAccessPermissions.READ)) sas_token = self.files.generate_shared_access_signature( blob_name=jobStoreFileID, shared_access_policy=sap) return self.files.make_blob_url( blob_name=jobStoreFileID) + '?' + sas_token def getSharedPublicUrl(self, sharedFileName): jobStoreFileID = self._newFileID(sharedFileName) return self.getPublicUrl(jobStoreFileID) def _newJobID(self): # raw UUIDs don't work for Azure property names because the '-' character is disallowed. return str(uuid.uuid4()).replace('-', '_') # A dummy job ID under which all shared files are stored. sharedFileJobID = uuid.UUID('891f7db6-e4d9-4221-a58e-ab6cc4395f94') def _newFileID(self, sharedFileName=None): if sharedFileName is None: ret = str(uuid.uuid4()) else: ret = str(uuid.uuid5(self.sharedFileJobID, str(sharedFileName))) return ret.replace('-', '_') def _associateFileWithJob(self, jobStoreFileID, jobStoreID=None): if jobStoreID is not None: self.jobFileIDs.insert_entity(entity={ 'PartitionKey': jobStoreID, 'RowKey': jobStoreFileID }) def _dissociateFileFromJob(self, jobStoreFileID): entities = self.jobFileIDs.query_entities(filter="RowKey eq '%s'" % jobStoreFileID) if entities: assert len(entities) == 1 jobStoreID = entities[0].PartitionKey self.jobFileIDs.delete_entity(partition_key=jobStoreID, row_key=jobStoreFileID) def _getOrCreateTable(self, tableName): # This will not fail if the table already exists. for attempt in retry_on_error(): with attempt: self.tableService.create_table(tableName) return AzureTable(self.tableService, tableName) def _getOrCreateBlobContainer(self, containerName): for attempt in retry_on_error(): with attempt: self.blobService.create_container(containerName) return AzureBlobContainer(self.blobService, containerName) def _sanitizeTableName(self, tableName): """ Azure table names must start with a letter and be alphanumeric. This will never cause a collision if uuids are used, but otherwise may not be safe. """ return 'a' + filter(lambda x: x.isalnum(), tableName) # Maximum bytes that can be in any block of an Azure block blob # https://github.com/Azure/azure-storage-python/blob/4c7666e05a9556c10154508335738ee44d7cb104/azure/storage/blob/blobservice.py#L106 _maxAzureBlockBytes = 4 * 1024 * 1024 @contextmanager def _uploadStream(self, jobStoreFileID, container, checkForModification=False, encrypted=None): """ :param encrypted: True to enforce encryption (will raise exception unless key is set), False to prevent encryption or None to encrypt if key is set. """ if checkForModification: try: expectedVersion = container.get_blob_properties( blob_name=jobStoreFileID)['etag'] except WindowsAzureMissingResourceError: expectedVersion = None if encrypted is None: encrypted = self.keyPath is not None elif encrypted: if self.keyPath is None: raise RuntimeError( 'Encryption requested but no key was provided') maxBlockSize = self._maxAzureBlockBytes if encrypted: # There is a small overhead for encrypted data. maxBlockSize -= encryption.overhead readable_fh, writable_fh = os.pipe() with os.fdopen(readable_fh, 'r') as readable: with os.fdopen(writable_fh, 'w') as writable: def reader(): blockIDs = [] try: while True: buf = readable.read(maxBlockSize) if len(buf) == 0: # We're safe to break here even if we never read anything, since # putting an empty block list creates an empty blob. break if encrypted: buf = encryption.encrypt(buf, self.keyPath) blockID = self._newFileID() container.put_block(blob_name=jobStoreFileID, block=buf, blockid=blockID) blockIDs.append(blockID) except: # This is guaranteed to delete any uncommitted # blocks. container.delete_blob(blob_name=jobStoreFileID) raise if checkForModification and expectedVersion is not None: # Acquire a (60-second) write lock, leaseID = container.lease_blob( blob_name=jobStoreFileID, x_ms_lease_action='acquire')['x-ms-lease-id'] # check for modification, blobProperties = container.get_blob_properties( blob_name=jobStoreFileID) if blobProperties['etag'] != expectedVersion: container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action='release', x_ms_lease_id=leaseID) raise ConcurrentFileModificationException( jobStoreFileID) # commit the file, container.put_block_list(blob_name=jobStoreFileID, block_list=blockIDs, x_ms_lease_id=leaseID, x_ms_meta_name_values=dict( encrypted=str(encrypted))) # then release the lock. container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action='release', x_ms_lease_id=leaseID) else: # No need to check for modification, just blindly write over whatever # was there. container.put_block_list(blob_name=jobStoreFileID, block_list=blockIDs, x_ms_meta_name_values=dict( encrypted=str(encrypted))) thread = ExceptionalThread(target=reader) thread.start() yield writable # The writable is now closed. This will send EOF to the readable and cause that # thread to finish. thread.join() @contextmanager def _downloadStream(self, jobStoreFileID, container): # The reason this is not in the writer is so we catch non-existant blobs early blobProps = container.get_blob_properties(blob_name=jobStoreFileID) encrypted = strict_bool(blobProps['x-ms-meta-encrypted']) if encrypted and self.keyPath is None: raise AssertionError( 'Content is encrypted but no key was provided.') readable_fh, writable_fh = os.pipe() with os.fdopen(readable_fh, 'r') as readable: with os.fdopen(writable_fh, 'w') as writable: def writer(): try: chunkStartPos = 0 fileSize = int(blobProps['Content-Length']) while chunkStartPos < fileSize: chunkEndPos = chunkStartPos + self._maxAzureBlockBytes - 1 buf = container.get_blob( blob_name=jobStoreFileID, x_ms_range="bytes=%d-%d" % (chunkStartPos, chunkEndPos)) if encrypted: buf = encryption.decrypt(buf, self.keyPath) writable.write(buf) chunkStartPos = chunkEndPos + 1 finally: # Ensure readers aren't left blocking if this thread crashes. # This close() will send EOF to the reading end and ultimately cause the # yield to return. It also makes the implict .close() done by the enclosing # "with" context redundant but that should be ok since .close() on file # objects are idempotent. writable.close() thread = ExceptionalThread(target=writer) thread.start() yield readable thread.join()
# Get settings from CustomScriptForLinux extension configurations waagent.LoggerInit('/var/log/waagent.log', '/dev/stdout') hutil = Util.HandlerUtility(waagent.Log, waagent.Error, "bosh-deploy-script") hutil.do_parse_context("enable") settings = hutil.get_public_settings() with open (os.path.join('bosh','settings'), "w") as tmpfile: tmpfile.write(json.dumps(settings, indent=4, sort_keys=True)) username = settings["username"] home_dir = os.path.join("/home", username) install_log = os.path.join(home_dir, "install.log") # Prepare the containers storage_account_name = settings["STORAGE-ACCOUNT-NAME"] storage_access_key = settings["STORAGE-ACCESS-KEY"] blob_service = BlobService(storage_account_name, storage_access_key) blob_service.create_container('bosh') blob_service.create_container(container_name='stemcell', x_ms_blob_public_access='blob' ) # Prepare the table for storing meta datas of storage account and stemcells table_service = TableService(storage_account_name, storage_access_key) table_service.create_table('stemcells') # Generate the private key and certificate call("sh create_cert.sh", shell=True) call("cp bosh.key ./bosh/bosh", shell=True) with open ('bosh_cert.pem', 'r') as tmpfile: ssh_cert = tmpfile.read() ssh_cert = "|\n" + ssh_cert ssh_cert="\n ".join([line for line in ssh_cert.split('\n')])
class AzureBackend(duplicity.backend.Backend): u""" Backend for Azure Blob Storage Service """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) # Import Microsoft Azure Storage SDK for Python library. try: import azure import azure.storage if hasattr(azure.storage, u'BlobService'): # v0.11.1 and below from azure.storage import BlobService self.AzureMissingResourceError = azure.WindowsAzureMissingResourceError self.AzureConflictError = azure.WindowsAzureConflictError else: # v1.0.0 and above import azure.storage.blob if hasattr(azure.storage.blob, u'BlobService'): from azure.storage.blob import BlobService else: from azure.storage.blob.blockblobservice import BlockBlobService as BlobService self.AzureMissingResourceError = azure.common.AzureMissingResourceHttpError self.AzureConflictError = azure.common.AzureConflictHttpError except ImportError as e: raise BackendException(u"""\ Azure backend requires Microsoft Azure Storage SDK for Python (https://pypi.python.org/pypi/azure-storage/). Exception: %s""" % str(e)) # TODO: validate container name self.container = parsed_url.path.lstrip(u'/') if u'AZURE_ACCOUNT_NAME' not in os.environ: raise BackendException(u'AZURE_ACCOUNT_NAME environment variable not set.') if u'AZURE_ACCOUNT_KEY' in os.environ: if u'AZURE_ENDPOINT_SUFFIX' in os.environ: self.blob_service = BlobService(account_name=os.environ[u'AZURE_ACCOUNT_NAME'], account_key=os.environ[u'AZURE_ACCOUNT_KEY'], endpoint_suffix=os.environ[u'AZURE_ENDPOINT_SUFFIX']) else: self.blob_service = BlobService(account_name=os.environ[u'AZURE_ACCOUNT_NAME'], account_key=os.environ[u'AZURE_ACCOUNT_KEY']) self._create_container() elif u'AZURE_SHARED_ACCESS_SIGNATURE' in os.environ: if u'AZURE_ENDPOINT_SUFFIX' in os.environ: self.blob_service = BlobService(account_name=os.environ[u'AZURE_ACCOUNT_NAME'], sas_token=os.environ[u'AZURE_SHARED_ACCESS_SIGNATURE'], endpoint_suffix=os.environ[u'AZURE_ENDPOINT_SUFFIX']) else: self.blob_service = BlobService(account_name=os.environ[u'AZURE_ACCOUNT_NAME'], sas_token=os.environ[u'AZURE_SHARED_ACCESS_SIGNATURE']) else: raise BackendException( u'Neither AZURE_ACCOUNT_KEY nor AZURE_SHARED_ACCESS_SIGNATURE environment variable not set.') if globals.azure_max_single_put_size: # check if we use azure-storage>=0.30.0 try: _ = self.blob_service.MAX_SINGLE_PUT_SIZE self.blob_service.MAX_SINGLE_PUT_SIZE = globals.azure_max_single_put_size # fallback for azure-storage<0.30.0 except AttributeError: self.blob_service._BLOB_MAX_DATA_SIZE = globals.azure_max_single_put_size if globals.azure_max_block_size: # check if we use azure-storage>=0.30.0 try: _ = self.blob_service.MAX_BLOCK_SIZE self.blob_service.MAX_BLOCK_SIZE = globals.azure_max_block_size # fallback for azure-storage<0.30.0 except AttributeError: self.blob_service._BLOB_MAX_CHUNK_DATA_SIZE = globals.azure_max_block_size def _create_container(self): try: self.blob_service.create_container(self.container, fail_on_exist=True) except self.AzureConflictError: # Indicates that the resource could not be created because it already exists. pass except Exception as e: log.FatalError(u"Could not create Azure container: %s" % str(e.message).split(u'\n', 1)[0], log.ErrorCode.connection_failed) def _put(self, source_path, remote_filename): remote_filename = fsdecode(remote_filename) kwargs = {} if globals.azure_max_connections: kwargs[u'max_connections'] = globals.azure_max_connections # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#upload-a-blob-into-a-container try: self.blob_service.create_blob_from_path(self.container, remote_filename, source_path.name, **kwargs) except AttributeError: # Old versions use a different method name self.blob_service.put_block_blob_from_path(self.container, remote_filename, source_path.name, **kwargs) self._set_tier(remote_filename) def _set_tier(self, remote_filename): if globals.azure_blob_tier is not None: try: self.blob_service.set_standard_blob_tier(self.container, remote_filename, globals.azure_blob_tier) except AttributeError: # might not be available in old API pass def _get(self, remote_filename, local_path): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#download-blobs self.blob_service.get_blob_to_path(self.container, fsdecode(remote_filename), local_path.name) def _list(self): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#list-the-blobs-in-a-container blobs = [] marker = None while True: batch = self.blob_service.list_blobs(self.container, marker=marker) blobs.extend(batch) if not batch.next_marker: break marker = batch.next_marker return [blob.name for blob in blobs] def _delete(self, filename): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#delete-blobs self.blob_service.delete_blob(self.container, fsdecode(filename)) def _query(self, filename): prop = self.blob_service.get_blob_properties(self.container, fsdecode(filename)) try: info = {u'size': int(prop.properties.content_length)} except AttributeError: # old versions directly returned the properties info = {u'size': int(prop[u'content-length'])} return info def _error_code(self, operation, e): if isinstance(e, self.AzureMissingResourceError): return log.ErrorCode.backend_not_found