def remove_image(album_name, username): gallery_db = connect_to_db() albums = gallery_db.albums requested_album = albums.find_one({"name": album_name}) if not requested_album: return redirect(url_for('albums', album =album_name, message="album not found")) if not username in requested_album["write"]: return redirect(url_for('albums', album = album_name, message="permission denied")) image = request.form.get('image', '') if not image: return redirect(url_for('albums', album=album_name, message="no image was chosen for removal")) blob_service = BlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY) try: blob_service.delete_blob(CONTAINER_NAME, image) except WindowsAzureMissingResourceError: # Even if the file is not in the blob storage, we want to remove it from the album pass gallery_db.albums.update({'name': album_name}, {'$pull': {'images': image}}) # increment the counter of the removed images stats_client.incr("images removed", 1) return redirect(url_for('albums', album=album_name))
def test_azure_call(request): import os try: from azure.storage import BlobService bs = BlobService(os.environ["AZURE_STORAGE_ACCOUNT"], os.environ["AZURE_STORAGE_ACCESS_KEY"]) import random container_name = hex(int(random.random() * 1000000000)) bs.create_container(container_name) bs.put_blob(container_name, 'testblob', 'hello world\n', 'BlockBlob') blob = bs.get_blob(container_name, 'testblob') if blob != 'hello world\n': return HttpResponse("Failed!", status = '404') bs.delete_blob(container_name, 'testblob') bs.delete_container(container_name) return HttpResponse("Succeeded!") except: try: import traceback return HttpResponse(traceback.format_exc() + str(os.environ.keys())) except: import traceback return HttpResponse(traceback.format_exc())
class FileService(Component): def __init__(self): self.blob_service = None def generate_blob_service(self): if self.blob_service is None: # if storage info doesn't exist in config.py upload file function stop working self.blob_service = BlobService(account_name=self.util.get_config("storage.azure.account_name"), account_key=self.util.get_config("storage.azure.account_key"), host_base=self.util.get_config("storage.azure.blob_service_host_base")) def create_container_in_storage(self, container_name, access): """ create a container if doesn't exist :param container_name: :param access: :return: """ self.generate_blob_service() try: names = map(lambda x: x.name, self.blob_service.list_containers()) if container_name not in names: self.blob_service.create_container(container_name, x_ms_blob_public_access=access) else: self.log.debug("container already exsit in storage") return True except Exception as e: self.log.error(e) return False def upload_file_to_azure(self, stream, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_file(container_name, blob_name, stream) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_path(self, path, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_path(container_name, blob_name, path) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def delete_file_from_azure(self, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.delete_blob(container_name, blob_name) except Exception as e: self.log.error(e) return None
def delete_video(id): vs_operator = videostoreoperator.VideoStoreOperator() video_detils = vs_operator.get(id) vs_operator.delete(id) count = len(video_detils.chuncks) for i in range(count): chk = video_detils.chuncks[i] blob_service = BlobService(account_name=chk.storagename, account_key=chk.key) try: blob_service.delete_blob(chk.container, chk.path) except azure.WindowsAzureMissingResourceError: pass
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, container_name): BaseTransfer.__init__(self) self.account_name = account_name self.account_key = account_key self.container_name = container_name self.conn = BlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized") def get_metadata_for_key(self, key): key = fix_path(key) return self.list_path(key)[0]['metadata'] def list_path(self, path): return_list = [] path = fix_path(path) self.log.info("Asking for listing of: %r", path) for r in self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata"): entry = {"name": r.name, "size": r.properties.content_length, "last_modified": dateutil.parser.parse(r.properties.last_modified), "metadata": r.metadata} return_list.append(entry) return return_list def delete_key(self, key_name): key_name = fix_path(key_name) self.log.debug("Deleting key: %r", key_name) return self.conn.delete_blob(self.container_name, key_name) def get_contents_to_file(self, obj_key, filepath_to_store_to): obj_key = fix_path(obj_key) self.log.debug("Starting to fetch the contents of: %r to: %r", obj_key, filepath_to_store_to) return self.conn.get_blob_to_path(self.container_name, obj_key, filepath_to_store_to) def get_contents_to_string(self, obj_key): obj_key = fix_path(obj_key) self.log.debug("Starting to fetch the contents of: %r", obj_key) return self.conn.get_blob_to_bytes(self.container_name, obj_key), self.get_metadata_for_key(obj_key) def store_file_from_memory(self, key, memstring, metadata=None): # For whatever reason Azure requires all values to be strings at the point of sending metadata_to_send = dict((str(k), str(v)) for k, v in metadata.items()) self.conn.put_block_blob_from_bytes(self.container_name, key, memstring, x_ms_meta_name_values=metadata_to_send) def store_file_from_disk(self, key, filepath, metadata=None): # For whatever reason Azure requires all values to be strings at the point of sending metadata_to_send = dict((str(k), str(v)) for k, v in metadata.items()) self.conn.put_block_blob_from_path(self.container_name, key, filepath, x_ms_meta_name_values=metadata_to_send) def get_or_create_container(self, container_name): start_time = time.time() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time) return container_name
class BlobCache: ''' Simplistic cache toolkit targetting an Azure Blob Service. name: the name of a storage account. key: the access key for the storage account. container: the name of the container to use. ''' def __init__(self, name, key, container): self.container = container self.blobstore = BlobService(name, key) self.blobstore.create_container(self.container) def getresponse(self, cachekey): ''' Get a value from the cache. cachekey: The key. Kilroy notes that this throws an exception rather than returning a value on failure. ''' return self.blobstore.get_blob_to_text(self.container, str2blobname(cachekey)) def putresponse(self, cachekey, value): ''' Put a value in the cache with the given key. cachekey: The key. value: The value to associate with the key. ''' return self.blobstore.put_block_blob_from_text(self.container, str2blobname(cachekey), value) def invalidate(self, cachekey): ''' Invalidate a value in the cache. Immediate. Permanent. cachekey: The key. ''' self.blobstore.delete_blob(self.container, str2blobname(cachekey))
class BlobCache: ''' Simplistic cache toolkit targetting an Azure Blob Service. name: the name of a storage account. key: the access key for the storage account. container: the name of the container to use. ''' def __init__(self, name, key, container): self.container = container self.blobstore = BlobService(name, key) self.blobstore.create_container(self.container) def getresponse(self, cachekey): ''' Get a value from the cache. cachekey: The key. Kilroy notes that this throws an exception rather than returning a value on failure. ''' return self.blobstore.get_blob_to_text(self.container,str2blobname(cachekey)) def putresponse(self, cachekey, value): ''' Put a value in the cache with the given key. cachekey: The key. value: The value to associate with the key. ''' return self.blobstore.put_block_blob_from_text(self.container, str2blobname(cachekey), value) def invalidate(self, cachekey): ''' Invalidate a value in the cache. Immediate. Permanent. cachekey: The key. ''' self.blobstore.delete_blob(self.container, str2blobname(cachekey))
def _deleteBlob(self, storageAccount, mediaLink): primary = self._getStorageAccountKey(storageAccount) # BlobServiceオブジェクトを作成 blobService = BlobService(storageAccount, primary) (container, blob) = self._getBlobFromMediaLink(blobService, mediaLink) rs = blobService.delete_blob(container_name=container.name, blob_name=blob.name) try: updatedBlob = blobService.get_blob_properties(container_name=container.name, blob_name=blob.name) except WindowsAzureMissingResourceError as e: return True return False
def rest_modify_image(album_name, username): gallery_db = connect_to_db() albums = gallery_db.albums requested_album = albums.find_one({"name": album_name}) if not requested_album: return jsonify({'error': "album does not exist"}) if not username in requested_album["write"]: return jsonify({'error': "no permission to post images"}) if request.method == 'POST': req_file = request.json.get('data', '') if not req_file: return jsonify({'error': "no images"}) file_name = uuid.uuid4().hex blob_service = BlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY) blob_service.put_block_blob_from_bytes(CONTAINER_NAME, file_name, req_file.decode("base64")) gallery_db.albums.update({'name': album_name}, {'$push': {'images': file_name}}) return jsonify({'success': "file uploaded", 'file_name': file_name}) else: # DELETE image = request.json.get('image', '') if not image: return jsonify({'error': 'no image name'}) blob_service = BlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY) try: blob_service.delete_blob(CONTAINER_NAME, image) except WindowsAzureMissingResourceError: # Even if the file is not in the blob storage, we want to remove it from the album pass gallery_db.albums.update({'name': album_name}, {'$pull': {'images': image}}) return jsonify({'success': "file deleted"})
def _deleteBlob(self, storageAccount, mediaLink): primary = self._getStorageAccountKey(storageAccount) # BlobServiceオブジェクトを作成 blobService = BlobService(storageAccount, primary) (container, blob) = self._getBlobFromMediaLink(blobService, mediaLink) rs = blobService.delete_blob(container_name=container.name, blob_name=blob.name) try: updatedBlob = blobService.get_blob_properties( container_name=container.name, blob_name=blob.name) except WindowsAzureMissingResourceError as e: return True return False
class BlobServiceAdapter(Component): """The :class:`BlobServiceAdapter` class is a thin wrapper over azure.storage.BlobService. All the attributes of the wrapper stream are proxied by the adapter so it's possible to do ``adapter.create_container()`` instead of the long form ``adapter.blob_service.adapter()``. """ def __init__(self): self.blob_service = BlobService( account_name=self.util.get_config("storage.azure.account_name"), account_key=self.util.get_config("storage.azure.account_key"), host_base=self.util.get_config("storage.azure.blob_service_host_base"), ) def __getattr__(self, name): return getattr(self.blob_service, name) def create_container_in_storage(self, container_name, access="container"): """create a container if doesn't exist :type container_name: str|unicode :param container_name: Name of container to create. :type access: str|unicode :param access: Optional. Possible values include: container, blob :return: """ try: names = [x.name for x in self.blob_service.list_containers()] if container_name not in names: return self.blob_service.create_container(container_name, x_ms_blob_public_access=access) else: self.log.debug("container already exists in storage") return True except Exception as e: self.log.error(e) return False def upload_file_to_azure(self, container_name, blob_name, stream): """ Creates a new block blob from a file/stream, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str | unicode :param blob_name: Name of blob to create or update. :type stream: file :param stream: Opened file/stream to upload as the blob content. """ try: if self.create_container_in_storage(container_name, "container"): self.blob_service.put_block_blob_from_file(container_name, blob_name, stream) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_bytes(self, container_name, blob_name, blob): """ Creates a new block blob from an array of bytes, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type blob: bytes :param blob: Content of blob as an array of bytes. """ try: if self.create_container_in_storage(container_name, "container"): self.blob_service.put_block_blob_from_bytes(container_name, blob_name, blob) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_text(self, container_name, blob_name, text): """ Creates a new block blob from str/unicode, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type text: str|unicode :param text: Text to upload to the blob. """ try: if self.create_container_in_storage(container_name, "container"): self.blob_service.put_block_blob_from_text(container_name, blob_name, text) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_path(self, container_name, blob_name, path): """ Creates a new page blob from a file path, or updates the content of an existing page blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type path: str|unicode :param path: Path of the file to upload as the blob content. """ try: if self.create_container_in_storage(container_name, "container"): self.blob_service.put_block_blob_from_path(container_name, blob_name, path) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def delete_file_from_azure(self, container_name, blob_name): try: if self.create_container_in_storage(container_name, "container"): self.blob_service.delete_blob(container_name, blob_name) except Exception as e: self.log.error(e) return None
class AzureStorageBlockDeviceAPI(object): """ An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices Current Support: Azure SMS API """ def __init__(self, **azure_config): """ :param ServiceManagement azure_client: an instance of the azure serivce managment api client. :param String service_name: The name of the cloud service :param names of Azure volumes to identify cluster :returns: A ``BlockDeviceVolume``. """ self._instance_id = self.compute_instance_id() self._azure_service_client = ServiceManagementService( azure_config['subscription_id'], azure_config['management_certificate_path']) self._service_name = azure_config['service_name'] self._azure_storage_client = BlobService( azure_config['storage_account_name'], azure_config['storage_account_key']) self._storage_account_name = azure_config['storage_account_name'] self._disk_container_name = azure_config['disk_container_name'] if azure_config['debug']: to_file(sys.stdout) def allocation_unit(self): """ 1GiB is the minimum allocation unit for azure disks return int: 1 GiB """ return int(GiB(1).to_Byte().value) def compute_instance_id(self): """ Azure Stored a UUID in the SDC kernel module. """ # Node host names should be unique within a vnet return unicode(socket.gethostname()) def create_volume(self, dataset_id, size): """ Create a new volume. :param UUID dataset_id: The Flocker dataset ID of the dataset on this volume. :param int size: The size of the new volume in bytes. :returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when the volume has been created. """ size_in_gb = Byte(size).to_GiB().value if size_in_gb % 1 != 0: raise UnsupportedVolumeSize(dataset_id) self._create_volume_blob(size, dataset_id) label = self._disk_label_for_dataset_id(str(dataset_id)) return BlockDeviceVolume( blockdevice_id=unicode(label), size=size, attached_to=None, dataset_id=self._dataset_id_for_disk_label(label)) def destroy_volume(self, blockdevice_id): """ Destroy an existing volume. :param unicode blockdevice_id: The unique identifier for the volume to destroy. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :return: ``None`` """ log_info('Destorying block device: ' + str(blockdevice_id)) (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) request = None if lun is not None: request = \ self._azure_service_client.delete_data_disk( service_name=self._service_name, deployment_name=self._service_name, role_name=target_disk.attached_to.role_name, lun=lun, delete_vhd=True) else: if target_disk.__class__.__name__ == 'Blob': # unregistered disk self._azure_storage_client.delete_blob( self._disk_container_name, target_disk.name) else: request = self._azure_service_client.delete_disk( target_disk.name, True) if request is not None: self._wait_for_async(request.request_id, 5000) self._wait_for_detach(blockdevice_id) def attach_volume(self, blockdevice_id, attach_to): """ Attach ``blockdevice_id`` to ``host``. :param unicode blockdevice_id: The unique identifier for the block device being attached. :param unicode attach_to: An identifier like the one returned by the ``compute_instance_id`` method indicating the node to which to attach the volume. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is already attached. :returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to ``host``. """ (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is not None: raise AlreadyAttachedVolume(blockdevice_id) log_info('Attempting to attach ' + str(blockdevice_id) + ' to ' + str(attach_to)) disk_size = self._attach_disk(blockdevice_id, target_disk, attach_to) self._wait_for_attach(blockdevice_id) log_info('disk attached') return self._blockdevicevolume_from_azure_volume(blockdevice_id, disk_size, attach_to) def detach_volume(self, blockdevice_id): """ Detach ``blockdevice_id`` from whatever host it is attached to. :param unicode blockdevice_id: The unique identifier for the block device being detached. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to anything. :returns: ``None`` """ (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) # contrary to function name it doesn't delete by default, just detachs request = \ self._azure_service_client.delete_data_disk( service_name=self._service_name, deployment_name=self._service_name, role_name=role_name, lun=lun) self._wait_for_async(request.request_id, 5000) self._wait_for_detach(blockdevice_id) def get_device_path(self, blockdevice_id): """ Return the device path that has been allocated to the block device on the host to which it is currently attached. :param unicode blockdevice_id: The unique identifier for the block device. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to a host. :returns: A ``FilePath`` for the device. """ (target_disk_or_blob, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk_or_blob is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) return Lun.get_device_path_for_lun(lun) def list_volumes(self): """ List all the block devices available via the back end API. :returns: A ``list`` of ``BlockDeviceVolume``s. """ media_url_prefix = 'https://' + self._storage_account_name \ + '.blob.core.windows.net/' + self._disk_container_name disks = self._azure_service_client.list_disks() disk_list = [] all_blobs = self._get_flocker_blobs() for d in disks: if media_url_prefix not in d.media_link or \ 'flocker-' not in d.label: continue role_name = None if d.attached_to is not None \ and d.attached_to.role_name is not None: role_name = d.attached_to.role_name disk_list.append(self._blockdevicevolume_from_azure_volume( d.label, self._gibytes_to_bytes(d.logical_disk_size_in_gb), role_name)) if d.label in all_blobs: del all_blobs[d.label] for key in all_blobs: # include unregistered 'disk' blobs disk_list.append(self._blockdevicevolume_from_azure_volume( all_blobs[key].name, all_blobs[key].properties.content_length, None)) return disk_list def _attach_disk( self, blockdevice_id, target_disk, attach_to): """ Attaches disk to specified VM :param string blockdevice_id: The identifier of the disk :param DataVirtualHardDisk/Blob target_disk: The Blob or Disk to be attached :returns int: The size of the attached disk """ lun = Lun.compute_next_lun( self._azure_service_client, self._service_name, str(attach_to)) common_params = { 'service_name': self._service_name, 'deployment_name': self._service_name, 'role_name': attach_to, 'lun': lun } disk_size = None if target_disk.__class__.__name__ == 'Blob': # exclude 512 byte footer disk_size = target_disk.properties.content_length common_params['source_media_link'] = \ 'https://' + self._storage_account_name \ + '.blob.core.windows.net/' + self._disk_container_name \ + '/' + blockdevice_id common_params['disk_label'] = blockdevice_id else: disk_size = self._gibytes_to_bytes( target_disk.logical_disk_size_in_gb) common_params['disk_name'] = target_disk.name request = self._azure_service_client.add_data_disk(**common_params) self._wait_for_async(request.request_id, 5000) return disk_size def _create_volume_blob(self, size, dataset_id): # Create a new page blob as a blank disk self._azure_storage_client.put_blob( container_name=self._disk_container_name, blob_name=self._disk_label_for_dataset_id(dataset_id), blob=None, x_ms_blob_type='PageBlob', x_ms_blob_content_type='application/octet-stream', x_ms_blob_content_length=size) # for disk to be a valid vhd it requires a vhd footer # on the last 512 bytes vhd_footer = Vhd.generate_vhd_footer(size) self._azure_storage_client.put_page( container_name=self._disk_container_name, blob_name=self._disk_label_for_dataset_id(dataset_id), page=vhd_footer, x_ms_page_write='update', x_ms_range='bytes=' + str((size - 512)) + '-' + str(size - 1)) def _disk_label_for_dataset_id(self, dataset_id): """ Returns a disk label for a given Dataset ID :param unicode dataset_id: The identifier of the dataset :returns string: A string representing the disk label """ label = 'flocker-' + str(dataset_id) return label def _dataset_id_for_disk_label(self, disk_label): """ Returns a UUID representing the Dataset ID for the given disk label :param string disk_label: The disk label :returns UUID: The UUID of the dataset """ return UUID(disk_label.replace('flocker-', '')) def _get_disk_vmname_lun(self, blockdevice_id): target_disk = None target_lun = None role_name = None disk_list = self._azure_service_client.list_disks() for d in disk_list: if 'flocker-' not in d.label: continue if d.label == str(blockdevice_id): target_disk = d break if target_disk is None: # check for unregisterd disk blobs = self._get_flocker_blobs() blob = None if str(blockdevice_id) in blobs: blob = blobs[str(blockdevice_id)] return blob, None, None vm_info = None if hasattr(target_disk.attached_to, 'role_name'): vm_info = self._azure_service_client.get_role( self._service_name, self._service_name, target_disk.attached_to.role_name) for d in vm_info.data_virtual_hard_disks: if d.disk_name == target_disk.name: target_lun = d.lun break role_name = target_disk.attached_to.role_name return (target_disk, role_name, target_lun) def _get_flocker_blobs(self): all_blobs = {} blobs = self._azure_storage_client.list_blobs( self._disk_container_name, prefix='flocker-') for b in blobs: # todo - this could be big! all_blobs[b.name] = b return all_blobs def _wait_for_detach(self, blockdevice_id): role_name = '' lun = -1 timeout_count = 0 log_info('waiting for azure to ' + 'report disk as detached...') while role_name is not None or lun is not None: (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) time.sleep(1) timeout_count += 1 if timeout_count > 5000: raise AsynchronousTimeout() log_info('Disk Detached') def _wait_for_attach(self, blockdevice_id): timeout_count = 0 lun = None log_info('waiting for azure to report disk as attached...') while lun is None: (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) time.sleep(.001) timeout_count += 1 if timeout_count > 5000: raise AsynchronousTimeout() def _wait_for_async(self, request_id, timeout): count = 0 result = self._azure_service_client.get_operation_status(request_id) while result.status == 'InProgress': count = count + 1 if count > timeout: log_error('Timed out waiting for async operation to complete.') raise AsynchronousTimeout() time.sleep(.001) log_info('.') result = self._azure_service_client.get_operation_status( request_id) if result.error: log_error(result.error.code) log_error(str(result.error.message)) log_error(result.status + ' in ' + str(count * 5) + 's') def _gibytes_to_bytes(self, size): return int(GiB(size).to_Byte().value) def _blockdevicevolume_from_azure_volume(self, label, size, attached_to_name): # azure will report the disk size excluding the 512 byte footer # however flocker expects the exact value it requested for disk size # so offset the reported size to flocker by 512 bytes return BlockDeviceVolume( blockdevice_id=unicode(label), size=int(size), attached_to=attached_to_name, dataset_id=self._dataset_id_for_disk_label(label) ) # disk labels are formatted as flocker-<data_set_id>
class AzureBackend(duplicity.backend.Backend): u""" Backend for Azure Blob Storage Service """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) # Import Microsoft Azure Storage SDK for Python library. try: import azure import azure.storage if hasattr(azure.storage, u'BlobService'): # v0.11.1 and below from azure.storage import BlobService self.AzureMissingResourceError = azure.WindowsAzureMissingResourceError self.AzureConflictError = azure.WindowsAzureConflictError else: # v1.0.0 and above import azure.storage.blob if hasattr(azure.storage.blob, u'BlobService'): from azure.storage.blob import BlobService else: from azure.storage.blob.blockblobservice import BlockBlobService as BlobService self.AzureMissingResourceError = azure.common.AzureMissingResourceHttpError self.AzureConflictError = azure.common.AzureConflictHttpError except ImportError as e: raise BackendException(u"""\ Azure backend requires Microsoft Azure Storage SDK for Python (https://pypi.python.org/pypi/azure-storage/). Exception: %s""" % str(e)) # TODO: validate container name self.container = parsed_url.path.lstrip(u'/') if u'AZURE_ACCOUNT_NAME' not in os.environ: raise BackendException(u'AZURE_ACCOUNT_NAME environment variable not set.') if u'AZURE_ACCOUNT_KEY' in os.environ: if u'AZURE_ENDPOINT_SUFFIX' in os.environ: self.blob_service = BlobService(account_name=os.environ[u'AZURE_ACCOUNT_NAME'], account_key=os.environ[u'AZURE_ACCOUNT_KEY'], endpoint_suffix=os.environ[u'AZURE_ENDPOINT_SUFFIX']) else: self.blob_service = BlobService(account_name=os.environ[u'AZURE_ACCOUNT_NAME'], account_key=os.environ[u'AZURE_ACCOUNT_KEY']) self._create_container() elif u'AZURE_SHARED_ACCESS_SIGNATURE' in os.environ: if u'AZURE_ENDPOINT_SUFFIX' in os.environ: self.blob_service = BlobService(account_name=os.environ[u'AZURE_ACCOUNT_NAME'], sas_token=os.environ[u'AZURE_SHARED_ACCESS_SIGNATURE'], endpoint_suffix=os.environ[u'AZURE_ENDPOINT_SUFFIX']) else: self.blob_service = BlobService(account_name=os.environ[u'AZURE_ACCOUNT_NAME'], sas_token=os.environ[u'AZURE_SHARED_ACCESS_SIGNATURE']) else: raise BackendException( u'Neither AZURE_ACCOUNT_KEY nor AZURE_SHARED_ACCESS_SIGNATURE environment variable not set.') if globals.azure_max_single_put_size: # check if we use azure-storage>=0.30.0 try: _ = self.blob_service.MAX_SINGLE_PUT_SIZE self.blob_service.MAX_SINGLE_PUT_SIZE = globals.azure_max_single_put_size # fallback for azure-storage<0.30.0 except AttributeError: self.blob_service._BLOB_MAX_DATA_SIZE = globals.azure_max_single_put_size if globals.azure_max_block_size: # check if we use azure-storage>=0.30.0 try: _ = self.blob_service.MAX_BLOCK_SIZE self.blob_service.MAX_BLOCK_SIZE = globals.azure_max_block_size # fallback for azure-storage<0.30.0 except AttributeError: self.blob_service._BLOB_MAX_CHUNK_DATA_SIZE = globals.azure_max_block_size def _create_container(self): try: self.blob_service.create_container(self.container, fail_on_exist=True) except self.AzureConflictError: # Indicates that the resource could not be created because it already exists. pass except Exception as e: log.FatalError(u"Could not create Azure container: %s" % str(e.message).split(u'\n', 1)[0], log.ErrorCode.connection_failed) def _put(self, source_path, remote_filename): remote_filename = fsdecode(remote_filename) kwargs = {} if globals.azure_max_connections: kwargs[u'max_connections'] = globals.azure_max_connections # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#upload-a-blob-into-a-container try: self.blob_service.create_blob_from_path(self.container, remote_filename, source_path.name, **kwargs) except AttributeError: # Old versions use a different method name self.blob_service.put_block_blob_from_path(self.container, remote_filename, source_path.name, **kwargs) self._set_tier(remote_filename) def _set_tier(self, remote_filename): if globals.azure_blob_tier is not None: try: self.blob_service.set_standard_blob_tier(self.container, remote_filename, globals.azure_blob_tier) except AttributeError: # might not be available in old API pass def _get(self, remote_filename, local_path): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#download-blobs self.blob_service.get_blob_to_path(self.container, fsdecode(remote_filename), local_path.name) def _list(self): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#list-the-blobs-in-a-container blobs = [] marker = None while True: batch = self.blob_service.list_blobs(self.container, marker=marker) blobs.extend(batch) if not batch.next_marker: break marker = batch.next_marker return [blob.name for blob in blobs] def _delete(self, filename): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#delete-blobs self.blob_service.delete_blob(self.container, fsdecode(filename)) def _query(self, filename): prop = self.blob_service.get_blob_properties(self.container, fsdecode(filename)) try: info = {u'size': int(prop.properties.content_length)} except AttributeError: # old versions directly returned the properties info = {u'size': int(prop[u'content-length'])} return info def _error_code(self, operation, e): if isinstance(e, self.AzureMissingResourceError): return log.ErrorCode.backend_not_found
class AzureStorageBlockDeviceAPI(object): """ An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices Current Support: Azure SMS API """ def __init__(self, **azure_config): """ :param ServiceManagement azure_client: an instance of the azure serivce managment api client. :param String service_name: The name of the cloud service :param names of Azure volumes to identify cluster :returns: A ``BlockDeviceVolume``. """ self._instance_id = self.compute_instance_id() self._azure_service_client = ServiceManagementService( azure_config['subscription_id'], azure_config['management_certificate_path']) self._service_name = azure_config['service_name'] self._azure_storage_client = BlobService( azure_config['storage_account_name'], azure_config['storage_account_key']) self._storage_account_name = azure_config['storage_account_name'] self._disk_container_name = azure_config['disk_container_name'] if azure_config['debug']: to_file(sys.stdout) def allocation_unit(self): """ 1GiB is the minimum allocation unit for azure disks return int: 1 GiB """ return int(GiB(1).to_Byte().value) def compute_instance_id(self): """ Azure Stored a UUID in the SDC kernel module. """ # Node host names should be unique within a vnet return unicode(socket.gethostname()) def create_volume(self, dataset_id, size): """ Create a new volume. :param UUID dataset_id: The Flocker dataset ID of the dataset on this volume. :param int size: The size of the new volume in bytes. :returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when the volume has been created. """ size_in_gb = Byte(size).to_GiB().value if size_in_gb % 1 != 0: raise UnsupportedVolumeSize(dataset_id) self._create_volume_blob(size, dataset_id) label = self._disk_label_for_dataset_id(str(dataset_id)) return BlockDeviceVolume( blockdevice_id=unicode(label), size=size, attached_to=None, dataset_id=self._dataset_id_for_disk_label(label)) def destroy_volume(self, blockdevice_id): """ Destroy an existing volume. :param unicode blockdevice_id: The unique identifier for the volume to destroy. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :return: ``None`` """ log_info('Destorying block device: ' + str(blockdevice_id)) (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) request = None if lun is not None: request = \ self._azure_service_client.delete_data_disk( service_name=self._service_name, deployment_name=self._service_name, role_name=target_disk.attached_to.role_name, lun=lun, delete_vhd=True) else: if target_disk.__class__.__name__ == 'Blob': # unregistered disk self._azure_storage_client.delete_blob( self._disk_container_name, target_disk.name) else: request = self._azure_service_client.delete_disk( target_disk.name, True) if request is not None: self._wait_for_async(request.request_id, 5000) self._wait_for_detach(blockdevice_id) def attach_volume(self, blockdevice_id, attach_to): """ Attach ``blockdevice_id`` to ``host``. :param unicode blockdevice_id: The unique identifier for the block device being attached. :param unicode attach_to: An identifier like the one returned by the ``compute_instance_id`` method indicating the node to which to attach the volume. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is already attached. :returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to ``host``. """ (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is not None: raise AlreadyAttachedVolume(blockdevice_id) log_info('Attempting to attach ' + str(blockdevice_id) + ' to ' + str(attach_to)) disk_size = self._attach_disk(blockdevice_id, target_disk, attach_to) self._wait_for_attach(blockdevice_id) log_info('disk attached') return self._blockdevicevolume_from_azure_volume( blockdevice_id, disk_size, attach_to) def detach_volume(self, blockdevice_id): """ Detach ``blockdevice_id`` from whatever host it is attached to. :param unicode blockdevice_id: The unique identifier for the block device being detached. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to anything. :returns: ``None`` """ (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) # contrary to function name it doesn't delete by default, just detachs request = \ self._azure_service_client.delete_data_disk( service_name=self._service_name, deployment_name=self._service_name, role_name=role_name, lun=lun) self._wait_for_async(request.request_id, 5000) self._wait_for_detach(blockdevice_id) def get_device_path(self, blockdevice_id): """ Return the device path that has been allocated to the block device on the host to which it is currently attached. :param unicode blockdevice_id: The unique identifier for the block device. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to a host. :returns: A ``FilePath`` for the device. """ (target_disk_or_blob, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk_or_blob is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) return Lun.get_device_path_for_lun(lun) def list_volumes(self): """ List all the block devices available via the back end API. :returns: A ``list`` of ``BlockDeviceVolume``s. """ media_url_prefix = 'https://' + self._storage_account_name \ + '.blob.core.windows.net/' + self._disk_container_name disks = self._azure_service_client.list_disks() disk_list = [] all_blobs = self._get_flocker_blobs() for d in disks: if media_url_prefix not in d.media_link or \ 'flocker-' not in d.label: continue role_name = None if d.attached_to is not None \ and d.attached_to.role_name is not None: role_name = d.attached_to.role_name disk_list.append( self._blockdevicevolume_from_azure_volume( d.label, self._gibytes_to_bytes(d.logical_disk_size_in_gb), role_name)) if d.label in all_blobs: del all_blobs[d.label] for key in all_blobs: # include unregistered 'disk' blobs disk_list.append( self._blockdevicevolume_from_azure_volume( all_blobs[key].name, all_blobs[key].properties.content_length, None)) return disk_list def _attach_disk(self, blockdevice_id, target_disk, attach_to): """ Attaches disk to specified VM :param string blockdevice_id: The identifier of the disk :param DataVirtualHardDisk/Blob target_disk: The Blob or Disk to be attached :returns int: The size of the attached disk """ lun = Lun.compute_next_lun(self._azure_service_client, self._service_name, str(attach_to)) common_params = { 'service_name': self._service_name, 'deployment_name': self._service_name, 'role_name': attach_to, 'lun': lun } disk_size = None if target_disk.__class__.__name__ == 'Blob': # exclude 512 byte footer disk_size = target_disk.properties.content_length common_params['source_media_link'] = \ 'https://' + self._storage_account_name \ + '.blob.core.windows.net/' + self._disk_container_name \ + '/' + blockdevice_id common_params['disk_label'] = blockdevice_id else: disk_size = self._gibytes_to_bytes( target_disk.logical_disk_size_in_gb) common_params['disk_name'] = target_disk.name request = self._azure_service_client.add_data_disk(**common_params) self._wait_for_async(request.request_id, 5000) return disk_size def _create_volume_blob(self, size, dataset_id): # Create a new page blob as a blank disk self._azure_storage_client.put_blob( container_name=self._disk_container_name, blob_name=self._disk_label_for_dataset_id(dataset_id), blob=None, x_ms_blob_type='PageBlob', x_ms_blob_content_type='application/octet-stream', x_ms_blob_content_length=size) # for disk to be a valid vhd it requires a vhd footer # on the last 512 bytes vhd_footer = Vhd.generate_vhd_footer(size) self._azure_storage_client.put_page( container_name=self._disk_container_name, blob_name=self._disk_label_for_dataset_id(dataset_id), page=vhd_footer, x_ms_page_write='update', x_ms_range='bytes=' + str((size - 512)) + '-' + str(size - 1)) def _disk_label_for_dataset_id(self, dataset_id): """ Returns a disk label for a given Dataset ID :param unicode dataset_id: The identifier of the dataset :returns string: A string representing the disk label """ label = 'flocker-' + str(dataset_id) return label def _dataset_id_for_disk_label(self, disk_label): """ Returns a UUID representing the Dataset ID for the given disk label :param string disk_label: The disk label :returns UUID: The UUID of the dataset """ return UUID(disk_label.replace('flocker-', '')) def _get_disk_vmname_lun(self, blockdevice_id): target_disk = None target_lun = None role_name = None disk_list = self._azure_service_client.list_disks() for d in disk_list: if 'flocker-' not in d.label: continue if d.label == str(blockdevice_id): target_disk = d break if target_disk is None: # check for unregisterd disk blobs = self._get_flocker_blobs() blob = None if str(blockdevice_id) in blobs: blob = blobs[str(blockdevice_id)] return blob, None, None vm_info = None if hasattr(target_disk.attached_to, 'role_name'): vm_info = self._azure_service_client.get_role( self._service_name, self._service_name, target_disk.attached_to.role_name) for d in vm_info.data_virtual_hard_disks: if d.disk_name == target_disk.name: target_lun = d.lun break role_name = target_disk.attached_to.role_name return (target_disk, role_name, target_lun) def _get_flocker_blobs(self): all_blobs = {} blobs = self._azure_storage_client.list_blobs( self._disk_container_name, prefix='flocker-') for b in blobs: # todo - this could be big! all_blobs[b.name] = b return all_blobs def _wait_for_detach(self, blockdevice_id): role_name = '' lun = -1 timeout_count = 0 log_info('waiting for azure to ' + 'report disk as detached...') while role_name is not None or lun is not None: (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) time.sleep(1) timeout_count += 1 if timeout_count > 5000: raise AsynchronousTimeout() log_info('Disk Detached') def _wait_for_attach(self, blockdevice_id): timeout_count = 0 lun = None log_info('waiting for azure to report disk as attached...') while lun is None: (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) time.sleep(.001) timeout_count += 1 if timeout_count > 5000: raise AsynchronousTimeout() def _wait_for_async(self, request_id, timeout): count = 0 result = self._azure_service_client.get_operation_status(request_id) while result.status == 'InProgress': count = count + 1 if count > timeout: log_error('Timed out waiting for async operation to complete.') raise AsynchronousTimeout() time.sleep(.001) log_info('.') result = self._azure_service_client.get_operation_status( request_id) if result.error: log_error(result.error.code) log_error(str(result.error.message)) log_error(result.status + ' in ' + str(count * 5) + 's') def _gibytes_to_bytes(self, size): return int(GiB(size).to_Byte().value) def _blockdevicevolume_from_azure_volume(self, label, size, attached_to_name): # azure will report the disk size excluding the 512 byte footer # however flocker expects the exact value it requested for disk size # so offset the reported size to flocker by 512 bytes return BlockDeviceVolume( blockdevice_id=unicode(label), size=int(size), attached_to=attached_to_name, dataset_id=self._dataset_id_for_disk_label( label)) # disk labels are formatted as flocker-<data_set_id>
class Storage(driver.Base): supports_bytes_range = True def __init__(self, path=None, config=None): self._config = config self._container = self._config.azure_storage_container protocol = 'https' if self._config.azure_use_https else 'http' acct_name = self._config.azure_storage_account_name acct_key = self._config.azure_storage_account_key self._blob = BlobService(account_name=acct_name, account_key=acct_key, protocol=protocol) self._init_container() logger.debug("Initialized azureblob storage driver") def _init_container(self): '''Initializes image container on Azure blob storage if the container does not exist. ''' created = self._blob.create_container(self._container, x_ms_blob_public_access='blob', fail_on_exist=False) if created: logger.info('Created blob container for image registry.') else: logger.debug('Registry container already exists.') return created @lru.get def get_content(self, path): try: return self._blob.get_blob(self._container, path) except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path) @lru.set def put_content(self, path, content): self._blob.put_blob(self._container, path, content, 'BlockBlob') return path def stream_read(self, path, bytes_range=None): try: f = io.BytesIO() self._blob.get_blob_to_file(self._container, path, f) if bytes_range: f.seek(bytes_range[0]) total_size = bytes_range[1] - bytes_range[0] + 1 else: f.seek(0) while True: buf = None if bytes_range: # Bytes Range is enabled buf_size = self.buffer_size if nb_bytes + buf_size > total_size: # We make sure we don't read out of the range buf_size = total_size - nb_bytes if buf_size > 0: buf = f.read(buf_size) nb_bytes += len(buf) else: # We're at the end of the range buf = '' else: buf = f.read(self.buffer_size) if not buf: break yield buf except IOError: raise exceptions.FileNotFoundError('%s is not there' % path) def stream_write(self, path, fp): self._blob.put_block_blob_from_file(self._container, path, fp) def list_directory(self, path=None): if not path.endswith('/'): path += '/' # path=a would list a/b.txt as well as 'abc.txt' blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) return [b.name for b in blobs] def exists(self, path): try: self._blob.get_blob_properties(self._container, path) return True except azure.WindowsAzureMissingResourceError: return False @lru.remove def remove(self, path): is_blob = self.exists(path) if is_blob: self._blob.delete_blob(self._container, path) return exists = False blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) for b in blobs: self._blob.delete_blob(self._container, b.name) def get_size(self, path): try: properties = self._blob.get_blob_properties(self._container, path) return int(properties['content-length']) # auto-converted to long except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path)
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, container_name, prefix=None): # NOTE: Azure wants all paths to start with a slash prefix = "/{}".format(prefix.lstrip("/") if prefix else "") super().__init__(prefix=prefix) self.account_name = account_name self.account_key = account_key self.container_name = container_name self.conn = BlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized") # XXX: AzureTransfer isn't actively tested and hasn't its error handling is probably lacking self.log.warning("AzureTransfer is experimental and has not been thoroughly tested") def get_metadata_for_key(self, key): key = self.format_key_for_backend(key) return self._list_blobs(key)[0]["metadata"] def _metadata_for_key(self, key): return self._list_blobs(key)[0]["metadata"] def list_path(self, key): path = self.format_key_for_backend(key, trailing_slash=True) return self._list_blobs(path) def _list_blobs(self, path): self.log.debug("Listing path %r", path) items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata") result = [] for item in items: result.append({ "last_modified": dateutil.parser.parse(item.properties.last_modified), "metadata": item.metadata, "name": self.format_key_from_backend(item.name), "size": item.properties.content_length, }) return result def delete_key(self, key): key = self.format_key_for_backend(key) self.log.debug("Deleting key: %r", key) return self.conn.delete_blob(self.container_name, key) def get_contents_to_file(self, key, filepath_to_store_to): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to) return self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to) def get_contents_to_fileobj(self, key, fileobj_to_store_to): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_file(self.container_name, key, fileobj_to_store_to) def get_contents_to_string(self, key): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_bytes(self.container_name, key), self._metadata_for_key(key) def store_file_from_memory(self, key, memstring, metadata=None): key = self.format_key_for_backend(key) # Azure requires all metadata keys and values to be strings metadata_to_send = {str(k): str(v) for k, v in metadata.items()} self.conn.put_block_blob_from_bytes(self.container_name, key, memstring, x_ms_meta_name_values=metadata_to_send) def store_file_from_disk(self, key, filepath, metadata=None, multipart=None): key = self.format_key_for_backend(key) # Azure requires all metadata keys and values to be strings metadata_to_send = {str(k): str(v) for k, v in metadata.items()} self.conn.put_block_blob_from_path(self.container_name, key, filepath, x_ms_meta_name_values=metadata_to_send) def get_or_create_container(self, container_name): start_time = time.time() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time) return container_name
try: blob = blob_service.get_blob(blob_container, imgBlobName) except azure.WindowsAzureMissingResourceError: queue_service.delete_message(imagesQueue, message.message_id, message.pop_receipt) continue image = blobToOpenCV(blob) # ADDED#2 ######################################## if image is None: print "GIF attempt in analyse" queue_service.delete_message(imagesQueue, message.message_id, message.pop_receipt) blob_service.delete_blob(blob_container, imgBlobName) table_service.delete_entity(tableName, tablePartitionKey, tableRowKey) continue # ADDED #################################################################### ## Decrease main image size # we need to keep in mind aspect ratio if image.shape[1] >= image.shape[0]: width = 800 r = (width * 1.0) / image.shape[1] dim = (width, int(image.shape[0] * r)) else: height = 800 r = (height * 1.0) / image.shape[0] dim = (int(image.shape[1] * r), height)
def deletefromazure (strPrefix): blob_service = BlobService(account_name='wanderight', account_key='gdmZeJOCx3HYlFPZZukUhHAfeGAu4cfHWGQZc3+HIpkBHjlznUDjhXMl5HWh5MgbjpJF09ZxRaET1JVF9S2MWQ==') blobsToDelete = blob_service.list_blobs(config['container'], prefix=strPrefix) for b in blobsToDelete: blob_service.delete_blob(config['container'], b.name)
class AzureBackend(duplicity.backend.Backend): """ Backend for Azure Blob Storage Service """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) # Import Microsoft Azure Storage SDK for Python library. try: import azure import azure.storage if hasattr(azure.storage, 'BlobService'): # v0.11.1 and below from azure.storage import BlobService self.AzureMissingResourceError = azure.WindowsAzureMissingResourceError self.AzureConflictError = azure.WindowsAzureConflictError else: # v1.0.0 and above from azure.storage.blob import BlobService self.AzureMissingResourceError = azure.common.AzureMissingResourceHttpError self.AzureConflictError = azure.common.AzureConflictHttpError except ImportError as e: raise BackendException("""\ Azure backend requires Microsoft Azure Storage SDK for Python (https://pypi.python.org/pypi/azure-storage/). Exception: %s""" % str(e)) if 'AZURE_ACCOUNT_NAME' not in os.environ: raise BackendException('AZURE_ACCOUNT_NAME environment variable not set.') if 'AZURE_ACCOUNT_KEY' not in os.environ: raise BackendException('AZURE_ACCOUNT_KEY environment variable not set.') self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], account_key=os.environ['AZURE_ACCOUNT_KEY']) # TODO: validate container name self.container = parsed_url.path.lstrip('/') try: self.blob_service.create_container(self.container, fail_on_exist=True) except self.AzureConflictError: # Indicates that the resource could not be created because it already exists. pass except Exception as e: log.FatalError("Could not create Azure container: %s" % unicode(e.message).split('\n', 1)[0], log.ErrorCode.connection_failed) def _put(self, source_path, remote_filename): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#upload-a-blob-into-a-container self.blob_service.put_block_blob_from_path(self.container, remote_filename, source_path.name) def _get(self, remote_filename, local_path): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#download-blobs self.blob_service.get_blob_to_path(self.container, remote_filename, local_path.name) def _list(self): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#list-the-blobs-in-a-container blobs = [] marker = None while True: batch = self.blob_service.list_blobs(self.container, marker=marker) blobs.extend(batch) if not batch.next_marker: break marker = batch.next_marker return [blob.name for blob in blobs] def _delete(self, filename): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#delete-blobs self.blob_service.delete_blob(self.container, filename) def _query(self, filename): prop = self.blob_service.get_blob_properties(self.container, filename) return {'size': int(prop['content-length'])} def _error_code(self, operation, e): if isinstance(e, self.AzureMissingResourceError): return log.ErrorCode.backend_not_found
class BlobServiceAdapter(Component): """The :class:`BlobServiceAdapter` class is a thin wrapper over azure.storage.BlobService. All the attributes of the wrapper stream are proxied by the adapter so it's possible to do ``adapter.create_container()`` instead of the long form ``adapter.blob_service.adapter()``. """ def __init__(self): self.blob_service = BlobService(account_name=self.util.get_config("storage.azure.account_name"), account_key=self.util.get_config("storage.azure.account_key"), host_base=self.util.get_config("storage.azure.blob_service_host_base")) def __getattr__(self, name): return getattr(self.blob_service, name) def create_container_in_storage(self, container_name, access="container"): """create a container if doesn't exist :type container_name: str|unicode :param container_name: Name of container to create. :type access: str|unicode :param access: Optional. Possible values include: container, blob :return: """ try: names = [x.name for x in self.blob_service.list_containers()] if container_name not in names: return self.blob_service.create_container(container_name, x_ms_blob_public_access=access) else: self.log.debug("container already exists in storage") return True except Exception as e: self.log.error(e) return False def upload_file_to_azure(self, container_name, blob_name, stream): """ Creates a new block blob from a file/stream, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str | unicode :param blob_name: Name of blob to create or update. :type stream: file :param stream: Opened file/stream to upload as the blob content. """ try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_file(container_name, blob_name, stream) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_bytes(self, container_name, blob_name, blob): """ Creates a new block blob from an array of bytes, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type blob: bytes :param blob: Content of blob as an array of bytes. """ try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_bytes(container_name, blob_name, blob) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_text(self, container_name, blob_name, text): """ Creates a new block blob from str/unicode, or updates the content of an existing block blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type text: str|unicode :param text: Text to upload to the blob. """ try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_text(container_name, blob_name, text) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_path(self, container_name, blob_name, path): """ Creates a new page blob from a file path, or updates the content of an existing page blob, with automatic chunking and progress notifications. :type container_name: str|unicode :param container_name: Name of existing container. :type blob_name: str|unicode :param blob_name: Name of blob to create or update. :type path: str|unicode :param path: Path of the file to upload as the blob content. """ try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_path(container_name, blob_name, path) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def delete_file_from_azure(self, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.delete_blob(container_name, blob_name) except Exception as e: self.log.error(e) return None
class AzureBlobStorage(Storage): def __init__(self, account='nyxstorage', container='pxo'): self.base_storage_uri = 'http://%s.blob.core.windows.net/%s/' % ( account, container) self.blob_service = BlobService( account, get_env_variable('AZURE_BLOB_STORAGE_KEY')) self.container = container def _open(self, name, mode='rb'): data = self.blob_service.get_blob(self.container, name) return ContentFile(data) def _save(self, name, content): _file = content.read() file_name = content.name[-35:] self.blob_service.put_blob( self.container, file_name, _file, x_ms_blob_type='BlockBlob') return self.base_storage_uri + file_name def create_container(self, container_name): result = self.blob_service.create_container( container_name, x_ms_blob_public_access='container') return result def delete(self, name): self.blob_service.delete_blob(self.container, name) def exists(self, name): try: self.blob_service.get_blob_properties(self.container, name) except: return False else: return True def get_available_name(self, name): return name def get_blobs(self): blobs = self.blob_service.list_blobs(self.container) return blobs def get_valid_name(self, name): return name def modified_time(self, name): metadata = self.blob_service.get_blob_metadata(self.container, name) modified_time = float(metadata.get('x-ms-meta-modified_time')) return datetime.fromtimestamp(modified_time) def set_public_container(self, container_name): result = self.blob_service.set_container_acl( container_name, x_ms_blob_public_access='container') return result def size(self, name): properties = self.blob_service.get_blob_properties( self.container, name) return properties.get('content-length') def url(self, name): blob = self.blob_service.list_blobs(self.container, prefix=name) return blob.blobs[0].url
class AzureBackend(duplicity.backend.Backend): """ Backend for Azure Blob Storage Service """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) # Import Microsoft Azure SDK for Python library. try: import azure from azure.storage import BlobService except ImportError: raise BackendException('Azure backend requires Microsoft Azure SDK for Python ' '(https://github.com/Azure/azure-sdk-for-python).') if 'AZURE_ACCOUNT_NAME' not in os.environ: raise BackendException('AZURE_ACCOUNT_NAME environment variable not set.') if 'AZURE_ACCOUNT_KEY' not in os.environ: raise BackendException('AZURE_ACCOUNT_KEY environment variable not set.') account_name = os.environ['AZURE_ACCOUNT_NAME'] account_key = os.environ['AZURE_ACCOUNT_KEY'] self.WindowsAzureMissingResourceError = azure.WindowsAzureMissingResourceError self.blob_service = BlobService(account_name=account_name, account_key=account_key) # TODO: validate container name self.container = parsed_url.path.lstrip('/') try: self.blob_service.create_container(self.container, fail_on_exist=True) except azure.WindowsAzureConflictError: # Indicates that the resource could not be created because it already exists. pass except Exception as e: log.FatalError("Could not create Azure container: %s" % unicode(e.message).split('\n', 1)[0], log.ErrorCode.connection_failed) def _put(self, source_path, remote_filename): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#upload-blob self.blob_service.put_block_blob_from_path(self.container, remote_filename, source_path.name) def _get(self, remote_filename, local_path): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#download-blobs self.blob_service.get_blob_to_path(self.container, remote_filename, local_path.name) def _list(self): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#list-blob blobs = self.blob_service.list_blobs(self.container) return [blob.name for blob in blobs] def _delete(self, filename): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#delete-blobs self.blob_service.delete_blob(self.container, filename) def _query(self, filename): prop = self.blob_service.get_blob_properties(self.container, filename) return {'size': int(prop['content-length'])} def _error_code(self, operation, e): if isinstance(e, self.WindowsAzureMissingResourceError): return log.ErrorCode.backend_not_found
class SAzure(SyncStorage): def __init__(self): super().__init__() self.msg_key_na = _('Key not available') try: import alxlib.key key = alxlib.key.Key() if os.path.isfile(key.get_path()): sys.path.insert(0, key.get_dir()) import alxkey self.key = alxkey.alxkey_azure """self.blob = BlobService(account_name=self.key['AZURE_STORAGE_ACCOUNT_NAME'], account_key=self.key['AZURE_ACCESS_KEY'])""" else: # raise (self.msg_key_na) self.key = None except: pass # raise (self.msg_key_na) def connect(self): try: self.blob = BlobService(account_name=self.key['AZURE_STORAGE_ACCOUNT_NAME'], account_key=self.key['AZURE_ACCESS_KEY']) return self.blob.list_containers(maxresults=1) except: return None def connect_blob(self, az_account_name=None, az_account_key=None): try: if az_account_name != None: self.key['AZURE_STORAGE_ACCOUNT_NAME'] = az_account_name self.key['AZURE_ACCESS_KEY'] = az_account_key return self.connect() except: return None def path_clean(self, path: str): try: i = path.index("//") + 2 self.container = path[0:i] if path[len(path) - 1] != "/": path += "/" return path[i:] except: print(_("Bad Path")) exit(1) def spath(self, container, root, b): spath = SyncPath() spath.BasePath = container if b.name[len(b.name)-1]=="/": spath.IsDir= True else: spath.IsFile= True spath.AbsPath = b.name if len(root)>0: spath.SPath = b.name[len(root) - 1:] else: spath.SPath=b.name spath.Size = b.properties.content_length import alxlib.time_help spath.ModifiedTS = alxlib.time_help.to_timestamp(b.properties.last_modified) spath.MD5 = b.properties.content_md5 spath.sys="azure" return spath def path_split(self, path: str): try: list = path.split("/") container = list[0] uri = "" if len(list) > 1: uri = "/".join(map(str, list[1:])) return container, uri except: print(_("Bad path")) exit(1) def path_list_blobs(self, container, uri): try: if len(uri)>0: blobs = self.blob.list_blobs(container, prefix=uri) else: blobs = self.blob.list_blobs(container) """for blob in blobs: print(blob.properties.__dict__) print(blob.name) print(blob.url)""" return blobs except Exception as e: print(_("Bad connection")) logging.warning("container {0}, path {1}".format(container, uri)) exit(1) def path_list(self, path): try: logging.debug("path_list {0}".format(path)) container, uri = self.path_split(path) logging.debug("Container: {0}, Uri: {1}".format(container, uri)) self.connect() self.blob.create_container(container) blobs = self.path_list_blobs(container, uri) d = {} for b in blobs: spath = self.spath(container, uri, b) # print(b.__dict__) #print(str(b.properties.last_modified.__dict__)) #print(str(spath.ModifiedTS)) d[spath.SPath] = spath # print(d) return d except Exception as e: print(e) def remove(self, src: SyncPath): try: logging.debug("Removing {0}".format(src.AbsPath)) self.connect() self.blob.create_container(src.BasePath) self.blob.delete_blob(src.BasePath, src.AbsPath) except: pass def copy_local2azure(self, src, base_dir): try: container, uri = self.path_split(base_dir) if len(src.SPath)>0 and src.SPath[0]=="/": path= uri+ src.SPath[1:] else: path= uri+src.SPath logging.debug("copy_local2azure Spath {0}. path:{1}".format(src.SPath, path)) self.connect() if not src.IsDir: self.blob.put_block_blob_from_path (container, path, src.AbsPath) else: self.blob.put_block_blob_from_text(container, path+"/", "") except Exception as e: print("Error Copying") print(e) def copy_azure2local(self, src, base_dir): try: if len(src.SPath)>0 and (src.SPath[0] == "/" or src.SPath[0] == "\\") : path = src.SPath[1:] else: path = src.SPath path= os.path.normpath(os.path.join(base_dir, path)) logging.debug("copy_azure2local basedir:{0} Spath {1}, path {2}, abs: {3}".format( base_dir, src.SPath, path, src.AbsPath)) if not os.path.isdir(path): os.makedirs(os.path.dirname(path), exist_ok=True) #print( os.path.dirname(path)+"***************") if not (len(src.AbsPath)>0 and src.AbsPath[len(src.AbsPath)-1]=="/"): self.blob.get_blob_to_path(src.BasePath, src.AbsPath, path) """container, uri = self.path_split(base_dir) if len(src.SPath)>0 and src.SPath[0]=="/": path= uri+ src.SPath[1:] else: path= uri+src.SPath self.connect() if not src.IsDir: self.blob.get_blob_to_path(src.BasePath, path, src.AbsPath) else: self.blob.put_block_blob_from_text(container, path, "")""" except Exception as e: print("Error copying") print(e)
class Command(BaseCommand): help = "Synchronizes static media to cloud files." option_list = BaseCommand.option_list + ( optparse.make_option( '-w', '--wipe', action='store_true', dest='wipe', default=False, help="Wipes out entire contents of container first."), optparse.make_option('-t', '--test-run', action='store_true', dest='test_run', default=False, help="Performs a test run of the sync."), optparse.make_option('-c', '--container', dest='container', help="Override STATIC_CONTAINER."), ) # settings from azurite.settings ACCOUNT_NAME = AZURITE['ACCOUNT_NAME'] ACCOUNT_KEY = AZURITE['ACCOUNT_KEY'] STATIC_CONTAINER = AZURITE['STATIC_CONTAINER'] # paths DIRECTORY = os.path.abspath(settings.STATIC_ROOT) STATIC_URL = settings.STATIC_URL if not DIRECTORY.endswith('/'): DIRECTORY = DIRECTORY + '/' if STATIC_URL.startswith('/'): STATIC_URL = STATIC_URL[1:] local_object_names = [] create_count = 0 upload_count = 0 update_count = 0 skip_count = 0 delete_count = 0 service = None def handle(self, *args, **options): self.wipe = options.get('wipe') self.test_run = options.get('test_run') self.verbosity = int(options.get('verbosity')) if hasattr(options, 'container'): self.STATIC_CONTAINER = options.get('container') self.sync_files() def sync_files(self): self.service = BlobService(account_name=self.ACCOUNT_NAME, account_key=self.ACCOUNT_KEY) try: self.service.get_container_properties(self.STATIC_CONTAINER) except WindowsAzureMissingResourceError: self.service.create_container(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') self.service.set_container_acl(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') # if -w option is provided, wipe out the contents of the container if self.wipe: blob_count = len(self.service.list_blobs(self.STATIC_CONTAINER)) if self.test_run: print "Wipe would delete %d objects." % blob_count else: print "Deleting %d objects..." % blob_count for blob in self.service.list_blobs(self.STATIC_CONTAINER): self.service.delete_blob(self.STATIC_CONTAINER, blob.name) # walk through the directory, creating or updating files on the cloud os.path.walk(self.DIRECTORY, self.upload_files, "foo") # remove any files on remote that don't exist locally self.delete_files() # print out the final tally to the cmd line self.update_count = self.upload_count - self.create_count print if self.test_run: print "Test run complete with the following results:" print "Skipped %d. Created %d. Updated %d. Deleted %d." % ( self.skip_count, self.create_count, self.update_count, self.delete_count) def upload_files(self, arg, dirname, names): # upload or skip items for item in names: file_path = os.path.join(dirname, item) if os.path.isdir(file_path): continue # Don't try to upload directories object_name = self.STATIC_URL + file_path.split(self.DIRECTORY)[1] self.local_object_names.append(object_name) try: properties = self.service.get_blob_properties( self.STATIC_CONTAINER, object_name) except WindowsAzureMissingResourceError: properties = {} self.create_count += 1 cloud_datetime = None if 'last-modified' in properties: cloud_datetime = ( properties['last-modified'] and datetime.datetime.strptime(properties['last-modified'], "%a, %d %b %Y %H:%M:%S %Z") or None) local_datetime = datetime.datetime.utcfromtimestamp( os.stat(file_path).st_mtime) if cloud_datetime and local_datetime < cloud_datetime: self.skip_count += 1 if self.verbosity > 1: print "Skipped %s: not modified." % object_name continue if not self.test_run: file_contents = open(file_path, 'r').read() content_type, encoding = mimetypes.guess_type(file_path) self.service.put_blob(self.STATIC_CONTAINER, object_name, file_contents, x_ms_blob_type='BlockBlob', x_ms_blob_content_type=content_type, content_encoding=encoding) # sync_headers(cloud_obj) self.upload_count += 1 if self.verbosity > 1: print "Uploaded", object_name def delete_files(self): # remove any objects in the container that don't exist locally for blob in self.service.list_blobs(self.STATIC_CONTAINER): if blob.name not in self.local_object_names: self.delete_count += 1 if self.verbosity > 1: print "Deleted %s" % blob.name if not self.test_run: self.service.delete_blob(self.STATIC_CONTAINER, blob.name)
class AzureBackend(duplicity.backend.Backend): """ Backend for Azure Blob Storage Service """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) # Import Microsoft Azure Storage SDK for Python library. try: import azure import azure.storage if hasattr(azure.storage, 'BlobService'): # v0.11.1 and below from azure.storage import BlobService self.AzureMissingResourceError = azure.WindowsAzureMissingResourceError self.AzureConflictError = azure.WindowsAzureConflictError else: # v1.0.0 and above import azure.storage.blob if hasattr(azure.storage.blob, 'BlobService'): from azure.storage.blob import BlobService else: from azure.storage.blob.blockblobservice import BlockBlobService as BlobService self.AzureMissingResourceError = azure.common.AzureMissingResourceHttpError self.AzureConflictError = azure.common.AzureConflictHttpError except ImportError as e: raise BackendException("""\ Azure backend requires Microsoft Azure Storage SDK for Python (https://pypi.python.org/pypi/azure-storage/). Exception: %s""" % str(e)) # TODO: validate container name self.container = parsed_url.path.lstrip('/') if 'AZURE_ACCOUNT_NAME' not in os.environ: raise BackendException('AZURE_ACCOUNT_NAME environment variable not set.') if 'AZURE_ACCOUNT_KEY' in os.environ: if 'AZURE_ENDPOINT_SUFFIX' in os.environ: self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], account_key=os.environ['AZURE_ACCOUNT_KEY'], endpoint_suffix=os.environ['AZURE_ENDPOINT_SUFFIX']) else: self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], account_key=os.environ['AZURE_ACCOUNT_KEY']) self._create_container() elif 'AZURE_SHARED_ACCESS_SIGNATURE' in os.environ: if 'AZURE_ENDPOINT_SUFFIX' in os.environ: self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], sas_token=os.environ['AZURE_SHARED_ACCESS_SIGNATURE'], endpoint_suffix=os.environ['AZURE_ENDPOINT_SUFFIX']) else: self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'], sas_token=os.environ['AZURE_SHARED_ACCESS_SIGNATURE']) else: raise BackendException( 'Neither AZURE_ACCOUNT_KEY nor AZURE_SHARED_ACCESS_SIGNATURE environment variable not set.') if globals.azure_max_single_put_size: # check if we use azure-storage>=0.30.0 try: _ = self.blob_service.MAX_SINGLE_PUT_SIZE self.blob_service.MAX_SINGLE_PUT_SIZE = globals.azure_max_single_put_size # fallback for azure-storage<0.30.0 except AttributeError: self.blob_service._BLOB_MAX_DATA_SIZE = globals.azure_max_single_put_size if globals.azure_max_block_size: # check if we use azure-storage>=0.30.0 try: _ = self.blob_service.MAX_BLOCK_SIZE self.blob_service.MAX_BLOCK_SIZE = globals.azure_max_block_size # fallback for azure-storage<0.30.0 except AttributeError: self.blob_service._BLOB_MAX_CHUNK_DATA_SIZE = globals.azure_max_block_size def _create_container(self): try: self.blob_service.create_container(self.container, fail_on_exist=True) except self.AzureConflictError: # Indicates that the resource could not be created because it already exists. pass except Exception as e: log.FatalError("Could not create Azure container: %s" % unicode(e.message).split('\n', 1)[0], log.ErrorCode.connection_failed) def _put(self, source_path, remote_filename): kwargs = {} if globals.azure_max_connections: kwargs['max_connections'] = globals.azure_max_connections # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#upload-a-blob-into-a-container try: self.blob_service.create_blob_from_path(self.container, remote_filename, source_path.name, **kwargs) except AttributeError: # Old versions use a different method name self.blob_service.put_block_blob_from_path(self.container, remote_filename, source_path.name, **kwargs) def _get(self, remote_filename, local_path): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#download-blobs self.blob_service.get_blob_to_path(self.container, remote_filename, local_path.name) def _list(self): # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#list-the-blobs-in-a-container blobs = [] marker = None while True: batch = self.blob_service.list_blobs(self.container, marker=marker) blobs.extend(batch) if not batch.next_marker: break marker = batch.next_marker return [blob.name for blob in blobs] def _delete(self, filename): # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#delete-blobs self.blob_service.delete_blob(self.container, filename) def _query(self, filename): prop = self.blob_service.get_blob_properties(self.container, filename) try: info = {'size': int(prop.properties.content_length)} except AttributeError: # old versions directly returned the properties info = {'size': int(prop['content-length'])} return info def _error_code(self, operation, e): if isinstance(e, self.AzureMissingResourceError): return log.ErrorCode.backend_not_found
class Command(BaseCommand): help = "Synchronizes static media to cloud files." option_list = BaseCommand.option_list + ( optparse.make_option('-w', '--wipe', action='store_true', dest='wipe', default=False, help="Wipes out entire contents of container first."), optparse.make_option('-t', '--test-run', action='store_true', dest='test_run', default=False, help="Performs a test run of the sync."), optparse.make_option('-c', '--container', dest='container', help="Override STATIC_CONTAINER."), ) # settings from azurite.settings ACCOUNT_NAME = AZURITE['ACCOUNT_NAME'] ACCOUNT_KEY = AZURITE['ACCOUNT_KEY'] STATIC_CONTAINER = AZURITE['STATIC_CONTAINER'] # paths DIRECTORY = os.path.abspath(settings.STATIC_ROOT) STATIC_URL = settings.STATIC_URL if not DIRECTORY.endswith('/'): DIRECTORY = DIRECTORY + '/' if STATIC_URL.startswith('/'): STATIC_URL = STATIC_URL[1:] local_object_names = [] create_count = 0 upload_count = 0 update_count = 0 skip_count = 0 delete_count = 0 service = None def handle(self, *args, **options): self.wipe = options.get('wipe') self.test_run = options.get('test_run') self.verbosity = int(options.get('verbosity')) if hasattr(options, 'container'): self.STATIC_CONTAINER = options.get('container') self.sync_files() def sync_files(self): self.service = BlobService(account_name=self.ACCOUNT_NAME, account_key=self.ACCOUNT_KEY) try: self.service.get_container_properties(self.STATIC_CONTAINER) except WindowsAzureMissingResourceError: self.service.create_container(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') self.service.set_container_acl(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') # if -w option is provided, wipe out the contents of the container if self.wipe: blob_count = len(self.service.list_blobs(self.STATIC_CONTAINER)) if self.test_run: print "Wipe would delete %d objects." % blob_count else: print "Deleting %d objects..." % blob_count for blob in self.service.list_blobs(self.STATIC_CONTAINER): self.service.delete_blob(self.STATIC_CONTAINER, blob.name) # walk through the directory, creating or updating files on the cloud os.path.walk(self.DIRECTORY, self.upload_files, "foo") # remove any files on remote that don't exist locally self.delete_files() # print out the final tally to the cmd line self.update_count = self.upload_count - self.create_count print if self.test_run: print "Test run complete with the following results:" print "Skipped %d. Created %d. Updated %d. Deleted %d." % ( self.skip_count, self.create_count, self.update_count, self.delete_count) def upload_files(self, arg, dirname, names): # upload or skip items for item in names: file_path = os.path.join(dirname, item) if os.path.isdir(file_path): continue # Don't try to upload directories object_name = self.STATIC_URL + file_path.split(self.DIRECTORY)[1] self.local_object_names.append(object_name) try: properties = self.service.get_blob_properties(self.STATIC_CONTAINER, object_name) except WindowsAzureMissingResourceError: properties = {} self.create_count += 1 cloud_datetime = None if 'last-modified' in properties: cloud_datetime = (properties['last-modified'] and datetime.datetime.strptime( properties['last-modified'], "%a, %d %b %Y %H:%M:%S %Z" ) or None) local_datetime = datetime.datetime.utcfromtimestamp( os.stat(file_path).st_mtime) if cloud_datetime and local_datetime < cloud_datetime: self.skip_count += 1 if self.verbosity > 1: print "Skipped %s: not modified." % object_name continue if not self.test_run: file_contents = open(file_path, 'r').read() content_type, encoding = mimetypes.guess_type(file_path) print "content-type", content_type print "encoding", encoding self.service.put_blob(self.STATIC_CONTAINER, object_name, file_contents, x_ms_blob_type='BlockBlob', x_ms_blob_content_type=content_type, content_encoding=encoding) # sync_headers(cloud_obj) self.upload_count += 1 if self.verbosity > 1: print "Uploaded", object_name def delete_files(self): # remove any objects in the container that don't exist locally for blob in self.service.list_blobs(self.STATIC_CONTAINER): if blob.name not in self.local_object_names: self.delete_count += 1 if self.verbosity > 1: print "Deleted %s" % blob.name if not self.test_run: self.service.delete_blob(self.STATIC_CONTAINER, blob.name)
class AzureFS(LoggingMixIn, Operations): """Azure Blob Storage filesystem""" blobs = None containers = dict() # <cname, dict(stat:dict, #files:None|dict<fname, stat>) fds = dict() # <fd, (path, bytes, dirty)> fd = 0 def __init__(self, account, key): self.blobs = BlobService(account, key) self.rebuild_container_list() def convert_to_epoch(self, date): """Converts Tue, 31 Jul 2012 07:17:34 GMT format to epoch""" return int(time.mktime(time.strptime(date, TIME_FORMAT))) def rebuild_container_list(self): cmap = dict() cnames = set() for c in self.blobs.list_containers(): date = c.properties.last_modified cstat = dict(st_mode=(S_IFDIR | 0755), st_uid=getuid(), st_size=0, st_mtime=self.convert_to_epoch(date)) cname = c.name cmap['/' + cname] = dict(stat=cstat, files=None) cnames.add(cname) cmap['/'] = dict(files={}, stat=dict(st_mode=(S_IFDIR | 0755), st_uid=getuid(), st_size=0, st_mtime=int(time.time()))) self.containers = cmap # destroys fs tree cache resistant to misses def _parse_path(self, path): # returns </dir, file(=None)> if path.count('/') > 1: # file return str(path[:path.rfind('/')]), str(path[path.rfind('/') + 1:]) else: # dir pos = path.rfind('/', 1) if pos == -1: return path, None else: return str(path[:pos]), None def parse_container(self, path): base_container = path[1:] # /abc/def/g --> abc if base_container.find('/') > -1: base_container = base_container[:base_container.find('/')] return str(base_container) def _get_dir(self, path, contents_required=False): if not self.containers: self.rebuild_container_list() if path in self.containers and not (contents_required and \ self.containers[path]['files'] is None): return self.containers[path] cname = self.parse_container(path) if '/' + cname not in self.containers: raise FuseOSError(ENOENT) else: if self.containers['/' + cname]['files'] is None: # fetch contents of container log.info("------> CONTENTS NOT FOUND: %s" % cname) blobs = self.blobs.list_blobs(cname) dirstat = dict(st_mode=(S_IFDIR | 0755), st_size=0, st_uid=getuid(), st_mtime=time.time()) if self.containers['/' + cname]['files'] is None: self.containers['/' + cname]['files'] = dict() for f in blobs: blob_name = f.name blob_date = f.properties.last_modified blob_size = long(f.properties.content_length) node = dict(st_mode=(S_IFREG | 0644), st_size=blob_size, st_mtime=self.convert_to_epoch(blob_date), st_uid=getuid()) if blob_name.find('/') == -1: # file just under container self.containers['/' + cname]['files'][blob_name] = node return self.containers['/' + cname] return None def _get_file(self, path): d, f = self._parse_path(path) dir = self._get_dir(d, True) if dir is not None and f in dir['files']: return dir['files'][f] def getattr(self, path, fh=None): d, f = self._parse_path(path) if f is None: dir = self._get_dir(d) return dir['stat'] else: file = self._get_file(path) if file: return file raise FuseOSError(ENOENT) # FUSE def mkdir(self, path, mode): if path.count('/') <= 1: # create on root name = path[1:] if not 3 <= len(name) <= 63: log.error("Container names can be 3 through 63 chars long.") raise FuseOSError(ENAMETOOLONG) if name is not name.lower(): log.error("Container names cannot contain uppercase \ characters.") raise FuseOSError(EACCES) if name.count('--') > 0: log.error('Container names cannot contain consecutive \ dashes (-).') raise FuseOSError(EAGAIN) #TODO handle all "-"s must be preceded by letter or numbers #TODO starts with only letter or number, can contain letter, nr,'-' resp = self.blobs.create_container(name) if resp: self.rebuild_container_list() log.info("CONTAINER %s CREATED" % name) else: raise FuseOSError(EACCES) log.error("Invalid container name or container already \ exists.") else: raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs def rmdir(self, path): if path.count('/') == 1: c_name = path[1:] resp = self.blobs.delete_container(c_name) if resp: if path in self.containers: del self.containers[path] else: raise FuseOSError(EACCES) else: raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs def create(self, path, mode): node = dict(st_mode=(S_IFREG | mode), st_size=0, st_nlink=1, st_uid=getuid(), st_mtime=time.time()) d, f = self._parse_path(path) if not f: log.error("Cannot create files on root level: /") raise FuseOSError(ENOSYS) dir = self._get_dir(d, True) if not dir: raise FuseOSError(EIO) dir['files'][f] = node return self.open(path, data='') # reusing handler provider def open(self, path, flags=0, data=None): if data == None: # download contents c_name = self.parse_container(path) f_name = path[path.find('/', 1) + 1:] try: data = self.blobs.get_blob(c_name, f_name) except WindowsAzureMissingResourceError: dir = self._get_dir('/' + c_name, True) if f_name in dir['files']: del dir['files'][f_name] raise FuseOSError(ENOENT) except WindowsAzureError as e: log.error("Read blob failed HTTP %d" % e.code) raise FuseOSError(EAGAIN) self.fd += 1 self.fds[self.fd] = (path, data, False) return self.fd def flush(self, path, fh=None): if not fh: raise FuseOSError(EIO) else: if fh not in self.fds: raise FuseOSError(EIO) path = self.fds[fh][0] data = self.fds[fh][1] dirty = self.fds[fh][2] if not dirty: return 0 # avoid redundant write d, f = self._parse_path(path) c_name = self.parse_container(path) if data is None: data = '' try: if len(data) < 64 * 1024 * 1024: # 64 mb self.blobs.put_blob(c_name, f, data, 'BlockBlob') else: # divide file by blocks and upload block_size = 8 * 1024 * 1024 num_blocks = int(math.ceil(len(data) * 1.0 / block_size)) rd = str(random.randint(1, 1e8)) block_ids = list() for i in range(num_blocks): part = data[i * block_size:min((i + 1) * block_size, len(data))] block_id = base64.encodestring('%s_%s' % (rd, (8 - len(str(i))) * '0' + str(i))) self.blobs.put_block(c_name, f, part, block_id) block_ids.append(block_id) self.blobs.put_block_list(c_name, f, block_ids) except WindowsAzureError: raise FuseOSError(EAGAIN) dir = self._get_dir(d, True) if not dir or f not in dir['files']: raise FuseOSError(EIO) # update local data dir['files'][f]['st_size'] = len(data) dir['files'][f]['st_mtime'] = time.time() self.fds[fh] = (path, data, False) # mark as not dirty return 0 def release(self, path, fh=None): if fh is not None and fh in self.fds: del self.fds[fh] def truncate(self, path, length, fh=None): return 0 # assume done, no need def write(self, path, data, offset, fh=None): if not fh or fh not in self.fds: raise FuseOSError(ENOENT) else: d = self.fds[fh][1] if d is None: d = "" self.fds[fh] = (self.fds[fh][0], d[:offset] + data, True) return len(data) def unlink(self, path): c_name = self.parse_container(path) d, f = self._parse_path(path) try: self.blobs.delete_blob(c_name, f) _dir = self._get_dir(path, True) if _dir and f in _dir['files']: del _dir['files'][f] return 0 except WindowsAzureMissingResourceError: raise FuseOSError(ENOENT) except Exception as e: raise FuseOSError(EAGAIN) def readdir(self, path, fh): if path == '/': return ['.', '..'] + [x[1:] for x in self.containers.keys() \ if x is not '/'] dir = self._get_dir(path, True) if not dir: raise FuseOSError(ENOENT) return ['.', '..'] + dir['files'].keys() def read(self, path, size, offset, fh): if not fh or fh not in self.fds: raise FuseOSError(ENOENT) f_name = path[path.find('/', 1) + 1:] c_name = path[1:path.find('/', 1)] try: data = self.blobs.get_blob(c_name, f_name) self.fds[fh] = (self.fds[fh][0], data, False) return data[offset:offset + size] except URLError, e: if e.code == 404: raise FuseOSError(ENOENT) elif e.code == 403: raise FUSEOSError(EPERM) else: log.error("Read blob failed HTTP %d" % e.code) raise FuseOSError(EAGAIN) data = self.fds[fh][1] if data is None: data = "" return data[offset:offset + size]
# クラウドサービス/デプロイメントが存在しない(_ERROR_NOT_FOUND) logger.error("deleteOSandDataDisk.py: OS Disk deleted.(%s)" % osHardDiskName) logger.error(traceback.format_exc()) sys.exit() except WindowsAzureError as e: # XXX: Blobがロックされている場合を想定しているが、他にもこのエラーにかかる可能性あり logger.error("deleteOSandDataDisk.py: OS Disk locked.(%s)" % osHardDiskName) logger.error(traceback.format_exc()) sys.exit() except Exception: # 予期せぬ例外 logger.error("deleteOSandDataDisk.py: OS Disk unknown exception error.(%s)" % osHardDiskName) logger.error(traceback.format_exc()) sys.exit() logger.debug("deleteOSandDataDisk.py: OS Disk deleted.(%s)" % osHardDiskName) # else: # logger.debug("osHardDiskName is None. OS disk deleting is skip.") logger.debug("sleeping... 60sec") time.sleep(60) #------------------ # Blobの削除 logger.debug("deleteOSandDataDisk.py: Blob deleting...(%s-->%s)" % (targetContainer.name, targetBlob.name)) rs = blobService.delete_blob(container_name=targetContainer.name, blob_name=targetBlob.name) logger.debug("deleteOSandDataDisk.py: Blob deleted.(%s-->%s)" % (targetContainer.name, targetBlob.name))
class FileService(Component): def __init__(self): self.blob_service = None def generate_blob_service(self): if self.blob_service is None: # if storage info doesn't exist in config.py upload file function stop working self.blob_service = BlobService( account_name=self.util.get_config( "storage.azure.account_name"), account_key=self.util.get_config("storage.azure.account_key"), host_base=self.util.get_config( "storage.azure.blob_service_host_base")) def create_container_in_storage(self, container_name, access): """ create a container if doesn't exist :param container_name: :param access: :return: """ self.generate_blob_service() try: names = map(lambda x: x.name, self.blob_service.list_containers()) if container_name not in names: self.blob_service.create_container( container_name, x_ms_blob_public_access=access) else: self.log.debug("container already exsit in storage") return True except Exception as e: self.log.error(e) return False def upload_file_to_azure(self, stream, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_file( container_name, blob_name, stream) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def upload_file_to_azure_from_path(self, path, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.put_block_blob_from_path( container_name, blob_name, path) return self.blob_service.make_blob_url(container_name, blob_name) else: return None except Exception as e: self.log.error(e) return None def delete_file_from_azure(self, container_name, blob_name): try: if self.create_container_in_storage(container_name, 'container'): self.blob_service.delete_blob(container_name, blob_name) except Exception as e: self.log.error(e) return None
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, container_name, prefix=None): # NOTE: Azure wants all paths to start with a slash prefix = "/{}".format(prefix.lstrip("/") if prefix else "") super().__init__(prefix=prefix) self.account_name = account_name self.account_key = account_key self.container_name = container_name self.conn = BlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized") # XXX: AzureTransfer isn't actively tested and hasn't its error handling is probably lacking self.log.warning( "AzureTransfer is experimental and has not been thoroughly tested") def get_metadata_for_key(self, key): key = self.format_key_for_backend(key) return self._list_blobs(key)[0]["metadata"] def _metadata_for_key(self, key): return self._list_blobs(key)[0]["metadata"] def list_path(self, key): path = self.format_key_for_backend(key, trailing_slash=True) return self._list_blobs(path) def _list_blobs(self, path): self.log.debug("Listing path %r", path) items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata") result = [] for item in items: result.append({ "last_modified": dateutil.parser.parse(item.properties.last_modified), "metadata": item.metadata, "name": self.format_key_from_backend(item.name), "size": item.properties.content_length, }) return result def delete_key(self, key): key = self.format_key_for_backend(key) self.log.debug("Deleting key: %r", key) return self.conn.delete_blob(self.container_name, key) def get_contents_to_file(self, key, filepath_to_store_to): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to) return self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to) def get_contents_to_fileobj(self, key, fileobj_to_store_to): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_file(self.container_name, key, fileobj_to_store_to) def get_contents_to_string(self, key): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_bytes(self.container_name, key), self._metadata_for_key(key) def store_file_from_memory(self, key, memstring, metadata=None): key = self.format_key_for_backend(key) # Azure requires all metadata keys and values to be strings metadata_to_send = {str(k): str(v) for k, v in metadata.items()} self.conn.put_block_blob_from_bytes( self.container_name, key, memstring, x_ms_meta_name_values=metadata_to_send) def store_file_from_disk(self, key, filepath, metadata=None, multipart=None): key = self.format_key_for_backend(key) # Azure requires all metadata keys and values to be strings metadata_to_send = {str(k): str(v) for k, v in metadata.items()} self.conn.put_block_blob_from_path( self.container_name, key, filepath, x_ms_meta_name_values=metadata_to_send) def get_or_create_container(self, container_name): start_time = time.time() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time) return container_name
class AzureBlobStorage(Storage): ''' classdocs ''' def __init__(self, azure_profile): ''' Constructor ''' if not azure_profile: raise Exception() else: container_name = azure_profile['container_name'] account_name = azure_profile['account_name'] account_key = azure_profile['key'] base_url = azure_profile['base_url'] self.blob = BlobService( account_name=account_name, account_key=account_key) self.container = container_name self.base_url = base_url def delete(self, name): """ Delete file. """ try: self.blob.delete_blob(self.container, name) except WindowsAzureMissingResourceError: return False else: return True def delete_files(self, files=None): """ Delete files in container. """ if not files: files = self.listdir(self.container)[1] for _file in files: self.delete(_file) def exists(self, name, with_properties=False): """ Existing check. """ result = False blob_properties = None try: blob_properties = self.blob.get_blob_properties( self.container, name) except WindowsAzureMissingResourceError: result = False else: result = True if with_properties: return result, blob_properties else: return result def get_available_name(self, name): return super(AzureBlobStorage, self).get_available_name(name.replace('\\', '/')) def get_valid_name(self, name): return name def _list(self, path, prefix, maxresults): result = [] blobs = self.blob.list_blobs(path, prefix, maxresults) for _blob in blobs: result.append(_blob.name) return result def listdir(self, path=None, prefix=None, maxresults=None): """ Catalog file list. """ if not path: path = self.container return [], self._list(path, prefix, maxresults) def size(self, name): """ File size. """ result, properties = self.exists(name, with_properties=True) if result: return int(properties['content-length']) else: return 0 def url(self, name, chk_exist=False): """ URL for file downloading. """ if chk_exist: if self.exists(name): return '%s%s/%s' % (self.base_url, self.container, name) else: return None else: return '%s%s/%s' % (self.base_url, self.container, name) def _open(self, name, mode='rb'): """ Open file. """ in_mem_file = StringIO.StringIO( self.blob.get_blob(self.container, name)) in_mem_file.name = name in_mem_file.mode = mode return File(in_mem_file) def _save(self, name, blob_to_upload, x_ms_blob_type='BlockBlob', content_type=None): """ Save file. """ if hasattr(blob_to_upload, 'content_type'): content_type = blob_to_upload.content_type or None if content_type is None: content_type = mimetypes.guess_type(name)[0] or None blob_to_upload.seek(0) self.blob.put_blob(self.container, name, blob_to_upload, x_ms_blob_type, x_ms_blob_content_type=content_type) return name def modified_time(self, name): """ Last modification time. """ result, properties = self.exists(name, with_properties=True) if result: date_string = properties['last-modified'] modified_dt = parser.parse(date_string) if timezone.is_naive(modified_dt): return modified_dt else: return timezone.make_naive(modified_dt, timezone.get_current_timezone()) else: return None created_time = accessed_time = modified_time
class Storage(driver.Base): supports_bytes_range = True def __init__(self, path=None, config=None): self._config = config self._container = self._config.azure_storage_container protocol = 'https' if self._config.azure_use_https else 'http' acct_name = self._config.azure_storage_account_name acct_key = self._config.azure_storage_account_key self._blob = BlobService( account_name=acct_name, account_key=acct_key, protocol=protocol) self._init_container() logger.debug("Initialized azureblob storage driver") def _init_container(self): '''Initializes image container on Azure blob storage if the container does not exist. ''' created = self._blob.create_container( self._container, x_ms_blob_public_access='blob', fail_on_exist=False) if created: logger.info('Created blob container for image registry.') else: logger.debug('Registry container already exists.') return created @lru.get def get_content(self, path): try: return self._blob.get_blob(self._container, path) except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path) @lru.set def put_content(self, path, content): self._blob.put_blob(self._container, path, content, 'BlockBlob') return path def stream_read(self, path, bytes_range=None): try: f = io.BytesIO() self._blob.get_blob_to_file(self._container, path, f) if bytes_range: f.seek(bytes_range[0]) total_size = bytes_range[1] - bytes_range[0] + 1 else: f.seek(0) while True: buf = None if bytes_range: # Bytes Range is enabled buf_size = self.buffer_size if nb_bytes + buf_size > total_size: # We make sure we don't read out of the range buf_size = total_size - nb_bytes if buf_size > 0: buf = f.read(buf_size) nb_bytes += len(buf) else: # We're at the end of the range buf = '' else: buf = f.read(self.buffer_size) if not buf: break yield buf except IOError: raise exceptions.FileNotFoundError('%s is not there' % path) def stream_write(self, path, fp): self._blob.put_block_blob_from_file(self._container, path, fp) def list_directory(self, path=None): if not path.endswith('/'): path += '/' # path=a would list a/b.txt as well as 'abc.txt' blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) return [b.name for b in blobs] def exists(self, path): try: self._blob.get_blob_properties(self._container, path) return True except azure.WindowsAzureMissingResourceError: return False @lru.remove def remove(self, path): is_blob = self.exists(path) if is_blob: self._blob.delete_blob(self._container, path) return exists = False blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) for b in blobs: self._blob.delete_blob(self._container, b.name) def get_size(self, path): try: properties = self._blob.get_blob_properties(self._container, path) return int(properties['content-length']) # auto-converted to long except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path)
print( imgBlobName ) tableRowKey = imgBlobName try: blob = blob_service.get_blob(blob_container, imgBlobName) except azure.WindowsAzureMissingResourceError: queue_service.delete_message( imagesQueue, message.message_id, message.pop_receipt ) continue image = blobToOpenCV(blob) # ADDED#2 ######################################## if image is None: print "GIF attempt in analyse" queue_service.delete_message( imagesQueue, message.message_id, message.pop_receipt ) blob_service.delete_blob( blob_container, imgBlobName) table_service.delete_entity( tableName, tablePartitionKey, tableRowKey) continue # ADDED #################################################################### ## Decrease main image size # we need to keep in mind aspect ratio if image.shape[1] >= image.shape[0]: width = 800 r = (width * 1.0) / image.shape[1] dim = ( width, int(image.shape[0] * r) ) else: height = 800 r = (height * 1.0) / image.shape[0] dim = ( int(image.shape[1] * r), height )
blob_service = BlobService(account_name, account_key) for container in blob_service.list_containers(): c = container.name if c == "heartbeat": continue if options.date and not ( c == "processed-"+options.date ): continue if debug: sys.stderr.write("Processing container: "+str(c)+"\n") for b in blob_service.list_blobs(c): if debug: sys.stderr.write("Processing blob: "+str(b.name)+"\n") data = blob_service.get_blob(c, b.name) cs = StringIO.StringIO(data) gzipstream = gzip.GzipFile(fileobj=cs) if output_format == "txt": print gzipstream.read() elif output_format == "json": d = {} i = 0 ds = gzipstream.read() # some DCU entries contains more than 28 values (outside the # definition of the headers) for x in ds.strip().split("\t")[:27]: d[headers[i]] = x i=i+1 print (json.dumps(d, sort_keys=True)) if options.clear: if debug: sys.stderr.write("Deleting blob: "+str(b.name)+"\n") blob_service.delete_blob(c, b.name) if options.clear: if debug: sys.stderr.write("Deleting container: "+str(c)+"\n") blob_service.delete_container(c)