def toItem(self): """ Convert this instance to an attribute dictionary suitable for SDB put_attributes(). :rtype: (dict,int) :return: the attributes dict and an integer specifying the the number of chunk attributes in the dictionary that are used for storing inlined content. """ if self.content is None: numChunks = 0 attributes = {} else: content = self.content if self.encrypted: sseKeyPath = self.outer.sseKeyPath if sseKeyPath is None: raise AssertionError('Encryption requested but no key was provided.') content = encryption.encrypt(content, sseKeyPath) attributes = self.binaryToAttributes(content) numChunks = len(attributes) attributes.update(dict(ownerID=self.ownerID, encrypted=self.encrypted, version=self.version or '')) return attributes, numChunks
def writeStatsAndLogging(self, statsAndLoggingString): # TODO: would be a great use case for the append blobs, once implemented in the Azure SDK jobStoreFileID = self._newFileID() encrypted = self.keyPath is not None if encrypted: statsAndLoggingString = encryption.encrypt(statsAndLoggingString, self.keyPath) self.statsFiles.create_blob_from_text(blob_name=str(jobStoreFileID), text=statsAndLoggingString, metadata=dict(encrypted=str(encrypted))) self.statsFileIDs.insert_entity(entity={'RowKey': jobStoreFileID})
def writeStatsAndLogging(self, statsAndLoggingString): # TODO: would be a great use case for the append blobs, once implemented in the Azure SDK jobStoreFileID = self._newFileID() encrypted = self.keyPath is not None if encrypted: statsAndLoggingString = encryption.encrypt(statsAndLoggingString, self.keyPath) self.statsFiles.put_block_blob_from_text( blob_name=jobStoreFileID, text=statsAndLoggingString, x_ms_meta_name_values=dict(encrypted=str(encrypted)) ) self.statsFileIDs.insert_entity(entity={"RowKey": jobStoreFileID})
def reader(): blockIDs = [] try: while True: buf = readable.read(maxBlockSize) if len(buf) == 0: # We're safe to break here even if we never read anything, since # putting an empty block list creates an empty blob. break if encrypted: buf = encryption.encrypt(buf, self.keyPath) blockID = self._newFileID() container.put_block(blob_name=jobStoreFileID, block=buf, blockid=blockID) blockIDs.append(blockID) except: # This is guaranteed to delete any uncommitted # blocks. container.delete_blob(blob_name=jobStoreFileID) raise if checkForModification and expectedVersion is not None: # Acquire a (60-second) write lock, leaseID = container.lease_blob( blob_name=jobStoreFileID, x_ms_lease_action='acquire')['x-ms-lease-id'] # check for modification, blobProperties = container.get_blob_properties( blob_name=jobStoreFileID) if blobProperties['etag'] != expectedVersion: container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action='release', x_ms_lease_id=leaseID) raise ConcurrentFileModificationException( jobStoreFileID) # commit the file, container.put_block_list(blob_name=jobStoreFileID, block_list=blockIDs, x_ms_lease_id=leaseID, x_ms_meta_name_values=dict( encrypted=str(encrypted))) # then release the lock. container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action='release', x_ms_lease_id=leaseID) else: # No need to check for modification, just blindly write over whatever # was there. container.put_block_list(blob_name=jobStoreFileID, block_list=blockIDs, x_ms_meta_name_values=dict( encrypted=str(encrypted)))
def reader(): blockIDs = [] try: while True: buf = readable.read(maxBlockSize) if len(buf) == 0: # We're safe to break here even if we never read anything, since # putting an empty block list creates an empty blob. break if encrypted: buf = encryption.encrypt(buf, self.keyPath) blockID = self._newFileID() container.put_block(blob_name=jobStoreFileID, block=buf, blockid=blockID) blockIDs.append(blockID) except: # This is guaranteed to delete any uncommitted # blocks. container.delete_blob(blob_name=jobStoreFileID) raise if checkForModification and expectedVersion is not None: # Acquire a (60-second) write lock, leaseID = container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action="acquire")[ "x-ms-lease-id" ] # check for modification, blobProperties = container.get_blob_properties(blob_name=jobStoreFileID) if blobProperties["etag"] != expectedVersion: container.lease_blob( blob_name=jobStoreFileID, x_ms_lease_action="release", x_ms_lease_id=leaseID ) raise ConcurrentFileModificationException(jobStoreFileID) # commit the file, container.put_block_list( blob_name=jobStoreFileID, block_list=blockIDs, x_ms_lease_id=leaseID, x_ms_meta_name_values=dict(encrypted=str(encrypted)), ) # then release the lock. container.lease_blob( blob_name=jobStoreFileID, x_ms_lease_action="release", x_ms_lease_id=leaseID ) else: # No need to check for modification, just blindly write over whatever # was there. container.put_block_list( blob_name=jobStoreFileID, block_list=blockIDs, x_ms_meta_name_values=dict(encrypted=str(encrypted)), )
def readFrom(self, readable): blocks = [] try: while True: buf = readable.read(maxBlockSize) if len(buf) == 0: # We're safe to break here even if we never read anything, since # putting an empty block list creates an empty blob. break if encrypted: buf = encryption.encrypt(buf, store.keyPath) blockID = store._newFileID() container.put_block(blob_name=bytes(jobStoreFileID), block=buf, block_id=blockID) blocks.append(BlobBlock(blockID)) except: with panic(log=logger): # This is guaranteed to delete any uncommitted blocks. container.delete_blob(blob_name=bytes(jobStoreFileID)) if checkForModification and expectedVersion is not None: # Acquire a (60-second) write lock, leaseID = container.acquire_blob_lease( blob_name=bytes(jobStoreFileID), lease_duration=60) # check for modification, blob = container.get_blob_properties( blob_name=bytes(jobStoreFileID)) if blob.properties.etag != expectedVersion: container.release_blob_lease( blob_name=bytes(jobStoreFileID), lease_id=leaseID) raise ConcurrentFileModificationException( jobStoreFileID) # commit the file, container.put_block_list( blob_name=bytes(jobStoreFileID), block_list=blocks, lease_id=leaseID, metadata=dict(encrypted=str(encrypted))) # then release the lock. container.release_blob_lease( blob_name=bytes(jobStoreFileID), lease_id=leaseID) else: # No need to check for modification, just blindly write over whatever # was there. container.put_block_list( blob_name=bytes(jobStoreFileID), block_list=blocks, metadata=dict(encrypted=str(encrypted)))