def toItem(self):
"""
Convert this instance to an attribute dictionary suitable for SDB put_attributes().
:rtype: (dict,int)
:return: the attributes dict and an integer specifying the the number of chunk
attributes in the dictionary that are used for storing inlined content.
"""
if self.content is None:
numChunks = 0
attributes = {}
else:
content = self.content
if self.encrypted:
sseKeyPath = self.outer.sseKeyPath
if sseKeyPath is None:
raise AssertionError('Encryption requested but no key was provided.')
content = encryption.encrypt(content, sseKeyPath)
attributes = self.binaryToAttributes(content)
numChunks = len(attributes)
attributes.update(dict(ownerID=self.ownerID,
encrypted=self.encrypted,
version=self.version or ''))
return attributes, numChunks
def toItem(self):
"""
Convert this instance to an attribute dictionary suitable for SDB put_attributes().
:rtype: (dict,int)
:return: the attributes dict and an integer specifying the the number of chunk
attributes in the dictionary that are used for storing inlined content.
"""
if self.content is None:
numChunks = 0
attributes = {}
else:
content = self.content
if self.encrypted:
sseKeyPath = self.outer.sseKeyPath
if sseKeyPath is None:
raise AssertionError('Encryption requested but no key was provided.')
content = encryption.encrypt(content, sseKeyPath)
attributes = self.binaryToAttributes(content)
numChunks = len(attributes)
attributes.update(dict(ownerID=self.ownerID,
encrypted=self.encrypted,
version=self.version or ''))
return attributes, numChunks
def writeStatsAndLogging(self, statsAndLoggingString):
# TODO: would be a great use case for the append blobs, once implemented in the Azure SDK
jobStoreFileID = self._newFileID()
encrypted = self.keyPath is not None
if encrypted:
statsAndLoggingString = encryption.encrypt(statsAndLoggingString, self.keyPath)
self.statsFiles.create_blob_from_text(blob_name=str(jobStoreFileID),
text=statsAndLoggingString,
metadata=dict(encrypted=str(encrypted)))
self.statsFileIDs.insert_entity(entity={'RowKey': jobStoreFileID})
def writeStatsAndLogging(self, statsAndLoggingString):
# TODO: would be a great use case for the append blobs, once implemented in the Azure SDK
jobStoreFileID = self._newFileID()
encrypted = self.keyPath is not None
if encrypted:
statsAndLoggingString = encryption.encrypt(statsAndLoggingString, self.keyPath)
self.statsFiles.put_block_blob_from_text(
blob_name=jobStoreFileID, text=statsAndLoggingString, x_ms_meta_name_values=dict(encrypted=str(encrypted))
)
self.statsFileIDs.insert_entity(entity={"RowKey": jobStoreFileID})
def reader():
blockIDs = []
try:
while True:
buf = readable.read(maxBlockSize)
if len(buf) == 0:
# We're safe to break here even if we never read anything, since
# putting an empty block list creates an empty blob.
break
if encrypted:
buf = encryption.encrypt(buf, self.keyPath)
blockID = self._newFileID()
container.put_block(blob_name=jobStoreFileID,
block=buf,
blockid=blockID)
blockIDs.append(blockID)
except:
# This is guaranteed to delete any uncommitted
# blocks.
container.delete_blob(blob_name=jobStoreFileID)
raise
if checkForModification and expectedVersion is not None:
# Acquire a (60-second) write lock,
leaseID = container.lease_blob(
blob_name=jobStoreFileID,
x_ms_lease_action='acquire')['x-ms-lease-id']
# check for modification,
blobProperties = container.get_blob_properties(
blob_name=jobStoreFileID)
if blobProperties['etag'] != expectedVersion:
container.lease_blob(blob_name=jobStoreFileID,
x_ms_lease_action='release',
x_ms_lease_id=leaseID)
raise ConcurrentFileModificationException(
jobStoreFileID)
# commit the file,
container.put_block_list(blob_name=jobStoreFileID,
block_list=blockIDs,
x_ms_lease_id=leaseID,
x_ms_meta_name_values=dict(
encrypted=str(encrypted)))
# then release the lock.
container.lease_blob(blob_name=jobStoreFileID,
x_ms_lease_action='release',
x_ms_lease_id=leaseID)
else:
# No need to check for modification, just blindly write over whatever
# was there.
container.put_block_list(blob_name=jobStoreFileID,
block_list=blockIDs,
x_ms_meta_name_values=dict(
encrypted=str(encrypted)))
def reader():
blockIDs = []
try:
while True:
buf = readable.read(maxBlockSize)
if len(buf) == 0:
# We're safe to break here even if we never read anything, since
# putting an empty block list creates an empty blob.
break
if encrypted:
buf = encryption.encrypt(buf, self.keyPath)
blockID = self._newFileID()
container.put_block(blob_name=jobStoreFileID, block=buf, blockid=blockID)
blockIDs.append(blockID)
except:
# This is guaranteed to delete any uncommitted
# blocks.
container.delete_blob(blob_name=jobStoreFileID)
raise
if checkForModification and expectedVersion is not None:
# Acquire a (60-second) write lock,
leaseID = container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action="acquire")[
"x-ms-lease-id"
]
# check for modification,
blobProperties = container.get_blob_properties(blob_name=jobStoreFileID)
if blobProperties["etag"] != expectedVersion:
container.lease_blob(
blob_name=jobStoreFileID, x_ms_lease_action="release", x_ms_lease_id=leaseID
)
raise ConcurrentFileModificationException(jobStoreFileID)
# commit the file,
container.put_block_list(
blob_name=jobStoreFileID,
block_list=blockIDs,
x_ms_lease_id=leaseID,
x_ms_meta_name_values=dict(encrypted=str(encrypted)),
)
# then release the lock.
container.lease_blob(
blob_name=jobStoreFileID, x_ms_lease_action="release", x_ms_lease_id=leaseID
)
else:
# No need to check for modification, just blindly write over whatever
# was there.
container.put_block_list(
blob_name=jobStoreFileID,
block_list=blockIDs,
x_ms_meta_name_values=dict(encrypted=str(encrypted)),
)
def readFrom(self, readable):
blocks = []
try:
while True:
buf = readable.read(maxBlockSize)
if len(buf) == 0:
# We're safe to break here even if we never read anything, since
# putting an empty block list creates an empty blob.
break
if encrypted:
buf = encryption.encrypt(buf, store.keyPath)
blockID = store._newFileID()
container.put_block(blob_name=bytes(jobStoreFileID),
block=buf,
block_id=blockID)
blocks.append(BlobBlock(blockID))
except:
with panic(log=logger):
# This is guaranteed to delete any uncommitted blocks.
container.delete_blob(blob_name=bytes(jobStoreFileID))
if checkForModification and expectedVersion is not None:
# Acquire a (60-second) write lock,
leaseID = container.acquire_blob_lease(
blob_name=bytes(jobStoreFileID), lease_duration=60)
# check for modification,
blob = container.get_blob_properties(
blob_name=bytes(jobStoreFileID))
if blob.properties.etag != expectedVersion:
container.release_blob_lease(
blob_name=bytes(jobStoreFileID), lease_id=leaseID)
raise ConcurrentFileModificationException(
jobStoreFileID)
# commit the file,
container.put_block_list(
blob_name=bytes(jobStoreFileID),
block_list=blocks,
lease_id=leaseID,
metadata=dict(encrypted=str(encrypted)))
# then release the lock.
container.release_blob_lease(
blob_name=bytes(jobStoreFileID), lease_id=leaseID)
else:
# No need to check for modification, just blindly write over whatever
# was there.
container.put_block_list(
blob_name=bytes(jobStoreFileID),
block_list=blocks,
metadata=dict(encrypted=str(encrypted)))
