def writer(): headers = self.encryptedHeaders if encrypt else self.headerValues if update: try: key.set_contents_from_stream(readable, headers=headers) except boto.exception.GSDataError: if encrypt: # https://github.com/boto/boto/issues/3518 # see self._writeFile for more pass else: raise else: try: # The if_condition kwarg insures that the existing key matches given # generation (version) before modifying anything. Setting # if_generation=0 insures key does not exist remotely key.set_contents_from_stream(readable, headers=headers, if_generation=0) except (boto.exception.GSResponseError, boto.exception.GSDataError) as e: if isinstance(e, boto.exception.GSResponseError): if e.status == 412: raise ConcurrentFileModificationException( key.name) else: raise e elif encrypt: # https://github.com/boto/boto/issues/3518 # see self._writeFile for more pass else: raise
def readFrom(self, readable): headers = store.encryptedHeaders if encrypt else store.headerValues if update: try: key.set_contents_from_stream(readable, headers=headers) except boto.exception.GSDataError: if encrypt: # https://github.com/boto/boto/issues/3518 # see self._writeFile for more pass else: raise else: try: # The if_generation argument insures that the existing key matches the # given generation, i.e. version, before modifying anything. Passing a # generation of 0 insures that the key does not exist remotely. key.set_contents_from_stream(readable, headers=headers, if_generation=0) except (boto.exception.GSResponseError, boto.exception.GSDataError) as e: if isinstance(e, boto.exception.GSResponseError): if e.status == 412: raise ConcurrentFileModificationException( key.name) else: raise e elif encrypt: # https://github.com/boto/boto/issues/3518 # see self._writeFile for more pass else: raise
def save(self): attributes, numNewContentChunks = self.toItem() # False stands for absence expected = ['version', False if self.previousVersion is None else self.previousVersion] try: for attempt in retry_sdb(): with attempt: assert self.outer.filesDomain.put_attributes(item_name=self.fileID, attributes=attributes, expected_value=expected) if self.previousVersion: self.outer.filesBucket.delete_key(self.fileID, version_id=self.previousVersion) self._previousVersion = self._version if numNewContentChunks < self._numContentChunks: residualChunks = xrange(numNewContentChunks, self._numContentChunks) attributes = [self._chunkName(i) for i in residualChunks] for attempt in retry_sdb(): with attempt: self.outer.filesDomain.delete_attributes(self.fileID, attributes=attributes) self._numContentChunks = numNewContentChunks except SDBResponseError as e: if e.error_code == 'ConditionalCheckFailed': raise ConcurrentFileModificationException(self.fileID) else: raise
def reader(): blockIDs = [] try: while True: buf = readable.read(maxBlockSize) if len(buf) == 0: # We're safe to break here even if we never read anything, since # putting an empty block list creates an empty blob. break if encrypted: buf = encryption.encrypt(buf, self.keyPath) blockID = self._newFileID() container.put_block(blob_name=jobStoreFileID, block=buf, blockid=blockID) blockIDs.append(blockID) except: # This is guaranteed to delete any uncommitted # blocks. container.delete_blob(blob_name=jobStoreFileID) raise if checkForModification and expectedVersion is not None: # Acquire a (60-second) write lock, leaseID = container.lease_blob( blob_name=jobStoreFileID, x_ms_lease_action='acquire')['x-ms-lease-id'] # check for modification, blobProperties = container.get_blob_properties( blob_name=jobStoreFileID) if blobProperties['etag'] != expectedVersion: container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action='release', x_ms_lease_id=leaseID) raise ConcurrentFileModificationException( jobStoreFileID) # commit the file, container.put_block_list(blob_name=jobStoreFileID, block_list=blockIDs, x_ms_lease_id=leaseID, x_ms_meta_name_values=dict( encrypted=str(encrypted))) # then release the lock. container.lease_blob(blob_name=jobStoreFileID, x_ms_lease_action='release', x_ms_lease_id=leaseID) else: # No need to check for modification, just blindly write over whatever # was there. container.put_block_list(blob_name=jobStoreFileID, block_list=blockIDs, x_ms_meta_name_values=dict( encrypted=str(encrypted)))
def readFrom(self, readable): blocks = [] try: while True: buf = readable.read(maxBlockSize) if len(buf) == 0: # We're safe to break here even if we never read anything, since # putting an empty block list creates an empty blob. break if encrypted: buf = encryption.encrypt(buf, store.keyPath) blockID = store._newFileID() container.put_block(blob_name=bytes(jobStoreFileID), block=buf, block_id=blockID) blocks.append(BlobBlock(blockID)) except: with panic(log=logger): # This is guaranteed to delete any uncommitted blocks. container.delete_blob(blob_name=bytes(jobStoreFileID)) if checkForModification and expectedVersion is not None: # Acquire a (60-second) write lock, leaseID = container.acquire_blob_lease( blob_name=bytes(jobStoreFileID), lease_duration=60) # check for modification, blob = container.get_blob_properties( blob_name=bytes(jobStoreFileID)) if blob.properties.etag != expectedVersion: container.release_blob_lease( blob_name=bytes(jobStoreFileID), lease_id=leaseID) raise ConcurrentFileModificationException( jobStoreFileID) # commit the file, container.put_block_list( blob_name=bytes(jobStoreFileID), block_list=blocks, lease_id=leaseID, metadata=dict(encrypted=str(encrypted))) # then release the lock. container.release_blob_lease( blob_name=bytes(jobStoreFileID), lease_id=leaseID) else: # No need to check for modification, just blindly write over whatever # was there. container.put_block_list( blob_name=bytes(jobStoreFileID), block_list=blocks, metadata=dict(encrypted=str(encrypted)))
def _registerFile(self, jobStoreFileID, bucketName='files', jobStoreID=None, newVersion=None, oldVersion=None): """ Register a a file in the store :param jobStoreFileID: the file's ID, mandatory :param bucketName: the name of the S3 bucket the file was placed in :param jobStoreID: the ID of the batchjob owning the file, only allowed for first version of file or when file is registered without content :param newVersion: the file's new version or None if the file is to be registered without content, in which case jobStoreId must be passed :param oldVersion: the expected previous version of the file or None if newVersion is the first version or file is registered without content """ # Must pass either jobStoreID or newVersion, or both assert jobStoreID is not None or newVersion is not None # Must pass newVersion if passing oldVersion assert oldVersion is None or newVersion is not None attributes = dict(bucketName=bucketName) if newVersion is not None: attributes['version'] = newVersion if jobStoreID is not None: attributes['jobStoreID'] = jobStoreID # False stands for absence expected = ['version', False if oldVersion is None else oldVersion] try: for attempt in retry_sdb(): with attempt: assert self.versions.put_attributes( item_name=jobStoreFileID, attributes=attributes, expected_value=expected) if oldVersion is not None: bucket = getattr(self, bucketName) bucket.delete_key(jobStoreFileID, version_id=oldVersion) except SDBResponseError as e: if e.error_code == 'ConditionalCheckFailed': raise ConcurrentFileModificationException(jobStoreFileID) else: raise