def _downloadStream(self, jobStoreFileID, container): # The reason this is not in the writer is so we catch non-existant blobs early blob = container.get_blob_properties(blob_name=bytes(jobStoreFileID)) encrypted = strict_bool(blob.metadata['encrypted']) if encrypted and self.keyPath is None: raise AssertionError( 'Content is encrypted but no key was provided.') outer_self = self class DownloadPipe(ReadablePipe): def writeTo(self, writable): chunkStart = 0 fileSize = blob.properties.content_length while chunkStart < fileSize: chunkEnd = chunkStart + outer_self._maxAzureBlockBytes - 1 buf = container.get_blob_to_bytes( blob_name=bytes(jobStoreFileID), start_range=chunkStart, end_range=chunkEnd).content if encrypted: buf = encryption.decrypt(buf, outer_self.keyPath) writable.write(buf) chunkStart = chunkEnd + 1 with DownloadPipe() as readable: yield readable
def _downloadStream(self, jobStoreFileID, container): # The reason this is not in the writer is so we catch non-existant blobs early blobProps = container.get_blob_properties(blob_name=jobStoreFileID) encrypted = strict_bool(blobProps['x-ms-meta-encrypted']) if encrypted and self.keyPath is None: raise AssertionError( 'Content is encrypted but no key was provided.') outer_self = self class DownloadPipe(ReadablePipe): def writeTo(self, writable): chunkStart = 0 fileSize = int(blobProps['Content-Length']) while chunkStart < fileSize: chunkEnd = chunkStart + outer_self._maxAzureBlockBytes - 1 buf = container.get_blob(blob_name=jobStoreFileID, x_ms_range="bytes=%d-%d" % (chunkStart, chunkEnd)) if encrypted: buf = encryption.decrypt(buf, outer_self.keyPath) writable.write(buf) chunkStart = chunkEnd + 1 with DownloadPipe() as readable: yield readable
def fromItem(cls, item): """ Convert an SDB item to an instance of this class. :type item: Item """ assert item is not None # Strings come back from SDB as unicode def strOrNone(s): return s if s is None else str(s) # ownerID and encrypted are the only mandatory attributes ownerID = strOrNone(item.get('ownerID')) encrypted = item.get('encrypted') if ownerID is None: assert encrypted is None return None else: version = strOrNone(item['version']) encrypted = strict_bool(encrypted) content, numContentChunks = cls.attributesToBinary(item) if encrypted: sseKeyPath = cls.outer.sseKeyPath if sseKeyPath is None: raise AssertionError('Content is encrypted but no key was provided.') if content is not None: content = encryption.decrypt(content, sseKeyPath) self = cls(fileID=item.name, ownerID=ownerID, encrypted=encrypted, version=version, content=content, numContentChunks=numContentChunks) return self
def _downloadStream(self, jobStoreFileID, container): # The reason this is not in the writer is so we catch non-existant blobs early blobProps = container.get_blob_properties(blob_name=bytes(jobStoreFileID)) encrypted = strict_bool(blobProps['x-ms-meta-encrypted']) if encrypted and self.keyPath is None: raise AssertionError('Content is encrypted but no key was provided.') outer_self = self class DownloadPipe(ReadablePipe): def writeTo(self, writable): chunkStart = 0 fileSize = int(blobProps['Content-Length']) while chunkStart < fileSize: chunkEnd = chunkStart + outer_self._maxAzureBlockBytes - 1 buf = container.get_blob(blob_name=bytes(jobStoreFileID), x_ms_range="bytes=%d-%d" % (chunkStart, chunkEnd)) if encrypted: buf = encryption.decrypt(buf, outer_self.keyPath) writable.write(buf) chunkStart = chunkEnd + 1 with DownloadPipe() as readable: yield readable
def __init__(self, region, namePrefix, config=None, partSize=defaultPartSize): """ Create a new job store in AWS or load an existing one from there. :param region: the AWS region to create the job store in, e.g. 'us-west-2' :param namePrefix: S3 bucket names and SDB tables will be prefixed with this :param config: the config object to written to this job store. Must be None for existing job stores. Must not be None for new job stores. """ log.debug("Instantiating %s for region %s and name prefix '%s'", self.__class__, region, namePrefix) self.region = region self.namePrefix = namePrefix self.jobsDomain = None self.filesDomain = None self.filesBucket = None self.db = self._connectSimpleDB() self.s3 = self._connectS3() self.partSize = partSize # Check global registry domain for existence of this job store. The first time this is # being executed in an AWS account, the registry domain will be created on the fly. create = config is not None self.registry_domain = self._getOrCreateDomain('toil-registry') for attempt in retry_sdb(): with attempt: attributes = self.registry_domain.get_attributes( item_name=namePrefix, attribute_name='exists', consistent_read=True) exists = strict_bool(attributes.get('exists', str(False))) self._checkJobStoreCreation(create, exists, region + ":" + namePrefix) def qualify(name): assert len(name) <= self.maxNameLen return self.namePrefix + self.nameSeparator + name self.jobsDomain = self._getOrCreateDomain(qualify('jobs')) self.filesDomain = self._getOrCreateDomain(qualify('files')) self.filesBucket = self._getOrCreateBucket(qualify('files'), versioning=True) # Now register this job store for attempt in retry_sdb(): with attempt: self.registry_domain.put_attributes( item_name=namePrefix, attributes=dict(exists='True')) super(AWSJobStore, self).__init__(config=config) self.sseKeyPath = self.config.sseKey
def __init__(self, region, namePrefix, config=None, partSize=defaultPartSize): """ Create a new job store in AWS or load an existing one from there. :param region: the AWS region to create the job store in, e.g. 'us-west-2' :param namePrefix: S3 bucket names and SDB tables will be prefixed with this :param config: the config object to written to this job store. Must be None for existing job stores. Must not be None for new job stores. """ log.debug("Instantiating %s for region %s and name prefix '%s'", self.__class__, region, namePrefix) self.region = region self.namePrefix = namePrefix self.jobsDomain = None self.filesDomain = None self.filesBucket = None self.db = self._connectSimpleDB() self.s3 = self._connectS3() self.partSize = partSize # Check global registry domain for existence of this job store. The first time this is # being executed in an AWS account, the registry domain will be created on the fly. create = config is not None self.registry_domain = self._getOrCreateDomain('toil-registry') for attempt in retry_sdb(): with attempt: attributes = self.registry_domain.get_attributes(item_name=namePrefix, attribute_name='exists', consistent_read=True) exists = strict_bool(attributes.get('exists', str(False))) self._checkJobStoreCreation(create, exists, region + ":" + namePrefix) def qualify(name): assert len(name) <= self.maxNameLen return self.namePrefix + self.nameSeparator + name self.jobsDomain = self._getOrCreateDomain(qualify('jobs')) self.filesDomain = self._getOrCreateDomain(qualify('files')) self.filesBucket = self._getOrCreateBucket(qualify('files'), versioning=True) # Now register this job store for attempt in retry_sdb(): with attempt: self.registry_domain.put_attributes(item_name=namePrefix, attributes=dict(exists='True')) super(AWSJobStore, self).__init__(config=config) self.sseKeyPath = self.config.sseKey
def _trackOfferedNodes(self, offers): for offer in offers: nodeAddress = socket.gethostbyname(offer.hostname) self._registerNode(nodeAddress, offer.slave_id.value) preemptable = False for attribute in offer.attributes: if attribute.name == 'preemptable': preemptable = strict_bool(attribute.text.value) if preemptable: try: self.nonPreemptableNodes.remove(offer.slave_id.value) except KeyError: pass else: self.nonPreemptableNodes.add(offer.slave_id.value)
def _parseOffer(self, offer): cores = 0 memory = 0 disk = 0 preemptable = None for attribute in offer.attributes: if attribute.name == 'preemptable': assert preemptable is None, "Attribute 'preemptable' occurs more than once." preemptable = strict_bool(attribute.text.value) if preemptable is None: log.debug('Slave not marked as either preemptable or not. Assuming non-preemptable.') preemptable = False for resource in offer.resources: if resource.name == "cpus": cores += resource.scalar.value elif resource.name == "mem": memory += resource.scalar.value elif resource.name == "disk": disk += resource.scalar.value return cores, memory, disk, preemptable
def _downloadStream(self, jobStoreFileID, container): # The reason this is not in the writer is so we catch non-existant blobs early blobProps = container.get_blob_properties(blob_name=jobStoreFileID) encrypted = strict_bool(blobProps['x-ms-meta-encrypted']) if encrypted and self.keyPath is None: raise AssertionError( 'Content is encrypted but no key was provided.') readable_fh, writable_fh = os.pipe() with os.fdopen(readable_fh, 'r') as readable: with os.fdopen(writable_fh, 'w') as writable: def writer(): try: chunkStartPos = 0 fileSize = int(blobProps['Content-Length']) while chunkStartPos < fileSize: chunkEndPos = chunkStartPos + self._maxAzureBlockBytes - 1 buf = container.get_blob( blob_name=jobStoreFileID, x_ms_range="bytes=%d-%d" % (chunkStartPos, chunkEndPos)) if encrypted: buf = encryption.decrypt(buf, self.keyPath) writable.write(buf) chunkStartPos = chunkEndPos + 1 finally: # Ensure readers aren't left blocking if this thread crashes. # This close() will send EOF to the reading end and ultimately cause the # yield to return. It also makes the implict .close() done by the enclosing # "with" context redundant but that should be ok since .close() on file # objects are idempotent. writable.close() thread = ExceptionalThread(target=writer) thread.start() yield readable thread.join()
def _downloadStream(self, jobStoreFileID, container): # The reason this is not in the writer is so we catch non-existant blobs early blobProps = container.get_blob_properties(blob_name=jobStoreFileID) encrypted = strict_bool(blobProps['x-ms-meta-encrypted']) if encrypted and self.keyPath is None: raise AssertionError('Content is encrypted but no key was provided.') readable_fh, writable_fh = os.pipe() with os.fdopen(readable_fh, 'r') as readable: with os.fdopen(writable_fh, 'w') as writable: def writer(): try: chunkStartPos = 0 fileSize = int(blobProps['Content-Length']) while chunkStartPos < fileSize: chunkEndPos = chunkStartPos + self._maxAzureBlockBytes - 1 buf = container.get_blob(blob_name=jobStoreFileID, x_ms_range="bytes=%d-%d" % (chunkStartPos, chunkEndPos)) if encrypted: buf = encryption.decrypt(buf, self.keyPath) writable.write(buf) chunkStartPos = chunkEndPos + 1 finally: # Ensure readers aren't left blocking if this thread crashes. # This close() will send EOF to the reading end and ultimately cause the # yield to return. It also makes the implict .close() done by the enclosing # "with" context redundant but that should be ok since .close() on file # objects are idempotent. writable.close() thread = ExceptionalThread(target=writer) thread.start() yield readable thread.join()
def fromCommand(cls, command): assert len(command) == 3 return cls(dirPath=command[0], name=command[1], fromVirtualEnv=strict_bool(command[2]))
def _set_instance_options( self, options ): super( AgentBox, self )._set_instance_options( options ) self._enable_agent = strict_bool( options.get( 'enable_agent', 'True' ) )