def _downloadStream(self, jobStoreFileID, container): # The reason this is not in the writer is so we catch non-existant blobs early blob = container.get_blob_properties(blob_name=str(jobStoreFileID)) encrypted = strict_bool(blob.metadata['encrypted']) if encrypted and self.keyPath is None: raise AssertionError('Content is encrypted but no key was provided.') outer_self = self class DownloadPipe(ReadablePipe): def writeTo(self, writable): chunkStart = 0 fileSize = blob.properties.content_length while chunkStart < fileSize: chunkEnd = chunkStart + outer_self._maxAzureBlockBytes - 1 buf = container.get_blob_to_bytes(blob_name=str(jobStoreFileID), start_range=chunkStart, end_range=chunkEnd).content if encrypted: buf = encryption.decrypt(buf, outer_self.keyPath) writable.write(buf) chunkStart = chunkEnd + 1 with DownloadPipe() as readable: yield readable
def _trackOfferedNodes(self, offers): for offer in offers: try: nodeAddress = socket.gethostbyname(offer.hostname) except: log.deug("Failed to resolve hostname %s" % offer.hostname) raise self._registerNode(nodeAddress, offer.slave_id.value) preemptable = False for attribute in offer.attributes: if attribute.name == 'preemptable': preemptable = strict_bool(attribute.text.value) if preemptable: try: self.nonPreemptableNodes.remove(offer.slave_id.value) except KeyError: pass else: self.nonPreemptableNodes.add(offer.slave_id.value)
def _parseOffer(self, offer): cores = 0 memory = 0 disk = 0 preemptable = None for attribute in offer.attributes: if attribute.name == 'preemptable': assert preemptable is None, "Attribute 'preemptable' occurs more than once." preemptable = strict_bool(attribute.text.value) if preemptable is None: log.debug('Agent not marked as either preemptable or not. Assuming non-preemptable.') preemptable = False for resource in offer.resources: if resource.name == "cpus": cores += resource.scalar.value elif resource.name == "mem": memory += resource.scalar.value elif resource.name == "disk": disk += resource.scalar.value return cores, memory, disk, preemptable
def _trackOfferedNodes(self, offers): for offer in offers: # All AgentID messages are required to have a value according to the Mesos Protobuf file. assert 'value' in offer.agent_id try: nodeAddress = socket.gethostbyname(offer.hostname) except: log.debug("Failed to resolve hostname %s" % offer.hostname) raise self._registerNode(nodeAddress, offer.agent_id.value) preemptable = False for attribute in offer.attributes: if attribute.name == 'preemptable': preemptable = strict_bool(attribute.text.value) if preemptable: try: self.nonPreemptableNodes.remove(offer.agent_id.value) except KeyError: pass else: self.nonPreemptableNodes.add(offer.agent_id.value)
def fromCommand(cls, command): assert len(command) == 3 return cls(dirPath=command[0], name=command[1], fromVirtualEnv=strict_bool(command[2]))