def _downloadStream(self, jobStoreFileID, container):
# The reason this is not in the writer is so we catch non-existant blobs early
blob = container.get_blob_properties(blob_name=str(jobStoreFileID))
encrypted = strict_bool(blob.metadata['encrypted'])
if encrypted and self.keyPath is None:
raise AssertionError('Content is encrypted but no key was provided.')
outer_self = self
class DownloadPipe(ReadablePipe):
def writeTo(self, writable):
chunkStart = 0
fileSize = blob.properties.content_length
while chunkStart < fileSize:
chunkEnd = chunkStart + outer_self._maxAzureBlockBytes - 1
buf = container.get_blob_to_bytes(blob_name=str(jobStoreFileID),
start_range=chunkStart,
end_range=chunkEnd).content
if encrypted:
buf = encryption.decrypt(buf, outer_self.keyPath)
writable.write(buf)
chunkStart = chunkEnd + 1
with DownloadPipe() as readable:
yield readable
def _trackOfferedNodes(self, offers):
for offer in offers:
try:
nodeAddress = socket.gethostbyname(offer.hostname)
except:
log.deug("Failed to resolve hostname %s" % offer.hostname)
raise
self._registerNode(nodeAddress, offer.slave_id.value)
preemptable = False
for attribute in offer.attributes:
if attribute.name == 'preemptable':
preemptable = strict_bool(attribute.text.value)
if preemptable:
try:
self.nonPreemptableNodes.remove(offer.slave_id.value)
except KeyError:
pass
else:
self.nonPreemptableNodes.add(offer.slave_id.value)
def _parseOffer(self, offer):
cores = 0
memory = 0
disk = 0
preemptable = None
for attribute in offer.attributes:
if attribute.name == 'preemptable':
assert preemptable is None, "Attribute 'preemptable' occurs more than once."
preemptable = strict_bool(attribute.text.value)
if preemptable is None:
log.debug('Agent not marked as either preemptable or not. Assuming non-preemptable.')
preemptable = False
for resource in offer.resources:
if resource.name == "cpus":
cores += resource.scalar.value
elif resource.name == "mem":
memory += resource.scalar.value
elif resource.name == "disk":
disk += resource.scalar.value
return cores, memory, disk, preemptable
def _trackOfferedNodes(self, offers):
for offer in offers:
# All AgentID messages are required to have a value according to the Mesos Protobuf file.
assert 'value' in offer.agent_id
try:
nodeAddress = socket.gethostbyname(offer.hostname)
except:
log.debug("Failed to resolve hostname %s" % offer.hostname)
raise
self._registerNode(nodeAddress, offer.agent_id.value)
preemptable = False
for attribute in offer.attributes:
if attribute.name == 'preemptable':
preemptable = strict_bool(attribute.text.value)
if preemptable:
try:
self.nonPreemptableNodes.remove(offer.agent_id.value)
except KeyError:
pass
else:
self.nonPreemptableNodes.add(offer.agent_id.value)
def fromCommand(cls, command):
assert len(command) == 3
return cls(dirPath=command[0],
name=command[1],
fromVirtualEnv=strict_bool(command[2]))
