def _downloadStream(self, jobStoreFileID, container):
# The reason this is not in the writer is so we catch non-existant blobs early
blob = container.get_blob_properties(blob_name=bytes(jobStoreFileID))
encrypted = strict_bool(blob.metadata['encrypted'])
if encrypted and self.keyPath is None:
raise AssertionError(
'Content is encrypted but no key was provided.')
outer_self = self
class DownloadPipe(ReadablePipe):
def writeTo(self, writable):
chunkStart = 0
fileSize = blob.properties.content_length
while chunkStart < fileSize:
chunkEnd = chunkStart + outer_self._maxAzureBlockBytes - 1
buf = container.get_blob_to_bytes(
blob_name=bytes(jobStoreFileID),
start_range=chunkStart,
end_range=chunkEnd).content
if encrypted:
buf = encryption.decrypt(buf, outer_self.keyPath)
writable.write(buf)
chunkStart = chunkEnd + 1
with DownloadPipe() as readable:
yield readable
def _downloadStream(self, jobStoreFileID, container):
# The reason this is not in the writer is so we catch non-existant blobs early
blobProps = container.get_blob_properties(blob_name=jobStoreFileID)
encrypted = strict_bool(blobProps['x-ms-meta-encrypted'])
if encrypted and self.keyPath is None:
raise AssertionError(
'Content is encrypted but no key was provided.')
outer_self = self
class DownloadPipe(ReadablePipe):
def writeTo(self, writable):
chunkStart = 0
fileSize = int(blobProps['Content-Length'])
while chunkStart < fileSize:
chunkEnd = chunkStart + outer_self._maxAzureBlockBytes - 1
buf = container.get_blob(blob_name=jobStoreFileID,
x_ms_range="bytes=%d-%d" %
(chunkStart, chunkEnd))
if encrypted:
buf = encryption.decrypt(buf, outer_self.keyPath)
writable.write(buf)
chunkStart = chunkEnd + 1
with DownloadPipe() as readable:
yield readable
def fromItem(cls, item):
"""
Convert an SDB item to an instance of this class.
:type item: Item
"""
assert item is not None
# Strings come back from SDB as unicode
def strOrNone(s):
return s if s is None else str(s)
# ownerID and encrypted are the only mandatory attributes
ownerID = strOrNone(item.get('ownerID'))
encrypted = item.get('encrypted')
if ownerID is None:
assert encrypted is None
return None
else:
version = strOrNone(item['version'])
encrypted = strict_bool(encrypted)
content, numContentChunks = cls.attributesToBinary(item)
if encrypted:
sseKeyPath = cls.outer.sseKeyPath
if sseKeyPath is None:
raise AssertionError('Content is encrypted but no key was provided.')
if content is not None:
content = encryption.decrypt(content, sseKeyPath)
self = cls(fileID=item.name, ownerID=ownerID, encrypted=encrypted, version=version,
content=content, numContentChunks=numContentChunks)
return self
def _downloadStream(self, jobStoreFileID, container):
# The reason this is not in the writer is so we catch non-existant blobs early
blobProps = container.get_blob_properties(blob_name=bytes(jobStoreFileID))
encrypted = strict_bool(blobProps['x-ms-meta-encrypted'])
if encrypted and self.keyPath is None:
raise AssertionError('Content is encrypted but no key was provided.')
outer_self = self
class DownloadPipe(ReadablePipe):
def writeTo(self, writable):
chunkStart = 0
fileSize = int(blobProps['Content-Length'])
while chunkStart < fileSize:
chunkEnd = chunkStart + outer_self._maxAzureBlockBytes - 1
buf = container.get_blob(blob_name=bytes(jobStoreFileID),
x_ms_range="bytes=%d-%d" % (chunkStart, chunkEnd))
if encrypted:
buf = encryption.decrypt(buf, outer_self.keyPath)
writable.write(buf)
chunkStart = chunkEnd + 1
with DownloadPipe() as readable:
yield readable
def fromItem(cls, item):
"""
Convert an SDB item to an instance of this class.
:type item: Item
"""
assert item is not None
# Strings come back from SDB as unicode
def strOrNone(s):
return s if s is None else str(s)
# ownerID and encrypted are the only mandatory attributes
ownerID = strOrNone(item.get('ownerID'))
encrypted = item.get('encrypted')
if ownerID is None:
assert encrypted is None
return None
else:
version = strOrNone(item['version'])
encrypted = strict_bool(encrypted)
content, numContentChunks = cls.attributesToBinary(item)
if encrypted:
sseKeyPath = cls.outer.sseKeyPath
if sseKeyPath is None:
raise AssertionError('Content is encrypted but no key was provided.')
if content is not None:
content = encryption.decrypt(content, sseKeyPath)
self = cls(fileID=item.name, ownerID=ownerID, encrypted=encrypted, version=version,
content=content, numContentChunks=numContentChunks)
return self
def __init__(self,
region,
namePrefix,
config=None,
partSize=defaultPartSize):
"""
Create a new job store in AWS or load an existing one from there.
:param region: the AWS region to create the job store in, e.g. 'us-west-2'
:param namePrefix: S3 bucket names and SDB tables will be prefixed with this
:param config: the config object to written to this job store. Must be None for existing
job stores. Must not be None for new job stores.
"""
log.debug("Instantiating %s for region %s and name prefix '%s'",
self.__class__, region, namePrefix)
self.region = region
self.namePrefix = namePrefix
self.jobsDomain = None
self.filesDomain = None
self.filesBucket = None
self.db = self._connectSimpleDB()
self.s3 = self._connectS3()
self.partSize = partSize
# Check global registry domain for existence of this job store. The first time this is
# being executed in an AWS account, the registry domain will be created on the fly.
create = config is not None
self.registry_domain = self._getOrCreateDomain('toil-registry')
for attempt in retry_sdb():
with attempt:
attributes = self.registry_domain.get_attributes(
item_name=namePrefix,
attribute_name='exists',
consistent_read=True)
exists = strict_bool(attributes.get('exists', str(False)))
self._checkJobStoreCreation(create, exists,
region + ":" + namePrefix)
def qualify(name):
assert len(name) <= self.maxNameLen
return self.namePrefix + self.nameSeparator + name
self.jobsDomain = self._getOrCreateDomain(qualify('jobs'))
self.filesDomain = self._getOrCreateDomain(qualify('files'))
self.filesBucket = self._getOrCreateBucket(qualify('files'),
versioning=True)
# Now register this job store
for attempt in retry_sdb():
with attempt:
self.registry_domain.put_attributes(
item_name=namePrefix, attributes=dict(exists='True'))
super(AWSJobStore, self).__init__(config=config)
self.sseKeyPath = self.config.sseKey
def __init__(self, region, namePrefix, config=None, partSize=defaultPartSize):
"""
Create a new job store in AWS or load an existing one from there.
:param region: the AWS region to create the job store in, e.g. 'us-west-2'
:param namePrefix: S3 bucket names and SDB tables will be prefixed with this
:param config: the config object to written to this job store. Must be None for existing
job stores. Must not be None for new job stores.
"""
log.debug("Instantiating %s for region %s and name prefix '%s'",
self.__class__, region, namePrefix)
self.region = region
self.namePrefix = namePrefix
self.jobsDomain = None
self.filesDomain = None
self.filesBucket = None
self.db = self._connectSimpleDB()
self.s3 = self._connectS3()
self.partSize = partSize
# Check global registry domain for existence of this job store. The first time this is
# being executed in an AWS account, the registry domain will be created on the fly.
create = config is not None
self.registry_domain = self._getOrCreateDomain('toil-registry')
for attempt in retry_sdb():
with attempt:
attributes = self.registry_domain.get_attributes(item_name=namePrefix,
attribute_name='exists',
consistent_read=True)
exists = strict_bool(attributes.get('exists', str(False)))
self._checkJobStoreCreation(create, exists, region + ":" + namePrefix)
def qualify(name):
assert len(name) <= self.maxNameLen
return self.namePrefix + self.nameSeparator + name
self.jobsDomain = self._getOrCreateDomain(qualify('jobs'))
self.filesDomain = self._getOrCreateDomain(qualify('files'))
self.filesBucket = self._getOrCreateBucket(qualify('files'), versioning=True)
# Now register this job store
for attempt in retry_sdb():
with attempt:
self.registry_domain.put_attributes(item_name=namePrefix,
attributes=dict(exists='True'))
super(AWSJobStore, self).__init__(config=config)
self.sseKeyPath = self.config.sseKey
def _trackOfferedNodes(self, offers):
for offer in offers:
nodeAddress = socket.gethostbyname(offer.hostname)
self._registerNode(nodeAddress, offer.slave_id.value)
preemptable = False
for attribute in offer.attributes:
if attribute.name == 'preemptable':
preemptable = strict_bool(attribute.text.value)
if preemptable:
try:
self.nonPreemptableNodes.remove(offer.slave_id.value)
except KeyError:
pass
else:
self.nonPreemptableNodes.add(offer.slave_id.value)
def _trackOfferedNodes(self, offers):
for offer in offers:
nodeAddress = socket.gethostbyname(offer.hostname)
self._registerNode(nodeAddress, offer.slave_id.value)
preemptable = False
for attribute in offer.attributes:
if attribute.name == 'preemptable':
preemptable = strict_bool(attribute.text.value)
if preemptable:
try:
self.nonPreemptableNodes.remove(offer.slave_id.value)
except KeyError:
pass
else:
self.nonPreemptableNodes.add(offer.slave_id.value)
def _parseOffer(self, offer):
cores = 0
memory = 0
disk = 0
preemptable = None
for attribute in offer.attributes:
if attribute.name == 'preemptable':
assert preemptable is None, "Attribute 'preemptable' occurs more than once."
preemptable = strict_bool(attribute.text.value)
if preemptable is None:
log.debug('Slave not marked as either preemptable or not. Assuming non-preemptable.')
preemptable = False
for resource in offer.resources:
if resource.name == "cpus":
cores += resource.scalar.value
elif resource.name == "mem":
memory += resource.scalar.value
elif resource.name == "disk":
disk += resource.scalar.value
return cores, memory, disk, preemptable
def _parseOffer(self, offer):
cores = 0
memory = 0
disk = 0
preemptable = None
for attribute in offer.attributes:
if attribute.name == 'preemptable':
assert preemptable is None, "Attribute 'preemptable' occurs more than once."
preemptable = strict_bool(attribute.text.value)
if preemptable is None:
log.debug('Slave not marked as either preemptable or not. Assuming non-preemptable.')
preemptable = False
for resource in offer.resources:
if resource.name == "cpus":
cores += resource.scalar.value
elif resource.name == "mem":
memory += resource.scalar.value
elif resource.name == "disk":
disk += resource.scalar.value
return cores, memory, disk, preemptable
def _downloadStream(self, jobStoreFileID, container):
# The reason this is not in the writer is so we catch non-existant blobs early
blobProps = container.get_blob_properties(blob_name=jobStoreFileID)
encrypted = strict_bool(blobProps['x-ms-meta-encrypted'])
if encrypted and self.keyPath is None:
raise AssertionError(
'Content is encrypted but no key was provided.')
readable_fh, writable_fh = os.pipe()
with os.fdopen(readable_fh, 'r') as readable:
with os.fdopen(writable_fh, 'w') as writable:
def writer():
try:
chunkStartPos = 0
fileSize = int(blobProps['Content-Length'])
while chunkStartPos < fileSize:
chunkEndPos = chunkStartPos + self._maxAzureBlockBytes - 1
buf = container.get_blob(
blob_name=jobStoreFileID,
x_ms_range="bytes=%d-%d" %
(chunkStartPos, chunkEndPos))
if encrypted:
buf = encryption.decrypt(buf, self.keyPath)
writable.write(buf)
chunkStartPos = chunkEndPos + 1
finally:
# Ensure readers aren't left blocking if this thread crashes.
# This close() will send EOF to the reading end and ultimately cause the
# yield to return. It also makes the implict .close() done by the enclosing
# "with" context redundant but that should be ok since .close() on file
# objects are idempotent.
writable.close()
thread = ExceptionalThread(target=writer)
thread.start()
yield readable
thread.join()
def _downloadStream(self, jobStoreFileID, container):
# The reason this is not in the writer is so we catch non-existant blobs early
blobProps = container.get_blob_properties(blob_name=jobStoreFileID)
encrypted = strict_bool(blobProps['x-ms-meta-encrypted'])
if encrypted and self.keyPath is None:
raise AssertionError('Content is encrypted but no key was provided.')
readable_fh, writable_fh = os.pipe()
with os.fdopen(readable_fh, 'r') as readable:
with os.fdopen(writable_fh, 'w') as writable:
def writer():
try:
chunkStartPos = 0
fileSize = int(blobProps['Content-Length'])
while chunkStartPos < fileSize:
chunkEndPos = chunkStartPos + self._maxAzureBlockBytes - 1
buf = container.get_blob(blob_name=jobStoreFileID,
x_ms_range="bytes=%d-%d" % (chunkStartPos,
chunkEndPos))
if encrypted:
buf = encryption.decrypt(buf, self.keyPath)
writable.write(buf)
chunkStartPos = chunkEndPos + 1
finally:
# Ensure readers aren't left blocking if this thread crashes.
# This close() will send EOF to the reading end and ultimately cause the
# yield to return. It also makes the implict .close() done by the enclosing
# "with" context redundant but that should be ok since .close() on file
# objects are idempotent.
writable.close()
thread = ExceptionalThread(target=writer)
thread.start()
yield readable
thread.join()
def fromCommand(cls, command):
assert len(command) == 3
return cls(dirPath=command[0], name=command[1], fromVirtualEnv=strict_bool(command[2]))
def _set_instance_options( self, options ):
super( AgentBox, self )._set_instance_options( options )
self._enable_agent = strict_bool( options.get( 'enable_agent', 'True' ) )
