def try_and_nested_panic_with_secondary(self):
try:
self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
raise ValueError("primary")
except:
with panic(log):
with panic(log):
raise RuntimeError("secondary")
def try_and_panic(self):
try:
self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
raise ValueError("primary")
except:
with panic(log):
pass
def chunkedFileUpload(readable, bucket, fileID, file_size, headers=None, partSize=50 << 20):
for attempt in retry_s3():
with attempt:
upload = bucket.initiate_multipart_upload(
key_name=compat_bytes(fileID),
headers=headers)
try:
start = 0
part_num = itertools.count()
while start < file_size:
end = min(start + partSize, file_size)
assert readable.tell() == start
for attempt in retry_s3():
with attempt:
upload.upload_part_from_file(fp=readable,
part_num=next(part_num) + 1,
size=end - start,
headers=headers)
start = end
assert readable.tell() == file_size == start
except:
with panic(log=log):
for attempt in retry_s3():
with attempt:
upload.cancel_upload()
else:
for attempt in retry_s3():
with attempt:
version = upload.complete_upload().version_id
return version
def readFrom(self, readable):
blocks = []
try:
while True:
buf = readable.read(maxBlockSize)
if len(buf) == 0:
# We're safe to break here even if we never read anything, since
# putting an empty block list creates an empty blob.
break
if encrypted:
buf = encryption.encrypt(buf, store.keyPath)
blockID = store._newFileID()
container.put_block(blob_name=bytes(jobStoreFileID),
block=buf,
block_id=blockID)
blocks.append(BlobBlock(blockID))
except:
with panic(log=logger):
# This is guaranteed to delete any uncommitted blocks.
container.delete_blob(blob_name=bytes(jobStoreFileID))
if checkForModification and expectedVersion is not None:
# Acquire a (60-second) write lock,
leaseID = container.acquire_blob_lease(
blob_name=bytes(jobStoreFileID), lease_duration=60)
# check for modification,
blob = container.get_blob_properties(
blob_name=bytes(jobStoreFileID))
if blob.properties.etag != expectedVersion:
container.release_blob_lease(
blob_name=bytes(jobStoreFileID), lease_id=leaseID)
raise ConcurrentFileModificationException(
jobStoreFileID)
# commit the file,
container.put_block_list(
blob_name=bytes(jobStoreFileID),
block_list=blocks,
lease_id=leaseID,
metadata=dict(encrypted=str(encrypted)))
# then release the lock.
container.release_blob_lease(
blob_name=bytes(jobStoreFileID), lease_id=leaseID)
else:
# No need to check for modification, just blindly write over whatever
# was there.
container.put_block_list(
blob_name=bytes(jobStoreFileID),
block_list=blocks,
metadata=dict(encrypted=str(encrypted)))
def wait_spot_requests_active(
ec2,
requests: Iterable[SpotInstanceRequest],
timeout: float = None,
tentative: bool = False) -> Iterable[List[SpotInstanceRequest]]:
"""
Wait until no spot request in the given iterator is in the 'open' state or, optionally,
a timeout occurs. Yield spot requests as soon as they leave the 'open' state.
:param requests: The requests to wait on.
:param timeout: Maximum time in seconds to spend waiting or None to wait forever. If a
timeout occurs, the remaining open requests will be cancelled.
:param tentative: if True, give up on a spot request at the earliest indication of it
not being fulfilled immediately
"""
if timeout is not None:
timeout = time.time() + timeout
active_ids = set()
other_ids = set()
open_ids = None
def cancel():
logger.warning('Cancelling remaining %i spot requests.', len(open_ids))
ec2.cancel_spot_instance_requests(list(open_ids))
def spot_request_not_found(e):
return get_error_code(e) == 'InvalidSpotInstanceRequestID.NotFound'
try:
while True:
open_ids, eval_ids, fulfill_ids = set(), set(), set()
batch = []
for r in requests:
if r.state == 'open':
open_ids.add(r.id)
if r.status.code == 'pending-evaluation':
eval_ids.add(r.id)
elif r.status.code == 'pending-fulfillment':
fulfill_ids.add(r.id)
else:
logger.info(
'Request %s entered status %s indicating that it will not be '
'fulfilled anytime soon.', r.id, r.status.code)
elif r.state == 'active':
assert r.id not in active_ids
active_ids.add(r.id)
batch.append(r)
else:
assert r.id not in other_ids
other_ids.add(r.id)
batch.append(r)
if batch:
yield batch
logger.info(
'%i spot requests(s) are open (%i of which are pending evaluation and %i '
'are pending fulfillment), %i are active and %i are in another state.',
*list(
map(len, (open_ids, eval_ids, fulfill_ids, active_ids,
other_ids))))
if not open_ids or tentative and not eval_ids and not fulfill_ids:
break
sleep_time = 2 * a_short_time
if timeout is not None and time.time() + sleep_time >= timeout:
logger.warning('Timed out waiting for spot requests.')
break
logger.info('Sleeping for %is', sleep_time)
time.sleep(sleep_time)
for attempt in retry_ec2(retry_while=spot_request_not_found):
with attempt:
requests = ec2.get_all_spot_instance_requests(
list(open_ids))
except BaseException:
if open_ids:
with panic(logger):
cancel()
raise
else:
if open_ids:
cancel()
