def openS3(keySize=None):
"""
Creates an AWS bucket. If keySize is given, a key of random bytes is created and its handle
is yielded. If no keySize is given an empty bucket handle is yielded. The bucket and all
created keys are cleaned up automatically.
:param int keySize: Size of key to be created.
"""
if keySize is not None and keySize < 0:
raise ValueError('Key size must be greater than zero')
with closing(boto.s3.connect_to_region(AWSMultipartCopyTest.region)) as s3:
bucket = s3.create_bucket('multipart-transfer-test-%s' % uuid.uuid4(),
location=region_to_bucket_location(
AWSMultipartCopyTest.region))
try:
keyName = 'test'
if keySize is None:
yield bucket
else:
key = bucket.new_key(keyName)
content = os.urandom(keySize)
key.set_contents_from_string(content)
yield bucket.get_key(keyName)
finally:
for key in bucket.list():
key.delete()
bucket.delete()
def openS3(keySize=None):
"""
Creates an AWS bucket. If keySize is given, a key of random bytes is created and its handle
is yielded. If no keySize is given an empty bucket handle is yielded. The bucket and all
created keys are cleaned up automatically.
:param int keySize: Size of key to be created.
"""
if keySize is not None and keySize < 0:
raise ValueError('Key size must be greater than zero')
with closing(boto.s3.connect_to_region(AWSMultipartCopyTest.region)) as s3:
bucket = s3.create_bucket('multipart-transfer-test-%s' % uuid.uuid4(),
location=region_to_bucket_location(AWSMultipartCopyTest.region))
try:
keyName = 'test'
if keySize is None:
yield bucket
else:
key = bucket.new_key(keyName)
content = os.urandom(keySize)
key.set_contents_from_string(content)
yield bucket.get_key(keyName)
finally:
for key in bucket.list():
key.delete()
bucket.delete()
def _createExternalStore(self):
import boto.s3
s3 = boto.s3.connect_to_region(self.awsRegion())
try:
return s3.create_bucket(bucket_name='import-export-test-%s' % uuid.uuid4(),
location=region_to_bucket_location(self.awsRegion()))
except:
with panic(log=logger):
s3.close()
def _createExternalStore(self):
import boto.s3
s3 = boto.s3.connect_to_region(self.testRegion)
try:
return s3.create_bucket(bucket_name='import-export-test-%s' % uuid.uuid4(),
location=region_to_bucket_location(self.testRegion))
except:
with panic(log=logger):
s3.close()
def _getOrCreateBucket(self, bucket_name, versioning=False):
"""
:rtype: Bucket
"""
assert self.minBucketNameLen <= len(bucket_name) <= self.maxBucketNameLen
assert self.bucketNameRe.match(bucket_name)
log.info("Setting up job store bucket '%s'.", bucket_name)
while True:
log.debug("Looking up job store bucket '%s'.", bucket_name)
try:
bucket = self.s3.get_bucket(bucket_name, validate=True)
assert self.__getBucketRegion(bucket) == self.region
assert versioning is self.__getBucketVersioning(bucket)
log.debug("Using existing job store bucket '%s'.", bucket_name)
return bucket
except S3ResponseError as e:
if e.error_code == 'NoSuchBucket':
log.debug("Bucket '%s' does not exist. Creating it.", bucket_name)
try:
location = region_to_bucket_location(self.region)
bucket = self.s3.create_bucket(bucket_name, location=location)
except S3CreateError as e:
if e.error_code == 'BucketAlreadyOwnedByYou':
# https://github.com/BD2KGenomics/toil/issues/955
log.warn('Got %s, retrying.', e)
# and loop
else:
raise
else:
assert self.__getBucketRegion(bucket) == self.region
if versioning:
bucket.configure_versioning(versioning)
log.debug("Created new job store bucket '%s'.", bucket_name)
return bucket
else:
raise
