def create_bucket_and_key(bucket_name=BUCKET_NAME,
key_name=KEY_NAME,
contents=None,
num_attempts=12,
sleep_time=5):
# fake (or not) connection, bucket and key
logger.debug('%r', locals())
s3 = boto3.resource('s3')
bucket_exist = cleanup_bucket(s3)
if not bucket_exist:
mybucket = s3.create_bucket(Bucket=bucket_name)
#
# In real life, it can take a few seconds for the bucket to become ready.
# If we try to write to the key while the bucket while it isn't ready, we
# will get a ClientError: NoSuchBucket.
#
for attempt in range(num_attempts):
try:
mybucket = s3.Bucket(bucket_name)
mykey = s3.Object(bucket_name, key_name)
if contents is not None:
mykey.put(Body=contents)
return mybucket, mykey
except botocore.exceptions.ClientError as err:
logger.error('caught %r, retrying', err)
time.sleep(sleep_time)
assert False, 'failed to create bucket after %d attempts' % num_attempts
def tearDownModule():
'''Called once by unittest when tearing down this module. Empties and
removes the test S3 bucket.
'''
s3 = boto3.resource('s3')
try:
cleanup_bucket()
s3.Bucket(BUCKET_NAME).delete()
except s3.meta.client.exceptions.NoSuchBucket:
pass
def populate_bucket(bucket_name=BUCKET_NAME, num_keys=10):
# fake (or not) connection, bucket and key
logger.debug('%r', locals())
s3 = boto3.resource('s3')
bucket_exist = cleanup_bucket(s3)
if not bucket_exist:
mybucket = s3.create_bucket(Bucket=bucket_name)
mybucket = s3.Bucket(bucket_name)
for key_number in range(num_keys):
key_name = 'key_%d' % key_number
s3.Object(bucket_name, key_name).put(Body=str(key_number))
def create_bucket_and_key(bucket_name=BUCKET_NAME,
key_name=KEY_NAME,
contents=None):
# fake (or not) connection, bucket and key
logger.debug('%r', locals())
s3 = boto3.resource('s3')
bucket_exist = cleanup_bucket(s3)
if not bucket_exist:
mybucket = s3.create_bucket(Bucket=bucket_name)
mybucket = s3.Bucket(bucket_name)
mykey = s3.Object(bucket_name, key_name)
if contents is not None:
mykey.put(Body=contents)
return mybucket, mykey
def tearDownModule():
'''Called once by unittest when tearing down this module. Empties and
removes the test S3 bucket.
'''
s3 = boto3.resource('s3')
bucket = s3.Bucket(BUCKET_NAME)
try:
cleanup_bucket()
bucket.delete()
except s3.meta.client.exceptions.NoSuchBucket:
pass
try:
bucket.wait_until_not_exists()
except Exception:
#
# This is bad, but not fatal, and should not cause the whole test run
# to explode. Either the bucket will get deleted by AWS eventually,
# or we can clean it up later ourselves.
#
pass
def test_accepts_boto3_bucket(self):
populate_bucket()
s3 = boto3.resource('s3')
bucket = s3.Bucket(BUCKET_NAME)
results = list(smart_open.s3.iter_bucket(bucket))
self.assertEqual(len(results), 10)
