def create_bucket_and_key(bucket_name=BUCKET_NAME, key_name=KEY_NAME, contents=None, num_attempts=12, sleep_time=5): # fake (or not) connection, bucket and key logger.debug('%r', locals()) s3 = boto3.resource('s3') bucket_exist = cleanup_bucket(s3) if not bucket_exist: mybucket = s3.create_bucket(Bucket=bucket_name) # # In real life, it can take a few seconds for the bucket to become ready. # If we try to write to the key while the bucket while it isn't ready, we # will get a ClientError: NoSuchBucket. # for attempt in range(num_attempts): try: mybucket = s3.Bucket(bucket_name) mykey = s3.Object(bucket_name, key_name) if contents is not None: mykey.put(Body=contents) return mybucket, mykey except botocore.exceptions.ClientError as err: logger.error('caught %r, retrying', err) time.sleep(sleep_time) assert False, 'failed to create bucket after %d attempts' % num_attempts
def tearDownModule(): '''Called once by unittest when tearing down this module. Empties and removes the test S3 bucket. ''' s3 = boto3.resource('s3') try: cleanup_bucket() s3.Bucket(BUCKET_NAME).delete() except s3.meta.client.exceptions.NoSuchBucket: pass
def populate_bucket(bucket_name=BUCKET_NAME, num_keys=10): # fake (or not) connection, bucket and key logger.debug('%r', locals()) s3 = boto3.resource('s3') bucket_exist = cleanup_bucket(s3) if not bucket_exist: mybucket = s3.create_bucket(Bucket=bucket_name) mybucket = s3.Bucket(bucket_name) for key_number in range(num_keys): key_name = 'key_%d' % key_number s3.Object(bucket_name, key_name).put(Body=str(key_number))
def create_bucket_and_key(bucket_name=BUCKET_NAME, key_name=KEY_NAME, contents=None): # fake (or not) connection, bucket and key logger.debug('%r', locals()) s3 = boto3.resource('s3') bucket_exist = cleanup_bucket(s3) if not bucket_exist: mybucket = s3.create_bucket(Bucket=bucket_name) mybucket = s3.Bucket(bucket_name) mykey = s3.Object(bucket_name, key_name) if contents is not None: mykey.put(Body=contents) return mybucket, mykey
def tearDownModule(): '''Called once by unittest when tearing down this module. Empties and removes the test S3 bucket. ''' s3 = boto3.resource('s3') bucket = s3.Bucket(BUCKET_NAME) try: cleanup_bucket() bucket.delete() except s3.meta.client.exceptions.NoSuchBucket: pass try: bucket.wait_until_not_exists() except Exception: # # This is bad, but not fatal, and should not cause the whole test run # to explode. Either the bucket will get deleted by AWS eventually, # or we can clean it up later ourselves. # pass
def test_accepts_boto3_bucket(self): populate_bucket() s3 = boto3.resource('s3') bucket = s3.Bucket(BUCKET_NAME) results = list(smart_open.s3.iter_bucket(bucket)) self.assertEqual(len(results), 10)