def test_single_object_operations(): file_name = "test_file" file_content = "test_content" bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE) assert bucket.create() assert bucket.put_object(file_name, file_content) assert not bucket.clean() assert not bucket.empty() file_content_s3 = bucket.get_object(file_name) if isinstance(file_content_s3, bytes): assert file_content == file_content_s3.decode("utf-8") else: assert file_content == file_content_s3 file_name_cpy = "test_file_copy" bucket.copy_object(file_name, file_name_cpy) file_content_s3_cpy = bucket.get_object(file_name_cpy) assert file_content_s3 == file_content_s3_cpy assert bucket.delete_object(file_name) assert bucket.get_object(file_name) is None assert bucket.delete_object(file_name_cpy) assert bucket.get_object(file_name_cpy) is None assert bucket.clean() assert bucket.empty()
def cluster_bucket(): bucket = AXS3Bucket(bucket_name="applatix-cluster-{}-0".format( AXCustomerId().get_customer_id()), region=TEST_AWS_REGION, aws_profile=TEST_AWS_PROFILE) bucket.create() yield bucket bucket.delete(force=True)
def test_cors_operation(): bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE) # Do the operations twice to ensure idempotency bucket.put_cors(TEST_CORS_CONFIG) bucket.put_cors(TEST_CORS_CONFIG) bucket.delete_cors() bucket.delete_cors()
def test_bucket_create(): bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE, region=TEST_AWS_REGION) assert bucket.create() assert bucket.exists() assert bucket.empty() assert bucket.clean() # Recreate should return True assert bucket.create()
def test_bucket_delete(): bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE) assert bucket.create() assert bucket.delete(force=True) assert bucket.clean() assert bucket.empty() # Re-delete should return True assert bucket.delete(force=True)
def test_bucket_clean(): file_name = BUCKET_CLEAN_KEYWORD + "{:05d}".format(random.randint(1, 99999)) file_content = "test_content" bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE) assert bucket.create() assert bucket.put_object(file_name, file_content) assert bucket.clean() if not bucket.delete_object(file_name): pytest.fail("Failed to delete object {}".format(file_name))
def test_generate_object_url(): file_name = "test_file_url" file_content = "test_content_url" bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE) assert bucket.put_object(file_name, file_content, ACL="public-read") url = bucket.get_object_url_from_key(key=file_name) data = requests.get(url, timeout=5).text assert data == file_content assert bucket.delete_object(file_name)
def test_list_objects(): file_name_prefix = "test_file-" file_content = "test_content" file_name_set = set() bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE) assert bucket.create() for i in range(50): file_name = file_name_prefix + "{:03d}".format(i) file_name_set.add(file_name) assert bucket.put_object(key=file_name, data=file_content) file_name_set_s3 = set(bucket.list_objects(keyword=file_name_prefix)) assert file_name_set_s3 == file_name_set
def test_s3_get_region(): for region in AWS_REGIONS: bucket_name = TEST_BUCKET_NAME_TEMPLATE.format(region=region) logger.info("Testing GetRegion for bucket %s", bucket_name) bucket = AXS3Bucket(bucket_name=bucket_name, aws_profile=TEST_AWS_PROFILE, region=region) bucket.create() assert bucket._get_bucket_region_from_aws() == region # Need to cool down a bit as bucket creation / deletion is very heavy weighted bucket.delete() time.sleep(5)
def test_delete_all_without_prefix(): bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE) assert bucket.create() file_name_prefix = "test_file-" file_content = "test_content" for i in range(50): file_name = file_name_prefix + "{:03d}".format(i) assert bucket.put_object(key=file_name, data=file_content) bucket.delete_all(use_prefix=False) assert bucket.clean() assert bucket.empty()
def _upload(s, d, meta_data): # To be consistent with file uploaded from container_outer_executor # - StorageClass is by default "STANDARD", and # - ContentLength is used only when file size cannot be automatically # determined, but as we upload .gz files after IN_CLOSE_WRITE, we # don't need to set it extra_args = { "Metadata": meta_data, "ContentDisposition": "attachment; filename={}".format(full_name) } if AXS3Bucket.supports_encryption(): extra_args["ACL"] = "bucket-owner-full-control" extra_args["ServerSideEncryption"] = "AES256" logger.info("about to upload log %s to %s (%s) to s3", s, d, artifact_uuid) if not bucket.put_file(local_file_name=s, s3_key=d, ExtraArgs=extra_args): raise AXPlatformException("Failed to put object {} to s3 {}".format(s, d)) logger.debug("upload %s done", artifact_uuid)
def get_bucket(self, bucket_name, **kwargs): """ Return a bucket object that contains methods to operate on a bucket of a cloud that current running code operates on (target cloud). :param bucket_name: :param kwargs: :return: """ if self.target_cloud_gcp(): from ax.cloud.gke.gcs import AXGCSBucket return AXGCSBucket(bucket_name) elif self.target_cloud_aws(): from ax.cloud.aws import AXS3Bucket aws_profile = kwargs.get('aws_profile', None) region = kwargs.get("region", None) return AXS3Bucket(bucket_name, aws_profile, region=region) else: assert 0, "Unsupported cloud provider {}".format(self._target_cloud)
def test_delete_all_with_prefix_and_exemption(): file_name_prefix = "test_file-" file_content = "test_content" bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE) assert bucket.create() for i in range(50): file_name = file_name_prefix + "{:03d}".format(i) assert bucket.put_object(key=file_name, data=file_content) bucket.put_object(key="special_file", data=file_content) bucket.delete_all(obj_prefix=file_name_prefix, exempt=["test_file-015"]) remaining_file_s3 = set(bucket.list_objects(list_all=True)) remaining_file = {"special_file", "test_file-015"} assert remaining_file_s3 == remaining_file bucket.delete_all(use_prefix=False) assert bucket.clean() assert bucket.empty()
def test_put_policy_invalid_content(): bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE) assert bucket.create() assert not bucket.put_policy(policy=TEST_INVALID_POLICY_CONTENT)
def test_delete_nonexist_bucket(): bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE, region=TEST_AWS_REGION) assert bucket.delete(force=True)
def test_put_policy_no_bucket(): bucket = AXS3Bucket(bucket_name=TEST_BUCKET_NAME, aws_profile=TEST_AWS_PROFILE, region=TEST_AWS_REGION) assert not bucket.put_policy(policy="")