def test_create_ingest_policy_volumetric(self, boss_util_fixtures): self._setup(boss_util_fixtures) policy = BossUtil.generate_ingest_policy( self.job_id, self.upload_queue, self.tile_index_queue, self.tile_bucket.bucket.name, ingest_type=VOLUMETRIC_INGEST, ) from ndingest.ndbucket.tilebucket import TileBucket try: assert settings.IAM_POLICY_PATH == policy.path assert policy.default_version is not None statements = policy.default_version.document["Statement"] assert 2 == len(statements) for stmt in statements: if stmt["Sid"] == "ClientUploadQueuePolicy": for perm in [ "sqs:ReceiveMessage", "sqs:GetQueueAttributes", "sqs:DeleteMessage", ]: assert perm in stmt["Action"] assert 3 == len(stmt["Action"]) assert self.upload_queue.arn == stmt["Resource"] elif stmt["Sid"] == "ClientTileBucketPolicy": assert "s3:PutObject" in stmt["Action"] assert len(stmt["Action"]) == 1 assert (TileBucket.buildArn( self.tile_bucket.bucket.name) == stmt["Resource"]) finally: policy.delete()
def test_buildArn_no_folder(): """Test buildArn with folder's default value.""" from ndingest.ndbucket.tilebucket import TileBucket expected = "arn:aws:s3:::my_bucket/*" actual = TileBucket.buildArn("my_bucket") assert expected == actual
def test_buildArn_with_folder_no_slashes(): """Test buildArn with a folder.""" from ndingest.ndbucket.tilebucket import TileBucket expected = "arn:aws:s3:::my_bucket/some/folder/*" actual = TileBucket.buildArn("my_bucket", "some/folder") assert expected == actual
def test_buildArn_with_folder_with_slashes(): """Test buildArn with folder with slashes at beginning and end.""" # Import here so S3 is properly mocked. from ndingest.ndbucket.tilebucket import TileBucket expected = "arn:aws:s3:::my_bucket/some/folder/*" actual = TileBucket.buildArn("my_bucket", "/some/folder/") assert expected == actual
def generate_ingest_policy(job_id, upload_queue, tile_bucket, region_name=settings.REGION_NAME, endpoint_url=None, description=''): """Generate the combined IAM policy. Policy allows receiving messages from the queue and writing to the tile bucket. Args: job_id (int): Id of ingest job. upload_queue (UploadQueue): tile_bucket (TileBucket): region_name (optional[string]): AWS region. endpoint_url (string|None): Alternative URL boto3 should use for testing instead of connecting to AWS. Returns: (iam.Policy) """ iam = boto3.resource( 'iam', region_name=region_name, endpoint_url=endpoint_url, aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY) if not settings.TEST_MODE: policy_name = INGEST_POLICY_NAME.format(settings.DOMAIN, job_id) else: if BossUtil.test_policy_id == -1: BossUtil.test_policy_id = random.randint(0, 999) policy_name = TEST_INGEST_POLICY_NAME.format( settings.DOMAIN, BossUtil.test_policy_id, job_id) policy = { "Version": "2012-10-17", "Id": policy_name, "Statement": [{ "Sid": "ClientQueuePolicy", "Effect": "Allow", "Action": ["sqs:ReceiveMessage", "sqs:GetQueueAttributes"], "Resource": upload_queue.arn }, { "Sid": "ClientTileBucketPolicy", "Effect": "Allow", "Action": ["s3:PutObject"], "Resource": TileBucket.buildArn(tile_bucket.bucket.name) }] } return iam.create_policy(PolicyName=policy['Id'], PolicyDocument=json.dumps(policy), Path=settings.IAM_POLICY_PATH, Description=description)
def test_buildArn_with_folder_with_slashes(self): """Test buildArn with folder with slashes at beginning and end.""" expected = 'arn:aws:s3:::my_bucket/some/folder/*' actual = TileBucket.buildArn('my_bucket', '/some/folder/') assert(expected == actual)
def test_buildArn_with_folder_no_slashes(self): """Test buildArn with a folder.""" expected = 'arn:aws:s3:::my_bucket/some/folder/*' actual = TileBucket.buildArn('my_bucket', 'some/folder') assert(expected == actual)
def test_buildArn_no_folder(self): """Test buildArn with folder's default value.""" expected = 'arn:aws:s3:::my_bucket/*' actual = TileBucket.buildArn('my_bucket') assert(expected == actual)
def generate_ingest_policy( job_id, upload_queue, tile_index_queue, bucket_name, region_name=settings.REGION_NAME, endpoint_url=None, description="", ingest_type=TILE_INGEST, ): """Generate the combined IAM policy. Policy allows receiving messages from the queue and writing to the tile bucket. Args: job_id (int): Id of ingest job. upload_queue (UploadQueue): tile_index_queue (TileIndexQueue|None): bucket_name (str): Name of bucket ingest client will upload to. region_name (optional[str]): AWS region. endpoint_url (optional[str|None]): Alternative URL boto3 should use for testing instead of connecting to AWS. description (optional[str]): Policy description. ingest_type (optional[int]): TILE_INGEST (default) | VOLUMETRIC_INGEST. Returns: (iam.Policy) Raises: (ValueError): if ingest_type invalid. """ iam = boto3.resource( "iam", region_name=region_name, endpoint_url=endpoint_url, aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, ) if not settings.TEST_MODE: policy_name = INGEST_POLICY_NAME.format(settings.DOMAIN, job_id) else: if BossUtil.test_policy_id == -1: BossUtil.test_policy_id = random.randint(0, 999) policy_name = TEST_INGEST_POLICY_NAME.format( settings.DOMAIN, BossUtil.test_policy_id, job_id) sqs_upload_actions = [ "sqs:DeleteMessage", "sqs:ReceiveMessage", "sqs:GetQueueAttributes", ] policy = { "Version": "2012-10-17", "Id": policy_name, "Statement": [ { "Sid": "ClientUploadQueuePolicy", "Effect": "Allow", "Action": sqs_upload_actions, "Resource": upload_queue.arn, }, { "Sid": "ClientTileBucketPolicy", "Effect": "Allow", "Action": ["s3:PutObject"], "Resource": TileBucket.buildArn(bucket_name), }, ], } if ingest_type == TILE_INGEST: sqs_index_actions = ["sqs:SendMessage"] policy["Statement"].append({ "Sid": "ClientIndexQueuePolicy", "Effect": "Allow", "Action": sqs_index_actions, "Resource": tile_index_queue.arn, }) elif ingest_type == VOLUMETRIC_INGEST: pass else: raise ValueError( "Got unknown ingest_type value: {}".format(ingest_type)) return iam.create_policy( PolicyName=policy["Id"], PolicyDocument=json.dumps(policy), Path=settings.IAM_POLICY_PATH, Description=description, )