def test_create_ingest_policy_volumetric(self, boss_util_fixtures):
self._setup(boss_util_fixtures)
policy = BossUtil.generate_ingest_policy(
self.job_id,
self.upload_queue,
self.tile_index_queue,
self.tile_bucket.bucket.name,
ingest_type=VOLUMETRIC_INGEST,
)
from ndingest.ndbucket.tilebucket import TileBucket
try:
assert settings.IAM_POLICY_PATH == policy.path
assert policy.default_version is not None
statements = policy.default_version.document["Statement"]
assert 2 == len(statements)
for stmt in statements:
if stmt["Sid"] == "ClientUploadQueuePolicy":
for perm in [
"sqs:ReceiveMessage",
"sqs:GetQueueAttributes",
"sqs:DeleteMessage",
]:
assert perm in stmt["Action"]
assert 3 == len(stmt["Action"])
assert self.upload_queue.arn == stmt["Resource"]
elif stmt["Sid"] == "ClientTileBucketPolicy":
assert "s3:PutObject" in stmt["Action"]
assert len(stmt["Action"]) == 1
assert (TileBucket.buildArn(
self.tile_bucket.bucket.name) == stmt["Resource"])
finally:
policy.delete()
def test_buildArn_no_folder():
"""Test buildArn with folder's default value."""
from ndingest.ndbucket.tilebucket import TileBucket
expected = "arn:aws:s3:::my_bucket/*"
actual = TileBucket.buildArn("my_bucket")
assert expected == actual
def test_buildArn_with_folder_no_slashes():
"""Test buildArn with a folder."""
from ndingest.ndbucket.tilebucket import TileBucket
expected = "arn:aws:s3:::my_bucket/some/folder/*"
actual = TileBucket.buildArn("my_bucket", "some/folder")
assert expected == actual
def test_buildArn_with_folder_with_slashes():
"""Test buildArn with folder with slashes at beginning and end."""
# Import here so S3 is properly mocked.
from ndingest.ndbucket.tilebucket import TileBucket
expected = "arn:aws:s3:::my_bucket/some/folder/*"
actual = TileBucket.buildArn("my_bucket", "/some/folder/")
assert expected == actual
def generate_ingest_policy(job_id,
upload_queue,
tile_bucket,
region_name=settings.REGION_NAME,
endpoint_url=None,
description=''):
"""Generate the combined IAM policy.
Policy allows receiving messages from the queue and writing to the tile bucket.
Args:
job_id (int): Id of ingest job.
upload_queue (UploadQueue):
tile_bucket (TileBucket):
region_name (optional[string]): AWS region.
endpoint_url (string|None): Alternative URL boto3 should use for testing instead of connecting to AWS.
Returns:
(iam.Policy)
"""
iam = boto3.resource(
'iam',
region_name=region_name,
endpoint_url=endpoint_url,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
if not settings.TEST_MODE:
policy_name = INGEST_POLICY_NAME.format(settings.DOMAIN, job_id)
else:
if BossUtil.test_policy_id == -1:
BossUtil.test_policy_id = random.randint(0, 999)
policy_name = TEST_INGEST_POLICY_NAME.format(
settings.DOMAIN, BossUtil.test_policy_id, job_id)
policy = {
"Version":
"2012-10-17",
"Id":
policy_name,
"Statement": [{
"Sid":
"ClientQueuePolicy",
"Effect":
"Allow",
"Action": ["sqs:ReceiveMessage", "sqs:GetQueueAttributes"],
"Resource":
upload_queue.arn
}, {
"Sid":
"ClientTileBucketPolicy",
"Effect":
"Allow",
"Action": ["s3:PutObject"],
"Resource":
TileBucket.buildArn(tile_bucket.bucket.name)
}]
}
return iam.create_policy(PolicyName=policy['Id'],
PolicyDocument=json.dumps(policy),
Path=settings.IAM_POLICY_PATH,
Description=description)
def test_buildArn_with_folder_with_slashes(self):
"""Test buildArn with folder with slashes at beginning and end."""
expected = 'arn:aws:s3:::my_bucket/some/folder/*'
actual = TileBucket.buildArn('my_bucket', '/some/folder/')
assert(expected == actual)
def test_buildArn_with_folder_no_slashes(self):
"""Test buildArn with a folder."""
expected = 'arn:aws:s3:::my_bucket/some/folder/*'
actual = TileBucket.buildArn('my_bucket', 'some/folder')
assert(expected == actual)
def test_buildArn_no_folder(self):
"""Test buildArn with folder's default value."""
expected = 'arn:aws:s3:::my_bucket/*'
actual = TileBucket.buildArn('my_bucket')
assert(expected == actual)
def generate_ingest_policy(
job_id,
upload_queue,
tile_index_queue,
bucket_name,
region_name=settings.REGION_NAME,
endpoint_url=None,
description="",
ingest_type=TILE_INGEST,
):
"""Generate the combined IAM policy.
Policy allows receiving messages from the queue and writing to the tile bucket.
Args:
job_id (int): Id of ingest job.
upload_queue (UploadQueue):
tile_index_queue (TileIndexQueue|None):
bucket_name (str): Name of bucket ingest client will upload to.
region_name (optional[str]): AWS region.
endpoint_url (optional[str|None]): Alternative URL boto3 should use for testing instead of connecting to AWS.
description (optional[str]): Policy description.
ingest_type (optional[int]): TILE_INGEST (default) | VOLUMETRIC_INGEST.
Returns:
(iam.Policy)
Raises:
(ValueError): if ingest_type invalid.
"""
iam = boto3.resource(
"iam",
region_name=region_name,
endpoint_url=endpoint_url,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
)
if not settings.TEST_MODE:
policy_name = INGEST_POLICY_NAME.format(settings.DOMAIN, job_id)
else:
if BossUtil.test_policy_id == -1:
BossUtil.test_policy_id = random.randint(0, 999)
policy_name = TEST_INGEST_POLICY_NAME.format(
settings.DOMAIN, BossUtil.test_policy_id, job_id)
sqs_upload_actions = [
"sqs:DeleteMessage",
"sqs:ReceiveMessage",
"sqs:GetQueueAttributes",
]
policy = {
"Version":
"2012-10-17",
"Id":
policy_name,
"Statement": [
{
"Sid": "ClientUploadQueuePolicy",
"Effect": "Allow",
"Action": sqs_upload_actions,
"Resource": upload_queue.arn,
},
{
"Sid": "ClientTileBucketPolicy",
"Effect": "Allow",
"Action": ["s3:PutObject"],
"Resource": TileBucket.buildArn(bucket_name),
},
],
}
if ingest_type == TILE_INGEST:
sqs_index_actions = ["sqs:SendMessage"]
policy["Statement"].append({
"Sid": "ClientIndexQueuePolicy",
"Effect": "Allow",
"Action": sqs_index_actions,
"Resource": tile_index_queue.arn,
})
elif ingest_type == VOLUMETRIC_INGEST:
pass
else:
raise ValueError(
"Got unknown ingest_type value: {}".format(ingest_type))
return iam.create_policy(
PolicyName=policy["Id"],
PolicyDocument=json.dumps(policy),
Path=settings.IAM_POLICY_PATH,
Description=description,
)