def generate_aws_account(arn=None, aws_account_id=None, user=None, name=None):
"""
Generate an AwsAccount for testing.
Any optional arguments not provided will be randomly generated.
Args:
arn (str): Optional ARN.
aws_account_id (12-digit string): Optional AWS account ID.
user (User): Optional Django auth User to be this account's owner.
name (str): Optional name for this account.
Returns:
AwsAccount: The created AwsAccount.
"""
if arn is None:
arn = helper.generate_dummy_arn(account_id=aws_account_id)
if user is None:
user = helper.generate_test_user()
return AwsAccount.objects.create(
account_arn=arn,
aws_account_id=aws.AwsArn(arn).account_id,
user=user,
name=name,
)
def validate_account_arn(self, value):
"""Validate the input account_arn."""
if self.instance is not None and value != self.instance.account_arn:
raise serializers.ValidationError(
_('You cannot change this field.'))
try:
aws.AwsArn(value)
except InvalidArn:
raise serializers.ValidationError(_('Invalid ARN.'))
return value
def validate_account_arn(self, value):
"""Validate the input account_arn."""
if (
self.instance is not None
and value != self.instance.content_object.account_arn
):
raise ValidationError(_("You cannot update account_arn."))
try:
aws.AwsArn(value)
except InvalidArn:
raise ValidationError(_("Invalid ARN."))
return value
def configure_customer_aws_and_create_cloud_account(username, customer_arn,
authentication_id,
application_id, source_id):
"""
Configure the customer's AWS account and create our CloudAccount.
This function is decorated to retry if an unhandled `RuntimeError` is
raised, which is the exception we raise in `rewrap_aws_errors` if we
encounter an unexpected error from AWS. This means it should keep retrying
if AWS is misbehaving.
Args:
username (string): Username of the user that will own the new cloud account
customer_arn (str): customer's ARN
authentication_id (str): Platform Sources' Authentication object id
application_id (str): Platform Sources' Application object id
source_id (str): Platform Sources' Source object id
"""
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
error = error_codes.CG1000
error.log_internal_message(logger, {
"application_id": application_id,
"username": username
})
error.notify(username, application_id)
return
try:
customer_aws_account_id = aws.AwsArn(customer_arn).account_id
except InvalidArn:
error = error_codes.CG1004
error.log_internal_message(logger, {"application_id": application_id})
error.notify(username, application_id)
return
cloud_account_name = get_standard_cloud_account_name(
"aws", customer_aws_account_id)
try:
create_aws_cloud_account(
user,
customer_arn,
cloud_account_name,
authentication_id,
application_id,
source_id,
)
except ValidationError as e:
logger.info("Unable to create cloud account: error %s", e.detail)
def validate_redrive_policy(source_queue_name, redrive_policy):
"""
Validate the queue redrive policy has an accessible DLQ.
Args:
source_queue_name (str): the queue name that should have a DLQ
redrive_policy (dict): the redrive policy
Returns:
bool: True if policy appears to be valid, else False.
"""
logger.info(
'SQS queue "%(queue)s" already has a redrive policy: '
"%(policy)s",
{
"queue": source_queue_name,
"policy": redrive_policy
},
)
dlq_queue_arn = redrive_policy.get("deadLetterTargetArn")
if not dlq_queue_arn:
return False
try:
dlq_queue_name = aws.AwsArn(dlq_queue_arn).resource
except InvalidArn:
return False
try:
region = settings.SQS_DEFAULT_REGION
sqs = boto3.client("sqs", region_name=region)
sqs.get_queue_url(QueueName=dlq_queue_name)["QueueUrl"]
queue_exists = True
except ClientError as e:
if e.response["Error"]["Code"].endswith(".NonExistentQueue"):
queue_exists = False
else:
raise
return queue_exists
def create(self, validated_data):
"""Create an AwsAccount."""
arn = aws.AwsArn(validated_data['account_arn'])
aws_account_id = arn.account_id
user = self.context['request'].user
account = AwsAccount(
account_arn=str(arn),
aws_account_id=aws_account_id,
user=user,
)
try:
session = aws.get_session(str(arn))
except ClientError as error:
if error.response.get('Error', {}).get('Code') == 'AccessDenied':
raise serializers.ValidationError(
detail={
'account_arn':
[_('Permission denied for ARN "{0}"').format(arn)]
})
raise
account_verified, failed_actions = aws.verify_account_access(session)
if account_verified:
instances_data = aws.get_running_instances(session)
with transaction.atomic():
account.save()
new_amis = create_new_machine_images(account, instances_data)
create_initial_aws_instance_events(account, instances_data)
messages = generate_aws_ami_messages(instances_data, new_amis)
for message in messages:
copy_ami_snapshot.delay(str(arn), message['image_id'],
message['region'])
else:
failure_details = [_('Account verification failed.')]
failure_details += [
_('Access denied for policy action "{0}".').format(action)
for action in failed_actions
]
raise serializers.ValidationError(
detail={'account_arn': failure_details})
return account
def create(self, validated_data):
"""Create an AwsAccount."""
arn = aws.AwsArn(validated_data['account_arn'])
aws_account_id = arn.account_id
account = AwsAccount.objects.filter(
aws_account_id=aws_account_id).first()
if account is not None:
raise serializers.ValidationError(
detail={
'account_arn': [
_('An ARN already exists for account "{0}"').format(
aws_account_id)
]
})
user = self.context['request'].user
name = validated_data.get('name')
account = AwsAccount(
account_arn=str(arn),
aws_account_id=aws_account_id,
name=name,
user=user,
)
try:
session = aws.get_session(str(arn))
except ClientError as error:
if error.response.get('Error', {}).get('Code') == 'AccessDenied':
raise serializers.ValidationError(
detail={
'account_arn':
[_('Permission denied for ARN "{0}"').format(arn)]
})
raise
account_verified, failed_actions = aws.verify_account_access(session)
if account_verified:
instances_data = aws.get_running_instances(session)
with transaction.atomic():
account.save()
try:
aws.configure_cloudtrail(session, aws_account_id)
except ClientError as error:
if error.response.get('Error', {}).get('Code') == \
'AccessDeniedException':
raise serializers.ValidationError(
detail={
'account_arn': [
_('Access denied to create CloudTrail for '
'ARN "{0}"').format(arn)
]
})
raise
new_amis = create_new_machine_images(account, instances_data)
create_initial_aws_instance_events(account, instances_data)
messages = generate_aws_ami_messages(instances_data, new_amis)
for message in messages:
image = start_image_inspection(str(arn), message['image_id'],
message['region'])
self.add_openshift_tag(session, message['image_id'],
message['region'], image)
else:
failure_details = [_('Account verification failed.')]
failure_details += [
_('Access denied for policy action "{0}".').format(action)
for action in failed_actions
]
raise serializers.ValidationError(
detail={'account_arn': failure_details})
return account
def test_generate_dummy_arn_random_account_id(self):
"""Assert generation of an ARN without a specified account ID."""
arn = helper.generate_dummy_arn(generate_account_id=True)
account_id = aws.AwsArn(arn).account_id
self.assertIn(account_id, arn)
def update_aws_cloud_account(
cloud_account,
customer_arn,
account_number,
authentication_id,
source_id,
):
"""
Update aws_cloud_account with the new arn.
Args:
cloud_account (api.models.CloudAccount)
customer_arn (str): customer's ARN
account_number (str): customer's account number
authentication_id (str): Platform Sources' Authentication object id
source_id (str): Platform Sources' Source object id
"""
logger.info(
_("Updating an AwsCloudAccount. "
"cloud_account=%(cloud_account)s, "
"customer_arn=%(customer_arn)s, "
"account_number=%(account_number)s, "
"authentication_id=%(authentication_id)s, "
"source_id=%(source_id)s"),
{
"cloud_account": cloud_account,
"customer_arn": customer_arn,
"account_number": account_number,
"authentication_id": authentication_id,
"source_id": source_id,
},
)
application_id = cloud_account.platform_application_id
try:
customer_aws_account_id = aws.AwsArn(customer_arn).account_id
except InvalidArn:
error = error_codes.CG1004
error.log_internal_message(logger, {"application_id": application_id})
error.notify(account_number, application_id)
return
# If the aws_account_id is different, then we disable the account,
# delete all related instances, and then enable the account.
# Otherwise just update the account_arn.
if cloud_account.content_object.aws_account_id != customer_aws_account_id:
logger.info(
_("Cloud Account with ID %(clount_id)s and aws_account_id "
"%(old_aws_account_id)s has received an update request for ARN "
"%(new_arn)s and aws_account_id %(new_aws_account_id)s. "
"Since the aws_account_id is different, Cloud Account ID "
"%(clount_id)s will be deleted. A new Cloud Account will be created "
"with aws_account_id %(new_aws_account_id)s and arn %(new_arn)s."
),
{
"clount_id": cloud_account.id,
"old_aws_account_id":
cloud_account.content_object.aws_account_id,
"new_aws_account_id": customer_aws_account_id,
"new_arn": customer_arn,
},
)
cloud_account.disable(power_off_instances=False)
# Remove instances associated with the clount
Instance.objects.filter(cloud_account=cloud_account).delete()
try:
customer_aws_account_id = aws.AwsArn(customer_arn).account_id
except InvalidArn:
error = error_codes.CG1004
error.log_internal_message(logger,
{"application_id": application_id})
error.notify(account_number, application_id)
return
# Verify that no AwsCloudAccount already exists with the same ARN.
if AwsCloudAccount.objects.filter(account_arn=customer_arn).exists():
error_code = error_codes.CG1001
existing_user_id = (AwsCloudAccount.objects.get(
account_arn=customer_arn).cloud_account.get().user.id)
# If the CloudAccount with the duplicate ARN belongs to the same user,
# we want to give the error code in addition to the generic message
_notify_error_with_generic_message_for_different_user(
error_code,
existing_user_id,
account_number,
application_id,
customer_arn,
)
return
# Verify that no AwsCloudAccount already exists with the same AWS Account ID.
if AwsCloudAccount.objects.filter(
aws_account_id=customer_aws_account_id).exists():
error_code = error_codes.CG1002
existing_user_id = (AwsCloudAccount.objects.get(
aws_account_id=customer_aws_account_id).cloud_account.get().
user.id)
_notify_error_with_generic_message_for_different_user(
error_code,
existing_user_id,
account_number,
application_id,
customer_arn,
)
return
cloud_account.content_object.account_arn = customer_arn
cloud_account.content_object.aws_account_id = customer_aws_account_id
cloud_account.content_object.save()
cloud_account.enable()
else:
try:
cloud_account.content_object.account_arn = customer_arn
cloud_account.content_object.save()
verify_permissions(customer_arn)
cloud_account.enable()
except ValidationError as e:
logger.info(
_("ARN %s failed validation. The Cloud Account will still be updated."
),
customer_arn,
)
# Tell the cloud account why we're disabling it
cloud_account.disable(message=str(e.detail))
logger.info(
_("Cloud Account with ID %s has been updated with arn %s. "),
cloud_account.id,
customer_arn,
)
def create_aws_cloud_account(
user,
customer_role_arn,
cloud_account_name,
platform_authentication_id,
platform_application_id,
platform_source_id,
):
"""
Create AwsCloudAccount for the customer user.
This function may raise ValidationError if certain verification steps fail.
We call CloudAccount.enable after creating it, and that effectively verifies AWS
permission and configures CloudTrail. If that fails, we must abort this creation.
That is why we put almost everything here in a transaction.atomic() context.
Args:
user (django.contrib.auth.models.User): user to own the CloudAccount
customer_role_arn (str): ARN to access the customer's AWS account
cloud_account_name (str): the name to use for our CloudAccount
platform_authentication_id (str): Platform Sources' Authentication object id
platform_application_id (str): Platform Sources' Application object id
platform_source_id (str): Platform Sources' Source object id
Returns:
CloudAccount the created cloud account.
"""
logger.info(
_("Creating an AwsCloudAccount. "
"user=%(user)s, "
"customer_role_arn=%(customer_role_arn)s, "
"cloud_account_name=%(cloud_account_name)s, "
"platform_authentication_id=%(platform_authentication_id)s, "
"platform_application_id=%(platform_application_id)s, "
"platform_source_id=%(platform_source_id)s"),
{
"user": user.username,
"customer_role_arn": customer_role_arn,
"cloud_account_name": cloud_account_name,
"platform_authentication_id": platform_authentication_id,
"platform_application_id": platform_application_id,
"platform_source_id": platform_source_id,
},
)
aws_account_id = aws.AwsArn(customer_role_arn).account_id
arn_str = str(customer_role_arn)
with transaction.atomic():
# Verify that no AwsCloudAccount already exists with the same ARN.
if AwsCloudAccount.objects.filter(account_arn=arn_str).exists():
error_code = error_codes.CG1001
existing_user_id = (AwsCloudAccount.objects.get(
account_arn=arn_str).cloud_account.get().user.id)
# If the CloudAccount with the duplicate ARN belongs to the same user,
# we want to give the error code in addition to the generic message
error_message = _notify_error_with_generic_message_for_different_user(
error_code, existing_user_id, user, platform_application_id,
arn_str)
raise ValidationError({"account_arn": error_message})
# Verify that no AwsCloudAccount already exists with the same AWS Account ID.
if AwsCloudAccount.objects.filter(
aws_account_id=aws_account_id).exists():
error_code = error_codes.CG1002
existing_user_id = (AwsCloudAccount.objects.get(
aws_account_id=aws_account_id).cloud_account.get().user.id)
error_message = _notify_error_with_generic_message_for_different_user(
error_code, existing_user_id, user, platform_application_id,
arn_str)
raise ValidationError({"account_arn": error_message})
# Verify that no CloudAccount exists with the same name.
if CloudAccount.objects.filter(user=user,
name=cloud_account_name).exists():
error_code = error_codes.CG1003
error_code.log_internal_message(
logger,
{
"application_id": platform_application_id,
"name": cloud_account_name,
},
)
error_code.notify(user.username, platform_application_id)
raise ValidationError({"name": error_code.get_message()})
try:
# Use get_or_create here in case there is another task running concurrently
# that created the AwsCloudAccount at the same time.
aws_cloud_account, created = AwsCloudAccount.objects.get_or_create(
aws_account_id=aws_account_id, account_arn=arn_str)
except IntegrityError:
# get_or_create can throw integrity error in the case that
# aws_account_id xor arn already exists in an account.
error_code = error_codes.CG1002
existing_user_id = (AwsCloudAccount.objects.get(
aws_account_id=aws_account_id).cloud_account.get().user.id)
error_message = _notify_error_with_generic_message_for_different_user(
error_code, existing_user_id, user, platform_application_id,
arn_str)
raise ValidationError({"account_arn": error_message})
if not created:
# If aws_account_id and arn already exist in an account because a
# another task created it, notify the user.
error_code = error_codes.CG1002
existing_user_id = (AwsCloudAccount.objects.get(
aws_account_id=aws_account_id).cloud_account.get().user.id)
error_message = _notify_error_with_generic_message_for_different_user(
error_code, existing_user_id, user, platform_application_id,
arn_str)
raise ValidationError({"account_arn": error_message})
cloud_account = CloudAccount.objects.create(
user=user,
name=cloud_account_name,
content_object=aws_cloud_account,
platform_application_id=platform_application_id,
platform_authentication_id=platform_authentication_id,
platform_source_id=platform_source_id,
)
# This enable call *must* be inside the transaction because we need to
# know to rollback the transaction if anything related to enabling fails.
# Yes, this means holding the transaction open while we wait on calls
# to AWS.
if cloud_account.enable() is False:
# Enabling of cloud account failed, rolling back.
transaction.set_rollback(True)
raise ValidationError({
"is_enabled":
"Could not enable cloud account. "
"Please check your credentials."
})
return cloud_account
def verify_permissions(customer_role_arn):
"""
Verify AWS permissions.
This function may raise ValidationError if certain verification steps fail.
Args:
customer_role_arn (str): ARN to access the customer's AWS account
Note:
This function not only verifies; it also has the side effect of configuring
the AWS CloudTrail. This should be refactored into a more explicit operation,
but at the time of this writing, there is no "dry run" check for CloudTrail
operations. Callers should be aware of the risk that we may configure CloudTrail
but somewhere else rollback our transaction, leaving that Trail orphaned.
Returns:
boolean indicating if the arn being verified is good.
"""
aws_account_id = aws.AwsArn(customer_role_arn).account_id
arn_str = str(customer_role_arn)
try:
session = aws.get_session(arn_str)
account_verified, failed_actions = aws.verify_account_access(session)
except ClientError as error:
if error.response.get("Error", {}).get("Code") in (
"AccessDenied",
"AccessDeniedException",
"UnrecognizedClientException",
):
raise ValidationError(
detail={
"account_arn":
[_('Permission denied for ARN "{0}"').format(arn_str)]
})
raise
if account_verified:
try:
aws.configure_cloudtrail(session, aws_account_id)
except ClientError as error:
if error.response.get("Error", {}).get("Code") in (
"AccessDenied",
"AccessDeniedException",
"UnrecognizedClientException",
):
logger.debug(_("Trying to throw a CG3000."))
cloud_account = CloudAccount.objects.get(
aws_cloud_account__aws_account_id=aws_account_id)
error_code = error_codes.CG3000
error_code.log_internal_message(logger, {
"cloud_account_id": aws_account_id,
"exception": error
})
error_code.notify(cloud_account.user.username,
cloud_account.platform_application_id)
logger.debug(
_("CG3000 notify called, raising ValidationError."))
raise ValidationError(
detail={
"account_arn": [
_("Access denied to create CloudTrail for "
'ARN "{0}"').format(arn_str)
]
})
raise
except MaximumNumberOfTrailsExceededException as error:
logger.debug(_("Trying to throw a CG3001."))
cloud_account = CloudAccount.objects.get(
aws_cloud_account__account_arn=arn_str)
error_code = error_codes.CG3001
error_code.log_internal_message(logger, {
"cloud_account_id": cloud_account.id,
"exception": error
})
error_code.notify(cloud_account.user.username,
cloud_account.platform_application_id)
logger.debug(_("CG3001 notify called, raising ValidationError."))
raise ValidationError(
detail={"account_arn": error_code.get_message()})
else:
failure_details = [_("Account verification failed.")]
failure_details += [
_('Access denied for policy action "{0}".').format(action)
for action in failed_actions
]
raise ValidationError(detail={"account_arn": failure_details})
return account_verified
def generate_cloud_account( # noqa: C901
arn=None,
aws_account_id=None,
user=None,
name=None,
created_at=None,
platform_authentication_id=None,
platform_application_id=None,
platform_source_id=None,
is_enabled=True,
enabled_at=None,
verify_task=None,
generate_verify_task=True,
cloud_type=AWS_PROVIDER_STRING,
azure_subscription_id=None,
azure_tenant_id=None,
):
"""
Generate an CloudAccount for testing.
Any optional arguments not provided will be randomly generated.
Args:
arn (str): Optional ARN.
aws_account_id (12-digit string): Optional AWS account ID.
user (User): Optional Django auth User to be this account's owner.
name (str): Optional name for this account.
created_at (datetime): Optional creation datetime for this account.
platform_authentication_id (int): Optional platform source authentication ID.
platform_application_id (int): Optional platform source application ID.
platform_source_id (int): Optional platform source source ID.
is_enabled (bool): Optional should the account be enabled.
enabled_at (datetime): Optional enabled datetime for this account.
verify_task (PeriodicTask): Optional Celery verify task for this account.
generate_verify_task (bool): Optional should a verify_task be generated here.
cloud_type (str): Str denoting cloud type, defaults to "aws"
azure_subscription_id (str): optional uuid str for azure subscription id
azure_tenant_id (str): optional uuid str for azure tenant id
Returns:
CloudAccount: The created Cloud Account.
"""
if user is None:
user = helper.generate_test_user()
if name is None:
name = str(uuid.uuid4())
if created_at is None:
created_at = get_now()
if enabled_at is None:
enabled_at = created_at
if platform_authentication_id is None:
platform_authentication_id = _faker.pyint()
if platform_application_id is None:
platform_application_id = _faker.pyint()
if platform_source_id is None:
platform_source_id = _faker.pyint()
if cloud_type == AZURE_PROVIDER_STRING:
if azure_subscription_id is None:
azure_subscription_id = uuid.uuid4()
if azure_tenant_id is None:
azure_tenant_id = uuid.uuid4()
cloud_provider_account = AzureCloudAccount.objects.create(
subscription_id=azure_subscription_id, tenant_id=azure_tenant_id)
# default to AWS
else:
if arn is None:
arn = helper.generate_dummy_arn(account_id=aws_account_id)
if verify_task is None and generate_verify_task:
schedule, _ = IntervalSchedule.objects.get_or_create(
every=settings.SCHEDULE_VERIFY_VERIFY_TASKS_INTERVAL,
period=IntervalSchedule.SECONDS,
)
verify_task, _ = PeriodicTask.objects.get_or_create(
interval=schedule,
name=f"Verify {arn}.",
task="api.clouds.aws.tasks.verify_account_permissions",
kwargs=json.dumps({
"account_arn": arn,
}),
defaults={"start_time": created_at},
)
cloud_provider_account = AwsCloudAccount.objects.create(
account_arn=arn,
aws_account_id=aws.AwsArn(arn).account_id,
verify_task=verify_task,
)
cloud_provider_account.created_at = created_at
cloud_provider_account.save()
cloud_account = CloudAccount.objects.create(
user=user,
name=name,
content_object=cloud_provider_account,
platform_authentication_id=platform_authentication_id,
platform_application_id=platform_application_id,
platform_source_id=platform_source_id,
is_enabled=is_enabled,
)
cloud_account.created_at = created_at
cloud_account.save()
if enabled_at:
cloud_account.enabled_at = enabled_at
cloud_account.save()
return cloud_account
