def copy_ami_snapshot(arn, ami_id, source_region):
"""
Copy an AWS Snapshot to the primary AWS account.
Args:
arn (str): The AWS Resource Number for the account with the snapshot
ami_id (str): The AWS ID for the machine image
source_region (str): The region the snapshot resides in
Returns:
None: Run as an asynchronous Celery task.
"""
session = aws.get_session(arn)
ami = aws.get_ami(session, ami_id, source_region)
snapshot_id = aws.get_ami_snapshot_id(ami)
snapshot = aws.get_snapshot(session, snapshot_id, source_region)
if snapshot.encrypted:
image = AwsMachineImage.objects.get(ec2_ami_id=ami_id)
image.is_encrypted = True
image.save()
raise AwsSnapshotEncryptedError
aws.add_snapshot_ownership(snapshot)
new_snapshot_id = aws.copy_snapshot(snapshot_id, source_region)
create_volume.delay(ami_id, new_snapshot_id)
def test_get_running_instances(self):
"""Assert we get expected instances in a dict keyed by regions."""
mock_arn = helper.generate_dummy_arn()
mock_regions = [f'region-{uuid.uuid4()}']
mock_role = helper.generate_dummy_role()
mock_running_instance = helper.generate_dummy_describe_instance(
state=aws.InstanceState.running)
mock_stopped_instance = helper.generate_dummy_describe_instance(
state=aws.InstanceState.stopped)
mock_described = {
'Reservations': [
{
'Instances': [
mock_running_instance,
mock_stopped_instance,
],
},
],
}
expected_found = {mock_regions[0]: [mock_running_instance]}
with patch.object(aws, 'get_regions') as mock_get_regions, \
patch.object(aws, 'boto3') as mock_boto3:
mock_assume_role = mock_boto3.client.return_value.assume_role
mock_assume_role.return_value = mock_role
mock_get_regions.return_value = mock_regions
mock_client = mock_boto3.Session.return_value.client.return_value
mock_client.describe_instances.return_value = mock_described
actual_found = aws.get_running_instances(aws.get_session(mock_arn))
self.assertDictEqual(expected_found, actual_found)
def test_verify_account_access_success(self):
"""Assert that account access via a IAM role is verified."""
mock_arn = helper.generate_dummy_arn()
mock_role = helper.generate_dummy_role()
mock_dry_run_exception = {
'Error': {
'Code':
'DryRunOperation',
'Message':
'Request would have succeeded, '
'but DryRun flag is set.'
}
}
with patch.object(aws, 'boto3') as mock_boto3:
mock_assume_role = mock_boto3.client.return_value.assume_role
mock_assume_role.return_value = mock_role
mock_client = mock_boto3.Session.return_value.client.return_value
mock_describe_images = mock_client.describe_images
mock_describe_instances = mock_client.describe_instances
mock_describe_snapshot_attribute = \
mock_client.describe_snapshot_attribute
mock_describe_snapshots = mock_client.describe_snapshots
mock_modify_snapshot_attribute = \
mock_client.modify_snapshot_attribute
mock_modify_image_attribute = mock_client.modify_image_attribute
mock_describe_images.side_effect = ClientError(
mock_dry_run_exception, 'DescribeImages')
mock_describe_instances.side_effect = ClientError(
mock_dry_run_exception, 'DescribeInstances')
mock_describe_snapshot_attribute.side_effect = ClientError(
mock_dry_run_exception, 'DescribeSnapshotAttribute')
mock_describe_snapshots.side_effect = ClientError(
mock_dry_run_exception, 'DescribeSnapshots')
mock_modify_snapshot_attribute.side_effect = ClientError(
mock_dry_run_exception, 'ModifySnapshotAttribute')
mock_modify_image_attribute.side_effect = ClientError(
mock_dry_run_exception, 'ModifyImageAttribute')
actual_verified = aws.verify_account_access(
aws.get_session(mock_arn))
mock_describe_images.assert_called_with(DryRun=True)
mock_describe_instances.assert_called_with(DryRun=True)
mock_describe_snapshot_attribute.assert_called_with(
DryRun=True, SnapshotId='string', Attribute='productCodes')
mock_describe_snapshots.assert_called_with(DryRun=True)
mock_modify_snapshot_attribute.assert_called_with(
SnapshotId='string',
DryRun=True,
Attribute='productCodes',
GroupNames=[
'string',
])
mock_modify_image_attribute.assert_called_with(
Attribute='description', ImageId='string', DryRun=True)
self.assertTrue(actual_verified)
def _parse_log_for_ec2_instance_events(log):
"""
Parse S3 log for EC2 on/off events.
Args:
log (str): The string contents of the log file.
Returns:
list(tuple): List of instance data seen in log.
list(tuple): List of instance_event data seen in log.
"""
instances = {}
log = json.loads(log)
# Each record is a single API call, but each call can
# be made against multiple EC2 instances
for record in log.get('Records', []):
if not _is_valid_event(record, ec2_instance_event_map.keys()):
continue
account_id = record.get('userIdentity', {}).get('accountId')
account = AwsAccount.objects.get(aws_account_id=account_id)
region = record.get('awsRegion')
session = aws.get_session(account.account_arn, region)
event_type = ec2_instance_event_map[record.get('eventName')]
occurred_at = record.get('eventTime')
ec2_info = record.get('responseElements', {})\
.get('instancesSet', {})\
.get('items', [])
# Collect the EC2 instances the API was called on
for item in ec2_info:
instance_id = item.get('instanceId')
instance = aws.get_ec2_instance(session, instance_id)
instances[instance_id] = {
'account_id': account_id,
'instance_details': instance,
'region': region
}
for __, data in instances.items():
event = {
'subnet': data['instance_details'].subnet_id,
'ec2_ami_id': data['instance_details'].image_id,
'instance_type': data['instance_details'].instance_type,
'event_type': event_type,
'occurred_at': occurred_at
}
if data.get('events'):
data['events'].append(event)
else:
data['events'] = [event]
return dict(instances)
def test_get_ec2_instance(self):
"""Assert that get_ec2_instance returns an instance object."""
mock_arn = helper.generate_dummy_arn()
mock_instance_id = str(uuid.uuid4())
mock_instance = helper.generate_mock_ec2_instance(mock_instance_id)
with patch.object(aws, 'boto3') as mock_boto3:
mock_session = mock_boto3.Session.return_value
resource = mock_session.resource.return_value
resource.Instance.return_value = mock_instance
actual_instance = aws.get_ec2_instance(aws.get_session(mock_arn),
mock_instance_id)
self.assertEqual(actual_instance, mock_instance)
def create(self, validated_data):
"""Create an AwsAccount."""
arn = validated_data['account_arn']
aws_account_id = aws.extract_account_id_from_arn(arn)
account = AwsAccount(account_arn=arn, aws_account_id=aws_account_id)
session = aws.get_session(arn)
if aws.verify_account_access(session):
instances_data = aws.get_running_instances(session)
with transaction.atomic():
account.save()
create_initial_aws_instance_events(account, instances_data)
else:
raise serializers.ValidationError(
_('AwsAccount verification failed. ARN Info Not Stored'))
return account
def copy_ami_to_customer_account(arn,
reference_ami_id,
snapshot_region,
maybe_marketplace=False):
"""
Copy an AWS Image to the customer's AWS account.
This is an intermediate step that we occasionally need to use when the
customer has an instance based on a image that has been privately shared
by a third party, and that means we cannot directly copy its snapshot. We
can, however, create a copy of the image in the customer's account and use
that copy for the remainder of the inspection process.
Args:
arn (str): The AWS Resource Number for the account with the snapshot
reference_ami_id (str): The AWS ID for the original image to copy
snapshot_region (str): The region the snapshot resides in
maybe_marketplace (bool): Set to True if we suspect this to be a
marketplace image.
Returns:
None: Run as an asynchronous Celery task.
"""
session = aws.get_session(arn)
reference_ami = aws.get_ami(session, reference_ami_id, snapshot_region)
try:
new_ami_id = aws.copy_ami(session, reference_ami.id, snapshot_region)
except ClientError as e:
if maybe_marketplace \
and e.response.get('Error').get('Code') == 'InvalidRequest' \
and 'Images with EC2 BillingProduct codes cannot be copied ' \
'to another AWS account' in \
e.response.get('Error').get('Message'):
# This appears to be a marketplace AMI, mark it as inspected.
logger.info(
_('Found a marketplace image "{0}", marking as '
'inspected').format(reference_ami_id))
ami = AwsMachineImage.objects.get(ec2_ami_id=reference_ami_id)
ami.status = ami.INSPECTED
ami.save()
return
raise e
copy_ami_snapshot.delay(arn, new_ami_id, snapshot_region, reference_ami_id)
def create(self, validated_data):
"""Create an AwsAccount."""
arn = aws.AwsArn(validated_data['account_arn'])
aws_account_id = arn.account_id
user = self.context['request'].user
account = AwsAccount(
account_arn=str(arn),
aws_account_id=aws_account_id,
user=user,
)
try:
session = aws.get_session(str(arn))
except ClientError as error:
if error.response.get('Error', {}).get('Code') == 'AccessDenied':
raise serializers.ValidationError(
detail={
'account_arn':
[_('Permission denied for ARN "{0}"').format(arn)]
})
raise
account_verified, failed_actions = aws.verify_account_access(session)
if account_verified:
instances_data = aws.get_running_instances(session)
with transaction.atomic():
account.save()
new_amis = create_new_machine_images(account, instances_data)
create_initial_aws_instance_events(account, instances_data)
messages = generate_aws_ami_messages(instances_data, new_amis)
for message in messages:
copy_ami_snapshot.delay(str(arn), message['image_id'],
message['region'])
else:
failure_details = [_('Account verification failed.')]
failure_details += [
_('Access denied for policy action "{0}".').format(action)
for action in failed_actions
]
raise serializers.ValidationError(
detail={'account_arn': failure_details})
return account
def test_get_session(self):
"""Assert get_session returns session object."""
mock_arn = helper.generate_dummy_arn()
mock_account_id = extract_account_id_from_arn(mock_arn)
mock_role = helper.generate_dummy_role()
with patch.object(aws.boto3, 'client') as mock_client:
mock_assume_role = mock_client.return_value.assume_role
mock_assume_role.return_value = mock_role
session = aws.get_session(mock_arn)
creds = session.get_credentials().get_frozen_credentials()
mock_client.assert_called_with('sts')
mock_assume_role.assert_called_with(
Policy=json.dumps(aws.cloudigrade_policy),
RoleArn=mock_arn,
RoleSessionName=f'cloudigrade-{mock_account_id}')
self.assertEqual(creds[0], mock_role['Credentials']['AccessKeyId'])
self.assertEqual(creds[1], mock_role['Credentials']['SecretAccessKey'])
self.assertEqual(creds[2], mock_role['Credentials']['SessionToken'])
def remove_snapshot_ownership(arn, customer_snapshot_id,
customer_snapshot_region, snapshot_copy_id):
"""
Remove cloudigrade ownership from customer snapshot.
Args:
arn (str): The AWS Resource Number for the account with the snapshot
customer_snapshot_id (str): The id of the snapshot to remove ownership
customer_snapshot_region (str): The region where
customer_snapshot_id resides
snapshot_copy_id (str): The id of the snapshot that must
be ready to continue
Returns:
None: Run as an asynchronous Celery task.
"""
ec2 = boto3.resource('ec2')
# Wait for snapshot to be ready
try:
snapshot_copy = ec2.Snapshot(snapshot_copy_id)
aws.check_snapshot_state(snapshot_copy)
except ClientError as error:
if error.response.get('Error',
{}).get('Code') == 'InvalidSnapshot.NotFound':
logger.info(
_('{0} detected snapshot_copy_id {1} already deleted.').format(
'remove_snapshot_ownership', snapshot_copy_id))
else:
raise
# Remove permissions from customer_snapshot
logger.info(
_('{0} remove ownership from customer snapshot {1}').format(
'remove_snapshot_ownership', customer_snapshot_id))
session = aws.get_session(arn)
customer_snapshot = aws.get_snapshot(session, customer_snapshot_id,
customer_snapshot_region)
aws.remove_snapshot_ownership(customer_snapshot)
def _load_missing_ami_data(instance_events, ami_tag_events):
"""
Load additional data so we can create the AMIs for the given events.
Args:
instance_events (list[CloudTrailInstanceEvent]): found instance events
ami_tag_events (list[CloudTrailImageTagEvent]): found AMI tag events
Returns:
dict: Dict of dicts from AWS describing AMIs that are referenced
by the input arguments but are not present in our database, with
the outer key being each AMI's ID.
"""
seen_ami_ids = set([
event.ec2_ami_id for event in instance_events + ami_tag_events
if event.ec2_ami_id is not None
])
known_images = AwsMachineImage.objects.filter(ec2_ami_id__in=seen_ami_ids)
known_ami_ids = set([image.ec2_ami_id for image in known_images])
new_ami_ids = seen_ami_ids.difference(known_ami_ids)
new_amis_keyed = set([(event.aws_account_id, event.region,
event.ec2_ami_id)
for event in instance_events + ami_tag_events
if event.ec2_ami_id in new_ami_ids])
described_amis = dict()
# Look up only the new AMIs that belong to each account+region group.
for (aws_account_id,
region), amis_keyed in itertools.groupby(new_amis_keyed,
key=lambda a: (a[0], a[1])):
amis_keyed = list(amis_keyed)
awsaccount = AwsCloudAccount.objects.get(aws_account_id=aws_account_id)
session = aws.get_session(awsaccount.account_arn, region)
ami_ids = [k[2] for k in amis_keyed]
# Get all relevant images in one API call for this account+region.
new_described_amis = aws.describe_images(session, ami_ids, region)
for described_ami in new_described_amis:
ami_id = described_ami["ImageId"]
logger.info(
_("Loading data for AMI %(ami_id)s for "
"ARN %(account_arn)s in region %(region)s"),
{
"ami_id": ami_id,
"account_arn": awsaccount.account_arn,
"region": region,
},
)
described_ami["found_in_region"] = region
described_ami["found_by_account_arn"] = awsaccount.account_arn
described_amis[ami_id] = described_ami
for aws_account_id, region, ec2_ami_id in new_amis_keyed:
if ec2_ami_id not in described_amis:
logger.info(
_("AMI %(ec2_ami_id)s could not be found in region "
"%(region)s for AWS account %(aws_account_id)s."),
{
"ec2_ami_id": ec2_ami_id,
"region": region,
"aws_account_id": aws_account_id,
},
)
return described_amis
def _load_missing_instance_data(instance_events): # noqa: C901
"""
Load additional data so we can create instances from Cloud Trail events.
We only get the necessary instance type and AMI ID from AWS Cloud Trail for
the "RunInstances" event upon first creation of an instance. If we didn't
get that (for example, if an instance already existed but was stopped when
we got the cloud account), that means we may not know about the instance
and need to describe it before we create our record of it.
However, there is an edge-case possibility when AWS gives us events out of
order. If we receive an instance event *before* its "RunInstances" that
would fully describe it, we have to describe it now even though the later
event should eventually give us that same information. There's a super edge
case here that means the AWS user could have also changed the type before
we receive that initial "RunInstances" event, but we are not explicitly
handling that scenario as of this writing.
Args:
instance_events (list[CloudTrailInstanceEvent]): found instance events
Side-effect:
instance_events input argument may have been updated with missing
image_id values from AWS.
Returns:
dict: dict of dicts from AWS describing EC2 instances that are
referenced by the input arguments but are not present in our
database, with the outer key being each EC2 instance's ID.
"""
all_ec2_instance_ids = set(
[instance_event.ec2_instance_id for instance_event in instance_events])
described_instances = dict()
defined_ec2_instance_ids = set()
# First identify which instances DON'T need to be described because we
# either already have them stored or at least one of instance_events has
# enough information for it.
for instance_event in instance_events:
ec2_instance_id = instance_event.ec2_instance_id
if (_instance_event_is_complete(instance_event)
or ec2_instance_id in defined_ec2_instance_ids):
# This means the incoming data is sufficiently populated so we
# should know the instance's image and type.
defined_ec2_instance_ids.add(ec2_instance_id)
elif (AwsInstance.objects.filter(
ec2_instance_id=instance_event.ec2_instance_id,
instance__machine_image__isnull=False,
).exists() and InstanceEvent.objects.filter(
instance__aws_instance__ec2_instance_id=ec2_instance_id,
aws_instance_event__instance_type__isnull=False,
).exists()):
# This means we already know the instance's image and at least once
# we have known the instance's type from an event.
defined_ec2_instance_ids.add(ec2_instance_id)
# Iterate through the instance events grouped by account and region in
# order to minimize the number of sessions and AWS API calls.
for (aws_account_id, region), grouped_instance_events in itertools.groupby(
instance_events, key=lambda e: (e.aws_account_id, e.region)):
grouped_instance_events = list(grouped_instance_events)
# Find the set of EC2 instance IDs that belong to this account+region.
ec2_instance_ids = set([
e.ec2_instance_id for e in grouped_instance_events
]).difference(defined_ec2_instance_ids)
if not ec2_instance_ids:
# Early continue if there are no instances we need to describe!
continue
awsaccount = AwsCloudAccount.objects.get(aws_account_id=aws_account_id)
session = aws.get_session(awsaccount.account_arn, region)
# Get all relevant instances in one API call for this account+region.
new_described_instances = aws.describe_instances(
session, ec2_instance_ids, region)
# How we found these instances will be important to save *later*.
# This wouldn't be necessary if we could save these here, but we don't
# want to mix DB transactions with external AWS API calls.
for (ec2_instance_id,
described_instance) in new_described_instances.items():
logger.info(
_("Loading data for EC2 Instance %(ec2_instance_id)s for "
"ARN %(account_arn)s in region %(region)s"),
{
"ec2_instance_id": ec2_instance_id,
"account_arn": awsaccount.account_arn,
"region": region,
},
)
described_instance["found_by_account_arn"] = awsaccount.account_arn
described_instance["found_in_region"] = region
described_instances[ec2_instance_id] = described_instance
# Add any missing image IDs to the instance_events from the describes.
for instance_event in instance_events:
ec2_instance_id = instance_event.ec2_instance_id
if instance_event.ec2_ami_id is None and ec2_instance_id in described_instances:
described_instance = described_instances[ec2_instance_id]
instance_event.ec2_ami_id = described_instance["ImageId"]
# We really *should* have what we need, but just in case...
for ec2_instance_id in all_ec2_instance_ids:
if (ec2_instance_id not in defined_ec2_instance_ids
and ec2_instance_id not in described_instances):
logger.info(
_("EC2 Instance %(ec2_instance_id)s could not be loaded "
"from database or AWS. It may have been terminated before "
"we processed it."),
{"ec2_instance_id": ec2_instance_id},
)
return described_instances
def create(self, validated_data):
"""Create an AwsAccount."""
arn = aws.AwsArn(validated_data['account_arn'])
aws_account_id = arn.account_id
account = AwsAccount.objects.filter(
aws_account_id=aws_account_id).first()
if account is not None:
raise serializers.ValidationError(
detail={
'account_arn': [
_('An ARN already exists for account "{0}"').format(
aws_account_id)
]
})
user = self.context['request'].user
name = validated_data.get('name')
account = AwsAccount(
account_arn=str(arn),
aws_account_id=aws_account_id,
name=name,
user=user,
)
try:
session = aws.get_session(str(arn))
except ClientError as error:
if error.response.get('Error', {}).get('Code') == 'AccessDenied':
raise serializers.ValidationError(
detail={
'account_arn':
[_('Permission denied for ARN "{0}"').format(arn)]
})
raise
account_verified, failed_actions = aws.verify_account_access(session)
if account_verified:
instances_data = aws.get_running_instances(session)
with transaction.atomic():
account.save()
try:
aws.configure_cloudtrail(session, aws_account_id)
except ClientError as error:
if error.response.get('Error', {}).get('Code') == \
'AccessDeniedException':
raise serializers.ValidationError(
detail={
'account_arn': [
_('Access denied to create CloudTrail for '
'ARN "{0}"').format(arn)
]
})
raise
new_amis = create_new_machine_images(account, instances_data)
create_initial_aws_instance_events(account, instances_data)
messages = generate_aws_ami_messages(instances_data, new_amis)
for message in messages:
image = start_image_inspection(str(arn), message['image_id'],
message['region'])
self.add_openshift_tag(session, message['image_id'],
message['region'], image)
else:
failure_details = [_('Account verification failed.')]
failure_details += [
_('Access denied for policy action "{0}".').format(action)
for action in failed_actions
]
raise serializers.ValidationError(
detail={'account_arn': failure_details})
return account
def copy_ami_snapshot(arn, ami_id, snapshot_region, reference_ami_id=None):
"""
Copy an AWS Snapshot to the primary AWS account.
Args:
arn (str): The AWS Resource Number for the account with the snapshot
ami_id (str): The AWS ID for the machine image
snapshot_region (str): The region the snapshot resides in
reference_ami_id (str): Optional. The id of the original image from
which this image was copied. We need to know this in some cases
where we create a copy of the image in the customer's account
before we can copy its snapshot, and we must pass this information
forward for appropriate reference and cleanup.
Returns:
None: Run as an asynchronous Celery task.
"""
session = aws.get_session(arn)
session_account_id = aws.get_session_account_id(session)
ami = aws.get_ami(session, ami_id, snapshot_region)
customer_snapshot_id = aws.get_ami_snapshot_id(ami)
try:
customer_snapshot = aws.get_snapshot(session, customer_snapshot_id,
snapshot_region)
if customer_snapshot.encrypted:
image = AwsMachineImage.objects.get(ec2_ami_id=ami_id)
image.is_encrypted = True
image.save()
raise AwsSnapshotEncryptedError
logger.info(
_('AWS snapshot "{snapshot_id}" for image "{image_id}" has owner '
'"{owner_id}"; current session is account "{account_id}"').
format(snapshot_id=customer_snapshot.snapshot_id,
image_id=ami.id,
owner_id=customer_snapshot.owner_id,
account_id=session_account_id))
if customer_snapshot.owner_id != session_account_id and \
reference_ami_id is None:
copy_ami_to_customer_account.delay(arn, ami_id, snapshot_region)
# Early return because we need to stop processing the current AMI.
# A future call will process this new copy of the
# current AMI instead.
return
except ClientError as e:
if e.response.get('Error').get('Code') == 'InvalidSnapshot.NotFound':
# Possibly a marketplace AMI, try to handle it by copying.
copy_ami_to_customer_account.delay(arn,
ami_id,
snapshot_region,
maybe_marketplace=True)
return
raise e
aws.add_snapshot_ownership(customer_snapshot)
snapshot_copy_id = aws.copy_snapshot(customer_snapshot_id, snapshot_region)
logger.info(
_('{0}: customer_snapshot_id={1}, snapshot_copy_id={2}').format(
'copy_ami_snapshot', customer_snapshot_id, snapshot_copy_id))
# Schedule removal of ownership on customer snapshot
remove_snapshot_ownership.delay(arn, customer_snapshot_id, snapshot_region,
snapshot_copy_id)
if reference_ami_id is not None:
# If a reference ami exists, that means we have been working with a
# copy in here. That means we need to remove that copy and pass the
# original reference AMI ID through the rest of the task chain so the
# results get reported for that original reference AMI, not our copy.
# TODO FIXME Do we or don't we clean up?
# If we do, we need permissions to include `ec2:DeregisterImage` and
# `ec2:DeleteSnapshot` but those are both somewhat scary...
#
# For now, since we are not deleting the copy image from AWS, we need
# to record a reference to our database that we can look at later to
# indicate the relationship between the original AMI and the copy.
create_aws_machine_image_copy(ami_id, reference_ami_id)
ami_id = reference_ami_id
# Create volume from snapshot copy
create_volume.delay(ami_id, snapshot_copy_id)
def verify_permissions(customer_role_arn):
"""
Verify AWS permissions.
This function may raise ValidationError if certain verification steps fail.
Args:
customer_role_arn (str): ARN to access the customer's AWS account
Note:
This function not only verifies; it also has the side effect of configuring
the AWS CloudTrail. This should be refactored into a more explicit operation,
but at the time of this writing, there is no "dry run" check for CloudTrail
operations. Callers should be aware of the risk that we may configure CloudTrail
but somewhere else rollback our transaction, leaving that Trail orphaned.
Returns:
boolean indicating if the arn being verified is good.
"""
aws_account_id = aws.AwsArn(customer_role_arn).account_id
arn_str = str(customer_role_arn)
try:
session = aws.get_session(arn_str)
account_verified, failed_actions = aws.verify_account_access(session)
except ClientError as error:
if error.response.get("Error", {}).get("Code") in (
"AccessDenied",
"AccessDeniedException",
"UnrecognizedClientException",
):
raise ValidationError(
detail={
"account_arn":
[_('Permission denied for ARN "{0}"').format(arn_str)]
})
raise
if account_verified:
try:
aws.configure_cloudtrail(session, aws_account_id)
except ClientError as error:
if error.response.get("Error", {}).get("Code") in (
"AccessDenied",
"AccessDeniedException",
"UnrecognizedClientException",
):
logger.debug(_("Trying to throw a CG3000."))
cloud_account = CloudAccount.objects.get(
aws_cloud_account__aws_account_id=aws_account_id)
error_code = error_codes.CG3000
error_code.log_internal_message(logger, {
"cloud_account_id": aws_account_id,
"exception": error
})
error_code.notify(cloud_account.user.username,
cloud_account.platform_application_id)
logger.debug(
_("CG3000 notify called, raising ValidationError."))
raise ValidationError(
detail={
"account_arn": [
_("Access denied to create CloudTrail for "
'ARN "{0}"').format(arn_str)
]
})
raise
except MaximumNumberOfTrailsExceededException as error:
logger.debug(_("Trying to throw a CG3001."))
cloud_account = CloudAccount.objects.get(
aws_cloud_account__account_arn=arn_str)
error_code = error_codes.CG3001
error_code.log_internal_message(logger, {
"cloud_account_id": cloud_account.id,
"exception": error
})
error_code.notify(cloud_account.user.username,
cloud_account.platform_application_id)
logger.debug(_("CG3001 notify called, raising ValidationError."))
raise ValidationError(
detail={"account_arn": error_code.get_message()})
else:
failure_details = [_("Account verification failed.")]
failure_details += [
_('Access denied for policy action "{0}".').format(action)
for action in failed_actions
]
raise ValidationError(detail={"account_arn": failure_details})
return account_verified
def initial_aws_describe_instances(account_id):
"""
Fetch and save instances data found upon AWS cloud account creation.
Args:
account_id (int): the AwsAccount id
"""
try:
aws_account = AwsCloudAccount.objects.get(pk=account_id)
except AwsCloudAccount.DoesNotExist:
logger.warning(
_("AwsCloudAccount id %s could not be found for initial describe"),
account_id,
)
# This can happen if a customer creates and then quickly deletes their
# cloud account before this async task has started to run. Early exit!
return
account = aws_account.cloud_account.get()
if not account.is_enabled:
logger.warning(
_("AwsCloudAccount id %s is not enabled; skipping initial describe"
),
account_id,
)
# This can happen if a customer creates and then quickly disabled their
# cloud account before this async task has started to run. Early exit!
return
arn = aws_account.account_arn
session = aws.get_session(arn)
instances_data = aws.describe_instances_everywhere(session)
try:
user_id = account.user.id
except User.DoesNotExist:
logger.info(
_("User for account id %s has already been deleted; "
"skipping initial describe."),
account_id,
)
# This can happen if a customer creates and then quickly deletes their
# cloud account before this async task has started to run. If the user has
# no other cloud accounts the user will also be deleted. Early exit!
return
# Lock the task at a user level. A user can only run one task at a time.
with lock_task_for_user_ids([user_id]):
try:
# Explicitly "get" the related AwsCloudAccount before proceeding.
# We do this at the start of this transaction in case the account has been
# deleted during the potentially slow describe_instances_everywhere above.
# If this fails, we'll jump to the except block to log an important warning.
AwsCloudAccount.objects.get(pk=account_id)
create_missing_power_off_aws_instance_events(
account, instances_data)
new_ami_ids = create_new_machine_images(session, instances_data)
logger.info(
_("Created new machine images include: %(new_ami_ids)s"),
{"new_ami_ids": new_ami_ids},
)
create_initial_aws_instance_events(account, instances_data)
except AwsCloudAccount.DoesNotExist:
logger.warning(
_("AwsCloudAccount id %s could not be found to save newly "
"discovered images and instances"),
account_id,
)
# This can happen if a customer deleted their cloud account between
# the start of this function and here. The AWS calls for
# describe_instances_everywhere may be slow and are not within this
# transaction. That's why we have to check again after it.
return
messages = generate_aws_ami_messages(instances_data, new_ami_ids)
for message in messages:
start_image_inspection(str(arn), message["image_id"],
message["region"])
def test_verify_account_access_failure(self):
"""Assert that account access via a IAM role is not verified."""
mock_arn = helper.generate_dummy_arn()
mock_role = helper.generate_dummy_role()
mock_unauthorized_exception = {
'Error': {
'Code': 'UnauthorizedOperation',
'Message': 'You are not authorized '
'to perform this operation.'
}
}
mock_garbage_exception = {
'Error': {
'Code':
'GarbageOperation',
'Message':
'You are not authorized '
'to perform this garbage operation.'
}
}
bad_policy = {
'Version':
'2012-10-17',
'Statement': [{
'Sid': 'CloudigradeGarbagePolicy',
'Effect': 'Allow',
'Action': ['ec2:DescribeGarbage'],
'Resource': '*'
}]
}
with patch.object(aws, 'boto3') as mock_boto3:
mock_assume_role = mock_boto3.client.return_value.assume_role
mock_assume_role.return_value = mock_role
mock_client = mock_boto3.Session.return_value.client.return_value
mock_describe_images = mock_client.describe_images
mock_describe_instances = mock_client.describe_instances
mock_describe_snapshot_attribute = \
mock_client.describe_snapshot_attribute
mock_describe_snapshots = mock_client.describe_snapshots
mock_modify_snapshot_attribute = \
mock_client.modify_snapshot_attribute
mock_modify_image_attribute = mock_client.modify_image_attribute
mock_describe_images.side_effect = ClientError(
mock_unauthorized_exception, 'DescribeImages')
mock_describe_instances.side_effect = ClientError(
mock_unauthorized_exception, 'DescribeInstances')
mock_describe_snapshot_attribute.side_effect = ClientError(
mock_unauthorized_exception, 'DescribeSnapshotAttribute')
mock_describe_snapshots.side_effect = ClientError(
mock_unauthorized_exception, 'DescribeSnapshots')
mock_modify_snapshot_attribute.side_effect = ClientError(
mock_unauthorized_exception, 'ModifySnapshotAttribute')
mock_modify_image_attribute.side_effect = ClientError(
mock_unauthorized_exception, 'ModifyImageAttribute')
session = aws.get_session(mock_arn)
actual_verified = aws.verify_account_access(session)
mock_describe_images.side_effect = ClientError(
mock_garbage_exception, 'DescribeImages')
with self.assertRaises(ClientError) as e:
aws.verify_account_access(session)
self.assertEqual(e.exception.response['Error']['Code'],
mock_garbage_exception['Error']['Code'])
self.assertEqual(e.exception.response['Error']['Message'],
mock_garbage_exception['Error']['Message'])
with patch.dict(aws.cloudigrade_policy, bad_policy):
aws.verify_account_access(session)
mock_describe_images.assert_called_with(DryRun=True)
mock_describe_instances.assert_called_with(DryRun=True)
mock_describe_snapshot_attribute.assert_called_with(
DryRun=True, SnapshotId='string', Attribute='productCodes')
mock_describe_snapshots.assert_called_with(DryRun=True)
mock_modify_snapshot_attribute.assert_called_with(
SnapshotId='string',
DryRun=True,
Attribute='productCodes',
GroupNames=[
'string',
])
mock_modify_image_attribute.assert_called_with(
Attribute='description', ImageId='string', DryRun=True)
self.assertFalse(actual_verified)
def delete_cloudtrail(aws_cloud_account):
"""
Delete an AwsCloudAccount's CloudTrail.
Note:
If the incoming AwsCloudAccount instance is being deleted, this call to
delete_cloudtrail may occur after the DB record has been deleted, and we are
only working with a shallow reference copy of the AwsCloudAccount. This means we
cannot reliably load related objects (e.g. aws_cloud_account.cloud_account).
Args:
aws_cloud_account (api.clouds.aws.models.AwsCloudAccount): the AwsCloudAccount
for which we should delete the CloudTrail
Returns:
bool True if CloudTrail was successfully deleted, else False.
"""
cloudtrail_name = aws.get_cloudtrail_name(
aws_cloud_account.cloud_account_id)
try:
session = aws.get_session(str(aws_cloud_account.account_arn))
cloudtrail_session = session.client("cloudtrail")
logger.info(
"attempting to delete cloudtrail '%(name)s' via ARN '%(arn)s'",
{
"name": cloudtrail_name,
"arn": aws_cloud_account.account_arn
},
)
aws.delete_cloudtrail(cloudtrail_session, cloudtrail_name)
return True
except ClientError as error:
error_code = error.response.get("Error", {}).get("Code")
if error_code == "TrailNotFoundException":
# If a cloudtrail does not exist, then we have nothing to do here!
return True
elif error_code in (
"AccessDenied",
"AccessDeniedException",
"UnrecognizedClientException",
):
# We may get AccessDenied if the user deletes the AWS account or role.
# We may get AccessDeniedException if the role or policy is broken.
# These could result in an orphaned cloudtrail writing to our s3 bucket.
logger.warning(
_("AwsCloudAccount ID %(aws_cloud_account_id)s for AWS account ID "
"%(aws_account_id)s encountered %(error_code)s and cannot "
"delete cloudtrail %(cloudtrail_name)s."),
{
"aws_cloud_account_id": aws_cloud_account.id,
"aws_account_id": aws_cloud_account.cloud_account_id,
"error_code": error_code,
"cloudtrail_name": cloudtrail_name,
},
)
logger.info(error)
else:
logger.exception(error)
logger.error(
_("Unexpected error %(error_code)s occurred disabling CloudTrail "
"%(cloudtrail_name)s for AwsCloudAccount ID "
"%(aws_cloud_account_id)s. "),
{
"error_code": error_code,
"cloudtrail_name": cloudtrail_name,
"aws_cloud_account_id": aws_cloud_account.id,
},
)
return False