def copy_ami_snapshot(arn, ami_id, source_region): """ Copy an AWS Snapshot to the primary AWS account. Args: arn (str): The AWS Resource Number for the account with the snapshot ami_id (str): The AWS ID for the machine image source_region (str): The region the snapshot resides in Returns: None: Run as an asynchronous Celery task. """ session = aws.get_session(arn) ami = aws.get_ami(session, ami_id, source_region) snapshot_id = aws.get_ami_snapshot_id(ami) snapshot = aws.get_snapshot(session, snapshot_id, source_region) if snapshot.encrypted: image = AwsMachineImage.objects.get(ec2_ami_id=ami_id) image.is_encrypted = True image.save() raise AwsSnapshotEncryptedError aws.add_snapshot_ownership(snapshot) new_snapshot_id = aws.copy_snapshot(snapshot_id, source_region) create_volume.delay(ami_id, new_snapshot_id)
def test_get_running_instances(self): """Assert we get expected instances in a dict keyed by regions.""" mock_arn = helper.generate_dummy_arn() mock_regions = [f'region-{uuid.uuid4()}'] mock_role = helper.generate_dummy_role() mock_running_instance = helper.generate_dummy_describe_instance( state=aws.InstanceState.running) mock_stopped_instance = helper.generate_dummy_describe_instance( state=aws.InstanceState.stopped) mock_described = { 'Reservations': [ { 'Instances': [ mock_running_instance, mock_stopped_instance, ], }, ], } expected_found = {mock_regions[0]: [mock_running_instance]} with patch.object(aws, 'get_regions') as mock_get_regions, \ patch.object(aws, 'boto3') as mock_boto3: mock_assume_role = mock_boto3.client.return_value.assume_role mock_assume_role.return_value = mock_role mock_get_regions.return_value = mock_regions mock_client = mock_boto3.Session.return_value.client.return_value mock_client.describe_instances.return_value = mock_described actual_found = aws.get_running_instances(aws.get_session(mock_arn)) self.assertDictEqual(expected_found, actual_found)
def test_verify_account_access_success(self): """Assert that account access via a IAM role is verified.""" mock_arn = helper.generate_dummy_arn() mock_role = helper.generate_dummy_role() mock_dry_run_exception = { 'Error': { 'Code': 'DryRunOperation', 'Message': 'Request would have succeeded, ' 'but DryRun flag is set.' } } with patch.object(aws, 'boto3') as mock_boto3: mock_assume_role = mock_boto3.client.return_value.assume_role mock_assume_role.return_value = mock_role mock_client = mock_boto3.Session.return_value.client.return_value mock_describe_images = mock_client.describe_images mock_describe_instances = mock_client.describe_instances mock_describe_snapshot_attribute = \ mock_client.describe_snapshot_attribute mock_describe_snapshots = mock_client.describe_snapshots mock_modify_snapshot_attribute = \ mock_client.modify_snapshot_attribute mock_modify_image_attribute = mock_client.modify_image_attribute mock_describe_images.side_effect = ClientError( mock_dry_run_exception, 'DescribeImages') mock_describe_instances.side_effect = ClientError( mock_dry_run_exception, 'DescribeInstances') mock_describe_snapshot_attribute.side_effect = ClientError( mock_dry_run_exception, 'DescribeSnapshotAttribute') mock_describe_snapshots.side_effect = ClientError( mock_dry_run_exception, 'DescribeSnapshots') mock_modify_snapshot_attribute.side_effect = ClientError( mock_dry_run_exception, 'ModifySnapshotAttribute') mock_modify_image_attribute.side_effect = ClientError( mock_dry_run_exception, 'ModifyImageAttribute') actual_verified = aws.verify_account_access( aws.get_session(mock_arn)) mock_describe_images.assert_called_with(DryRun=True) mock_describe_instances.assert_called_with(DryRun=True) mock_describe_snapshot_attribute.assert_called_with( DryRun=True, SnapshotId='string', Attribute='productCodes') mock_describe_snapshots.assert_called_with(DryRun=True) mock_modify_snapshot_attribute.assert_called_with( SnapshotId='string', DryRun=True, Attribute='productCodes', GroupNames=[ 'string', ]) mock_modify_image_attribute.assert_called_with( Attribute='description', ImageId='string', DryRun=True) self.assertTrue(actual_verified)
def _parse_log_for_ec2_instance_events(log): """ Parse S3 log for EC2 on/off events. Args: log (str): The string contents of the log file. Returns: list(tuple): List of instance data seen in log. list(tuple): List of instance_event data seen in log. """ instances = {} log = json.loads(log) # Each record is a single API call, but each call can # be made against multiple EC2 instances for record in log.get('Records', []): if not _is_valid_event(record, ec2_instance_event_map.keys()): continue account_id = record.get('userIdentity', {}).get('accountId') account = AwsAccount.objects.get(aws_account_id=account_id) region = record.get('awsRegion') session = aws.get_session(account.account_arn, region) event_type = ec2_instance_event_map[record.get('eventName')] occurred_at = record.get('eventTime') ec2_info = record.get('responseElements', {})\ .get('instancesSet', {})\ .get('items', []) # Collect the EC2 instances the API was called on for item in ec2_info: instance_id = item.get('instanceId') instance = aws.get_ec2_instance(session, instance_id) instances[instance_id] = { 'account_id': account_id, 'instance_details': instance, 'region': region } for __, data in instances.items(): event = { 'subnet': data['instance_details'].subnet_id, 'ec2_ami_id': data['instance_details'].image_id, 'instance_type': data['instance_details'].instance_type, 'event_type': event_type, 'occurred_at': occurred_at } if data.get('events'): data['events'].append(event) else: data['events'] = [event] return dict(instances)
def test_get_ec2_instance(self): """Assert that get_ec2_instance returns an instance object.""" mock_arn = helper.generate_dummy_arn() mock_instance_id = str(uuid.uuid4()) mock_instance = helper.generate_mock_ec2_instance(mock_instance_id) with patch.object(aws, 'boto3') as mock_boto3: mock_session = mock_boto3.Session.return_value resource = mock_session.resource.return_value resource.Instance.return_value = mock_instance actual_instance = aws.get_ec2_instance(aws.get_session(mock_arn), mock_instance_id) self.assertEqual(actual_instance, mock_instance)
def create(self, validated_data): """Create an AwsAccount.""" arn = validated_data['account_arn'] aws_account_id = aws.extract_account_id_from_arn(arn) account = AwsAccount(account_arn=arn, aws_account_id=aws_account_id) session = aws.get_session(arn) if aws.verify_account_access(session): instances_data = aws.get_running_instances(session) with transaction.atomic(): account.save() create_initial_aws_instance_events(account, instances_data) else: raise serializers.ValidationError( _('AwsAccount verification failed. ARN Info Not Stored')) return account
def copy_ami_to_customer_account(arn, reference_ami_id, snapshot_region, maybe_marketplace=False): """ Copy an AWS Image to the customer's AWS account. This is an intermediate step that we occasionally need to use when the customer has an instance based on a image that has been privately shared by a third party, and that means we cannot directly copy its snapshot. We can, however, create a copy of the image in the customer's account and use that copy for the remainder of the inspection process. Args: arn (str): The AWS Resource Number for the account with the snapshot reference_ami_id (str): The AWS ID for the original image to copy snapshot_region (str): The region the snapshot resides in maybe_marketplace (bool): Set to True if we suspect this to be a marketplace image. Returns: None: Run as an asynchronous Celery task. """ session = aws.get_session(arn) reference_ami = aws.get_ami(session, reference_ami_id, snapshot_region) try: new_ami_id = aws.copy_ami(session, reference_ami.id, snapshot_region) except ClientError as e: if maybe_marketplace \ and e.response.get('Error').get('Code') == 'InvalidRequest' \ and 'Images with EC2 BillingProduct codes cannot be copied ' \ 'to another AWS account' in \ e.response.get('Error').get('Message'): # This appears to be a marketplace AMI, mark it as inspected. logger.info( _('Found a marketplace image "{0}", marking as ' 'inspected').format(reference_ami_id)) ami = AwsMachineImage.objects.get(ec2_ami_id=reference_ami_id) ami.status = ami.INSPECTED ami.save() return raise e copy_ami_snapshot.delay(arn, new_ami_id, snapshot_region, reference_ami_id)
def create(self, validated_data): """Create an AwsAccount.""" arn = aws.AwsArn(validated_data['account_arn']) aws_account_id = arn.account_id user = self.context['request'].user account = AwsAccount( account_arn=str(arn), aws_account_id=aws_account_id, user=user, ) try: session = aws.get_session(str(arn)) except ClientError as error: if error.response.get('Error', {}).get('Code') == 'AccessDenied': raise serializers.ValidationError( detail={ 'account_arn': [_('Permission denied for ARN "{0}"').format(arn)] }) raise account_verified, failed_actions = aws.verify_account_access(session) if account_verified: instances_data = aws.get_running_instances(session) with transaction.atomic(): account.save() new_amis = create_new_machine_images(account, instances_data) create_initial_aws_instance_events(account, instances_data) messages = generate_aws_ami_messages(instances_data, new_amis) for message in messages: copy_ami_snapshot.delay(str(arn), message['image_id'], message['region']) else: failure_details = [_('Account verification failed.')] failure_details += [ _('Access denied for policy action "{0}".').format(action) for action in failed_actions ] raise serializers.ValidationError( detail={'account_arn': failure_details}) return account
def test_get_session(self): """Assert get_session returns session object.""" mock_arn = helper.generate_dummy_arn() mock_account_id = extract_account_id_from_arn(mock_arn) mock_role = helper.generate_dummy_role() with patch.object(aws.boto3, 'client') as mock_client: mock_assume_role = mock_client.return_value.assume_role mock_assume_role.return_value = mock_role session = aws.get_session(mock_arn) creds = session.get_credentials().get_frozen_credentials() mock_client.assert_called_with('sts') mock_assume_role.assert_called_with( Policy=json.dumps(aws.cloudigrade_policy), RoleArn=mock_arn, RoleSessionName=f'cloudigrade-{mock_account_id}') self.assertEqual(creds[0], mock_role['Credentials']['AccessKeyId']) self.assertEqual(creds[1], mock_role['Credentials']['SecretAccessKey']) self.assertEqual(creds[2], mock_role['Credentials']['SessionToken'])
def remove_snapshot_ownership(arn, customer_snapshot_id, customer_snapshot_region, snapshot_copy_id): """ Remove cloudigrade ownership from customer snapshot. Args: arn (str): The AWS Resource Number for the account with the snapshot customer_snapshot_id (str): The id of the snapshot to remove ownership customer_snapshot_region (str): The region where customer_snapshot_id resides snapshot_copy_id (str): The id of the snapshot that must be ready to continue Returns: None: Run as an asynchronous Celery task. """ ec2 = boto3.resource('ec2') # Wait for snapshot to be ready try: snapshot_copy = ec2.Snapshot(snapshot_copy_id) aws.check_snapshot_state(snapshot_copy) except ClientError as error: if error.response.get('Error', {}).get('Code') == 'InvalidSnapshot.NotFound': logger.info( _('{0} detected snapshot_copy_id {1} already deleted.').format( 'remove_snapshot_ownership', snapshot_copy_id)) else: raise # Remove permissions from customer_snapshot logger.info( _('{0} remove ownership from customer snapshot {1}').format( 'remove_snapshot_ownership', customer_snapshot_id)) session = aws.get_session(arn) customer_snapshot = aws.get_snapshot(session, customer_snapshot_id, customer_snapshot_region) aws.remove_snapshot_ownership(customer_snapshot)
def _load_missing_ami_data(instance_events, ami_tag_events): """ Load additional data so we can create the AMIs for the given events. Args: instance_events (list[CloudTrailInstanceEvent]): found instance events ami_tag_events (list[CloudTrailImageTagEvent]): found AMI tag events Returns: dict: Dict of dicts from AWS describing AMIs that are referenced by the input arguments but are not present in our database, with the outer key being each AMI's ID. """ seen_ami_ids = set([ event.ec2_ami_id for event in instance_events + ami_tag_events if event.ec2_ami_id is not None ]) known_images = AwsMachineImage.objects.filter(ec2_ami_id__in=seen_ami_ids) known_ami_ids = set([image.ec2_ami_id for image in known_images]) new_ami_ids = seen_ami_ids.difference(known_ami_ids) new_amis_keyed = set([(event.aws_account_id, event.region, event.ec2_ami_id) for event in instance_events + ami_tag_events if event.ec2_ami_id in new_ami_ids]) described_amis = dict() # Look up only the new AMIs that belong to each account+region group. for (aws_account_id, region), amis_keyed in itertools.groupby(new_amis_keyed, key=lambda a: (a[0], a[1])): amis_keyed = list(amis_keyed) awsaccount = AwsCloudAccount.objects.get(aws_account_id=aws_account_id) session = aws.get_session(awsaccount.account_arn, region) ami_ids = [k[2] for k in amis_keyed] # Get all relevant images in one API call for this account+region. new_described_amis = aws.describe_images(session, ami_ids, region) for described_ami in new_described_amis: ami_id = described_ami["ImageId"] logger.info( _("Loading data for AMI %(ami_id)s for " "ARN %(account_arn)s in region %(region)s"), { "ami_id": ami_id, "account_arn": awsaccount.account_arn, "region": region, }, ) described_ami["found_in_region"] = region described_ami["found_by_account_arn"] = awsaccount.account_arn described_amis[ami_id] = described_ami for aws_account_id, region, ec2_ami_id in new_amis_keyed: if ec2_ami_id not in described_amis: logger.info( _("AMI %(ec2_ami_id)s could not be found in region " "%(region)s for AWS account %(aws_account_id)s."), { "ec2_ami_id": ec2_ami_id, "region": region, "aws_account_id": aws_account_id, }, ) return described_amis
def _load_missing_instance_data(instance_events): # noqa: C901 """ Load additional data so we can create instances from Cloud Trail events. We only get the necessary instance type and AMI ID from AWS Cloud Trail for the "RunInstances" event upon first creation of an instance. If we didn't get that (for example, if an instance already existed but was stopped when we got the cloud account), that means we may not know about the instance and need to describe it before we create our record of it. However, there is an edge-case possibility when AWS gives us events out of order. If we receive an instance event *before* its "RunInstances" that would fully describe it, we have to describe it now even though the later event should eventually give us that same information. There's a super edge case here that means the AWS user could have also changed the type before we receive that initial "RunInstances" event, but we are not explicitly handling that scenario as of this writing. Args: instance_events (list[CloudTrailInstanceEvent]): found instance events Side-effect: instance_events input argument may have been updated with missing image_id values from AWS. Returns: dict: dict of dicts from AWS describing EC2 instances that are referenced by the input arguments but are not present in our database, with the outer key being each EC2 instance's ID. """ all_ec2_instance_ids = set( [instance_event.ec2_instance_id for instance_event in instance_events]) described_instances = dict() defined_ec2_instance_ids = set() # First identify which instances DON'T need to be described because we # either already have them stored or at least one of instance_events has # enough information for it. for instance_event in instance_events: ec2_instance_id = instance_event.ec2_instance_id if (_instance_event_is_complete(instance_event) or ec2_instance_id in defined_ec2_instance_ids): # This means the incoming data is sufficiently populated so we # should know the instance's image and type. defined_ec2_instance_ids.add(ec2_instance_id) elif (AwsInstance.objects.filter( ec2_instance_id=instance_event.ec2_instance_id, instance__machine_image__isnull=False, ).exists() and InstanceEvent.objects.filter( instance__aws_instance__ec2_instance_id=ec2_instance_id, aws_instance_event__instance_type__isnull=False, ).exists()): # This means we already know the instance's image and at least once # we have known the instance's type from an event. defined_ec2_instance_ids.add(ec2_instance_id) # Iterate through the instance events grouped by account and region in # order to minimize the number of sessions and AWS API calls. for (aws_account_id, region), grouped_instance_events in itertools.groupby( instance_events, key=lambda e: (e.aws_account_id, e.region)): grouped_instance_events = list(grouped_instance_events) # Find the set of EC2 instance IDs that belong to this account+region. ec2_instance_ids = set([ e.ec2_instance_id for e in grouped_instance_events ]).difference(defined_ec2_instance_ids) if not ec2_instance_ids: # Early continue if there are no instances we need to describe! continue awsaccount = AwsCloudAccount.objects.get(aws_account_id=aws_account_id) session = aws.get_session(awsaccount.account_arn, region) # Get all relevant instances in one API call for this account+region. new_described_instances = aws.describe_instances( session, ec2_instance_ids, region) # How we found these instances will be important to save *later*. # This wouldn't be necessary if we could save these here, but we don't # want to mix DB transactions with external AWS API calls. for (ec2_instance_id, described_instance) in new_described_instances.items(): logger.info( _("Loading data for EC2 Instance %(ec2_instance_id)s for " "ARN %(account_arn)s in region %(region)s"), { "ec2_instance_id": ec2_instance_id, "account_arn": awsaccount.account_arn, "region": region, }, ) described_instance["found_by_account_arn"] = awsaccount.account_arn described_instance["found_in_region"] = region described_instances[ec2_instance_id] = described_instance # Add any missing image IDs to the instance_events from the describes. for instance_event in instance_events: ec2_instance_id = instance_event.ec2_instance_id if instance_event.ec2_ami_id is None and ec2_instance_id in described_instances: described_instance = described_instances[ec2_instance_id] instance_event.ec2_ami_id = described_instance["ImageId"] # We really *should* have what we need, but just in case... for ec2_instance_id in all_ec2_instance_ids: if (ec2_instance_id not in defined_ec2_instance_ids and ec2_instance_id not in described_instances): logger.info( _("EC2 Instance %(ec2_instance_id)s could not be loaded " "from database or AWS. It may have been terminated before " "we processed it."), {"ec2_instance_id": ec2_instance_id}, ) return described_instances
def create(self, validated_data): """Create an AwsAccount.""" arn = aws.AwsArn(validated_data['account_arn']) aws_account_id = arn.account_id account = AwsAccount.objects.filter( aws_account_id=aws_account_id).first() if account is not None: raise serializers.ValidationError( detail={ 'account_arn': [ _('An ARN already exists for account "{0}"').format( aws_account_id) ] }) user = self.context['request'].user name = validated_data.get('name') account = AwsAccount( account_arn=str(arn), aws_account_id=aws_account_id, name=name, user=user, ) try: session = aws.get_session(str(arn)) except ClientError as error: if error.response.get('Error', {}).get('Code') == 'AccessDenied': raise serializers.ValidationError( detail={ 'account_arn': [_('Permission denied for ARN "{0}"').format(arn)] }) raise account_verified, failed_actions = aws.verify_account_access(session) if account_verified: instances_data = aws.get_running_instances(session) with transaction.atomic(): account.save() try: aws.configure_cloudtrail(session, aws_account_id) except ClientError as error: if error.response.get('Error', {}).get('Code') == \ 'AccessDeniedException': raise serializers.ValidationError( detail={ 'account_arn': [ _('Access denied to create CloudTrail for ' 'ARN "{0}"').format(arn) ] }) raise new_amis = create_new_machine_images(account, instances_data) create_initial_aws_instance_events(account, instances_data) messages = generate_aws_ami_messages(instances_data, new_amis) for message in messages: image = start_image_inspection(str(arn), message['image_id'], message['region']) self.add_openshift_tag(session, message['image_id'], message['region'], image) else: failure_details = [_('Account verification failed.')] failure_details += [ _('Access denied for policy action "{0}".').format(action) for action in failed_actions ] raise serializers.ValidationError( detail={'account_arn': failure_details}) return account
def copy_ami_snapshot(arn, ami_id, snapshot_region, reference_ami_id=None): """ Copy an AWS Snapshot to the primary AWS account. Args: arn (str): The AWS Resource Number for the account with the snapshot ami_id (str): The AWS ID for the machine image snapshot_region (str): The region the snapshot resides in reference_ami_id (str): Optional. The id of the original image from which this image was copied. We need to know this in some cases where we create a copy of the image in the customer's account before we can copy its snapshot, and we must pass this information forward for appropriate reference and cleanup. Returns: None: Run as an asynchronous Celery task. """ session = aws.get_session(arn) session_account_id = aws.get_session_account_id(session) ami = aws.get_ami(session, ami_id, snapshot_region) customer_snapshot_id = aws.get_ami_snapshot_id(ami) try: customer_snapshot = aws.get_snapshot(session, customer_snapshot_id, snapshot_region) if customer_snapshot.encrypted: image = AwsMachineImage.objects.get(ec2_ami_id=ami_id) image.is_encrypted = True image.save() raise AwsSnapshotEncryptedError logger.info( _('AWS snapshot "{snapshot_id}" for image "{image_id}" has owner ' '"{owner_id}"; current session is account "{account_id}"'). format(snapshot_id=customer_snapshot.snapshot_id, image_id=ami.id, owner_id=customer_snapshot.owner_id, account_id=session_account_id)) if customer_snapshot.owner_id != session_account_id and \ reference_ami_id is None: copy_ami_to_customer_account.delay(arn, ami_id, snapshot_region) # Early return because we need to stop processing the current AMI. # A future call will process this new copy of the # current AMI instead. return except ClientError as e: if e.response.get('Error').get('Code') == 'InvalidSnapshot.NotFound': # Possibly a marketplace AMI, try to handle it by copying. copy_ami_to_customer_account.delay(arn, ami_id, snapshot_region, maybe_marketplace=True) return raise e aws.add_snapshot_ownership(customer_snapshot) snapshot_copy_id = aws.copy_snapshot(customer_snapshot_id, snapshot_region) logger.info( _('{0}: customer_snapshot_id={1}, snapshot_copy_id={2}').format( 'copy_ami_snapshot', customer_snapshot_id, snapshot_copy_id)) # Schedule removal of ownership on customer snapshot remove_snapshot_ownership.delay(arn, customer_snapshot_id, snapshot_region, snapshot_copy_id) if reference_ami_id is not None: # If a reference ami exists, that means we have been working with a # copy in here. That means we need to remove that copy and pass the # original reference AMI ID through the rest of the task chain so the # results get reported for that original reference AMI, not our copy. # TODO FIXME Do we or don't we clean up? # If we do, we need permissions to include `ec2:DeregisterImage` and # `ec2:DeleteSnapshot` but those are both somewhat scary... # # For now, since we are not deleting the copy image from AWS, we need # to record a reference to our database that we can look at later to # indicate the relationship between the original AMI and the copy. create_aws_machine_image_copy(ami_id, reference_ami_id) ami_id = reference_ami_id # Create volume from snapshot copy create_volume.delay(ami_id, snapshot_copy_id)
def verify_permissions(customer_role_arn): """ Verify AWS permissions. This function may raise ValidationError if certain verification steps fail. Args: customer_role_arn (str): ARN to access the customer's AWS account Note: This function not only verifies; it also has the side effect of configuring the AWS CloudTrail. This should be refactored into a more explicit operation, but at the time of this writing, there is no "dry run" check for CloudTrail operations. Callers should be aware of the risk that we may configure CloudTrail but somewhere else rollback our transaction, leaving that Trail orphaned. Returns: boolean indicating if the arn being verified is good. """ aws_account_id = aws.AwsArn(customer_role_arn).account_id arn_str = str(customer_role_arn) try: session = aws.get_session(arn_str) account_verified, failed_actions = aws.verify_account_access(session) except ClientError as error: if error.response.get("Error", {}).get("Code") in ( "AccessDenied", "AccessDeniedException", "UnrecognizedClientException", ): raise ValidationError( detail={ "account_arn": [_('Permission denied for ARN "{0}"').format(arn_str)] }) raise if account_verified: try: aws.configure_cloudtrail(session, aws_account_id) except ClientError as error: if error.response.get("Error", {}).get("Code") in ( "AccessDenied", "AccessDeniedException", "UnrecognizedClientException", ): logger.debug(_("Trying to throw a CG3000.")) cloud_account = CloudAccount.objects.get( aws_cloud_account__aws_account_id=aws_account_id) error_code = error_codes.CG3000 error_code.log_internal_message(logger, { "cloud_account_id": aws_account_id, "exception": error }) error_code.notify(cloud_account.user.username, cloud_account.platform_application_id) logger.debug( _("CG3000 notify called, raising ValidationError.")) raise ValidationError( detail={ "account_arn": [ _("Access denied to create CloudTrail for " 'ARN "{0}"').format(arn_str) ] }) raise except MaximumNumberOfTrailsExceededException as error: logger.debug(_("Trying to throw a CG3001.")) cloud_account = CloudAccount.objects.get( aws_cloud_account__account_arn=arn_str) error_code = error_codes.CG3001 error_code.log_internal_message(logger, { "cloud_account_id": cloud_account.id, "exception": error }) error_code.notify(cloud_account.user.username, cloud_account.platform_application_id) logger.debug(_("CG3001 notify called, raising ValidationError.")) raise ValidationError( detail={"account_arn": error_code.get_message()}) else: failure_details = [_("Account verification failed.")] failure_details += [ _('Access denied for policy action "{0}".').format(action) for action in failed_actions ] raise ValidationError(detail={"account_arn": failure_details}) return account_verified
def initial_aws_describe_instances(account_id): """ Fetch and save instances data found upon AWS cloud account creation. Args: account_id (int): the AwsAccount id """ try: aws_account = AwsCloudAccount.objects.get(pk=account_id) except AwsCloudAccount.DoesNotExist: logger.warning( _("AwsCloudAccount id %s could not be found for initial describe"), account_id, ) # This can happen if a customer creates and then quickly deletes their # cloud account before this async task has started to run. Early exit! return account = aws_account.cloud_account.get() if not account.is_enabled: logger.warning( _("AwsCloudAccount id %s is not enabled; skipping initial describe" ), account_id, ) # This can happen if a customer creates and then quickly disabled their # cloud account before this async task has started to run. Early exit! return arn = aws_account.account_arn session = aws.get_session(arn) instances_data = aws.describe_instances_everywhere(session) try: user_id = account.user.id except User.DoesNotExist: logger.info( _("User for account id %s has already been deleted; " "skipping initial describe."), account_id, ) # This can happen if a customer creates and then quickly deletes their # cloud account before this async task has started to run. If the user has # no other cloud accounts the user will also be deleted. Early exit! return # Lock the task at a user level. A user can only run one task at a time. with lock_task_for_user_ids([user_id]): try: # Explicitly "get" the related AwsCloudAccount before proceeding. # We do this at the start of this transaction in case the account has been # deleted during the potentially slow describe_instances_everywhere above. # If this fails, we'll jump to the except block to log an important warning. AwsCloudAccount.objects.get(pk=account_id) create_missing_power_off_aws_instance_events( account, instances_data) new_ami_ids = create_new_machine_images(session, instances_data) logger.info( _("Created new machine images include: %(new_ami_ids)s"), {"new_ami_ids": new_ami_ids}, ) create_initial_aws_instance_events(account, instances_data) except AwsCloudAccount.DoesNotExist: logger.warning( _("AwsCloudAccount id %s could not be found to save newly " "discovered images and instances"), account_id, ) # This can happen if a customer deleted their cloud account between # the start of this function and here. The AWS calls for # describe_instances_everywhere may be slow and are not within this # transaction. That's why we have to check again after it. return messages = generate_aws_ami_messages(instances_data, new_ami_ids) for message in messages: start_image_inspection(str(arn), message["image_id"], message["region"])
def test_verify_account_access_failure(self): """Assert that account access via a IAM role is not verified.""" mock_arn = helper.generate_dummy_arn() mock_role = helper.generate_dummy_role() mock_unauthorized_exception = { 'Error': { 'Code': 'UnauthorizedOperation', 'Message': 'You are not authorized ' 'to perform this operation.' } } mock_garbage_exception = { 'Error': { 'Code': 'GarbageOperation', 'Message': 'You are not authorized ' 'to perform this garbage operation.' } } bad_policy = { 'Version': '2012-10-17', 'Statement': [{ 'Sid': 'CloudigradeGarbagePolicy', 'Effect': 'Allow', 'Action': ['ec2:DescribeGarbage'], 'Resource': '*' }] } with patch.object(aws, 'boto3') as mock_boto3: mock_assume_role = mock_boto3.client.return_value.assume_role mock_assume_role.return_value = mock_role mock_client = mock_boto3.Session.return_value.client.return_value mock_describe_images = mock_client.describe_images mock_describe_instances = mock_client.describe_instances mock_describe_snapshot_attribute = \ mock_client.describe_snapshot_attribute mock_describe_snapshots = mock_client.describe_snapshots mock_modify_snapshot_attribute = \ mock_client.modify_snapshot_attribute mock_modify_image_attribute = mock_client.modify_image_attribute mock_describe_images.side_effect = ClientError( mock_unauthorized_exception, 'DescribeImages') mock_describe_instances.side_effect = ClientError( mock_unauthorized_exception, 'DescribeInstances') mock_describe_snapshot_attribute.side_effect = ClientError( mock_unauthorized_exception, 'DescribeSnapshotAttribute') mock_describe_snapshots.side_effect = ClientError( mock_unauthorized_exception, 'DescribeSnapshots') mock_modify_snapshot_attribute.side_effect = ClientError( mock_unauthorized_exception, 'ModifySnapshotAttribute') mock_modify_image_attribute.side_effect = ClientError( mock_unauthorized_exception, 'ModifyImageAttribute') session = aws.get_session(mock_arn) actual_verified = aws.verify_account_access(session) mock_describe_images.side_effect = ClientError( mock_garbage_exception, 'DescribeImages') with self.assertRaises(ClientError) as e: aws.verify_account_access(session) self.assertEqual(e.exception.response['Error']['Code'], mock_garbage_exception['Error']['Code']) self.assertEqual(e.exception.response['Error']['Message'], mock_garbage_exception['Error']['Message']) with patch.dict(aws.cloudigrade_policy, bad_policy): aws.verify_account_access(session) mock_describe_images.assert_called_with(DryRun=True) mock_describe_instances.assert_called_with(DryRun=True) mock_describe_snapshot_attribute.assert_called_with( DryRun=True, SnapshotId='string', Attribute='productCodes') mock_describe_snapshots.assert_called_with(DryRun=True) mock_modify_snapshot_attribute.assert_called_with( SnapshotId='string', DryRun=True, Attribute='productCodes', GroupNames=[ 'string', ]) mock_modify_image_attribute.assert_called_with( Attribute='description', ImageId='string', DryRun=True) self.assertFalse(actual_verified)
def delete_cloudtrail(aws_cloud_account): """ Delete an AwsCloudAccount's CloudTrail. Note: If the incoming AwsCloudAccount instance is being deleted, this call to delete_cloudtrail may occur after the DB record has been deleted, and we are only working with a shallow reference copy of the AwsCloudAccount. This means we cannot reliably load related objects (e.g. aws_cloud_account.cloud_account). Args: aws_cloud_account (api.clouds.aws.models.AwsCloudAccount): the AwsCloudAccount for which we should delete the CloudTrail Returns: bool True if CloudTrail was successfully deleted, else False. """ cloudtrail_name = aws.get_cloudtrail_name( aws_cloud_account.cloud_account_id) try: session = aws.get_session(str(aws_cloud_account.account_arn)) cloudtrail_session = session.client("cloudtrail") logger.info( "attempting to delete cloudtrail '%(name)s' via ARN '%(arn)s'", { "name": cloudtrail_name, "arn": aws_cloud_account.account_arn }, ) aws.delete_cloudtrail(cloudtrail_session, cloudtrail_name) return True except ClientError as error: error_code = error.response.get("Error", {}).get("Code") if error_code == "TrailNotFoundException": # If a cloudtrail does not exist, then we have nothing to do here! return True elif error_code in ( "AccessDenied", "AccessDeniedException", "UnrecognizedClientException", ): # We may get AccessDenied if the user deletes the AWS account or role. # We may get AccessDeniedException if the role or policy is broken. # These could result in an orphaned cloudtrail writing to our s3 bucket. logger.warning( _("AwsCloudAccount ID %(aws_cloud_account_id)s for AWS account ID " "%(aws_account_id)s encountered %(error_code)s and cannot " "delete cloudtrail %(cloudtrail_name)s."), { "aws_cloud_account_id": aws_cloud_account.id, "aws_account_id": aws_cloud_account.cloud_account_id, "error_code": error_code, "cloudtrail_name": cloudtrail_name, }, ) logger.info(error) else: logger.exception(error) logger.error( _("Unexpected error %(error_code)s occurred disabling CloudTrail " "%(cloudtrail_name)s for AwsCloudAccount ID " "%(aws_cloud_account_id)s. "), { "error_code": error_code, "cloudtrail_name": cloudtrail_name, "aws_cloud_account_id": aws_cloud_account.id, }, ) return False