def test_start_image_inspection_cloud_access_skips(self, mock_copy):
"""Test that inspection skips for Cloud Access images."""
image = api_helper.generate_image(is_cloud_access=True)
util.start_image_inspection(None, image.content_object.ec2_ami_id,
None)
mock_copy.delay.assert_not_called()
image.refresh_from_db()
self.assertEqual(image.status, image.INSPECTED)
def test_start_image_inspection_rhel_tagged_skips(self, mock_copy):
"""Test that inspection skips for RHEL-tagged images."""
image = api_helper.generate_image(rhel_detected_by_tag=True)
util.start_image_inspection(None, image.content_object.ec2_ami_id,
None)
mock_copy.delay.assert_not_called()
image.refresh_from_db()
self.assertEqual(image.status, image.INSPECTED)
self.assertTrue(image.rhel_detected_by_tag)
def test_start_image_inspection_exceed_max_allowed(self, mock_copy):
"""Test that inspection stops when max allowed attempts is exceeded."""
image = api_helper.generate_image()
for _ in range(0, settings.MAX_ALLOWED_INSPECTION_ATTEMPTS + 1):
MachineImageInspectionStart.objects.create(machineimage=image)
util.start_image_inspection(None, image.content_object.ec2_ami_id,
None)
mock_copy.delay.assert_not_called()
image.refresh_from_db()
self.assertEqual(image.status, image.ERROR)
def test_start_image_inspection_runs(self, mock_copy):
"""Test that inspection skips for marketplace images."""
image = api_helper.generate_image()
mock_arn = Mock()
mock_region = Mock()
util.start_image_inspection(mock_arn, image.content_object.ec2_ami_id,
mock_region)
mock_copy.delay.assert_called_with(mock_arn,
image.content_object.ec2_ami_id,
mock_region)
image.refresh_from_db()
self.assertEqual(image.status, image.PREPARING)
self.assertTrue(
MachineImageInspectionStart.objects.filter(
machineimage__id=image.id).exists())
def inspect_pending_images():
"""
(Re)start inspection of images in PENDING, PREPARING, or INSPECTING status.
This generally should not be necessary for most images, but if an image
inspection fails to proceed normally, this function will attempt to run it
through inspection again.
This function runs atomically in a transaction to protect against the risk
of it being called multiple times simultaneously which could result in the
same image being found and getting multiple inspection tasks.
"""
updated_since = get_now() - timedelta(
seconds=settings.INSPECT_PENDING_IMAGES_MIN_AGE)
restartable_statuses = [
MachineImage.PENDING,
MachineImage.PREPARING,
MachineImage.INSPECTING,
]
images = MachineImage.objects.filter(
status__in=restartable_statuses,
instance__aws_instance__region__isnull=False,
updated_at__lt=updated_since,
).distinct()
logger.info(
_("Found %(number)s images for inspection that have not updated "
"since %(updated_time)s"),
{
"number": images.count(),
"updated_time": updated_since
},
)
for image in images:
instance = image.instance_set.filter(
aws_instance__region__isnull=False).first()
arn = instance.cloud_account.content_object.account_arn
ami_id = image.content_object.ec2_ami_id
region = instance.content_object.region
start_image_inspection(arn, ami_id, region)
def _process_cloudtrail_message(message):
"""
Process a single CloudTrail log update's SQS message.
This may have the side-effect of starting the inspection process for newly
discovered images.
Args:
message (Message): the SQS Message object to process
Returns:
bool: True only if message processing completed without error.
"""
logs = []
extracted_messages = aws.extract_sqs_message(message)
# Get the S3 objects referenced by the SQS messages
for extracted_message in extracted_messages:
bucket = extracted_message["bucket"]["name"]
key = extracted_message["object"]["key"]
raw_content = aws.get_object_content_from_s3(bucket, key)
content = json.loads(raw_content)
logs.append((content, bucket, key))
logger.info(
_("Read CloudTrail log file from bucket %(bucket)s object key %(key)s"
),
{
"bucket": bucket,
"key": key
},
)
# Extract actionable details from each of the S3 log files
instance_events = []
ami_tag_events = []
for content, bucket, key in logs:
for record in content.get("Records", []):
instance_events.extend(extract_ec2_instance_events(record))
ami_tag_events.extend(extract_ami_tag_events(record))
# Get supporting details from AWS so we can save our models.
# Note: It's important that we do all AWS API loading calls here before
# saving anything to the database. We don't want to leave database write
# transactions open while waiting on external APIs.
described_instances = _load_missing_instance_data(instance_events)
described_amis = _load_missing_ami_data(instance_events, ami_tag_events)
try:
# Save the results
new_images = _save_cloudtrail_activity(
instance_events,
ami_tag_events,
described_instances,
described_amis,
)
# Starting image inspection MUST come after all other database writes
# so that we are confident the atomic transaction will complete.
for ami_id, awsimage in new_images.items():
# Is it even possible to get here when status is *not* PENDING?
# I don't think so, but just in case, we only want inspection to
# start if status == PENDING.
image = awsimage.machine_image.get()
if image.status == image.PENDING:
start_image_inspection(
described_amis[ami_id]["found_by_account_arn"],
ami_id,
described_amis[ami_id]["found_in_region"],
)
logger.debug(_("Saved instances and/or events to the DB."))
return True
except: # noqa: E722 because we don't know what could go wrong yet.
logger.exception(
_("Failed to save instances and/or events to the DB. "
"Instance events: %(instance_events)s AMI tag events: "
"%(ami_tag_events)s"),
{
"instance_events": instance_events,
"ami_tag_events": ami_tag_events
},
)
return False
def initial_aws_describe_instances(account_id):
"""
Fetch and save instances data found upon AWS cloud account creation.
Args:
account_id (int): the AwsAccount id
"""
try:
aws_account = AwsCloudAccount.objects.get(pk=account_id)
except AwsCloudAccount.DoesNotExist:
logger.warning(
_("AwsCloudAccount id %s could not be found for initial describe"),
account_id,
)
# This can happen if a customer creates and then quickly deletes their
# cloud account before this async task has started to run. Early exit!
return
account = aws_account.cloud_account.get()
if not account.is_enabled:
logger.warning(
_("AwsCloudAccount id %s is not enabled; skipping initial describe"
),
account_id,
)
# This can happen if a customer creates and then quickly disabled their
# cloud account before this async task has started to run. Early exit!
return
arn = aws_account.account_arn
session = aws.get_session(arn)
instances_data = aws.describe_instances_everywhere(session)
try:
user_id = account.user.id
except User.DoesNotExist:
logger.info(
_("User for account id %s has already been deleted; "
"skipping initial describe."),
account_id,
)
# This can happen if a customer creates and then quickly deletes their
# cloud account before this async task has started to run. If the user has
# no other cloud accounts the user will also be deleted. Early exit!
return
# Lock the task at a user level. A user can only run one task at a time.
with lock_task_for_user_ids([user_id]):
try:
# Explicitly "get" the related AwsCloudAccount before proceeding.
# We do this at the start of this transaction in case the account has been
# deleted during the potentially slow describe_instances_everywhere above.
# If this fails, we'll jump to the except block to log an important warning.
AwsCloudAccount.objects.get(pk=account_id)
create_missing_power_off_aws_instance_events(
account, instances_data)
new_ami_ids = create_new_machine_images(session, instances_data)
logger.info(
_("Created new machine images include: %(new_ami_ids)s"),
{"new_ami_ids": new_ami_ids},
)
create_initial_aws_instance_events(account, instances_data)
except AwsCloudAccount.DoesNotExist:
logger.warning(
_("AwsCloudAccount id %s could not be found to save newly "
"discovered images and instances"),
account_id,
)
# This can happen if a customer deleted their cloud account between
# the start of this function and here. The AWS calls for
# describe_instances_everywhere may be slow and are not within this
# transaction. That's why we have to check again after it.
return
messages = generate_aws_ami_messages(instances_data, new_ami_ids)
for message in messages:
start_image_inspection(str(arn), message["image_id"],
message["region"])