def analyze_log():
"""Read SQS Queue for log location, and parse log for events."""
queue_url = settings.CLOUDTRAIL_EVENT_URL
logs = []
extracted_messages = []
instances = {}
# Get messages off of an SQS queue
messages = aws.receive_message_from_queue(queue_url)
# Parse the SQS messages to get S3 object locations
for message in messages:
extracted_messages.extend(aws.extract_sqs_message(message))
# Grab the object contents from S3
for extracted_message in extracted_messages:
bucket = extracted_message['bucket']['name']
key = extracted_message['object']['key']
logs.append(aws.get_object_content_from_s3(bucket, key))
# Parse logs for on/off events
for log in logs:
if log:
instances = _parse_log_for_ec2_instance_events(log)
_parse_log_for_ami_tag_events(log)
if instances:
_save_results(instances)
logger.debug(_('Saved instances and/or events to the DB.'))
else:
logger.debug(_('No instances or events to save to the DB.'))
aws.delete_message_from_queue(queue_url, messages)
def test_get_object_content_from_s3_gzipped(self):
"""Assert that gzipped content is handled."""
mock_bucket = 'test_bucket'
mock_key = '/path/to/file'
mock_content_bytes = b'{"Key": "Value"}'
mock_byte_stream = io.BytesIO(gzip.compress(mock_content_bytes))
mock_object_body = {'Body': mock_byte_stream}
with patch.object(aws, 'boto3') as mock_boto3:
mock_resource = mock_boto3.resource.return_value
mock_s3_object = mock_resource.Object.return_value
mock_s3_object.get.return_value = mock_object_body
actual_content = aws.get_object_content_from_s3(
mock_bucket, mock_key)
self.assertEqual(mock_content_bytes.decode('utf-8'), actual_content)
def test_get_object_content_from_s3_unsupported_compression(self):
"""Assert that unhandled compression does not return content."""
mock_bucket = 'test_bucket'
mock_key = '/path/to/file'
mock_content_bytes = b'{"Key": "Value"}'
mock_byte_stream = io.BytesIO(mock_content_bytes)
mock_object_body = {'Body': mock_byte_stream}
with patch.object(aws, 'boto3') as mock_boto3:
mock_resource = mock_boto3.resource.return_value
mock_s3_object = mock_resource.Object.return_value
mock_s3_object.get.return_value = mock_object_body
actual_content = aws.get_object_content_from_s3(mock_bucket,
mock_key,
compression='bzip')
self.assertIsNone(actual_content)
def _process_cloudtrail_message(message):
"""
Process a single CloudTrail log update's SQS message.
This may have the side-effect of starting the inspection process for newly
discovered images.
Args:
message (Message): the SQS Message object to process
Returns:
bool: True only if message processing completed without error.
"""
logs = []
extracted_messages = aws.extract_sqs_message(message)
# Get the S3 objects referenced by the SQS messages
for extracted_message in extracted_messages:
bucket = extracted_message["bucket"]["name"]
key = extracted_message["object"]["key"]
raw_content = aws.get_object_content_from_s3(bucket, key)
content = json.loads(raw_content)
logs.append((content, bucket, key))
logger.info(
_("Read CloudTrail log file from bucket %(bucket)s object key %(key)s"
),
{
"bucket": bucket,
"key": key
},
)
# Extract actionable details from each of the S3 log files
instance_events = []
ami_tag_events = []
for content, bucket, key in logs:
for record in content.get("Records", []):
instance_events.extend(extract_ec2_instance_events(record))
ami_tag_events.extend(extract_ami_tag_events(record))
# Get supporting details from AWS so we can save our models.
# Note: It's important that we do all AWS API loading calls here before
# saving anything to the database. We don't want to leave database write
# transactions open while waiting on external APIs.
described_instances = _load_missing_instance_data(instance_events)
described_amis = _load_missing_ami_data(instance_events, ami_tag_events)
try:
# Save the results
new_images = _save_cloudtrail_activity(
instance_events,
ami_tag_events,
described_instances,
described_amis,
)
# Starting image inspection MUST come after all other database writes
# so that we are confident the atomic transaction will complete.
for ami_id, awsimage in new_images.items():
# Is it even possible to get here when status is *not* PENDING?
# I don't think so, but just in case, we only want inspection to
# start if status == PENDING.
image = awsimage.machine_image.get()
if image.status == image.PENDING:
start_image_inspection(
described_amis[ami_id]["found_by_account_arn"],
ami_id,
described_amis[ami_id]["found_in_region"],
)
logger.debug(_("Saved instances and/or events to the DB."))
return True
except: # noqa: E722 because we don't know what could go wrong yet.
logger.exception(
_("Failed to save instances and/or events to the DB. "
"Instance events: %(instance_events)s AMI tag events: "
"%(ami_tag_events)s"),
{
"instance_events": instance_events,
"ami_tag_events": ami_tag_events
},
)
return False