def test_delete_message_from_queue(self):
"""Assert that messages are deleted from SQS queue."""
mock_queue_url = 'https://123.abc'
mock_messages_to_delete = [
helper.generate_mock_sqs_message(str(uuid.uuid4()),
'',
str(uuid.uuid4())),
helper.generate_mock_sqs_message(str(uuid.uuid4()),
'',
str(uuid.uuid4()))
]
mock_response = {
'ResponseMetadata': {
'HTTPHeaders': {
'connection': 'keep-alive',
'content-length': '1358',
'content-type': 'text/xml',
'date': 'Mon, 19 Feb 2018 20:31:09 GMT',
'server': 'Server',
'x-amzn-requestid': '1234'
},
'HTTPStatusCode': 200,
'RequestId': '123456',
'RetryAttempts': 0
},
'Successful': [
{
'Id': 'fe3b9df2-416c-4ee2-a04e-7ba8b80490ca'
},
{
'Id': '3dc419e6-b841-48ad-ae4d-57da10a4315a'
}
]
}
with patch.object(sqs, 'boto3') as mock_boto3:
mock_resource = mock_boto3.resource.return_value
mock_queue = mock_resource.Queue.return_value
mock_queue.delete_messages.return_value = mock_response
actual_response = sqs.delete_message_from_queue(
mock_queue_url,
mock_messages_to_delete
)
self.assertEqual(mock_response, actual_response)
def test_delete_message_from_queue(self):
"""Assert that messages are deleted from SQS queue."""
mock_queue_url = "https://123.abc"
mock_messages_to_delete = [
helper.generate_mock_sqs_message(str(uuid.uuid4()), "", str(uuid.uuid4())),
helper.generate_mock_sqs_message(str(uuid.uuid4()), "", str(uuid.uuid4())),
]
mock_response = {
"ResponseMetadata": {
"HTTPHeaders": {
"connection": "keep-alive",
"content-length": "1358",
"content-type": "text/xml",
"date": "Mon, 19 Feb 2018 20:31:09 GMT",
"server": "Server",
"x-amzn-requestid": "1234",
},
"HTTPStatusCode": 200,
"RequestId": "123456",
"RetryAttempts": 0,
},
"Successful": [
{"Id": "fe3b9df2-416c-4ee2-a04e-7ba8b80490ca"},
{"Id": "3dc419e6-b841-48ad-ae4d-57da10a4315a"},
],
}
with patch.object(sqs, "boto3") as mock_boto3:
mock_resource = mock_boto3.resource.return_value
mock_queue = mock_resource.Queue.return_value
mock_queue.delete_messages.return_value = mock_response
actual_response = sqs.delete_messages_from_queue(
mock_queue_url, mock_messages_to_delete
)
self.assertEqual(mock_response, actual_response)
def test_persist_inspect_results_aws_cloud_no_images(
self, mock_scale_down, _, mock_receive, mock_delete):
"""Assert message is not deleted if it is missing images."""
receipt_handle = str(uuid.uuid4())
message_id = str(uuid.uuid4())
body_dict = {"cloud": "aws"}
sqs_message = util_helper.generate_mock_sqs_message(
message_id, json.dumps(body_dict), receipt_handle)
mock_receive.return_value = [sqs_message]
s, f = tasks.persist_inspection_cluster_results_task()
mock_delete.assert_not_called()
mock_scale_down.delay.assert_called_once()
self.assertEqual([], s)
self.assertIn(sqs_message, f)
def test_command_output_non_on_off_events(self, mock_receive, mock_s3,
mock_del):
"""Test that non on/off events are not processed."""
mock_instance_id = util_helper.generate_dummy_instance_id()
mock_queue_url = 'https://sqs.queue.url'
mock_receipt_handle = str(uuid.uuid4())
mock_sqs_message_body = {
'Records': [{
's3': {
'bucket': {
'name': 'test-bucket',
},
'object': {
'key': 'path/to/log',
},
},
}]
}
mock_message = util_helper.generate_mock_sqs_message(
mock_queue_url, json.dumps(mock_sqs_message_body),
mock_receipt_handle)
mock_cloudtrail_log = {
'Records': [{
'eventSource': 'null.amazonaws.com',
}, {
'errorCode': 123,
}, {
'eventName': 'InvalidEvent'
}]
}
mock_receive.return_value = [mock_message]
mock_s3.return_value = json.dumps(mock_cloudtrail_log)
mock_del.return_value = 'Success'
tasks.analyze_log()
instances = list(
AwsInstance.objects.filter(ec2_instance_id=mock_instance_id).all())
instance_events = list(
AwsInstanceEvent.objects.filter(
instance=instances[0]).all()) if instances else []
self.assertListEqual(instances, [])
self.assertListEqual(instance_events, [])
def test_persist_inspect_results_aws_cloud_image_not_found(
self, mock_scale_down, _, mock_receive, mock_delete):
"""
Assert message is still deleted when our image is not found.
See also: test_persist_aws_inspection_cluster_results_our_model_is_gone
"""
body_dict = {"cloud": "aws", "images": {"fake_image": {}}}
receipt_handle = str(uuid.uuid4())
message_id = str(uuid.uuid4())
sqs_message = util_helper.generate_mock_sqs_message(
message_id, json.dumps(body_dict), receipt_handle)
mock_receive.return_value = [sqs_message]
s, f = tasks.persist_inspection_cluster_results_task()
mock_delete.assert_called_once()
mock_scale_down.delay.assert_called_once()
self.assertIn(sqs_message, s)
self.assertEqual([], f)
def generate_mock_cloudtrail_sqs_message(
bucket_name="analyzer-test-bucket",
object_key="path/to/file.json.gz",
receipt_handle=None,
message_id=None,
):
"""
Generate a Mock object that behaves like a CloudTrail SQS message.
Args:
bucket_name (str): optional name of the S3 bucket
object_key (str): optional path to the S3 object
receipt_handle (str): optional SQS message receipt handle
message_id (str): id of the message
Returns:
Mock: populated to look and behave like a CloudTrail SQS message
"""
if not receipt_handle:
receipt_handle = str(uuid.uuid4())
if not message_id:
message_id = str(uuid.uuid4())
mock_sqs_message_body = {
"Records": [{
"s3": {
"bucket": {
"name": bucket_name,
},
"object": {
"key": object_key,
},
},
}]
}
mock_message = helper.generate_mock_sqs_message(
message_id, json.dumps(mock_sqs_message_body), receipt_handle)
return mock_message
def test_persist_inspect_results_task_aws_success(self, mock_persist,
mock_scale_down, _,
mock_receive,
mock_delete):
"""Assert that a valid message is correctly handled and deleted."""
receipt_handle = str(uuid.uuid4())
message_id = str(uuid.uuid4())
body_dict = {
"cloud": "aws",
"images": {
"ami-12345": {
"rhel_found": False,
"rhel_version": None,
"syspurpose": None,
"drive": {
"partition": {
"facts": [{
"release_file": "/centos-release",
"release_file_contents": "CentOS\n",
"rhel_found": False,
}]
}
},
}
},
}
sqs_message = util_helper.generate_mock_sqs_message(
message_id, json.dumps(body_dict), receipt_handle)
mock_receive.return_value = [sqs_message]
s, f = tasks.persist_inspection_cluster_results_task()
mock_persist.assert_called_once_with(body_dict)
mock_delete.assert_called_once()
mock_scale_down.delay.assert_called_once()
self.assertIn(sqs_message, s)
self.assertEqual([], f)
def test_other_tags_ignored(self, mock_receive, mock_s3, mock_del):
"""Test processing a CloudTrail log for other tags ignored."""
mock_instance_id = util_helper.generate_dummy_instance_id()
mock_sqs_message_body = {
'Records': [{
's3': {
'bucket': {
'name': 'test-bucket',
},
'object': {
'key': 'path/to/log/log.json.gz',
},
},
}]
}
mock_queue_url = 'https://sqs.queue.url'
mock_receipt_handle = str(uuid.uuid4())
mock_region = random.choice(util_helper.SOME_AWS_REGIONS)
now = datetime.datetime.utcnow()
mock_occurred_at = datetime.datetime(year=now.year,
month=now.month,
day=now.day,
hour=now.hour,
minute=now.minute,
second=now.second,
tzinfo=tz.tzutc())
mock_message = util_helper.generate_mock_sqs_message(
mock_queue_url, json.dumps(mock_sqs_message_body),
mock_receipt_handle)
mock_cloudtrail_log = {
'Records': [{
'eventTime':
mock_occurred_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
'eventSource':
'ec2.amazonaws.com',
'awsRegion':
mock_region,
'eventName':
tasks.DELETE_TAG,
'requestParameters': {
'resourcesSet': {
'items': [{
'resourceId': mock_instance_id
}]
},
'tagSet': {
'items': [{
'key': tasks.AWS_OPENSHIFT_TAG,
'value': tasks.AWS_OPENSHIFT_TAG
}]
}
}
}]
}
mock_receive.return_value = [mock_message]
mock_s3.return_value = json.dumps(mock_cloudtrail_log)
try:
tasks.analyze_log()
except Exception:
self.fail('Should not raise exceptions for ignored tag events.')
def test_ami_tags_removed_success(self, mock_receive, mock_s3, mock_del):
"""Test processing a CloudTrail log for ami tags removed."""
mock_user = util_helper.generate_test_user()
mock_arn = util_helper.generate_dummy_arn()
aws_account = AwsAccount(user=mock_user,
name='test',
aws_account_id='1234',
account_arn=mock_arn)
aws_account.save()
mock_ec2_ami_id = util_helper.generate_dummy_image_id()
aws_machine_image = AwsMachineImage(account=aws_account,
ec2_ami_id=mock_ec2_ami_id)
aws_machine_image.save()
openshift_tag = ImageTag.objects.filter(
description=tasks.OPENSHIFT_MODEL_TAG).first()
aws_machine_image.tags.add(openshift_tag)
mock_sqs_message_body = {
'Records': [{
's3': {
'bucket': {
'name': 'test-bucket',
},
'object': {
'key': 'path/to/log/log.json.gz',
},
},
}]
}
mock_queue_url = 'https://sqs.queue.url'
mock_receipt_handle = str(uuid.uuid4())
mock_region = random.choice(util_helper.SOME_AWS_REGIONS)
now = datetime.datetime.utcnow()
mock_occurred_at = datetime.datetime(year=now.year,
month=now.month,
day=now.day,
hour=now.hour,
minute=now.minute,
second=now.second,
tzinfo=tz.tzutc())
mock_message = util_helper.generate_mock_sqs_message(
mock_queue_url, json.dumps(mock_sqs_message_body),
mock_receipt_handle)
mock_cloudtrail_log = {
'Records': [{
'eventTime':
mock_occurred_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
'eventSource':
'ec2.amazonaws.com',
'awsRegion':
mock_region,
'eventName':
tasks.DELETE_TAG,
'requestParameters': {
'resourcesSet': {
'items': [{
'resourceId': mock_ec2_ami_id
}]
},
'tagSet': {
'items': [{
'key': tasks.AWS_OPENSHIFT_TAG,
'value': tasks.AWS_OPENSHIFT_TAG
}]
}
}
}]
}
mock_receive.return_value = [mock_message]
mock_s3.return_value = json.dumps(mock_cloudtrail_log)
tasks.analyze_log()
self.assertEqual(
None,
aws_machine_image.tags.filter(
description=tasks.OPENSHIFT_MODEL_TAG).first())
def test_command_output_success_ec2_attributes_included(
self, mock_receive, mock_s3, mock_del, mock_session, mock_ec2,
mock_inspection):
"""Test processing a CloudTrail log with all data included."""
mock_queue_url = 'https://sqs.queue.url'
mock_receipt_handle = str(uuid.uuid4())
mock_instance_id = util_helper.generate_dummy_instance_id()
mock_instance_id2 = util_helper.generate_dummy_instance_id()
mock_region = random.choice(util_helper.SOME_AWS_REGIONS)
mock_event_type = 'RunInstances'
now = datetime.datetime.utcnow()
# Work around for sqlite handling of microseconds
mock_occurred_at = datetime.datetime(year=now.year,
month=now.month,
day=now.day,
hour=now.hour,
minute=now.minute,
second=now.second,
tzinfo=tz.tzutc())
mock_subnet = 'subnet-9000'
mock_ec2_ami_id = util_helper.generate_dummy_image_id()
mock_instance_type = 't2.nano'
mock_instance = util_helper.generate_mock_ec2_instance(
mock_instance_id, mock_ec2_ami_id, mock_subnet, None,
mock_instance_type, 'windows')
mock_inspection.delay.return_value = True
mock_sqs_message_body = {
'Records': [{
's3': {
'bucket': {
'name': 'test-bucket',
},
'object': {
'key': 'path/to/log/log.json.gz',
},
},
}]
}
mock_message = util_helper.generate_mock_sqs_message(
mock_queue_url, json.dumps(mock_sqs_message_body),
mock_receipt_handle)
mock_cloudtrail_log = {
'Records': [{
'awsRegion':
mock_region,
'eventName':
mock_event_type,
'eventSource':
'ec2.amazonaws.com',
'eventTime':
mock_occurred_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
'eventType':
'AwsApiCall',
'responseElements': {
'instancesSet': {
'items': [{
'imageId': mock_ec2_ami_id,
'instanceId': mock_instance_id,
'instanceType': mock_instance_type,
'subnetId': mock_subnet
}]
},
},
'userIdentity': {
'accountId': self.mock_account_id
}
}, {
'awsRegion':
mock_region,
'eventName':
mock_event_type,
'eventSource':
'ec2.amazonaws.com',
'eventTime':
mock_occurred_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
'eventType':
'AwsApiCall',
'responseElements': {
'instancesSet': {
'items': [{
'imageId': mock_ec2_ami_id,
'instanceId': mock_instance_id,
'instanceType': mock_instance_type,
'subnetId': mock_subnet
}]
},
},
'userIdentity': {
'accountId': self.mock_account_id
}
}, {
'awsRegion':
mock_region,
'eventName':
mock_event_type,
'eventSource':
'ec2.amazonaws.com',
'eventTime':
mock_occurred_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
'eventType':
'DifferentAwsApiCall',
'responseElements': {
'instancesSet': {
'items': [{
'imageId': mock_ec2_ami_id,
'instanceId': mock_instance_id2,
'instanceType': mock_instance_type,
'subnetId': mock_subnet
}]
},
},
'userIdentity': {
'accountId': self.mock_account_id
}
}]
}
mock_receive.return_value = [mock_message]
mock_s3.return_value = json.dumps(mock_cloudtrail_log)
mock_del.return_value = 'Success'
mock_session.return_value = 'Session'
mock_ec2.return_value = mock_instance
tasks.analyze_log()
instances = list(
AwsInstance.objects.filter(ec2_instance_id=mock_instance_id).all())
instance_events = list(
AwsInstanceEvent.objects.filter(
instance=instances[0]).all()) if instances else []
for instance in instances:
self.assertEqual(instance.account, self.mock_account)
self.assertEqual(instance.ec2_instance_id, mock_instance_id)
self.assertEqual(instance.region, mock_region)
for event in instance_events:
self.assertEqual(event.instance, instances[0])
self.assertEqual(event.event_type, InstanceEvent.TYPE.power_on)
self.assertEqual(event.occurred_at, mock_occurred_at)
self.assertEqual(event.subnet, mock_subnet)
self.assertEqual(event.machineimage.ec2_ami_id, mock_ec2_ami_id)
self.assertEqual(event.instance_type, mock_instance_type)
