def _mock_resources(scan_initiator):
mock_plan_table, mock_info_table = (MagicMock(), MagicMock())
scan_initiator.dynamo_resource.Table.side_effect = iter([mock_plan_table, mock_info_table])
scan_initiator.dynamo_resource.close.return_value = coroutine_of(None)
scan_initiator.ssm_client.close.return_value = coroutine_of(None)
scan_initiator.sqs_client.close.return_value = coroutine_of(None)
return mock_info_table, mock_plan_table
async def test_publish_no_data():
mock_sns_client = MagicMock()
mock_sns_client.publish.return_value = coroutine_of({"MessageId": "Msg32"})
context = ResultsContext(
"PubTopic",
{"address": "123.123.123.123"},
"scan_12",
iso_date_string_from_timestamp(123456),
iso_date_string_from_timestamp(789123),
"scan_name",
mock_sns_client
)
await context.publish_results()
# it should publish the top level info parent and temporal key
mock_sns_client.publish.assert_called_with(
TopicArn="PubTopic",
Subject="scan_name",
Message=dumps(
{
"scan_id": "scan_12",
"scan_start_time": iso_date_string_from_timestamp(123456),
"scan_end_time": iso_date_string_from_timestamp(789123),
"__docs": {}
}
),
MessageAttributes={
"ParentKey": {"StringValue": ResultsContext._hash_of({"address": "123.123.123.123"}), "DataType": "String"},
"TemporalKey": {"StringValue": ResultsContext._hash_of(iso_date_string_from_timestamp(789123)),
"DataType": "String"}
}
)
def setup_mocks(ssm_failure=False):
with patch("aioboto3.client"), patch.dict(os.environ, TEST_ENV):
scanner = NmapScanner()
scanner.ensure_initialised()
scanner.ecs_client.run_task.side_effect = (coroutine_of({}) for _ in count(0, 1))
scanner.ssm_client.get_parameters.return_value = ssm_return_vals(ssm_failure)
return scanner
def execute_test_using_results_archive(filename):
with patch("aioboto3.client"), \
patch.dict(os.environ, TEST_ENV):
results_parser = NmapResultsParser()
results_context_constructor.return_value = mock_results_context = MagicMock()
mock_mgr = Mock()
mock_mgr.attach_mock(results_context_constructor, "ResultsContext")
mock_mgr.attach_mock(mock_results_context.push_context, "push_context")
mock_mgr.attach_mock(mock_results_context.pop_context, "pop_context")
mock_mgr.attach_mock(mock_results_context.post_results, "post_results")
mock_mgr.attach_mock(mock_results_context.publish_results, "publish_results")
mock_mgr.attach_mock(mock_results_context.add_summaries, "add_summaries")
mock_mgr.attach_mock(mock_results_context.add_summary, "add_summary")
results_parser.ensure_initialised()
results_parser.sns_client.publish.return_value = coroutine_of({"MessageId": "foo"})
results_parser.ssm_client.get_parameters = MagicMock()
results_parser.ssm_client.get_parameters.return_value = ssm_return_vals()
mock_results_context.publish_results.side_effect = (coroutine_of(MagicMock()) for _ in count(0, 1))
# load sample results file and make mock return it
sample_file_name = f"{TEST_DIR}{filename}"
with open(sample_file_name, "rb") as sample_data:
class AsyncReader:
async def read(self):
return StreamingBody(sample_data, os.stat(sample_file_name).st_size).read()
results_parser.s3_client.get_object.return_value = coroutine_of({
"Body": AsyncReader()
})
results_parser.invoke(
{
"Records": [
{"s3": {
"bucket": {"name": "test_bucket"},
"object": {"key": filename}
}}
]
},
MagicMock()
)
return results_parser, mock_mgr, mock_results_context
def test_paginates_scan_results(_, scan_initiator):
# ssm params don"t matter much in this test
set_ssm_return_vals(scan_initiator.ssm_client, 40, 10)
# access mock for dynamodb table
mock_info_table, mock_plan_table = _mock_resources(scan_initiator)
# return a single result but with a last evaluated key present, second result wont have
# that key
mock_plan_table.scan.side_effect = iter([
coroutine_of({
"Items": [{
"Address": "123.456.123.456",
"DnsIngestTime": 12345,
"PlannedScanTime": 67890
}],
"LastEvaluatedKey": "SomeKey"
}),
coroutine_of({
"Items": [{
"Address": "456.345.123.123",
"DnsIngestTime": 123456,
"PlannedScanTime": 67890
}]
}),
])
mock_info_table.update_item.side_effect = iter([coroutine_of(None), coroutine_of(None)])
# pretend the sqs messages are all successfully dispatched
scan_initiator.sqs_client.send_message_batch.side_effect = [
coroutine_of(None),
coroutine_of(None)
]
# pretend the delete item calls are all successful too
writer = _mock_delete_responses(mock_plan_table, [coroutine_of(None), coroutine_of(None)])
# actually do the test
scan_initiator.initiate_scans({}, MagicMock())
# check the scan happens twice, searching for planned scans earlier than 1984 + 40/10 i.e. now + bucket_length
assert mock_plan_table.scan.call_args_list == [
call(
IndexName="MyIndexName",
FilterExpression=Key("PlannedScanTime").lte(Decimal(1988))
),
call(
IndexName="MyIndexName",
FilterExpression=Key("PlannedScanTime").lte(Decimal(1988)),
ExclusiveStartKey="SomeKey"
)
]
# Doesn"t batch across pages
assert scan_initiator.sqs_client.send_message_batch.call_count == 2
assert writer.delete_item.call_count == 2
def test_batches_sqs_writes(_, scan_initiator):
# ssm params don"t matter much in this test
set_ssm_return_vals(scan_initiator.ssm_client, 100, 4)
# access mock for dynamodb table
mock_info_table, mock_plan_table = _mock_resources(scan_initiator)
# send 32 responses in a single scan result, will be batched into groups of 10 for
# sqs
mock_plan_table.scan.side_effect = iter([
coroutine_of({
"Items": [
{
"Address": f"123.456.123.{item_num}",
"DnsIngestTime": 12345,
"PlannedScanTime": 67890
}
for item_num in range(0, 32)
]
})
])
mock_info_table.update_item.side_effect = iter([coroutine_of(None) for _ in range(0, 32)])
# pretend the sqs and dynamo deletes are all ok, there are 4 calls to sqs
# and
scan_initiator.sqs_client.send_message_batch.side_effect = [
coroutine_of(None) for _ in range(0, 4)
]
writer = _mock_delete_responses(mock_plan_table, [
coroutine_of(None) for _ in range(0, 32)
])
# actually do the test
scan_initiator.initiate_scans({}, MagicMock())
# There will be 4 calls to sqs
assert scan_initiator.sqs_client.send_message_batch.call_count == 4
# The last batch will have 2 remaining items in it N.B. a call object is a tuple of the
# positional args and then the kwags
assert len(scan_initiator.sqs_client.send_message_batch.call_args_list[3][1]["Entries"]) == 2
# There will be individual deletes for each address i.e. 32 of them
assert writer.delete_item.call_count == 32
async def test_summary_info_published():
mock_sns_client = MagicMock()
mock_sns_client.publish.return_value = coroutine_of({"MessageId": "Msg32"})
context = ResultsContext(
"PubTopic",
{"address": "123.456.123.456"},
"scan_9",
iso_date_string_from_timestamp(4),
iso_date_string_from_timestamp(5),
"scan_name",
mock_sns_client
)
context.add_summaries({"foo": "bar", "boo": "baz"})
context.add_summary("banana", "man")
context.post_results("host_info", {"uptime": "1234567"}, include_summaries=True)
await context.publish_results()
mock_sns_client.publish.assert_called_with(
TopicArn="PubTopic",
Subject="scan_name",
Message=dumps(
{
"scan_id": "scan_9",
"scan_start_time": iso_date_string_from_timestamp(4),
"scan_end_time": iso_date_string_from_timestamp(5),
"__docs": {
"host_info": [
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
}),
"Data": {
"address": "123.456.123.456",
"uptime": "1234567",
"summary_foo": "bar",
"summary_boo": "baz",
"summary_banana": "man",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
}
]
}
}
),
MessageAttributes={
"ParentKey": {
"StringValue": ResultsContext._hash_of({"address": "123.456.123.456"}),
"DataType": "String"
},
"TemporalKey": {
"StringValue": ResultsContext._hash_of(iso_date_string_from_timestamp(5)),
"DataType": "String"
}
}
)
def ssm_return_vals():
stage = os.environ["STAGE"]
app_name = os.environ["APP_NAME"]
ssm_prefix = f"/{app_name}/{stage}"
return coroutine_of({
"Parameters": [{
"Name": f"{ssm_prefix}/analytics/elastic/es_endpoint/url",
"Value": "elastic.url.com"
}]
})
def test_raises_when_ecs_failures_present():
with patch("aioboto3.client"), patch.dict(os.environ, TEST_ENV):
scanner = NmapScanner()
scanner.ensure_initialised()
scanner.ssm_client.get_parameters.return_value = ssm_return_vals(False)
scanner.ecs_client.run_task.return_value = coroutine_of({"failures": [
{"arn": "arn::some:::arn", "reason": "failed miserably"}
]})
with pytest.raises(RuntimeError, match=r"\{\"arn\": \"arn::some:::arn\", \"reason\": \"failed miserably\"\}"):
scanner.invoke({"Records": [{"body": "some.host", "messageId": "13"}]}, MagicMock())
def ssm_return_vals():
stage = os.environ["STAGE"]
app_name = os.environ["APP_NAME"]
task_name = os.environ["TASK_NAME"]
ssm_prefix = f"/{app_name}/{stage}"
return coroutine_of({
"Parameters": [
{"Name": f"{ssm_prefix}/tasks/{task_name}/results/arn", "Value": "test_topic_arn"},
{"Name": f"{ssm_prefix}/tasks/{task_name}/s3/results/id", "Value": "test_topic_id"}
]
})
def set_ssm_return_vals(ssm_client, period, buckets):
stage = os.environ["STAGE"]
app_name = os.environ["APP_NAME"]
ssm_prefix = f"/{app_name}/{stage}"
ssm_client.get_parameters.return_value = coroutine_of({
"Parameters": [
{"Name": f"{ssm_prefix}/scheduler/dynamodb/scans_planned/id", "Value": "MyTableId"},
{"Name": f"{ssm_prefix}/scheduler/dynamodb/scans_planned/plan_index", "Value": "MyIndexName"},
{"Name": f"{ssm_prefix}/scheduler/dynamodb/address_info/id", "Value": "MyIndexName"},
{"Name": f"{ssm_prefix}/scheduler/config/period", "Value": str(period)},
{"Name": f"{ssm_prefix}/scheduler/config/buckets", "Value": str(buckets)},
{"Name": f"{ssm_prefix}/scheduler/scan_delay_queue", "Value": "MyDelayQueue"}
]
})
def ssm_return_vals(using_private):
stage = os.environ["STAGE"]
app_name = os.environ["APP_NAME"]
task_name = os.environ["TASK_NAME"]
ssm_prefix = f"/{app_name}/{stage}"
return coroutine_of({
"Parameters": [
{"Name": f"{ssm_prefix}/vpc/using_private_subnets", "Value": "true" if using_private else "false"},
{"Name": f"{ssm_prefix}/tasks/{task_name}/security_group/id", "Value": "sg-123"},
{"Name": f"{ssm_prefix}/tasks/{task_name}/image/id", "Value": "imagination"},
{"Name": f"{ssm_prefix}/tasks/{task_name}/s3/results/id", "Value": "bid"},
{"Name": f"{ssm_prefix}/vpc/subnets/instance", "Value": "subnet-123,subnet-456"},
{"Name": f"{ssm_prefix}/ecs/cluster", "Value": "cid"}
]
})
def test_replace_punctuation_in_address_ids(_, scan_initiator):
# ssm params don"t matter much in this test
set_ssm_return_vals(scan_initiator.ssm_client, 100, 4)
# access mock for dynamodb table
mock_info_table, mock_plan_table = _mock_resources(scan_initiator)
# return a single result with ip4 and another with ip6
mock_plan_table.scan.side_effect = iter([
coroutine_of({
"Items": [
{
"Address": "123.456.123.456",
"DnsIngestTime": 12345,
"PlannedScanTime": 67890
},
{
"Address": "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
"DnsIngestTime": 12345,
"PlannedScanTime": 67890
}
]
})
])
mock_info_table.update_item.side_effect = iter([coroutine_of(None), coroutine_of(None)])
# pretend the sqs and dynamo deletes are all ok
scan_initiator.sqs_client.send_message_batch.side_effect = [coroutine_of(None)]
_mock_delete_responses(mock_plan_table, [coroutine_of(None), coroutine_of(None)])
# actually do the test
scan_initiator.initiate_scans({}, MagicMock())
# check both addresses have : and . replaced with -
scan_initiator.sqs_client.send_message_batch.assert_called_once_with(
QueueUrl="MyDelayQueue",
Entries=[
{
"Id": "123-456-123-456",
"DelaySeconds": 67890-1984, # planned scan time minus now time
"MessageBody": "{\"AddressToScan\":\"123.456.123.456\"}"
},
{
"Id": "2001-0db8-85a3-0000-0000-8a2e-0370-7334",
"DelaySeconds": 67890-1984, # planned scan time minus now time
"MessageBody": "{\"AddressToScan\":\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\"}"
}
]
)
def test_no_deletes_until_all_sqs_success(_, scan_initiator):
# ssm params don"t matter much in this test
set_ssm_return_vals(scan_initiator.ssm_client, 100, 4)
# access mock for dynamodb table
mock_info_table, mock_plan_table = _mock_resources(scan_initiator)
# send a single response
mock_plan_table.scan.side_effect = [
coroutine_of({
"Items": [
{
"Address": f"123.456.123.5",
"DnsIngestTime": 12345,
"PlannedScanTime": 67890
}
]
})
]
# pretend the sqs and dynamo deletes are all ok, there are 4 calls to sqs
# and
scan_initiator.sqs_client.send_message_batch.side_effect = [
Exception("test error")
]
writer = _mock_delete_responses(mock_plan_table, [])
# actually do the test
with pytest.raises(Exception):
scan_initiator.initiate_scans({}, MagicMock())
# There will be 1 call to sqs
assert scan_initiator.sqs_client.send_message_batch.call_count == 1
# and none to dynamo
assert writer.delete_item.call_count == 0
async def test_context_push_and_pop():
mock_sns_client = MagicMock()
mock_sns_client.publish.return_value = coroutine_of({"MessageId": "Msg32"})
context = ResultsContext(
"PubTopic",
{"address": "123.456.123.456"},
"scan_2",
iso_date_string_from_timestamp(4),
iso_date_string_from_timestamp(5),
"scan_name",
mock_sns_client
)
context.push_context({"port": "22"})
context.post_results("port_info", {"open": "false"})
context.push_context({"vulnerability": "cve4"})
context.post_results("vuln_info", {"severity": "5"})
context.pop_context()
context.push_context({"vulnerability": "cve5"})
context.post_results("vuln_info", {"severity": "2"})
context.pop_context()
context.pop_context()
context.push_context({"port": "80"})
context.post_results("port_info", {"open": "true"})
context.pop_context()
context.post_results("host_info", {"uptime": "1234567"})
await context.publish_results()
# it should publish the top level info parent and temporal key
mock_sns_client.publish.assert_called_with(
TopicArn="PubTopic",
Subject="scan_name",
Message=dumps(
{
"scan_id": "scan_2",
"scan_start_time": iso_date_string_from_timestamp(4),
"scan_end_time": iso_date_string_from_timestamp(5),
"__docs": {
"port_info": [
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
"port": "22"
}),
"Data": {
"address": "123.456.123.456",
"port": "22",
"open": "false",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
},
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
"port": "80"
}),
"Data": {
"address": "123.456.123.456",
"port": "80",
"open": "true",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
}
],
"vuln_info": [
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve4"
}),
"Data": {
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve4",
"severity": "5",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
},
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve5"
}),
"Data": {
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve5",
"severity": "2",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
}
],
"host_info": [
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
}),
"Data": {
"address": "123.456.123.456",
"uptime": "1234567",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
}
]
}
}
),
MessageAttributes={
"ParentKey": {
"StringValue": ResultsContext._hash_of({"address": "123.456.123.456"}),
"DataType": "String"
},
"TemporalKey": {
"StringValue": ResultsContext._hash_of(iso_date_string_from_timestamp(5)),
"DataType": "String"
}
}
)