async def test_publish_no_data():
mock_sns_client = MagicMock()
mock_sns_client.publish.return_value = coroutine_of({"MessageId": "Msg32"})
context = ResultsContext(
"PubTopic",
{"address": "123.123.123.123"},
"scan_12",
iso_date_string_from_timestamp(123456),
iso_date_string_from_timestamp(789123),
"scan_name",
mock_sns_client
)
await context.publish_results()
# it should publish the top level info parent and temporal key
mock_sns_client.publish.assert_called_with(
TopicArn="PubTopic",
Subject="scan_name",
Message=dumps(
{
"scan_id": "scan_12",
"scan_start_time": iso_date_string_from_timestamp(123456),
"scan_end_time": iso_date_string_from_timestamp(789123),
"__docs": {}
}
),
MessageAttributes={
"ParentKey": {"StringValue": ResultsContext._hash_of({"address": "123.123.123.123"}), "DataType": "String"},
"TemporalKey": {"StringValue": ResultsContext._hash_of(iso_date_string_from_timestamp(789123)),
"DataType": "String"}
}
)
async def test_summary_info_published():
mock_sns_client = MagicMock()
mock_sns_client.publish.return_value = coroutine_of({"MessageId": "Msg32"})
context = ResultsContext(
"PubTopic",
{"address": "123.456.123.456"},
"scan_9",
iso_date_string_from_timestamp(4),
iso_date_string_from_timestamp(5),
"scan_name",
mock_sns_client
)
context.add_summaries({"foo": "bar", "boo": "baz"})
context.add_summary("banana", "man")
context.post_results("host_info", {"uptime": "1234567"}, include_summaries=True)
await context.publish_results()
mock_sns_client.publish.assert_called_with(
TopicArn="PubTopic",
Subject="scan_name",
Message=dumps(
{
"scan_id": "scan_9",
"scan_start_time": iso_date_string_from_timestamp(4),
"scan_end_time": iso_date_string_from_timestamp(5),
"__docs": {
"host_info": [
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
}),
"Data": {
"address": "123.456.123.456",
"uptime": "1234567",
"summary_foo": "bar",
"summary_boo": "baz",
"summary_banana": "man",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
}
]
}
}
),
MessageAttributes={
"ParentKey": {
"StringValue": ResultsContext._hash_of({"address": "123.456.123.456"}),
"DataType": "String"
},
"TemporalKey": {
"StringValue": ResultsContext._hash_of(iso_date_string_from_timestamp(5)),
"DataType": "String"
}
}
)
async def report_letters(event, _):
es_queue = event['ssm_params'][ES_SQS]
writes = []
for record in event["Records"]:
s3_object = objectify(record["s3"])
bucket = s3_object.bucket.name
key = unquote_plus(s3_object.object.key)
print(f"Loading new dead letter file: {(bucket, key)}")
obj = await s3_client.get_object(Bucket=bucket, Key=key)
dead_letter_details = obj["Metadata"]
print(f"Wring new dead letter with metadata: {dead_letter_details}")
ensure_essential_metadata(dead_letter_details, [
("deadletterqueuename", "Metadata missing"),
("deadletterkey", "Metadata missing"),
("deadlettersenttime",
str(iso_date_string_from_timestamp(datetime.now().timestamp())))
])
writes.append(
sqs_client.send_message(QueueUrl=es_queue,
MessageBody=dumps({
"Subject":
"dead_letter:data:write",
"Message":
dumps(dead_letter_details)
})))
print(f"Gathering writes")
await gather(*writes)
print(f"Written successfully")
def test_exception_on_no_parent_key(post_mock):
queue_ingestor.ssm_client.get_parameters.return_value = ssm_return_vals()
# mock response contains error
post_mock.return_value = response_mock = MagicMock()
response_mock.json.return_value = {}
response_mock.text = "Walk"
test_event = {
"Records": [{
"body":
dumps({
"Subject": "my_scan",
"Message": dumps({"some_field": "some_value"}),
"MessageAttributes": {
"TemporalKey": {
"Value":
ResultsContext._hash_of(
iso_date_string_from_timestamp(5)),
"DataType":
"String"
}
}
})
}]
}
with pytest.raises(
ValueError,
match=
"Analytics ingestor requires the ParentKey message attribute be present"
):
queue_ingestor.ingest(test_event, MagicMock())
async def scan(self, scan_request_id, scan_request):
scan_request = loads(scan_request)
msg = loads(scan_request["Message"])
if msg["port_id"] == "443" or msg["service"] == "https":
print(f"address {msg['address']}")
host_names = self.get_hosts(
msg["address"], self.get_ssm_param(self._dynamodb_param))
print(host_names)
for host in host_names:
target = f"{host}:{msg['port_id']}"
msg["target"] = target
scan_start_time = iso_date_string_from_timestamp(
datetime.now().timestamp())
print(
f"open ssl scan: {target} for request id {scan_request_id}"
)
print("running openssl")
cmd = f"openssl s_client -showcerts -connect {target}"
merged_file = io.BytesIO()
try:
# openssl outputs on two tty streams, so merge the two together and put in S3 for processing later
out = subprocess.check_output(
f"{cmd} </dev/null 1>/tmp/tty1.txt 2>/tmp/tty2.txt",
shell=True)
except subprocess.CalledProcessError as e:
# openssl will generate an error if there"s a problem in the chain
# that is used to source the error info and we do want to suppress this exception
pass
scan_end_time = iso_date_string_from_timestamp(
datetime.now().timestamp())
with open("/tmp/tty2.txt", "r") as f:
merged_file.write(f.read().encode("UTF-8"))
with open("/tmp/tty1.txt", "r") as f:
merged_file.write(f.read().encode("UTF-8"))
merged_file.seek(0)
await self.write_results_set(f"{scan_request_id}-{host}",
merged_file, ".txt", msg,
scan_start_time, scan_end_time)
else:
print("Ignoring non 443 or https service")
def test_snapshot_only_mode(post_mock):
queue_ingestor.ssm_client.get_parameters.return_value = ssm_return_vals()
post_mock.return_value = post_response_mock = MagicMock()
post_response_mock.json.return_value = {}
# Using as a sample event the expected output of the test_scan_results.py test
test_event = {
"Records": [{
"body":
dumps({
"Subject": "scan_name",
"Message": SAMPLE_DOC_COLLECTION,
"MessageAttributes": {
# N.B. No TemporalKey here
"ParentKey": {
"Value":
ResultsContext._hash_of({"address":
"123.456.123.456"}),
"DataType":
"String"
}
}
})
}]
}
queue_ingestor.ingest(test_event, MagicMock())
# There will be 2 port info, 2 vuln info, and one host info docs posted, but only for the snapshot collections
# There will be 3 delete old snapshot requests made, one for each doc_type, and each using the parent key
assert post_mock.call_count == 5 + 3
expected_deletes = {
doc_type: call(
f"https://elastic.url.com/scan_name:{doc_type}_snapshot:write/_doc/_delete_by_query?conflicts=proceed",
auth=auth_mock,
data=dumps({
"query": {
"term": {
"__ParentKey":
ResultsContext._hash_of({"address": "123.456.123.456"})
}
}
}),
headers={"content-type": "application/json"})
for doc_type in ["port_info", "vuln_info", "host_info"]
}
parent_key = ResultsContext._hash_of({"address": "123.456.123.456"})
assert post_mock.call_args_list == [
# info for port 22 and 80
expected_deletes["port_info"],
_expected_snapshot_write(
"port_info", {
'address': '123.456.123.456',
'port': '22'
}, auth_mock, {
"scan_id": "scan_2",
"scan_start_time": iso_date_string_from_timestamp(4),
"scan_end_time": iso_date_string_from_timestamp(5),
"address": "123.456.123.456",
"port": "22",
"open": "false",
"__ParentKey": parent_key,
}),
_expected_snapshot_write(
"port_info", {
'address': '123.456.123.456',
'port': '80'
}, auth_mock, {
"scan_id": "scan_2",
"scan_start_time": iso_date_string_from_timestamp(4),
"scan_end_time": iso_date_string_from_timestamp(5),
"address": "123.456.123.456",
"port": "80",
"open": "true",
"__ParentKey": parent_key,
}),
# info for the two cves
expected_deletes["vuln_info"],
_expected_snapshot_write(
"vuln_info", {
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve4"
}, auth_mock, {
"scan_id": "scan_2",
"scan_start_time": iso_date_string_from_timestamp(4),
"scan_end_time": iso_date_string_from_timestamp(5),
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve4",
"severity": "5",
"__ParentKey": parent_key,
}),
_expected_snapshot_write(
"vuln_info", {
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve5"
}, auth_mock, {
"scan_id": "scan_2",
"scan_start_time": iso_date_string_from_timestamp(4),
"scan_end_time": iso_date_string_from_timestamp(5),
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve5",
"severity": "2",
"__ParentKey": parent_key,
}),
# Host info
expected_deletes["host_info"],
_expected_snapshot_write(
"host_info", {"address": "123.456.123.456"}, auth_mock, {
"scan_id": "scan_2",
"scan_start_time": iso_date_string_from_timestamp(4),
"scan_end_time": iso_date_string_from_timestamp(5),
"address": "123.456.123.456",
"uptime": "1234567",
"__ParentKey": parent_key,
}),
]
}]
}
with pytest.raises(
ValueError,
match=
"Analytics ingestor requires the ParentKey message attribute be present"
):
queue_ingestor.ingest(test_event, MagicMock())
SAMPLE_DOC_COLLECTION = dumps({
"scan_id":
"scan_2",
"scan_start_time":
iso_date_string_from_timestamp(4),
"scan_end_time":
iso_date_string_from_timestamp(5),
"__docs": {
"port_info": [{
"NonTemporalKey":
ResultsContext._hash_of({
"address": "123.456.123.456",
"port": "22"
}),
"Data": {
"address":
"123.456.123.456",
"port":
"22",
"open":
async def test_context_push_and_pop():
mock_sns_client = MagicMock()
mock_sns_client.publish.return_value = coroutine_of({"MessageId": "Msg32"})
context = ResultsContext(
"PubTopic",
{"address": "123.456.123.456"},
"scan_2",
iso_date_string_from_timestamp(4),
iso_date_string_from_timestamp(5),
"scan_name",
mock_sns_client
)
context.push_context({"port": "22"})
context.post_results("port_info", {"open": "false"})
context.push_context({"vulnerability": "cve4"})
context.post_results("vuln_info", {"severity": "5"})
context.pop_context()
context.push_context({"vulnerability": "cve5"})
context.post_results("vuln_info", {"severity": "2"})
context.pop_context()
context.pop_context()
context.push_context({"port": "80"})
context.post_results("port_info", {"open": "true"})
context.pop_context()
context.post_results("host_info", {"uptime": "1234567"})
await context.publish_results()
# it should publish the top level info parent and temporal key
mock_sns_client.publish.assert_called_with(
TopicArn="PubTopic",
Subject="scan_name",
Message=dumps(
{
"scan_id": "scan_2",
"scan_start_time": iso_date_string_from_timestamp(4),
"scan_end_time": iso_date_string_from_timestamp(5),
"__docs": {
"port_info": [
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
"port": "22"
}),
"Data": {
"address": "123.456.123.456",
"port": "22",
"open": "false",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
},
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
"port": "80"
}),
"Data": {
"address": "123.456.123.456",
"port": "80",
"open": "true",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
}
],
"vuln_info": [
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve4"
}),
"Data": {
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve4",
"severity": "5",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
},
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve5"
}),
"Data": {
"address": "123.456.123.456",
"port": "22",
"vulnerability": "cve5",
"severity": "2",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
}
],
"host_info": [
{
"NonTemporalKey": ResultsContext._hash_of({
"address": "123.456.123.456",
}),
"Data": {
"address": "123.456.123.456",
"uptime": "1234567",
"__ParentKey": ResultsContext._hash_of({"address": "123.456.123.456"}),
}
}
]
}
}
),
MessageAttributes={
"ParentKey": {
"StringValue": ResultsContext._hash_of({"address": "123.456.123.456"}),
"DataType": "String"
},
"TemporalKey": {
"StringValue": ResultsContext._hash_of(iso_date_string_from_timestamp(5)),
"DataType": "String"
}
}
)