def test_strangerealintel_feed_processing_is_successful(self): self.run_bot() self.assertOutputQueueLen(len(EXAMPLE_STRANGERINTEL_FILE_JSON)) sha256_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy() sha256_event['malware.hash.sha256'] = EXAMPLE_STRANGERINTEL_FILE_JSON[ 0]['Indicator'] sha256_event[ 'event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[0][ 'Description'] sha256_event['classification.taxonomy'] = 'malicious code' sha256_event['classification.type'] = 'malware' sha256_event['raw'] = utils.base64_encode( str(EXAMPLE_STRANGERINTEL_FILE_JSON[0])) self.assertMessageEqual(0, sha256_event) md5_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy() md5_event['malware.hash.md5'] = EXAMPLE_STRANGERINTEL_FILE_JSON[1][ 'Indicator'] md5_event['event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[ 1]['Description'] md5_event['classification.taxonomy'] = 'malicious code' md5_event['classification.type'] = 'malware' md5_event['raw'] = utils.base64_encode( str(EXAMPLE_STRANGERINTEL_FILE_JSON[1])) self.assertMessageEqual(1, md5_event) domain_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy() domain_event['source.fqdn'] = EXAMPLE_STRANGERINTEL_FILE_JSON[2][ 'Indicator'] # description text is empty so no field is created # domain_event['event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[2]['Description'] domain_event['raw'] = utils.base64_encode( str(EXAMPLE_STRANGERINTEL_FILE_JSON[2])) self.assertMessageEqual(2, domain_event) ip_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy() ip_event['source.ip'] = EXAMPLE_STRANGERINTEL_FILE_JSON[3]['Indicator'] ip_event['event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[ 3]['Description'] ip_event['raw'] = utils.base64_encode( str(EXAMPLE_STRANGERINTEL_FILE_JSON[3])) self.assertMessageEqual(3, ip_event) url_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy() url_event['source.url'] = EXAMPLE_STRANGERINTEL_FILE_JSON[4][ 'Indicator'] url_event['event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[ 4]['Description'] url_event['raw'] = utils.base64_encode( str(EXAMPLE_STRANGERINTEL_FILE_JSON[4])) self.assertMessageEqual(4, url_event)
def test_events(self): """ Test if correct Events have been produced. """ self.run_bot() for position, event in enumerate(EXAMPLE_EVENTS_PARTS): event_ = EVENT_TEMPL.copy() event_.update(event) event_['raw'] = utils.base64_encode('\n'.join( (FILE_LINES[0], FILE_LINES[1 + position]))) self.assertMessageEqual(position, event_)
def test_events(self): """ Test if correct Events have been produced. """ self.run_bot() for position, event in enumerate(EXAMPLE_EVENTS_PARTS): event_ = EVENT_TEMPL.copy() event_.update(event) event_['raw'] = utils.base64_encode(FILE) self.assertMessageEqual(position, event_) self.assertOutputQFILE_LINES = FILE.splitlines()
def test_events(self): """ Test if correct Events have been produced. """ self.run_bot() for position, event in enumerate(EXAMPLE_EVENTS_PARTS): event_ = EVENT_TEMPL.copy() event_.update(event) event_['raw'] = utils.base64_encode('\n'.join((FILE_LINES[0], FILE_LINES[1+position]))) self.assertMessageEqual(position, event_)
def test_events(self): """ Test if correct Events have been produced. """ self.input_message = REPORT self.run_bot() for i, event in enumerate(EVENTS): event.update(EVENT_TEMPLATE) event['raw'] = utils.base64_encode(RAWS[i]) self.assertMessageEqual(i, event) self.assertEqual(len(self.get_output_queue()), len(EVENTS), msg='Length of EVENTS does not match length of ' 'output queue.')
def test_adding_extra_fields_should_warn(self, requests_get_mock): requests_get_mock.side_effect = print_requests_get_parameters custom_config = SHOULD_PASS_WITH_TXT_FILES_AND_EXTRA_FIELD_SIZE_TEST[ 'CONFIG'].copy() custom_config['extra_fields'] = 'aaa,bbb' self.allowed_warning_count = 2 self.run_bot(parameters=custom_config, prepare=True) self.assertRegexpMatchesLog( pattern=".*Field 'aaa' does not exist in the Github file data.*") self.assertRegexpMatchesLog( pattern=".*Field 'bbb' does not exist in the Github file data.*") self.assertMessageEqual( 0, { "__type": "Report", "feed.name": "Github feed", "feed.accuracy": 100., "feed.url": JSON_CONTENTS[1]['download_url'], "raw": utils.base64_encode(EXAMPLE_CONTENT_STR) })
import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.nothink.parser_dns_attacks import NothinkDNSAttackParserBot with open(os.path.join(os.path.dirname(__file__), 'honeypot_dns_attacks.txt')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_REPORT = { 'feed.url': 'http://www.nothink.org/honeypot_dns_attacks.txt', 'feed.name': 'DNS Attacks', '__type': 'Report', 'raw': utils.base64_encode(EXAMPLE_FILE), 'time.observation': '2016-11-22T08:26:00+00:00' } EXAMPLE_EVENTS = [ { 'feed.url': 'http://www.nothink.org/honeypot_dns_attacks.txt', 'feed.name': 'DNS Attacks', '__type': 'Event', 'raw': 'MjAxNi0xMS0xMSAxNToxMzoyMCwxODYuMi4xNjcuMTQsMjYyMjU0LERBTkNPTSBMVEQsLCwgQlosZGRvcy1ndWFyZC5uZXQsQlo=', 'time.source': '2016-11-11T15:13:20+00:00',
from intelmq.bots.parsers.bambenek.parser import BambenekParserBot with open(os.path.join(os.path.dirname(__file__), 'c2-dommasterlist.txt')) as handle: DOMAIN_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'c2-ipmasterlist.txt')) as handle: IP_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'dga-feed.txt')) as handle: DGA_FILE = handle.read() DOMAIN_REPORT = {'feed.url': 'http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt', 'feed.name': 'Bambenek C2 Domain Feed', '__type': 'Report', 'raw': utils.base64_encode(DOMAIN_FILE), 'time.observation': '2016-01-01T00:00:00+00:00' } DOMAIN_EVENTS = {'feed.url': 'http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt', 'feed.name': 'Bambenek C2 Domain Feed', '__type': 'Event', 'time.observation': '2016-01-01T00:00:00+00:00', 'raw': 'c2ZraDIxb2tycm5sdS5jb20sRG9tYWluIHVzZWQgYnkgc2hpb3RvYi91cmx6b25lL2JlYmxvaCwyMDE2LTExLTEyIDE1OjAyLGh0dHA6Ly9vc2ludC5iYW1iZW5la2NvbnN1bHRpbmcuY29tL21hbnVhbC9iZWJsb2gudHh0', 'time.source': '2016-11-12T15:02:00+00:00', 'source.fqdn': 'sfkh21okrrnlu.com', 'malware.name': 'bebloh', 'classification.type': 'c2server', 'status': 'online', 'event_description.text': 'Domain used by shiotob/urlzone/bebloh', 'event_description.url': 'http://osint.bambenekconsulting.com/manual/bebloh.txt'
# -*- coding: utf-8 -*- import os import unittest import intelmq.lib.utils as utils import intelmq.lib.test as test from intelmq.bots.parsers.generic.parser_csv import GenericCsvParserBot with open(os.path.join(os.path.dirname(__file__), 'invalid_sample_report.csv')) as handle: SAMPLE_FILE = handle.read() SAMPLE_SPLIT = SAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"feed.name": "Sample CSV Feed", "__type": "Report", "raw": utils.base64_encode(SAMPLE_FILE), "time.observation": "2015-01-01T00:00:00+00:00", } EXAMPLE_EVENT = {"feed.name": "Sample CSV Feed", "__type": "Event", "time.source": "2018-08-01T07:49:41+00:00", "classification.type": "malware", "source.ip": "127.0.0.1", "source.fqdn": "example.com", "raw": utils.base64_encode(SAMPLE_SPLIT[1]+'\r\n'), "time.observation": "2015-01-01T00:00:00+00:00", } EXAMPLE_EVENT2 = EXAMPLE_EVENT.copy() del EXAMPLE_EVENT2['source.fqdn'] EXAMPLE_EVENT2["raw"] = utils.base64_encode(SAMPLE_SPLIT[0]+'\r\n')
import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), "snmp.csv")) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() with open(os.path.join(os.path.dirname(__file__), "snmp_RECONSTRUCTED.csv")) as handle: RECONSTRUCTED_FILE = handle.read() RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines() EXAMPLE_REPORT = { "feed.name": "ShadowServer SNMP", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [ { "__type": "Event", "feed.name": "ShadowServer SNMP", "classification.identifier": "opensnmp", "classification.taxonomy": "Vulnerable", "classification.type": "vulnerable service", "extra": '{"sysdesc": "Hardware: x86 Family 6 Model 8 Stepping 6 AT/AT COMPATIBLE - Software: Windows 2000 Version 5.0 (Build 2195 Uniprocessor Free)", "sysname": "ORSONKA", "version": 2}', "protocol.application": "snmp", "protocol.transport": "udp", "raw": utils.base64_encode("\n".join([RECONSTRUCTED_LINES[0], RECONSTRUCTED_LINES[1], ""])), "source.asn": 22864,
reader = csv.DictReader(fop, restval="") for row in reader: datalist.append(row) return json.dumps(datalist, indent=4) CSVREPORTS = {} JSONREPORTS = {} testdata = pathlib.Path(__file__).parent / 'testdata' for filename in testdata.glob('*.csv'): EXAMPLE_FILE = filename.read_text() shortname = filename.stem CSVREPORTS[shortname] = { "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-{}-test-test.csv".format(shortname), } JSONREPORTS[shortname] = { "raw": utils.base64_encode(csvtojson(filename)), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-{}-test-test.csv".format(shortname), } def generate_feed_function(feedname, reports): def test_feed(self): """ Test if no errors happen for feed %s. """ % feedname
import unittest import intelmq.lib.test as test from intelmq.bots.parsers.webinspektor.parser import WebinspektorParserBot from intelmq.lib import utils with open(os.path.join(os.path.dirname(__file__), 'test_webinspektor.data')) as handle: REPORT_DATA = handle.read() REPORT_DATA_SPLIT = REPORT_DATA.splitlines() REPORT = { "__type": "Report", "feed.name": "Webinspektor", "feed.url": "https://app.webinspector.com/public/recent_detections", "raw": utils.base64_encode(REPORT_DATA), "time.observation": "2018-01-22T14:38:24+00:00", } EVENT1 = { "raw": utils.base64_encode(REPORT_DATA_SPLIT[102].strip() + REPORT_DATA_SPLIT[104].strip()), "__type": "Event", "time.observation": "2018-01-22T14:38:24+00:00", "feed.url": "https://app.webinspector.com/public/recent_detections", "source.url": "https://cummins.inhance.io", "classification.taxonomy":
# -*- coding: utf-8 -*- import unittest import os import intelmq.lib.utils as utils import intelmq.lib.test as test from intelmq.bots.parsers.generic.parser_csv import GenericCsvParserBot with open(os.path.join(os.path.dirname(__file__), 'empty_src_url.csv')) as handle: SAMPLE_FILE = handle.read() SAMPLE_SPLIT = SAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"feed.name": "Sample CSV Feed", "raw": utils.base64_encode(SAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EXAMPLE_EVENT = {"feed.name": "Sample CSV Feed", "__type": "Event", "raw": utils.base64_encode(SAMPLE_SPLIT[0] + '\r\n' + SAMPLE_SPLIT[1].replace('"', '')+'\r\n'), "time.observation": "2015-01-01T00:00:00+00:00", "classification.type": "infected-system", "source.ip": "11.11.11.11", "feed.name": "Sample CSV Feed", "source.port": 9001, 'time.source': '2017-05-26T07:00:36+00:00', "source.url": "http://test.com" }
# -*- coding: utf-8 -*- import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.abusech.parser_ip import AbusechIPParserBot with open(os.path.join(os.path.dirname(__file__), 'zeusips.txt')) as handle: EXAMPLE_ZEUS_FILE = handle.read() EXAMPLE_ZEUS_REPORT = {"feed.url": "https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist", "feed.name": "AbuseCH Zeustracker", "__type": "Report", "raw": utils.base64_encode(EXAMPLE_ZEUS_FILE), "time.observation": "2015-11-02T13:11:44+00:00" } EXAMPLE_ZEUS_EVENT = {"feed.url": "https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist", "feed.name": "AbuseCH Zeustracker", "source.ip": "101.200.81.187", "raw": utils.base64_encode("101.200.81.187"), "time.observation": "2015-11-02T13:11:44+00:00", "classification.taxonomy": "malicious code", "classification.type": "c2server", "malware.name": "zeus", "__type": "Event" }
# -*- coding: utf-8 -*- import unittest import os import intelmq.lib.utils as utils import intelmq.lib.test as test from intelmq.bots.parsers.generic.parser_csv import GenericCsvParserBot with open(os.path.join(os.path.dirname(__file__), 'data_type.csv')) as handle: SAMPLE_FILE = handle.read() SAMPLE_SPLIT = SAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"feed.name": "Sample CSV Feed", "raw": utils.base64_encode(SAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00"} EXAMPLE_EVENT = {"feed.name": "Sample CSV Feed", "__type": "Event", "raw": utils.base64_encode(SAMPLE_FILE.replace('\n', '\r\n')), "time.observation": "2015-01-01T00:00:00+00:00", "classification.type": "infected-system", "source.ip": "11.11.11.11", 'extra.tags': ["t1", "t2", "t3"], 'source.url': 'http://test.com'} class TestGenericCsvParserBot(test.BotTestCase, unittest.TestCase): """ A TestCase for a GenericCsvParserBot with extra, column_regex_search and windows_nt time format. """
import intelmq.lib.bot as bot import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.lib.message import Event RAW = """# ignore this 2015/06/04 13:37 +00,example.org,192.0.2.3,reverse.example.net,example description,[email protected],0 2015/06/04 13:38 +00,example.org,19d2.0.2.3,reverse.example.net,example description,[email protected],0 #ending line""" RAW_SPLIT = RAW.splitlines() EXAMPLE_REPORT = {"feed.url": "http://www.example.com/", "time.observation": "2015-08-11T13:03:40+00:00", "raw": utils.base64_encode(RAW), "__type": "Report", "feed.name": "Example"} EXAMPLE_EVENT = {"feed.url": "http://www.example.com/", "source.ip": "192.0.2.3", "time.source": "2015-06-04T13:37:00+00:00", "source.reverse_dns": "reverse.example.net", "source.fqdn": "example.org", "source.account": "*****@*****.**", "time.observation": "2015-08-11T13:03:40+00:00", "__type": "Event", "classification.type": "malware", "event_description.text": "example description", "source.asn": 0, "feed.name": "Example", "raw": utils.base64_encode('\n'.join(RAW_SPLIT[:2]))}
"time.observation": "2015-11-17T12:17:27.043452Z", "source.geolocation.cc": "US", "raw": "eyJjYXRlZ29yeSI6ICJib3RzIiwgIm9yaWdpbiI6ICJzaW5raG9sZSIsICJjb25maWRlbmNlIjogImhpZ2giLCAibmFtZSI6ICJzYWxpdHkiLCAicHJvdG8iOiAidGNwIiwgImFkZHJlc3MiOiBbeyJjYyI6ICJVUyIsICJpcCI6ICI2OC42OC45Ni4yMzUiLCAiYXNuIjogMTg3Nzl9XSwgInNvdXJjZSI6ICJjZXJ0LXBsLnNpbmtob2xlIiwgImFkaXAiOiAieC54LjExMS45OSIsICJ0aW1lIjogIjIwMTUtMTEtMTdUMTI6MTc6MjcuMDQzNDUyWiIsICJkcG9ydCI6IDgwLCAic3BvcnQiOiAyMjMwOCwgInR5cGUiOiAiZXZlbnQiLCAiaWQiOiAiZDc3YWU4Y2Y2ODFkY2RiYjZlMjAwMTQ1ODE0MDFlZDUifQ==", # noqa "classification.identifier": "sality", "malware.name": "sality", "classification.type": "infected-system", "destination.port": 80, "__type": "Event", "protocol.transport": "tcp", "source.asn": 18779, 'extra.confidence': 'high', 'extra.feed_source': 'cert-pl.sinkhole', } BLACKLIST_REPORT = {"__type": "Report", "raw": utils.base64_encode(""" {"category": "other", "confidence": "low", "expires": "2019-02-03T20:12:03Z", "fqdn": "example.com", "source": "example", "time": "2019-01-12T23:12:02Z", "type": "bl-update", "id": "some hex string", "address": [{"cc": "AT", "ip": "10.0.0.1", "asn": 65536}]} """), "time.observation": "2015-11-17T12:17:27.043452Z", } BLACKLIST_EVENT = {"__type": "Event", "classification.taxonomy": "other", "classification.type": "blacklist", "extra.confidence": "low", "extra.expires": "2019-02-03T20:12:03+00:00", "source.fqdn": "example.com", "extra.feed_source": "example", "time.source": "2019-01-12T23:12:02+00:00", "extra.feed_id": "some hex string", "source.geolocation.cc": "AT", "source.ip": "10.0.0.1", "source.asn": 65536,
import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), 'chargen.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() with open(os.path.join(os.path.dirname(__file__), 'chargen_short.csv')) as handle: EXAMPLE_FILE_SHORT = handle.read() EXAMPLE_LINE_SHORT = EXAMPLE_FILE_SHORT.splitlines() EXAMPLE_REPORT = {"feed.name": "ShadowServer Chargen", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EXAMPLE_REPORT_SHORT = {"feed.name": "ShadowServer Chargen", "raw": utils.base64_encode(EXAMPLE_FILE_SHORT), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', 'feed.name': 'ShadowServer Chargen', 'classification.type': 'vulnerable service', 'classification.identifier': 'open-chargen', 'classification.taxonomy': 'vulnerable', 'extra.response_size': 116, 'extra.naics': 123456,
from intelmq.bots.parsers.malc0de.parser import Malc0deParserBot with open(os.path.join(os.path.dirname(__file__), 'BOOT')) as handle: BOOT_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'ZONE')) as handle: ZONE_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'IP_Blacklist.txt')) as handle: IP_BLACKLIST_FILE = handle.read() BOOT_REPORT = {'feed.name': 'Windows Format', 'feed.url': 'http://malc0de.com/bl/BOOT', '__type': 'Report', 'time.observation': '2016-12-4T14:16:25+00:00', 'raw': utils.base64_encode(BOOT_FILE) } BOOT_EVENT = [{'feed.name': 'Windows Format', 'feed.url': 'http://malc0de.com/bl/BOOT', '__type': 'Event', 'time.observation': '2016-12-4T14:16:25+00:00', 'time.source': '2016-11-10T00:00:00+00:00', 'raw': 'UFJJTUFSWSBleGFtcGxlLmNvbSBibG9ja2VkZG9tYWluLmhvc3Rz', 'source.fqdn': 'example.com', 'event_description.url': 'http://malc0de.com/database/index.php?search=example.com', 'classification.type': 'malware' }, {'feed.name': 'Windows Format', 'feed.url': 'http://malc0de.com/bl/BOOT', '__type': 'Event',
# -*- coding: utf-8 -*- import os import unittest import intelmq.lib.test as test from intelmq.bots.parsers.webinspektor.parser import WebinspektorParserBot from intelmq.lib import utils with open(os.path.join(os.path.dirname(__file__), 'test_webinspektor.data')) as handle: REPORT_DATA = handle.read() REPORT_DATA_SPLIT = REPORT_DATA.splitlines() REPORT = {"__type": "Report", "feed.name": "Webinspektor", "feed.url": "https://app.webinspector.com/public/recent_detections", "raw": utils.base64_encode(REPORT_DATA), "time.observation": "2018-01-22T14:38:24+00:00", } EVENT1 = {"raw": utils.base64_encode(REPORT_DATA_SPLIT[102].strip() + REPORT_DATA_SPLIT[104].strip()), "__type": "Event", "time.observation": "2018-01-22T14:38:24+00:00", "feed.url": "https://app.webinspector.com/public/recent_detections", "source.url": "https://cummins.inhance.io", "classification.taxonomy": "other", "classification.type": "blacklist", "classification.identifier": "Suspicious", "time.source": "2018-02-13T08:44:30+00:00", "feed.name": "Webinspektor"} EVENT2 = {"raw": utils.base64_encode(REPORT_DATA_SPLIT[111].strip() + REPORT_DATA_SPLIT[113].strip()), "__type": "Event", "time.observation": "2018-01-22T14:38:24+00:00",
import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), 'accessible-cisco-smart-install.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() with open(os.path.join(os.path.dirname(__file__), 'accessible-cisco-smart-install-reconstructed.csv')) as handle: RECONSTRUCTED_FILE = handle.read() RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines() EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible Cisco Smart Install", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', 'feed.name': 'ShadowServer Accessible Cisco Smart Install', 'classification.identifier': 'accessible-cisco-smart-install', 'classification.type': 'vulnerable service', 'protocol.application': 'cisco-smart-install', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], RECONSTRUCTED_LINES[1], ''])), 'source.asn': 8559, 'source.geolocation.cc': 'AT', 'source.geolocation.city': 'VIENNA', 'source.geolocation.region': 'WIEN',
from intelmq.lib import test from intelmq.lib.utils import base64_encode with open(os.path.join(os.path.dirname(__file__), 'misp_event.json')) as handle: EXAMPLE_MISP_EVENT = handle.read() with open(os.path.join(os.path.dirname(__file__), 'misp_attribute.json')) as handle: EXAMPLE_MISP_ATTR = handle.read() EXAMPLE_REPORT = { "__type": "Report", "feed.accuracy": 100.0, "feed.name": "misp_test", "feed.url": "http://192.168.56.102/", "time.observation": "2016-06-15T09:25:26+00:00", "raw": base64_encode(EXAMPLE_MISP_EVENT) } EXAMPLE_EVENT = { "__type": "Event", "feed.accuracy": 100.0, "feed.name": "misp_test", "feed.url": "http://192.168.56.102/", "time.observation": "2016-06-15T09:25:26+00:00", "time.source": "2016-06-11T21:41:44+00:00", "source.url": "http://fake.website.com/malware/is/here", "event_description.text": "Payload delivery", "event_description.url": "http://192.168.56.102/event/view/2", "classification.type": "ransomware", "malware.name": "locky", 'misp.attribute_uuid': '575c8598-f1f0-4c16-a94a-0612c0a83866',
# -*- coding: utf-8 -*- import base64 import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.openphish.parser_commercial import OpenPhishCommercialParserBot with open(os.path.join(os.path.dirname(__file__), 'feed_commercial.txt'), 'r') as fh: FILE = fh.read() RAW = utils.base64_encode(FILE.encode()) SPLITTED = FILE.splitlines() OUTPUT1 = {"source.url": "http://example.com/glossy/zip/secure/d553c33636b465c21554b757e48bcf04/", "source.ip": "104.24.119.70", "time.source": "2018-02-06T15:16:06+00:00", "source.as_name": "Example Pete", "source.asn": 13335, "source.fqdn": "example.com", "source.geolocation.cc": "US", "raw": utils.base64_encode(SPLITTED[0]), "extra.brand": "Webmail Providers", "extra.country_name": "United States", "extra.discover_time": "06-02-2018 15:16:06 UTC", "extra.family_id": "922070603a96d81be0e354099d62f54e", "extra.sector": "Email Provider", "extra.tld": "ga", "classification.type": "phishing", "__type": "Event", }
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), 'chargen.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() with open(os.path.join(os.path.dirname(__file__), 'chargen_short.csv')) as handle: EXAMPLE_FILE_SHORT = handle.read() EXAMPLE_LINE_SHORT = EXAMPLE_FILE_SHORT.splitlines() with open(os.path.join(os.path.dirname(__file__), 'chargen_RECONSTRUCTED.csv')) as handle: RECONSTRUCTED_FILE = handle.read() RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines() EXAMPLE_REPORT = {"feed.name": "ShadowServer Chargen", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EXAMPLE_REPORT_SHORT = {"feed.name": "ShadowServer Chargen", "raw": utils.base64_encode(EXAMPLE_FILE_SHORT), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', 'feed.name': 'ShadowServer Chargen', 'classification.type': 'vulnerable service', 'classification.identifier': 'open-chargen', 'classification.taxonomy': 'vulnerable', 'extra.response_size': 116, 'extra.naics': 123456,
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), 'drone-hadoop.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() with open( os.path.join(os.path.dirname(__file__), 'drone-hadoop_RECONSTRUCTED.csv')) as handle: RECONSTRUCTED_FILE = handle.read() RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines() EXAMPLE_REPORT = { "feed.name": "ShadowServer Botnet Drone Hadoop", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{ '__type': 'Event', 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'malicious code', 'classification.type': 'botnet drone', 'classification.identifier': 'infected system', 'destination.asn':
with open(os.path.join(os.path.dirname(__file__), 'sipquery.txt')) as handle: SIP_QUERY_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'sipregistration.txt')) as handle: SIP_REGISTER_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'sshclient.txt')) as handle: SSH_CLIENT_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'sshpwauth.txt')) as handle: SSH_AUTH_FILE = handle.read() SIP_INVITE_REPORT = {'feed.url': 'http://dataplane.org/sipinvitation.txt', 'feed.name': 'SIP Invitation', '__type': 'Report', 'raw': utils.base64_encode(SIP_INVITE_FILE), 'time.observation': '2016-12-07T06:27:26+00:00' } SIP_INVITE_EVENT = {'feed.url': 'http://dataplane.org/sipinvitation.txt', 'feed.name': 'SIP Invitation', '__type': 'Event', 'time.observation': '2016-12-07T06:27:26+00:00', 'raw': 'ODU2MCAgICAgICAgIHwgIE9ORUFORE9ORS1BUyBCcmF1ZXJzdHJhc3NlIDQ4LCAgfCAgIDc0LjIwOC4xNDkuMjMxICB8ICAyMDE2LTEyLTA1IDE2OjA4OjI5ICB8ICBzaXBpbnZpdGF0aW9u', 'event_description.text': 'Address has been seen initiating a SIP INVITE operation to a remote host. ' 'The source report lists hosts that are suspicious of more than just port ' 'scanning. The host may be SIP client cataloging or conducting various forms ' 'of telephony abuse.', 'source.asn': 8560, 'source.ip': '74.208.149.231', 'source.as_name': 'ONEANDONE-AS',
# -*- coding: utf-8 -*- import unittest import os import intelmq.lib.utils as utils import intelmq.lib.test as test from intelmq.bots.parsers.generic.parser_csv import GenericCsvParserBot with open(os.path.join(os.path.dirname(__file__), 'test_filter_whitelist_ipset.csv')) as handle: SAMPLE_FILE = handle.read() SAMPLE_SPLIT = SAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"feed.name": "Sample CSV Feed", "raw": utils.base64_encode(SAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00"} EXAMPLE_EVENT = {"feed.name": "Sample CSV Feed", "__type": "Event", "raw": "aXBzZXQsYWRkLHRlc3RJUCwxMC4xLjEuMQ0K", "time.observation": "2015-01-01T00:00:00+00:00", "classification.type": "blacklist", "source.ip": "10.1.1.1"} class TestGenericCsvParserBot(test.BotTestCase, unittest.TestCase): """ A TestCase for a GenericCsvParserBot with extra, column_regex_search and windows_nt time format. """ @classmethod
# -*- coding: utf-8 -*- import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.spamhaus.parser_cert import SpamhausCERTParserBot with open(os.path.join(os.path.dirname(__file__), 'cert.txt')) as handle: FILE = handle.read() FILE_LINES = FILE.splitlines() EXAMPLE_REPORT = { "feed.url": "https://portal.spamhaus.org/cert/api.php?cert=" "<CERTNAME>&key=<APIKEY>", 'raw': utils.base64_encode(FILE), "__type": "Report", "feed.name": "Spamhaus Cert", "time.observation": "2015-01-01T00:00:00+00:00", } EVENT_TEMPL = { "feed.url": "https://portal.spamhaus.org/cert/api.php?cert=" "<CERTNAME>&", "feed.name": "Spamhaus Cert", "__type": "Event", "time.observation": "2015-01-01T00:00:00+00:00", } EXAMPLE_EVENTS_PARTS = [ { 'source.ip': '109.126.64.2', 'source.asn': 12635,
RAW_REPORTS = [ 'srcip=192.0.2.1 type=test', 'srcip=192.0.2.1 type=test', 'srcip=192.0.2.1 dstip=192.0.2.2 type=test', 'header srcip=192.0.2.1 type=test trailer', 'srcip="192.0.2.1" type="test"', 'comment="Quoted" srcip=192.0.2.1 type=test', 'srcip:192.0.2.1 type:test', 'srcip=192.0.2.1|type=test', 'srcip=192.0.2.1 type=test timestamp=1234567890', 'srcip=192.0.2.1 type=test timestamp="2009-02-14T00:31:30+01:00"' ] REPORTS = [{ 'feed.url': 'file://localhost/test', 'feed.name': 'Key-value parser test', '__type': 'Report', 'raw': utils.base64_encode(raw), 'time.observation': '2020-09-11T10:40:18+02:00' } for raw in RAW_REPORTS] KEYS = {'srcip': 'source.ip', 'type': 'classification.type'} PARAMETERS = [{ 'keys': KEYS }, { 'keys': { **KEYS, **{ 'dstip': 'destination.ip' } } }, {
# -*- coding: utf-8 -*- import os.path import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.calidog.parser_certstream import CertStreamParserBot with open(os.path.join(os.path.dirname(__file__), 'data.json')) as handle: RAW = handle.read() REPORT = { '__type': 'Report', 'raw': utils.base64_encode(RAW), 'time.observation': '2015-11-01T00:01:45+00:05', } EVENTS = [ { '__type': 'Event', 'classification.type': 'other', 'source.fqdn': 'fishpondcabins.com', 'time.source': '2018-06-15T16:10:21+00:00', 'raw': utils.base64_encode(RAW), }, { '__type': 'Event', 'classification.type': 'other', 'source.fqdn': 'mail.fishpondcabins.com', 'time.source': '2018-06-15T16:10:21+00:00', 'raw': utils.base64_encode(RAW), },
import intelmq.lib.utils as utils import intelmq.lib.test as test from intelmq.bots.parsers.cleanmx.parser import \ CleanMXParserBot with open(os.path.join(os.path.dirname(__file__), 'xmlphishing')) as handle: PHISHING_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'xmlviruses')) as handle: VIRUSES_FILE = handle.read() PHISHING_REPORT = {"feed.url": "http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&format=csv&domain=", "feed.name": "CleanMX Phishing", "__type": "Report", "raw": utils.base64_encode(PHISHING_FILE), "time.observation": "2015-11-02T13:11:43+00:00" } PHISHING_EVENTS = [{'__type': 'Event', 'classification.type': 'phishing', 'event_description.target': 'DHL', 'extra': '{"ddescr": "Example Layer", "id": "9377142", "inetnum": ' '"198.18.0.0 - 198.19.255.255", "netname": "EXAMPLE-NETWORK-15", ' '"ns1": "ns2.example.com", "ns2": "ns1.example.com", "phishtank": ' '"4647345", "response": "alive", "review": "198.18.0.1"}', 'feed.name': 'CleanMX Phishing', 'feed.url': 'http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&format=csv&domain=', 'raw': 'bGluZSxpZCxmaXJzdHRpbWUsbGFzdHRpbWUscGhpc2h0YW5rLHZpcnVzbmFtZSx1cmwscmVjZW50LHJlc3BvbnNlLGlwLHJldmlldyxkb21haW4sY291bnRyeSxzb3VyY2UsZW1haWwsaW5ldG51bSxuZXRuYW1lLGRkZXNjcixuczEsbnMyLG5zMyxuczQsbnM1DQoxLDkzNzcxNDIsMjAxNi0xMS0yOSAxMDozMTo0NSwxOTcwLTAxLTAxIDAxOjAwOjAwLDQ2NDczNDUsREhMLGh0dHA6Ly9leGFtcGxlLmNvbS9kZWhsJTIwcGFja2FnZS9jb25maXJtLyxkb3duLGFsaXZlLDE5OC4xOC4wLjEsMTk4LjE4LjAuMSwxOTguMTguMC4xLFVTLEFSSU4sYWJ1c2VAZXhhbXBsZS5jb20sMTk4LjE4LjAuMCAtIDE5OC4xOS4yNTUuMjU1LEVYQU1QTEUtTkVUV09SSy0xNSxFeGFtcGxlIExheWVyLG5zMi5leGFtcGxlLmNvbSxuczEuZXhhbXBsZS5jb20sLCwNCg==', 'source.abuse_contact': '*****@*****.**', 'source.geolocation.cc': 'US', 'source.ip': '198.18.0.1',
# -*- coding: utf-8 -*- import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.mcafee.parser_atd import ATDParserBot with open(os.path.join(os.path.dirname(__file__), 'atdreport.txt')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_REPORT = {"feed.name": "ATD", "__type": "Report", "raw": utils.base64_encode(EXAMPLE_FILE), "time.observation": "2015-11-02T13:11:43+00:00" } EXAMPLE_EVENT = {"feed.name": "ATD", "time.observation": "2015-11-02T13:11:44+00:00", "classification.taxonomy": "malicious code", "classification.type": "infected-system", "raw": utils.base64_encode(EXAMPLE_FILE), 'malware.hash.md5': '6C3F06652A4868E005EB42DAAF1CEE43', 'malware.hash.sha1': '6B53023EE7E6E336913DEBAD5BAD7E633362407A', 'malware.hash.sha256': 'DDBBB3C3141024A17E1F20C09C75D8913809062A3D326E2AC81626E65215C430', 'malware.name': 'install-tmetrade-trust-2f9aec.exe', "__type": "Event" }
# -*- coding: utf-8 -*- import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), 'microsoft-sinkhole.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"feed.name": "ShadowServer Microsoft-Sinkhole", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'malicious code', 'classification.type': 'infected-system', 'destination.asn': 8075, 'destination.geolocation.cc': 'SG', 'destination.ip': '168.63.184.224', 'destination.port': 16470, 'malware.name': 'b68-zeroaccess-1-64bit', 'protocol.application': 'http', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), 'accessible-hadoop.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() with open(os.path.join(os.path.dirname(__file__), 'accessible-hadoop-reconstructed.csv')) as handle: RECONSTRUCTED_FILE = handle.read() RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines() EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible-Hadoop", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', 'feed.name': 'ShadowServer Accessible-Hadoop', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable service', 'classification.identifier': 'accessible-hadoop', 'extra.version': '2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff', 'extra.server_type': 'namenode', 'extra.clusterid': 'CID-64471a53-60cb-4302-9832-92f321f111fe', 'extra.total_disk': 41567956992, 'extra.used_disk': 53248, 'extra.free_disk': 25160089600, 'extra.livenodes': 'edmonton:50010',
SNMP_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'blacklist_ssh_day.txt')) as handle: SSH_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'blacklist_telnet_day.txt')) as handle: TELNET_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'honeypot_dns_attacks.txt')) as handle: DNS_ATTACK_FILE = handle.read() SNMP_REPORT = {'feed.name': 'SNMP Blacklist', 'feed.url': 'http://www.nothink.org/blacklist/blacklist_snmp_day.txt', '__type': 'Report', 'time.observation': '2016-12-05T09:23:46+00:00', 'raw': utils.base64_encode(SNMP_FILE) } SNMP_EVENT = {'feed.name': 'SNMP Blacklist', 'feed.url': 'http://www.nothink.org/blacklist/blacklist_snmp_day.txt', '__type': 'Event', 'time.observation': '2016-12-05T09:23:46+00:00', 'raw': 'MTg1LjEyOC40MC4xNjI=', 'time.source': '2016-11-14T23:02:04+00:00', 'source.ip': '185.128.40.162', 'classification.type': 'scanner', 'protocol.application': 'snmp', } SSH_REPORT = {'feed.name': 'SSH Blacklist', 'feed.url': 'http://www.nothink.org/blacklist/blacklist_ssh_day.txt', '__type': 'Report',
# -*- coding: utf-8 -*- import os.path import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.cymru.parser_cap_program import CymruCAPProgramParserBot with open(os.path.join(os.path.dirname(__file__), 'infected_20171031.txt')) as handle: RAW = handle.read() RAW_LINES = RAW.splitlines() REPORT = { '__type': 'Report', 'raw': utils.base64_encode(RAW), 'time.observation': '2015-11-01T00:01:45+00:05', } EVENT0 = { '__type': 'Event', 'time.source': '2017-10-31 10:00:00', 'time.observation': '2015-11-01T00:01:45+00:05', 'classification.identifier': 'ssh', 'classification.type': 'brute-force', 'protocol.application': 'ssh', 'raw': utils.base64_encode('\n'.join(RAW_LINES[:3])), 'source.as_name': 'Example AS Name', 'source.asn': 64496, 'source.ip': '172.16.0.21', 'time.source': '2017-10-31T10:00:00+00:00', }
from intelmq.bots.parsers.microsoft.parser_ctip import MicrosoftCTIPParserBot from intelmq.lib import test from intelmq.lib.utils import base64_encode with open(os.path.join(os.path.dirname(__file__), 'ctip.txt')) as handle: EXAMPLE_DATA = handle.read() EXAMPLE_PARSED = json.loads(EXAMPLE_DATA) EXAMPLE_REPORT = { "__type": "Report", "feed.accuracy": 100.0, "feed.name": "misp_test", "feed.url": "http://192.168.56.102/", "time.observation": "2016-06-15T09:25:26+00:00", "raw": base64_encode(EXAMPLE_DATA) } EXAMPLE_EVENTS = [ { "__type": "Event", 'classification.type': 'infected system', 'destination.ip': '198.18.18.18', 'destination.port': 443, 'event_description.text': 'Host is a member of Botnet B85-R2S', 'extra.ispartnershareable': 'true', 'extra.isproductlicensed': 'true', 'tlp': 'GREEN', 'feed.accuracy': 20.0, 'feed.name': 'misp_test', 'feed.url': 'http://192.168.56.102/',
# -*- coding: utf-8 -*- import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.spamhaus.parser_cert import SpamhausCERTParserBot with open(os.path.join(os.path.dirname(__file__), 'cert.txt')) as handle: FILE = handle.read() FILE_LINES = FILE.splitlines() EXAMPLE_REPORT = {"feed.url": "https://portal.spamhaus.org/cert/api.php?cert=" "<CERTNAME>&key=<APIKEY>", 'raw': utils.base64_encode(FILE), "__type": "Report", "feed.name": "Spamhaus Cert", "time.observation": "2015-01-01T00:00:00+00:00", } EVENT_TEMPL = {"feed.url": "https://portal.spamhaus.org/cert/api.php?cert=" "<CERTNAME>&", "feed.name": "Spamhaus Cert", "__type": "Event", "time.observation": "2015-01-01T00:00:00+00:00", } EXAMPLE_EVENTS_PARTS = [{'source.ip': '109.126.64.2', 'source.asn': 12635, 'classification.type': 'infected-system', 'time.source': '2015-08-31T08:16:10+00:00', 'malware.name': 'asprox',
from intelmq.bots.parsers.netlab_360.parser import Netlab360ParserBot with open(os.path.join(os.path.dirname(__file__), 'dga.txt')) as handle: DGA_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'magnitude.txt')) as handle: MAGNITUDE_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'mirai.txt')) as handle: MIRAI_FILE = handle.read() DGA_REPORT = {'feed.name': 'Netlab 360 DGA', 'feed.url': 'http://data.netlab.360.com/feeds/dga/dga.txt', '__type': 'Report', 'time.observation': '2016-01-01T00:00:00+00:00', 'raw': utils.base64_encode(DGA_FILE), } DGA_EVENTS = {'feed.name': 'Netlab 360 DGA', 'feed.url': 'http://data.netlab.360.com/feeds/dga/dga.txt', '__type': 'Event', 'time.observation': '2016-01-01T00:00:00+00:00', 'time.source': '2016-11-13T00:04:15+00:00', 'source.fqdn': 'difficultdress.net', 'classification.type': 'c&c', 'classification.identifier': 'suppobox', 'event_description.url': 'http://data.netlab.360.com/dga', 'raw': 'c3VwcG9ib3gJZGlmZmljdWx0ZHJlc3MubmV0CTIwMTYtMTEtMTIgMTE6NTg6NTYJMjAxNi0xMS0xMyAwMDowNDoxNQ==', } MAGNITUDE_REPORT = {'feed.name': 'Netlab 360 Magnitude',
with open(os.path.join(os.path.dirname(__file__), 'chargen.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() with open(os.path.join(os.path.dirname(__file__), 'chargen_short.csv')) as handle: EXAMPLE_FILE_SHORT = handle.read() EXAMPLE_LINE_SHORT = EXAMPLE_FILE_SHORT.splitlines() with open(os.path.join(os.path.dirname(__file__), 'chargen_RECONSTRUCTED.csv')) as handle: RECONSTRUCTED_FILE = handle.read() RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines() EXAMPLE_REPORT = { "feed.name": "ShadowServer Chargen", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EXAMPLE_REPORT_SHORT = { "feed.name": "ShadowServer Chargen", "raw": utils.base64_encode(EXAMPLE_FILE_SHORT), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{ '__type': 'Event', 'classification.type': 'vulnerable service', 'classification.identifier':
import codecs import os import unittest import intelmq.lib.test as test from intelmq.bots.parsers.sucuri.parser import SucuriParserBot from intelmq.lib import utils with codecs.open(os.path.join(os.path.dirname(__file__), 'test_sucuri.data'), encoding='UTF-8') as handle: REPORT_DATA = handle.read() REPORT_DATA_SPLIT = REPORT_DATA.splitlines() REPORT = {"__type": "Report", "feed.name": "Sucuri Hidden Iframes", "feed.url": "http://labs.sucuri.net/?malware", "raw": utils.base64_encode(REPORT_DATA), "time.observation": "2018-01-22T14:38:24+00:00", } EVENT1 = {"__type": "Event", "classification.identifier": "hidden-iframe", "classification.type": "blacklist", "feed.name": "Sucuri Hidden Iframes", "feed.url": "http://labs.sucuri.net/?malware", "raw": utils.base64_encode(REPORT_DATA_SPLIT[616]), "source.url": "http://poseyhumane.org/stats.php", "time.observation": "2018-01-24T14:23:34+00:00", } EVENT2 = {"classification.identifier": "hidden-iframe", "feed.url": "http://labs.sucuri.net/?malware", "time.observation": "2018-01-24T15:58:48+00:00", "__type": "Event",
import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.collectors.tcp.collector import TCPCollectorBot from intelmq.bots.outputs.tcp.output import TCPOutputBot from intelmq.lib.message import Event from intelmq.lib.message import MessageFactory from intelmq.lib.utils import base64_decode PORT = 5011 SEPARATOR = '\n' INPUT1 = {'classification.taxonomy': 'malicious code', 'classification.type': 'c2server', 'feed.name': 'Example feed', 'feed.accuracy': 100.0, 'feed.url': 'http://localhost/two_files.tar.gz', 'raw': utils.base64_encode('bar text\n'), 'time.observation': '2018-03-20T14:05:03+00:00'} REPORT1 = { 'feed.name': 'Example feed', 'feed.accuracy': 100.0} # '__type': 'Report' # 'raw': 'eyJjbGFzc2lmaWNhdGlvbi50YXhvbm9teSI6ICJtYWxpY2lvdXMgY29kZSIsICJjbGFzc2lmaWNhdGlvbi50eXBlIjogImMmYyIsICJmZWVkLm5hbWUiOiAiRXhhbXBsZSBmZWVkIiwgImZlZWQuYWNjdXJhY3kiOiAxMDAuMCwgImZlZWQudXJsIjogImh0dHA6Ly9sb2NhbGhvc3QvdHdvX2ZpbGVzLnRhci5neiIsICJyYXciOiAiWW1GeUlIUmxlSFFLIiwgInRpbWUub2JzZXJ2YXRpb24iOiAiMjAxOC0wMy0yMFQxNDowNTowMyswMDowMCJ9', INPUT2 = {'feed.name': 'Example feed 2', 'feed.accuracy': 100.0, 'feed.url': 'http://localhost/two_files.tar.gz', 'raw': utils.base64_encode('foo text\n')} ORIGINAL_DATA = ('some random input{}another line').format(SEPARATOR) class Client: """ You find here an example of a non-intelmq client that might connect to the bot. """
import intelmq.lib.bot as bot import intelmq.lib.test as test import intelmq.lib.utils as utils RAW = """# ignore this 2015/06/04 13:37 +00,example.org,192.0.2.3,reverse.example.net,example description,[email protected],1 2015/06/04 13:38 +00,example.org,19d2.0.2.3,reverse.example.net,example description,[email protected],1 #ending line""" RAW_SPLIT = RAW.splitlines() EXAMPLE_REPORT = { "feed.url": "http://www.example.com/", "time.observation": "2015-08-11T13:03:40+00:00", "raw": utils.base64_encode(RAW), "__type": "Report", "feed.name": "Example" } EXAMPLE_EVENT = { "feed.url": "http://www.example.com/", "source.ip": "192.0.2.3", "time.source": "2015-06-04T13:37:00+00:00", "source.reverse_dns": "reverse.example.net", "source.fqdn": "example.org", "source.account": "*****@*****.**", "time.observation": "2015-08-11T13:03:40+00:00", "__type": "Event", "classification.type": "malware", "event_description.text": "example description", "source.asn": 1,
import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.spamhaus.parser_drop import SpamhausDropParserBot with open(os.path.join(os.path.dirname(__file__), 'drop.lasso')) as handle: EXAMPLE_FILE = handle.read() REPORT = {'__type': 'Report', 'feed.name': 'Spamhaus Drop', 'feed.url': 'https://www.spamhaus.org/drop/drop.lasso', 'raw': utils.base64_encode(EXAMPLE_FILE), 'time.observation': '2015-09-14T12:00:00+02:00', } TEMPLATE = {'__type': 'Event', 'feed.name': 'Spamhaus Drop', 'feed.url': 'https://www.spamhaus.org/drop/drop.lasso', 'time.observation': '2015-09-14T12:00:00+02:00', 'time.source': '2015-09-14T04:39:22+00:00', 'classification.type': 'spam', } EVENTS = ({'source.network': '1.4.0.0/17', 'extra': '{"blocklist": "SBL256893"}', 'raw': 'MS40LjAuMC8xNyA7IFNCTDI1Njg5Mw=='}, {'source.network': '1.10.16.0/20', 'extra': '{"blocklist": "SBL256894"}', 'raw': 'MS4xMC4xNi4wLzIwIDsgU0JMMjU2ODk0'},
# -*- coding: utf-8 -*- import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), 'drone-hadoop.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"feed.name": "ShadowServer Botnet Drone Hadoop", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'destination.asn': 8560, 'destination.geolocation.cc': 'US', 'destination.ip': '74.208.164.166', 'destination.port': 80, 'extra': '{"connection_count": 1, "os.name": "Windows", "os.version": "2000 SP4, XP SP1+"}', 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1], ''])),
rt_url = os.getenv('INTELMQ_TEST_RT_URL') rt_username = os.getenv('INTELMQ_TEST_RT_USERNAME') rt_password = os.getenv('INTELMQ_TEST_RT_PASSWORD') rt_subject = os.getenv('INTELMQ_TEST_RT_SUBJECT') rt_attachment_ticket = os.getenv('INTELMQ_TEST_RT_ATTACHMENT_TICKET') rt_url_ticket = os.getenv('INTELMQ_TEST_RT_URL_TICKET') REPORT = { "__type": "Report", "feed.name": "Example feed", "feed.accuracy": 100., "raw": utils.base64_encode('bar text\n'), "extra.file_name": "foobar", "extra.email_subject": "Incoming IntelMQ Test Report", "extra.ticket_subject": "Incoming IntelMQ Test Report", "extra.ticket_requestors": "*****@*****.**", "extra.email_from": "*****@*****.**", "extra.ticket_queue": "Incident Reports", "extra.ticket_status": "new", "extra.ticket_owner":
# -*- coding: utf-8 -*- import os.path import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.cymru.parser_cap_program import CymruCAPProgramParserBot with open(os.path.join(os.path.dirname(__file__), 'certname_20190327.txt')) as handle: RAW = handle.read() RAW_LINES = RAW.splitlines() REPORT = {'__type': 'Report', 'raw': utils.base64_encode(RAW), 'time.observation': '2015-11-01T00:01:45+00:05', } EVENT0 = {'__type': 'Event', 'time.observation': '2015-11-01T00:01:45+00:05', 'source.as_name': 'Example AS Name', 'source.asn': 64496, 'source.ip': '172.16.0.21', 'time.source': '2019-03-22T11:18:52+00:00', 'raw': utils.base64_encode('\n'.join(RAW_LINES[:2] + [RAW_LINES[3]])), 'classification.type': 'infected-system', 'classification.identifier': 'conficker', 'malware.name': 'conficker', } EVENT1 = {'__type': 'Event', 'time.observation': '2015-11-01T00:01:45+00:05',
import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open( os.path.join(os.path.dirname(__file__), "testdata/event4_ip_spoofer.csv")) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = { "feed.name": "CAIDA", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2020-08-19T00:00:00+00:00", "extra.file_name": "2020-08-19-event4_ip_spoofer.csv", } EVENTS = [ { "__type": "Event", "feed.name": "CAIDA", "time.source": "2021-03-28T00:42:59+00:00", "source.ip": "98.191.250.0", "source.asn": 22898, "source.geolocation.cc": "US", "source.geolocation.region": "OKLAHOMA", "source.geolocation.city": "OKLAHOMA CITY",
def sanitize(value): value = utils.base64_encode(value) return value
import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.cert_eu.parser_csv import CertEUCSVParserBot with open(os.path.join(os.path.dirname(__file__), 'example.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_FILE_SPLIT = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"feed.url": "https://www.cert-eu.eu", "feed.name": "CERT-EU Feed", "__type": "Report", "time.observation": "2018-01-01T01:00:00+00:00", "raw": utils.base64_encode(EXAMPLE_FILE), "feed.accuracy": 50, } EXAMPLE_EVENT1 = { "feed.url": "https://www.cert-eu.eu", "feed.name": "CERT-EU Feed", "source.ip": "109.125.93.10", "raw": utils.base64_encode('\r\n'.join(EXAMPLE_FILE_SPLIT[:2])), "time.observation": "2018-01-01T01:00:00+00:00", "extra.cert_eu_time_observation": "2019-04-01T10:13:23+00:00", "classification.type": "malware", "time.source": "2019-04-01T03:13:20+00:00", "tlp": "AMBER", "event_description.text": "A URL is the most common resource with reference to malware binary distribution.", "source.geolocation.country": "Germany",
SNMP_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'blacklist_ssh_day.txt')) as handle: SSH_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'blacklist_telnet_day.txt')) as handle: TELNET_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'honeypot_dns_attacks.txt'), encoding='utf-8') as handle: DNS_ATTACK_FILE = handle.read() SNMP_REPORT = {'feed.name': 'SNMP Blacklist', 'feed.url': 'http://www.nothink.org/blacklist/blacklist_snmp_day.txt', '__type': 'Report', 'time.observation': '2016-12-05T09:23:46+00:00', 'raw': utils.base64_encode(SNMP_FILE) } SNMP_EVENT = {'feed.name': 'SNMP Blacklist', 'feed.url': 'http://www.nothink.org/blacklist/blacklist_snmp_day.txt', '__type': 'Event', 'time.observation': '2016-12-05T09:23:46+00:00', 'raw': 'MTg1LjEyOC40MC4xNjI=', 'time.source': '2016-11-14T23:02:04+00:00', 'source.ip': '185.128.40.162', 'classification.type': 'scanner', 'protocol.application': 'snmp', } SSH_REPORT = {'feed.name': 'SSH Blacklist', 'feed.url': 'http://www.nothink.org/blacklist/blacklist_ssh_day.txt', '__type': 'Report',
with open(os.path.join(os.path.dirname(__file__), 'misp_event.json')) as handle: EXAMPLE_MISP_EVENT = handle.read() with open(os.path.join(os.path.dirname(__file__), 'misp_attribute.json')) as handle: EXAMPLE_MISP_ATTR = handle.read().strip() EXAMPLE_REPORT = { "__type": "Report", "feed.accuracy": 100.0, "feed.name": "misp_test", "feed.url": "http://192.168.56.102/", "time.observation": "2016-06-15T09:25:26+00:00", "raw": base64_encode(EXAMPLE_MISP_EVENT) } EXAMPLE_EVENT = { "__type": "Event", "feed.accuracy": 100.0, "feed.name": "misp_test", "feed.url": "http://192.168.56.102/", "time.observation": "2016-06-15T09:25:26+00:00", "time.source": "2016-06-11T21:41:44+00:00", "source.url": "http://fake.example.com/malware/is/here", "event_description.text": "Payload delivery", "event_description.url": "http://192.168.56.102/events/view/2", "classification.type": "infected-system", "malware.name": "locky", 'misp.attribute_uuid': '575c8598-f1f0-4c16-a94a-0612c0a83866',
# -*- coding: utf-8 -*- """ Data is from the HIBP Documentation """ import json import unittest import pkg_resources import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.hibp.parser_callback import HIBPCallbackParserBot BREACHREQUEST = json.load(open(pkg_resources.resource_filename('intelmq', 'tests/bots/parsers/hibp/breach_callbacktest.json'))) BREACHRAW = utils.base64_encode(json.dumps(BREACHREQUEST, sort_keys=True)) BR_REP = {"feed.name": "HIBP Enterprise", "time.observation": "2019-03-01T01:01:01+00:00", "__type": "Report", "raw": BREACHRAW } BR_EV = {"feed.name": "HIBP Enterprise", "raw": BREACHRAW, "time.observation": "2019-03-01T01:01:01+00:00", "extra.domain_emails": BREACHREQUEST["DomainEmails"], "extra.breach": BREACHREQUEST["Breach"], "classification.taxonomy": "information content security", "classification.type": "leak", "classification.identifier": "breach", "source.account": "*****@*****.**", "source.fqdn": "example.com", "__type": "Event" }
import os import unittest import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open( os.path.join(os.path.dirname(__file__), 'testdata/outdated_dnssec_key.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = { "feed.name": "ShadowServer Outdated DNSSEC Key", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-outdated_dnssec_key-test-geo.csv", } EVENTS = [{ '__type': 'Event', 'classification.identifier': 'outdated-dnssec-key', 'classification.taxonomy': 'availability', 'classification.type': 'other', 'destination.asn': 65537,
import intelmq.lib.utils as utils import intelmq.lib.test as test from intelmq.bots.parsers.cleanmx.parser import \ CleanMXParserBot with open(os.path.join(os.path.dirname(__file__), 'xmlphishing')) as handle: PHISHING_FILE = handle.read() with open(os.path.join(os.path.dirname(__file__), 'xmlviruses')) as handle: VIRUSES_FILE = handle.read() PHISHING_REPORT = { "feed.url": "http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&format=csv&domain=", "feed.name": "CleanMX Phishing", "__type": "Report", "raw": utils.base64_encode(PHISHING_FILE), "time.observation": "2015-11-02T13:11:43+00:00" } PHISHING_EVENTS = [{ '__type': 'Event', 'classification.type': 'phishing', 'event_description.target': 'DHL', 'extra': '{"ddescr": "Example Layer", "id": "9377142", "inetnum": ' '"198.18.0.0 - 198.19.255.255", "netname": "EXAMPLE-NETWORK-15", ' '"ns1": "ns2.example.com", "ns2": "ns1.example.com", "phishtank": ' '"4647345", "response": "alive", "review": "198.18.0.1"}', 'feed.name':