def test_strangerealintel_feed_processing_is_successful(self):
self.run_bot()
self.assertOutputQueueLen(len(EXAMPLE_STRANGERINTEL_FILE_JSON))
sha256_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy()
sha256_event['malware.hash.sha256'] = EXAMPLE_STRANGERINTEL_FILE_JSON[
0]['Indicator']
sha256_event[
'event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[0][
'Description']
sha256_event['classification.taxonomy'] = 'malicious code'
sha256_event['classification.type'] = 'malware'
sha256_event['raw'] = utils.base64_encode(
str(EXAMPLE_STRANGERINTEL_FILE_JSON[0]))
self.assertMessageEqual(0, sha256_event)
md5_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy()
md5_event['malware.hash.md5'] = EXAMPLE_STRANGERINTEL_FILE_JSON[1][
'Indicator']
md5_event['event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[
1]['Description']
md5_event['classification.taxonomy'] = 'malicious code'
md5_event['classification.type'] = 'malware'
md5_event['raw'] = utils.base64_encode(
str(EXAMPLE_STRANGERINTEL_FILE_JSON[1]))
self.assertMessageEqual(1, md5_event)
domain_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy()
domain_event['source.fqdn'] = EXAMPLE_STRANGERINTEL_FILE_JSON[2][
'Indicator']
# description text is empty so no field is created
# domain_event['event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[2]['Description']
domain_event['raw'] = utils.base64_encode(
str(EXAMPLE_STRANGERINTEL_FILE_JSON[2]))
self.assertMessageEqual(2, domain_event)
ip_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy()
ip_event['source.ip'] = EXAMPLE_STRANGERINTEL_FILE_JSON[3]['Indicator']
ip_event['event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[
3]['Description']
ip_event['raw'] = utils.base64_encode(
str(EXAMPLE_STRANGERINTEL_FILE_JSON[3]))
self.assertMessageEqual(3, ip_event)
url_event = EXAMPLE_STRANGEREALINTEL_EVENT.copy()
url_event['source.url'] = EXAMPLE_STRANGERINTEL_FILE_JSON[4][
'Indicator']
url_event['event_description.text'] = EXAMPLE_STRANGERINTEL_FILE_JSON[
4]['Description']
url_event['raw'] = utils.base64_encode(
str(EXAMPLE_STRANGERINTEL_FILE_JSON[4]))
self.assertMessageEqual(4, url_event)
def test_events(self):
""" Test if correct Events have been produced. """
self.run_bot()
for position, event in enumerate(EXAMPLE_EVENTS_PARTS):
event_ = EVENT_TEMPL.copy()
event_.update(event)
event_['raw'] = utils.base64_encode('\n'.join(
(FILE_LINES[0], FILE_LINES[1 + position])))
self.assertMessageEqual(position, event_)
def test_events(self):
""" Test if correct Events have been produced. """
self.run_bot()
for position, event in enumerate(EXAMPLE_EVENTS_PARTS):
event_ = EVENT_TEMPL.copy()
event_.update(event)
event_['raw'] = utils.base64_encode(FILE)
self.assertMessageEqual(position, event_)
self.assertOutputQFILE_LINES = FILE.splitlines()
def test_events(self):
""" Test if correct Events have been produced. """
self.run_bot()
for position, event in enumerate(EXAMPLE_EVENTS_PARTS):
event_ = EVENT_TEMPL.copy()
event_.update(event)
event_['raw'] = utils.base64_encode('\n'.join((FILE_LINES[0],
FILE_LINES[1+position])))
self.assertMessageEqual(position, event_)
def test_events(self):
""" Test if correct Events have been produced. """
self.input_message = REPORT
self.run_bot()
for i, event in enumerate(EVENTS):
event.update(EVENT_TEMPLATE)
event['raw'] = utils.base64_encode(RAWS[i])
self.assertMessageEqual(i, event)
self.assertEqual(len(self.get_output_queue()),
len(EVENTS),
msg='Length of EVENTS does not match length of '
'output queue.')
def test_adding_extra_fields_should_warn(self, requests_get_mock):
requests_get_mock.side_effect = print_requests_get_parameters
custom_config = SHOULD_PASS_WITH_TXT_FILES_AND_EXTRA_FIELD_SIZE_TEST[
'CONFIG'].copy()
custom_config['extra_fields'] = 'aaa,bbb'
self.allowed_warning_count = 2
self.run_bot(parameters=custom_config, prepare=True)
self.assertRegexpMatchesLog(
pattern=".*Field 'aaa' does not exist in the Github file data.*")
self.assertRegexpMatchesLog(
pattern=".*Field 'bbb' does not exist in the Github file data.*")
self.assertMessageEqual(
0, {
"__type": "Report",
"feed.name": "Github feed",
"feed.accuracy": 100.,
"feed.url": JSON_CONTENTS[1]['download_url'],
"raw": utils.base64_encode(EXAMPLE_CONTENT_STR)
})
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.nothink.parser_dns_attacks import NothinkDNSAttackParserBot
with open(os.path.join(os.path.dirname(__file__),
'honeypot_dns_attacks.txt')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_REPORT = {
'feed.url': 'http://www.nothink.org/honeypot_dns_attacks.txt',
'feed.name': 'DNS Attacks',
'__type': 'Report',
'raw': utils.base64_encode(EXAMPLE_FILE),
'time.observation': '2016-11-22T08:26:00+00:00'
}
EXAMPLE_EVENTS = [
{
'feed.url':
'http://www.nothink.org/honeypot_dns_attacks.txt',
'feed.name':
'DNS Attacks',
'__type':
'Event',
'raw':
'MjAxNi0xMS0xMSAxNToxMzoyMCwxODYuMi4xNjcuMTQsMjYyMjU0LERBTkNPTSBMVEQsLCwgQlosZGRvcy1ndWFyZC5uZXQsQlo=',
'time.source':
'2016-11-11T15:13:20+00:00',
from intelmq.bots.parsers.bambenek.parser import BambenekParserBot
with open(os.path.join(os.path.dirname(__file__), 'c2-dommasterlist.txt')) as handle:
DOMAIN_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'c2-ipmasterlist.txt')) as handle:
IP_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'dga-feed.txt')) as handle:
DGA_FILE = handle.read()
DOMAIN_REPORT = {'feed.url': 'http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt',
'feed.name': 'Bambenek C2 Domain Feed',
'__type': 'Report',
'raw': utils.base64_encode(DOMAIN_FILE),
'time.observation': '2016-01-01T00:00:00+00:00'
}
DOMAIN_EVENTS = {'feed.url': 'http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt',
'feed.name': 'Bambenek C2 Domain Feed',
'__type': 'Event',
'time.observation': '2016-01-01T00:00:00+00:00',
'raw': 'c2ZraDIxb2tycm5sdS5jb20sRG9tYWluIHVzZWQgYnkgc2hpb3RvYi91cmx6b25lL2JlYmxvaCwyMDE2LTExLTEyIDE1OjAyLGh0dHA6Ly9vc2ludC5iYW1iZW5la2NvbnN1bHRpbmcuY29tL21hbnVhbC9iZWJsb2gudHh0',
'time.source': '2016-11-12T15:02:00+00:00',
'source.fqdn': 'sfkh21okrrnlu.com',
'malware.name': 'bebloh',
'classification.type': 'c2server',
'status': 'online',
'event_description.text': 'Domain used by shiotob/urlzone/bebloh',
'event_description.url': 'http://osint.bambenekconsulting.com/manual/bebloh.txt'
# -*- coding: utf-8 -*-
import os
import unittest
import intelmq.lib.utils as utils
import intelmq.lib.test as test
from intelmq.bots.parsers.generic.parser_csv import GenericCsvParserBot
with open(os.path.join(os.path.dirname(__file__), 'invalid_sample_report.csv')) as handle:
SAMPLE_FILE = handle.read()
SAMPLE_SPLIT = SAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "Sample CSV Feed",
"__type": "Report",
"raw": utils.base64_encode(SAMPLE_FILE),
"time.observation": "2015-01-01T00:00:00+00:00",
}
EXAMPLE_EVENT = {"feed.name": "Sample CSV Feed",
"__type": "Event",
"time.source": "2018-08-01T07:49:41+00:00",
"classification.type": "malware",
"source.ip": "127.0.0.1",
"source.fqdn": "example.com",
"raw": utils.base64_encode(SAMPLE_SPLIT[1]+'\r\n'),
"time.observation": "2015-01-01T00:00:00+00:00",
}
EXAMPLE_EVENT2 = EXAMPLE_EVENT.copy()
del EXAMPLE_EVENT2['source.fqdn']
EXAMPLE_EVENT2["raw"] = utils.base64_encode(SAMPLE_SPLIT[0]+'\r\n')
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(os.path.join(os.path.dirname(__file__), "snmp.csv")) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
with open(os.path.join(os.path.dirname(__file__), "snmp_RECONSTRUCTED.csv")) as handle:
RECONSTRUCTED_FILE = handle.read()
RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines()
EXAMPLE_REPORT = {
"feed.name": "ShadowServer SNMP",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [
{
"__type": "Event",
"feed.name": "ShadowServer SNMP",
"classification.identifier": "opensnmp",
"classification.taxonomy": "Vulnerable",
"classification.type": "vulnerable service",
"extra": '{"sysdesc": "Hardware: x86 Family 6 Model 8 Stepping 6 AT/AT COMPATIBLE - Software: Windows 2000 Version 5.0 (Build 2195 Uniprocessor Free)", "sysname": "ORSONKA", "version": 2}',
"protocol.application": "snmp",
"protocol.transport": "udp",
"raw": utils.base64_encode("\n".join([RECONSTRUCTED_LINES[0], RECONSTRUCTED_LINES[1], ""])),
"source.asn": 22864,
reader = csv.DictReader(fop, restval="")
for row in reader:
datalist.append(row)
return json.dumps(datalist, indent=4)
CSVREPORTS = {}
JSONREPORTS = {}
testdata = pathlib.Path(__file__).parent / 'testdata'
for filename in testdata.glob('*.csv'):
EXAMPLE_FILE = filename.read_text()
shortname = filename.stem
CSVREPORTS[shortname] = {
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
"extra.file_name": "2019-01-01-{}-test-test.csv".format(shortname),
}
JSONREPORTS[shortname] = {
"raw": utils.base64_encode(csvtojson(filename)),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
"extra.file_name": "2019-01-01-{}-test-test.csv".format(shortname),
}
def generate_feed_function(feedname, reports):
def test_feed(self):
""" Test if no errors happen for feed %s. """ % feedname
import unittest
import intelmq.lib.test as test
from intelmq.bots.parsers.webinspektor.parser import WebinspektorParserBot
from intelmq.lib import utils
with open(os.path.join(os.path.dirname(__file__),
'test_webinspektor.data')) as handle:
REPORT_DATA = handle.read()
REPORT_DATA_SPLIT = REPORT_DATA.splitlines()
REPORT = {
"__type": "Report",
"feed.name": "Webinspektor",
"feed.url": "https://app.webinspector.com/public/recent_detections",
"raw": utils.base64_encode(REPORT_DATA),
"time.observation": "2018-01-22T14:38:24+00:00",
}
EVENT1 = {
"raw":
utils.base64_encode(REPORT_DATA_SPLIT[102].strip() +
REPORT_DATA_SPLIT[104].strip()),
"__type":
"Event",
"time.observation":
"2018-01-22T14:38:24+00:00",
"feed.url":
"https://app.webinspector.com/public/recent_detections",
"source.url":
"https://cummins.inhance.io",
"classification.taxonomy":
# -*- coding: utf-8 -*-
import unittest
import os
import intelmq.lib.utils as utils
import intelmq.lib.test as test
from intelmq.bots.parsers.generic.parser_csv import GenericCsvParserBot
with open(os.path.join(os.path.dirname(__file__), 'empty_src_url.csv')) as handle:
SAMPLE_FILE = handle.read()
SAMPLE_SPLIT = SAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "Sample CSV Feed",
"raw": utils.base64_encode(SAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EXAMPLE_EVENT = {"feed.name": "Sample CSV Feed",
"__type": "Event",
"raw": utils.base64_encode(SAMPLE_SPLIT[0] + '\r\n' +
SAMPLE_SPLIT[1].replace('"', '')+'\r\n'),
"time.observation": "2015-01-01T00:00:00+00:00",
"classification.type": "infected-system",
"source.ip": "11.11.11.11",
"feed.name": "Sample CSV Feed",
"source.port": 9001,
'time.source': '2017-05-26T07:00:36+00:00',
"source.url": "http://test.com"
}
# -*- coding: utf-8 -*-
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.abusech.parser_ip import AbusechIPParserBot
with open(os.path.join(os.path.dirname(__file__), 'zeusips.txt')) as handle:
EXAMPLE_ZEUS_FILE = handle.read()
EXAMPLE_ZEUS_REPORT = {"feed.url": "https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist",
"feed.name": "AbuseCH Zeustracker",
"__type": "Report",
"raw": utils.base64_encode(EXAMPLE_ZEUS_FILE),
"time.observation": "2015-11-02T13:11:44+00:00"
}
EXAMPLE_ZEUS_EVENT = {"feed.url": "https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist",
"feed.name": "AbuseCH Zeustracker",
"source.ip": "101.200.81.187",
"raw": utils.base64_encode("101.200.81.187"),
"time.observation": "2015-11-02T13:11:44+00:00",
"classification.taxonomy": "malicious code",
"classification.type": "c2server",
"malware.name": "zeus",
"__type": "Event"
}
# -*- coding: utf-8 -*-
import unittest
import os
import intelmq.lib.utils as utils
import intelmq.lib.test as test
from intelmq.bots.parsers.generic.parser_csv import GenericCsvParserBot
with open(os.path.join(os.path.dirname(__file__), 'data_type.csv')) as handle:
SAMPLE_FILE = handle.read()
SAMPLE_SPLIT = SAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "Sample CSV Feed",
"raw": utils.base64_encode(SAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00"}
EXAMPLE_EVENT = {"feed.name": "Sample CSV Feed",
"__type": "Event",
"raw": utils.base64_encode(SAMPLE_FILE.replace('\n', '\r\n')),
"time.observation": "2015-01-01T00:00:00+00:00",
"classification.type": "infected-system",
"source.ip": "11.11.11.11",
'extra.tags': ["t1", "t2", "t3"],
'source.url': 'http://test.com'}
class TestGenericCsvParserBot(test.BotTestCase, unittest.TestCase):
"""
A TestCase for a GenericCsvParserBot with extra, column_regex_search and windows_nt time format.
"""
import intelmq.lib.bot as bot import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.lib.message import Event RAW = """# ignore this 2015/06/04 13:37 +00,example.org,192.0.2.3,reverse.example.net,example description,[email protected],0 2015/06/04 13:38 +00,example.org,19d2.0.2.3,reverse.example.net,example description,[email protected],0 #ending line""" RAW_SPLIT = RAW.splitlines() EXAMPLE_REPORT = {"feed.url": "http://www.example.com/", "time.observation": "2015-08-11T13:03:40+00:00", "raw": utils.base64_encode(RAW), "__type": "Report", "feed.name": "Example"} EXAMPLE_EVENT = {"feed.url": "http://www.example.com/", "source.ip": "192.0.2.3", "time.source": "2015-06-04T13:37:00+00:00", "source.reverse_dns": "reverse.example.net", "source.fqdn": "example.org", "source.account": "*****@*****.**", "time.observation": "2015-08-11T13:03:40+00:00", "__type": "Event", "classification.type": "malware", "event_description.text": "example description", "source.asn": 0, "feed.name": "Example", "raw": utils.base64_encode('\n'.join(RAW_SPLIT[:2]))}
"time.observation": "2015-11-17T12:17:27.043452Z",
"source.geolocation.cc": "US",
"raw": "eyJjYXRlZ29yeSI6ICJib3RzIiwgIm9yaWdpbiI6ICJzaW5raG9sZSIsICJjb25maWRlbmNlIjogImhpZ2giLCAibmFtZSI6ICJzYWxpdHkiLCAicHJvdG8iOiAidGNwIiwgImFkZHJlc3MiOiBbeyJjYyI6ICJVUyIsICJpcCI6ICI2OC42OC45Ni4yMzUiLCAiYXNuIjogMTg3Nzl9XSwgInNvdXJjZSI6ICJjZXJ0LXBsLnNpbmtob2xlIiwgImFkaXAiOiAieC54LjExMS45OSIsICJ0aW1lIjogIjIwMTUtMTEtMTdUMTI6MTc6MjcuMDQzNDUyWiIsICJkcG9ydCI6IDgwLCAic3BvcnQiOiAyMjMwOCwgInR5cGUiOiAiZXZlbnQiLCAiaWQiOiAiZDc3YWU4Y2Y2ODFkY2RiYjZlMjAwMTQ1ODE0MDFlZDUifQ==", # noqa
"classification.identifier": "sality",
"malware.name": "sality",
"classification.type": "infected-system",
"destination.port": 80,
"__type": "Event",
"protocol.transport": "tcp",
"source.asn": 18779,
'extra.confidence': 'high',
'extra.feed_source': 'cert-pl.sinkhole',
}
BLACKLIST_REPORT = {"__type": "Report",
"raw": utils.base64_encode("""
{"category": "other", "confidence": "low", "expires": "2019-02-03T20:12:03Z", "fqdn": "example.com", "source": "example", "time": "2019-01-12T23:12:02Z", "type": "bl-update", "id": "some hex string", "address": [{"cc": "AT", "ip": "10.0.0.1", "asn": 65536}]}
"""),
"time.observation": "2015-11-17T12:17:27.043452Z",
}
BLACKLIST_EVENT = {"__type": "Event",
"classification.taxonomy": "other",
"classification.type": "blacklist",
"extra.confidence": "low",
"extra.expires": "2019-02-03T20:12:03+00:00",
"source.fqdn": "example.com",
"extra.feed_source": "example",
"time.source": "2019-01-12T23:12:02+00:00",
"extra.feed_id": "some hex string",
"source.geolocation.cc": "AT",
"source.ip": "10.0.0.1",
"source.asn": 65536,
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(os.path.join(os.path.dirname(__file__), 'chargen.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
with open(os.path.join(os.path.dirname(__file__), 'chargen_short.csv')) as handle:
EXAMPLE_FILE_SHORT = handle.read()
EXAMPLE_LINE_SHORT = EXAMPLE_FILE_SHORT.splitlines()
EXAMPLE_REPORT = {"feed.name": "ShadowServer Chargen",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EXAMPLE_REPORT_SHORT = {"feed.name": "ShadowServer Chargen",
"raw": utils.base64_encode(EXAMPLE_FILE_SHORT),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [{'__type': 'Event',
'feed.name': 'ShadowServer Chargen',
'classification.type': 'vulnerable service',
'classification.identifier': 'open-chargen',
'classification.taxonomy': 'vulnerable',
'extra.response_size': 116,
'extra.naics': 123456,
from intelmq.bots.parsers.malc0de.parser import Malc0deParserBot
with open(os.path.join(os.path.dirname(__file__), 'BOOT')) as handle:
BOOT_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'ZONE')) as handle:
ZONE_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'IP_Blacklist.txt')) as handle:
IP_BLACKLIST_FILE = handle.read()
BOOT_REPORT = {'feed.name': 'Windows Format',
'feed.url': 'http://malc0de.com/bl/BOOT',
'__type': 'Report',
'time.observation': '2016-12-4T14:16:25+00:00',
'raw': utils.base64_encode(BOOT_FILE)
}
BOOT_EVENT = [{'feed.name': 'Windows Format',
'feed.url': 'http://malc0de.com/bl/BOOT',
'__type': 'Event',
'time.observation': '2016-12-4T14:16:25+00:00',
'time.source': '2016-11-10T00:00:00+00:00',
'raw': 'UFJJTUFSWSBleGFtcGxlLmNvbSBibG9ja2VkZG9tYWluLmhvc3Rz',
'source.fqdn': 'example.com',
'event_description.url': 'http://malc0de.com/database/index.php?search=example.com',
'classification.type': 'malware'
},
{'feed.name': 'Windows Format',
'feed.url': 'http://malc0de.com/bl/BOOT',
'__type': 'Event',
# -*- coding: utf-8 -*-
import os
import unittest
import intelmq.lib.test as test
from intelmq.bots.parsers.webinspektor.parser import WebinspektorParserBot
from intelmq.lib import utils
with open(os.path.join(os.path.dirname(__file__), 'test_webinspektor.data')) as handle:
REPORT_DATA = handle.read()
REPORT_DATA_SPLIT = REPORT_DATA.splitlines()
REPORT = {"__type": "Report",
"feed.name": "Webinspektor",
"feed.url": "https://app.webinspector.com/public/recent_detections",
"raw": utils.base64_encode(REPORT_DATA),
"time.observation": "2018-01-22T14:38:24+00:00",
}
EVENT1 = {"raw": utils.base64_encode(REPORT_DATA_SPLIT[102].strip() + REPORT_DATA_SPLIT[104].strip()),
"__type": "Event",
"time.observation": "2018-01-22T14:38:24+00:00",
"feed.url": "https://app.webinspector.com/public/recent_detections",
"source.url": "https://cummins.inhance.io",
"classification.taxonomy": "other",
"classification.type": "blacklist",
"classification.identifier": "Suspicious",
"time.source": "2018-02-13T08:44:30+00:00",
"feed.name": "Webinspektor"}
EVENT2 = {"raw": utils.base64_encode(REPORT_DATA_SPLIT[111].strip() + REPORT_DATA_SPLIT[113].strip()),
"__type": "Event",
"time.observation": "2018-01-22T14:38:24+00:00",
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(os.path.join(os.path.dirname(__file__), 'accessible-cisco-smart-install.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
with open(os.path.join(os.path.dirname(__file__),
'accessible-cisco-smart-install-reconstructed.csv')) as handle:
RECONSTRUCTED_FILE = handle.read()
RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible Cisco Smart Install",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [{'__type': 'Event',
'feed.name': 'ShadowServer Accessible Cisco Smart Install',
'classification.identifier': 'accessible-cisco-smart-install',
'classification.type': 'vulnerable service',
'protocol.application': 'cisco-smart-install',
'protocol.transport': 'tcp',
'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0],
RECONSTRUCTED_LINES[1], ''])),
'source.asn': 8559,
'source.geolocation.cc': 'AT',
'source.geolocation.city': 'VIENNA',
'source.geolocation.region': 'WIEN',
from intelmq.lib import test
from intelmq.lib.utils import base64_encode
with open(os.path.join(os.path.dirname(__file__), 'misp_event.json')) as handle:
EXAMPLE_MISP_EVENT = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'misp_attribute.json')) as handle:
EXAMPLE_MISP_ATTR = handle.read()
EXAMPLE_REPORT = {
"__type": "Report",
"feed.accuracy": 100.0,
"feed.name": "misp_test",
"feed.url": "http://192.168.56.102/",
"time.observation": "2016-06-15T09:25:26+00:00",
"raw": base64_encode(EXAMPLE_MISP_EVENT)
}
EXAMPLE_EVENT = {
"__type": "Event",
"feed.accuracy": 100.0,
"feed.name": "misp_test",
"feed.url": "http://192.168.56.102/",
"time.observation": "2016-06-15T09:25:26+00:00",
"time.source": "2016-06-11T21:41:44+00:00",
"source.url": "http://fake.website.com/malware/is/here",
"event_description.text": "Payload delivery",
"event_description.url": "http://192.168.56.102/event/view/2",
"classification.type": "ransomware",
"malware.name": "locky",
'misp.attribute_uuid': '575c8598-f1f0-4c16-a94a-0612c0a83866',
# -*- coding: utf-8 -*-
import base64
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.openphish.parser_commercial import OpenPhishCommercialParserBot
with open(os.path.join(os.path.dirname(__file__), 'feed_commercial.txt'), 'r') as fh:
FILE = fh.read()
RAW = utils.base64_encode(FILE.encode())
SPLITTED = FILE.splitlines()
OUTPUT1 = {"source.url": "http://example.com/glossy/zip/secure/d553c33636b465c21554b757e48bcf04/",
"source.ip": "104.24.119.70",
"time.source": "2018-02-06T15:16:06+00:00",
"source.as_name": "Example Pete",
"source.asn": 13335,
"source.fqdn": "example.com",
"source.geolocation.cc": "US",
"raw": utils.base64_encode(SPLITTED[0]),
"extra.brand": "Webmail Providers",
"extra.country_name": "United States",
"extra.discover_time": "06-02-2018 15:16:06 UTC",
"extra.family_id": "922070603a96d81be0e354099d62f54e",
"extra.sector": "Email Provider",
"extra.tld": "ga",
"classification.type": "phishing",
"__type": "Event",
}
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(os.path.join(os.path.dirname(__file__), 'chargen.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
with open(os.path.join(os.path.dirname(__file__), 'chargen_short.csv')) as handle:
EXAMPLE_FILE_SHORT = handle.read()
EXAMPLE_LINE_SHORT = EXAMPLE_FILE_SHORT.splitlines()
with open(os.path.join(os.path.dirname(__file__),
'chargen_RECONSTRUCTED.csv')) as handle:
RECONSTRUCTED_FILE = handle.read()
RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "ShadowServer Chargen",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EXAMPLE_REPORT_SHORT = {"feed.name": "ShadowServer Chargen",
"raw": utils.base64_encode(EXAMPLE_FILE_SHORT),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [{'__type': 'Event',
'feed.name': 'ShadowServer Chargen',
'classification.type': 'vulnerable service',
'classification.identifier': 'open-chargen',
'classification.taxonomy': 'vulnerable',
'extra.response_size': 116,
'extra.naics': 123456,
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(os.path.join(os.path.dirname(__file__),
'drone-hadoop.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
with open(
os.path.join(os.path.dirname(__file__),
'drone-hadoop_RECONSTRUCTED.csv')) as handle:
RECONSTRUCTED_FILE = handle.read()
RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines()
EXAMPLE_REPORT = {
"feed.name": "ShadowServer Botnet Drone Hadoop",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [{
'__type':
'Event',
'feed.name':
'ShadowServer Botnet Drone Hadoop',
'classification.taxonomy':
'malicious code',
'classification.type':
'botnet drone',
'classification.identifier':
'infected system',
'destination.asn':
with open(os.path.join(os.path.dirname(__file__), 'sipquery.txt')) as handle:
SIP_QUERY_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'sipregistration.txt')) as handle:
SIP_REGISTER_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'sshclient.txt')) as handle:
SSH_CLIENT_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'sshpwauth.txt')) as handle:
SSH_AUTH_FILE = handle.read()
SIP_INVITE_REPORT = {'feed.url': 'http://dataplane.org/sipinvitation.txt',
'feed.name': 'SIP Invitation',
'__type': 'Report',
'raw': utils.base64_encode(SIP_INVITE_FILE),
'time.observation': '2016-12-07T06:27:26+00:00'
}
SIP_INVITE_EVENT = {'feed.url': 'http://dataplane.org/sipinvitation.txt',
'feed.name': 'SIP Invitation',
'__type': 'Event',
'time.observation': '2016-12-07T06:27:26+00:00',
'raw': 'ODU2MCAgICAgICAgIHwgIE9ORUFORE9ORS1BUyBCcmF1ZXJzdHJhc3NlIDQ4LCAgfCAgIDc0LjIwOC4xNDkuMjMxICB8ICAyMDE2LTEyLTA1IDE2OjA4OjI5ICB8ICBzaXBpbnZpdGF0aW9u',
'event_description.text': 'Address has been seen initiating a SIP INVITE operation to a remote host. '
'The source report lists hosts that are suspicious of more than just port '
'scanning. The host may be SIP client cataloging or conducting various forms '
'of telephony abuse.',
'source.asn': 8560,
'source.ip': '74.208.149.231',
'source.as_name': 'ONEANDONE-AS',
# -*- coding: utf-8 -*-
import unittest
import os
import intelmq.lib.utils as utils
import intelmq.lib.test as test
from intelmq.bots.parsers.generic.parser_csv import GenericCsvParserBot
with open(os.path.join(os.path.dirname(__file__), 'test_filter_whitelist_ipset.csv')) as handle:
SAMPLE_FILE = handle.read()
SAMPLE_SPLIT = SAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "Sample CSV Feed",
"raw": utils.base64_encode(SAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00"}
EXAMPLE_EVENT = {"feed.name": "Sample CSV Feed",
"__type": "Event",
"raw": "aXBzZXQsYWRkLHRlc3RJUCwxMC4xLjEuMQ0K",
"time.observation": "2015-01-01T00:00:00+00:00",
"classification.type": "blacklist",
"source.ip": "10.1.1.1"}
class TestGenericCsvParserBot(test.BotTestCase, unittest.TestCase):
"""
A TestCase for a GenericCsvParserBot with extra, column_regex_search and windows_nt time format.
"""
@classmethod
# -*- coding: utf-8 -*-
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.spamhaus.parser_cert import SpamhausCERTParserBot
with open(os.path.join(os.path.dirname(__file__), 'cert.txt')) as handle:
FILE = handle.read()
FILE_LINES = FILE.splitlines()
EXAMPLE_REPORT = {
"feed.url": "https://portal.spamhaus.org/cert/api.php?cert="
"<CERTNAME>&key=<APIKEY>",
'raw': utils.base64_encode(FILE),
"__type": "Report",
"feed.name": "Spamhaus Cert",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENT_TEMPL = {
"feed.url": "https://portal.spamhaus.org/cert/api.php?cert="
"<CERTNAME>&",
"feed.name": "Spamhaus Cert",
"__type": "Event",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EXAMPLE_EVENTS_PARTS = [
{
'source.ip': '109.126.64.2',
'source.asn': 12635,
RAW_REPORTS = [
'srcip=192.0.2.1 type=test', 'srcip=192.0.2.1 type=test',
'srcip=192.0.2.1 dstip=192.0.2.2 type=test',
'header srcip=192.0.2.1 type=test trailer',
'srcip="192.0.2.1" type="test"',
'comment="Quoted" srcip=192.0.2.1 type=test', 'srcip:192.0.2.1 type:test',
'srcip=192.0.2.1|type=test',
'srcip=192.0.2.1 type=test timestamp=1234567890',
'srcip=192.0.2.1 type=test timestamp="2009-02-14T00:31:30+01:00"'
]
REPORTS = [{
'feed.url': 'file://localhost/test',
'feed.name': 'Key-value parser test',
'__type': 'Report',
'raw': utils.base64_encode(raw),
'time.observation': '2020-09-11T10:40:18+02:00'
} for raw in RAW_REPORTS]
KEYS = {'srcip': 'source.ip', 'type': 'classification.type'}
PARAMETERS = [{
'keys': KEYS
}, {
'keys': {
**KEYS,
**{
'dstip': 'destination.ip'
}
}
}, {
# -*- coding: utf-8 -*-
import os.path
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.calidog.parser_certstream import CertStreamParserBot
with open(os.path.join(os.path.dirname(__file__), 'data.json')) as handle:
RAW = handle.read()
REPORT = {
'__type': 'Report',
'raw': utils.base64_encode(RAW),
'time.observation': '2015-11-01T00:01:45+00:05',
}
EVENTS = [
{
'__type': 'Event',
'classification.type': 'other',
'source.fqdn': 'fishpondcabins.com',
'time.source': '2018-06-15T16:10:21+00:00',
'raw': utils.base64_encode(RAW),
},
{
'__type': 'Event',
'classification.type': 'other',
'source.fqdn': 'mail.fishpondcabins.com',
'time.source': '2018-06-15T16:10:21+00:00',
'raw': utils.base64_encode(RAW),
},
import intelmq.lib.utils as utils
import intelmq.lib.test as test
from intelmq.bots.parsers.cleanmx.parser import \
CleanMXParserBot
with open(os.path.join(os.path.dirname(__file__), 'xmlphishing')) as handle:
PHISHING_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'xmlviruses')) as handle:
VIRUSES_FILE = handle.read()
PHISHING_REPORT = {"feed.url": "http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&format=csv&domain=",
"feed.name": "CleanMX Phishing",
"__type": "Report",
"raw": utils.base64_encode(PHISHING_FILE),
"time.observation": "2015-11-02T13:11:43+00:00"
}
PHISHING_EVENTS = [{'__type': 'Event',
'classification.type': 'phishing',
'event_description.target': 'DHL',
'extra': '{"ddescr": "Example Layer", "id": "9377142", "inetnum": '
'"198.18.0.0 - 198.19.255.255", "netname": "EXAMPLE-NETWORK-15", '
'"ns1": "ns2.example.com", "ns2": "ns1.example.com", "phishtank": '
'"4647345", "response": "alive", "review": "198.18.0.1"}',
'feed.name': 'CleanMX Phishing',
'feed.url': 'http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&format=csv&domain=',
'raw': 'bGluZSxpZCxmaXJzdHRpbWUsbGFzdHRpbWUscGhpc2h0YW5rLHZpcnVzbmFtZSx1cmwscmVjZW50LHJlc3BvbnNlLGlwLHJldmlldyxkb21haW4sY291bnRyeSxzb3VyY2UsZW1haWwsaW5ldG51bSxuZXRuYW1lLGRkZXNjcixuczEsbnMyLG5zMyxuczQsbnM1DQoxLDkzNzcxNDIsMjAxNi0xMS0yOSAxMDozMTo0NSwxOTcwLTAxLTAxIDAxOjAwOjAwLDQ2NDczNDUsREhMLGh0dHA6Ly9leGFtcGxlLmNvbS9kZWhsJTIwcGFja2FnZS9jb25maXJtLyxkb3duLGFsaXZlLDE5OC4xOC4wLjEsMTk4LjE4LjAuMSwxOTguMTguMC4xLFVTLEFSSU4sYWJ1c2VAZXhhbXBsZS5jb20sMTk4LjE4LjAuMCAtIDE5OC4xOS4yNTUuMjU1LEVYQU1QTEUtTkVUV09SSy0xNSxFeGFtcGxlIExheWVyLG5zMi5leGFtcGxlLmNvbSxuczEuZXhhbXBsZS5jb20sLCwNCg==',
'source.abuse_contact': '*****@*****.**',
'source.geolocation.cc': 'US',
'source.ip': '198.18.0.1',
# -*- coding: utf-8 -*-
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.mcafee.parser_atd import ATDParserBot
with open(os.path.join(os.path.dirname(__file__), 'atdreport.txt')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_REPORT = {"feed.name": "ATD",
"__type": "Report",
"raw": utils.base64_encode(EXAMPLE_FILE),
"time.observation": "2015-11-02T13:11:43+00:00"
}
EXAMPLE_EVENT = {"feed.name": "ATD",
"time.observation": "2015-11-02T13:11:44+00:00",
"classification.taxonomy": "malicious code",
"classification.type": "infected-system",
"raw": utils.base64_encode(EXAMPLE_FILE),
'malware.hash.md5': '6C3F06652A4868E005EB42DAAF1CEE43',
'malware.hash.sha1': '6B53023EE7E6E336913DEBAD5BAD7E633362407A',
'malware.hash.sha256': 'DDBBB3C3141024A17E1F20C09C75D8913809062A3D326E2AC81626E65215C430',
'malware.name': 'install-tmetrade-trust-2f9aec.exe',
"__type": "Event"
}
# -*- coding: utf-8 -*-
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(os.path.join(os.path.dirname(__file__),
'microsoft-sinkhole.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "ShadowServer Microsoft-Sinkhole",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [{'__type': 'Event',
'feed.name': 'ShadowServer Microsoft-Sinkhole',
'classification.taxonomy': 'malicious code',
'classification.type': 'infected-system',
'destination.asn': 8075,
'destination.geolocation.cc': 'SG',
'destination.ip': '168.63.184.224',
'destination.port': 16470,
'malware.name': 'b68-zeroaccess-1-64bit',
'protocol.application': 'http',
'protocol.transport': 'tcp',
'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(os.path.join(os.path.dirname(__file__),
'accessible-hadoop.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
with open(os.path.join(os.path.dirname(__file__),
'accessible-hadoop-reconstructed.csv')) as handle:
RECONSTRUCTED_FILE = handle.read()
RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible-Hadoop",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [{'__type': 'Event',
'feed.name': 'ShadowServer Accessible-Hadoop',
'classification.taxonomy': 'vulnerable',
'classification.type': 'vulnerable service',
'classification.identifier': 'accessible-hadoop',
'extra.version': '2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff',
'extra.server_type': 'namenode',
'extra.clusterid': 'CID-64471a53-60cb-4302-9832-92f321f111fe',
'extra.total_disk': 41567956992,
'extra.used_disk': 53248,
'extra.free_disk': 25160089600,
'extra.livenodes': 'edmonton:50010',
SNMP_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'blacklist_ssh_day.txt')) as handle:
SSH_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'blacklist_telnet_day.txt')) as handle:
TELNET_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'honeypot_dns_attacks.txt')) as handle:
DNS_ATTACK_FILE = handle.read()
SNMP_REPORT = {'feed.name': 'SNMP Blacklist',
'feed.url': 'http://www.nothink.org/blacklist/blacklist_snmp_day.txt',
'__type': 'Report',
'time.observation': '2016-12-05T09:23:46+00:00',
'raw': utils.base64_encode(SNMP_FILE)
}
SNMP_EVENT = {'feed.name': 'SNMP Blacklist',
'feed.url': 'http://www.nothink.org/blacklist/blacklist_snmp_day.txt',
'__type': 'Event',
'time.observation': '2016-12-05T09:23:46+00:00',
'raw': 'MTg1LjEyOC40MC4xNjI=',
'time.source': '2016-11-14T23:02:04+00:00',
'source.ip': '185.128.40.162',
'classification.type': 'scanner',
'protocol.application': 'snmp',
}
SSH_REPORT = {'feed.name': 'SSH Blacklist',
'feed.url': 'http://www.nothink.org/blacklist/blacklist_ssh_day.txt',
'__type': 'Report',
# -*- coding: utf-8 -*-
import os.path
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.cymru.parser_cap_program import CymruCAPProgramParserBot
with open(os.path.join(os.path.dirname(__file__),
'infected_20171031.txt')) as handle:
RAW = handle.read()
RAW_LINES = RAW.splitlines()
REPORT = {
'__type': 'Report',
'raw': utils.base64_encode(RAW),
'time.observation': '2015-11-01T00:01:45+00:05',
}
EVENT0 = {
'__type': 'Event',
'time.source': '2017-10-31 10:00:00',
'time.observation': '2015-11-01T00:01:45+00:05',
'classification.identifier': 'ssh',
'classification.type': 'brute-force',
'protocol.application': 'ssh',
'raw': utils.base64_encode('\n'.join(RAW_LINES[:3])),
'source.as_name': 'Example AS Name',
'source.asn': 64496,
'source.ip': '172.16.0.21',
'time.source': '2017-10-31T10:00:00+00:00',
}
from intelmq.bots.parsers.microsoft.parser_ctip import MicrosoftCTIPParserBot
from intelmq.lib import test
from intelmq.lib.utils import base64_encode
with open(os.path.join(os.path.dirname(__file__), 'ctip.txt')) as handle:
EXAMPLE_DATA = handle.read()
EXAMPLE_PARSED = json.loads(EXAMPLE_DATA)
EXAMPLE_REPORT = {
"__type": "Report",
"feed.accuracy": 100.0,
"feed.name": "misp_test",
"feed.url": "http://192.168.56.102/",
"time.observation": "2016-06-15T09:25:26+00:00",
"raw": base64_encode(EXAMPLE_DATA)
}
EXAMPLE_EVENTS = [
{
"__type": "Event",
'classification.type': 'infected system',
'destination.ip': '198.18.18.18',
'destination.port': 443,
'event_description.text': 'Host is a member of Botnet B85-R2S',
'extra.ispartnershareable': 'true',
'extra.isproductlicensed': 'true',
'tlp': 'GREEN',
'feed.accuracy': 20.0,
'feed.name': 'misp_test',
'feed.url': 'http://192.168.56.102/',
# -*- coding: utf-8 -*-
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.spamhaus.parser_cert import SpamhausCERTParserBot
with open(os.path.join(os.path.dirname(__file__), 'cert.txt')) as handle:
FILE = handle.read()
FILE_LINES = FILE.splitlines()
EXAMPLE_REPORT = {"feed.url": "https://portal.spamhaus.org/cert/api.php?cert="
"<CERTNAME>&key=<APIKEY>",
'raw': utils.base64_encode(FILE),
"__type": "Report",
"feed.name": "Spamhaus Cert",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENT_TEMPL = {"feed.url": "https://portal.spamhaus.org/cert/api.php?cert="
"<CERTNAME>&",
"feed.name": "Spamhaus Cert",
"__type": "Event",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EXAMPLE_EVENTS_PARTS = [{'source.ip': '109.126.64.2',
'source.asn': 12635,
'classification.type': 'infected-system',
'time.source': '2015-08-31T08:16:10+00:00',
'malware.name': 'asprox',
from intelmq.bots.parsers.netlab_360.parser import Netlab360ParserBot
with open(os.path.join(os.path.dirname(__file__), 'dga.txt')) as handle:
DGA_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'magnitude.txt')) as handle:
MAGNITUDE_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'mirai.txt')) as handle:
MIRAI_FILE = handle.read()
DGA_REPORT = {'feed.name': 'Netlab 360 DGA',
'feed.url': 'http://data.netlab.360.com/feeds/dga/dga.txt',
'__type': 'Report',
'time.observation': '2016-01-01T00:00:00+00:00',
'raw': utils.base64_encode(DGA_FILE),
}
DGA_EVENTS = {'feed.name': 'Netlab 360 DGA',
'feed.url': 'http://data.netlab.360.com/feeds/dga/dga.txt',
'__type': 'Event',
'time.observation': '2016-01-01T00:00:00+00:00',
'time.source': '2016-11-13T00:04:15+00:00',
'source.fqdn': 'difficultdress.net',
'classification.type': 'c&c',
'classification.identifier': 'suppobox',
'event_description.url': 'http://data.netlab.360.com/dga',
'raw': 'c3VwcG9ib3gJZGlmZmljdWx0ZHJlc3MubmV0CTIwMTYtMTEtMTIgMTE6NTg6NTYJMjAxNi0xMS0xMyAwMDowNDoxNQ==',
}
MAGNITUDE_REPORT = {'feed.name': 'Netlab 360 Magnitude',
with open(os.path.join(os.path.dirname(__file__), 'chargen.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
with open(os.path.join(os.path.dirname(__file__),
'chargen_short.csv')) as handle:
EXAMPLE_FILE_SHORT = handle.read()
EXAMPLE_LINE_SHORT = EXAMPLE_FILE_SHORT.splitlines()
with open(os.path.join(os.path.dirname(__file__),
'chargen_RECONSTRUCTED.csv')) as handle:
RECONSTRUCTED_FILE = handle.read()
RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines()
EXAMPLE_REPORT = {
"feed.name": "ShadowServer Chargen",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EXAMPLE_REPORT_SHORT = {
"feed.name": "ShadowServer Chargen",
"raw": utils.base64_encode(EXAMPLE_FILE_SHORT),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [{
'__type':
'Event',
'classification.type':
'vulnerable service',
'classification.identifier':
import codecs
import os
import unittest
import intelmq.lib.test as test
from intelmq.bots.parsers.sucuri.parser import SucuriParserBot
from intelmq.lib import utils
with codecs.open(os.path.join(os.path.dirname(__file__), 'test_sucuri.data'), encoding='UTF-8') as handle:
REPORT_DATA = handle.read()
REPORT_DATA_SPLIT = REPORT_DATA.splitlines()
REPORT = {"__type": "Report",
"feed.name": "Sucuri Hidden Iframes",
"feed.url": "http://labs.sucuri.net/?malware",
"raw": utils.base64_encode(REPORT_DATA),
"time.observation": "2018-01-22T14:38:24+00:00",
}
EVENT1 = {"__type": "Event",
"classification.identifier": "hidden-iframe",
"classification.type": "blacklist",
"feed.name": "Sucuri Hidden Iframes",
"feed.url": "http://labs.sucuri.net/?malware",
"raw": utils.base64_encode(REPORT_DATA_SPLIT[616]),
"source.url": "http://poseyhumane.org/stats.php",
"time.observation": "2018-01-24T14:23:34+00:00",
}
EVENT2 = {"classification.identifier": "hidden-iframe",
"feed.url": "http://labs.sucuri.net/?malware",
"time.observation": "2018-01-24T15:58:48+00:00",
"__type": "Event",
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.collectors.tcp.collector import TCPCollectorBot
from intelmq.bots.outputs.tcp.output import TCPOutputBot
from intelmq.lib.message import Event
from intelmq.lib.message import MessageFactory
from intelmq.lib.utils import base64_decode
PORT = 5011
SEPARATOR = '\n'
INPUT1 = {'classification.taxonomy': 'malicious code',
'classification.type': 'c2server',
'feed.name': 'Example feed',
'feed.accuracy': 100.0,
'feed.url': 'http://localhost/two_files.tar.gz',
'raw': utils.base64_encode('bar text\n'),
'time.observation': '2018-03-20T14:05:03+00:00'}
REPORT1 = {
'feed.name': 'Example feed',
'feed.accuracy': 100.0}
# '__type': 'Report'
# 'raw': 'eyJjbGFzc2lmaWNhdGlvbi50YXhvbm9teSI6ICJtYWxpY2lvdXMgY29kZSIsICJjbGFzc2lmaWNhdGlvbi50eXBlIjogImMmYyIsICJmZWVkLm5hbWUiOiAiRXhhbXBsZSBmZWVkIiwgImZlZWQuYWNjdXJhY3kiOiAxMDAuMCwgImZlZWQudXJsIjogImh0dHA6Ly9sb2NhbGhvc3QvdHdvX2ZpbGVzLnRhci5neiIsICJyYXciOiAiWW1GeUlIUmxlSFFLIiwgInRpbWUub2JzZXJ2YXRpb24iOiAiMjAxOC0wMy0yMFQxNDowNTowMyswMDowMCJ9',
INPUT2 = {'feed.name': 'Example feed 2',
'feed.accuracy': 100.0,
'feed.url': 'http://localhost/two_files.tar.gz',
'raw': utils.base64_encode('foo text\n')}
ORIGINAL_DATA = ('some random input{}another line').format(SEPARATOR)
class Client:
""" You find here an example of a non-intelmq client that might connect to the bot. """
import intelmq.lib.bot as bot import intelmq.lib.test as test import intelmq.lib.utils as utils RAW = """# ignore this 2015/06/04 13:37 +00,example.org,192.0.2.3,reverse.example.net,example description,[email protected],1 2015/06/04 13:38 +00,example.org,19d2.0.2.3,reverse.example.net,example description,[email protected],1 #ending line""" RAW_SPLIT = RAW.splitlines() EXAMPLE_REPORT = { "feed.url": "http://www.example.com/", "time.observation": "2015-08-11T13:03:40+00:00", "raw": utils.base64_encode(RAW), "__type": "Report", "feed.name": "Example" } EXAMPLE_EVENT = { "feed.url": "http://www.example.com/", "source.ip": "192.0.2.3", "time.source": "2015-06-04T13:37:00+00:00", "source.reverse_dns": "reverse.example.net", "source.fqdn": "example.org", "source.account": "*****@*****.**", "time.observation": "2015-08-11T13:03:40+00:00", "__type": "Event", "classification.type": "malware", "event_description.text": "example description", "source.asn": 1,
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.spamhaus.parser_drop import SpamhausDropParserBot
with open(os.path.join(os.path.dirname(__file__),
'drop.lasso')) as handle:
EXAMPLE_FILE = handle.read()
REPORT = {'__type': 'Report',
'feed.name': 'Spamhaus Drop',
'feed.url': 'https://www.spamhaus.org/drop/drop.lasso',
'raw': utils.base64_encode(EXAMPLE_FILE),
'time.observation': '2015-09-14T12:00:00+02:00',
}
TEMPLATE = {'__type': 'Event',
'feed.name': 'Spamhaus Drop',
'feed.url': 'https://www.spamhaus.org/drop/drop.lasso',
'time.observation': '2015-09-14T12:00:00+02:00',
'time.source': '2015-09-14T04:39:22+00:00',
'classification.type': 'spam',
}
EVENTS = ({'source.network': '1.4.0.0/17',
'extra': '{"blocklist": "SBL256893"}',
'raw': 'MS40LjAuMC8xNyA7IFNCTDI1Njg5Mw=='},
{'source.network': '1.10.16.0/20',
'extra': '{"blocklist": "SBL256894"}',
'raw': 'MS4xMC4xNi4wLzIwIDsgU0JMMjU2ODk0'},
# -*- coding: utf-8 -*-
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(os.path.join(os.path.dirname(__file__), 'drone-hadoop.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "ShadowServer Botnet Drone Hadoop",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [{'__type': 'Event',
'classification.taxonomy': 'Malicious Code',
'classification.type': 'botnet drone',
'destination.asn': 8560,
'destination.geolocation.cc': 'US',
'destination.ip': '74.208.164.166',
'destination.port': 80,
'extra': '{"connection_count": 1, "os.name": "Windows", "os.version": "2000 SP4, XP SP1+"}',
'feed.name': 'ShadowServer Botnet Drone Hadoop',
'malware.name': 'sinkhole',
'protocol.transport': 'tcp',
'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
EXAMPLE_LINES[1], ''])),
rt_url = os.getenv('INTELMQ_TEST_RT_URL')
rt_username = os.getenv('INTELMQ_TEST_RT_USERNAME')
rt_password = os.getenv('INTELMQ_TEST_RT_PASSWORD')
rt_subject = os.getenv('INTELMQ_TEST_RT_SUBJECT')
rt_attachment_ticket = os.getenv('INTELMQ_TEST_RT_ATTACHMENT_TICKET')
rt_url_ticket = os.getenv('INTELMQ_TEST_RT_URL_TICKET')
REPORT = {
"__type":
"Report",
"feed.name":
"Example feed",
"feed.accuracy":
100.,
"raw":
utils.base64_encode('bar text\n'),
"extra.file_name":
"foobar",
"extra.email_subject":
"Incoming IntelMQ Test Report",
"extra.ticket_subject":
"Incoming IntelMQ Test Report",
"extra.ticket_requestors":
"*****@*****.**",
"extra.email_from":
"*****@*****.**",
"extra.ticket_queue":
"Incident Reports",
"extra.ticket_status":
"new",
"extra.ticket_owner":
# -*- coding: utf-8 -*-
import os.path
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.cymru.parser_cap_program import CymruCAPProgramParserBot
with open(os.path.join(os.path.dirname(__file__),
'certname_20190327.txt')) as handle:
RAW = handle.read()
RAW_LINES = RAW.splitlines()
REPORT = {'__type': 'Report',
'raw': utils.base64_encode(RAW),
'time.observation': '2015-11-01T00:01:45+00:05',
}
EVENT0 = {'__type': 'Event',
'time.observation': '2015-11-01T00:01:45+00:05',
'source.as_name': 'Example AS Name',
'source.asn': 64496,
'source.ip': '172.16.0.21',
'time.source': '2019-03-22T11:18:52+00:00',
'raw': utils.base64_encode('\n'.join(RAW_LINES[:2] + [RAW_LINES[3]])),
'classification.type': 'infected-system',
'classification.identifier': 'conficker',
'malware.name': 'conficker',
}
EVENT1 = {'__type': 'Event',
'time.observation': '2015-11-01T00:01:45+00:05',
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(
os.path.join(os.path.dirname(__file__),
"testdata/event4_ip_spoofer.csv")) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {
"feed.name": "CAIDA",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2020-08-19T00:00:00+00:00",
"extra.file_name": "2020-08-19-event4_ip_spoofer.csv",
}
EVENTS = [
{
"__type": "Event",
"feed.name": "CAIDA",
"time.source": "2021-03-28T00:42:59+00:00",
"source.ip": "98.191.250.0",
"source.asn": 22898,
"source.geolocation.cc": "US",
"source.geolocation.region": "OKLAHOMA",
"source.geolocation.city": "OKLAHOMA CITY",
def sanitize(value):
value = utils.base64_encode(value)
return value
def sanitize(value):
value = utils.base64_encode(value)
return value
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.cert_eu.parser_csv import CertEUCSVParserBot
with open(os.path.join(os.path.dirname(__file__), 'example.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_FILE_SPLIT = EXAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {"feed.url": "https://www.cert-eu.eu",
"feed.name": "CERT-EU Feed",
"__type": "Report",
"time.observation": "2018-01-01T01:00:00+00:00",
"raw": utils.base64_encode(EXAMPLE_FILE),
"feed.accuracy": 50,
}
EXAMPLE_EVENT1 = {
"feed.url": "https://www.cert-eu.eu",
"feed.name": "CERT-EU Feed",
"source.ip": "109.125.93.10",
"raw": utils.base64_encode('\r\n'.join(EXAMPLE_FILE_SPLIT[:2])),
"time.observation": "2018-01-01T01:00:00+00:00",
"extra.cert_eu_time_observation": "2019-04-01T10:13:23+00:00",
"classification.type": "malware",
"time.source": "2019-04-01T03:13:20+00:00",
"tlp": "AMBER",
"event_description.text": "A URL is the most common resource with reference to malware binary distribution.",
"source.geolocation.country": "Germany",
SNMP_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'blacklist_ssh_day.txt')) as handle:
SSH_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'blacklist_telnet_day.txt')) as handle:
TELNET_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'honeypot_dns_attacks.txt'), encoding='utf-8') as handle:
DNS_ATTACK_FILE = handle.read()
SNMP_REPORT = {'feed.name': 'SNMP Blacklist',
'feed.url': 'http://www.nothink.org/blacklist/blacklist_snmp_day.txt',
'__type': 'Report',
'time.observation': '2016-12-05T09:23:46+00:00',
'raw': utils.base64_encode(SNMP_FILE)
}
SNMP_EVENT = {'feed.name': 'SNMP Blacklist',
'feed.url': 'http://www.nothink.org/blacklist/blacklist_snmp_day.txt',
'__type': 'Event',
'time.observation': '2016-12-05T09:23:46+00:00',
'raw': 'MTg1LjEyOC40MC4xNjI=',
'time.source': '2016-11-14T23:02:04+00:00',
'source.ip': '185.128.40.162',
'classification.type': 'scanner',
'protocol.application': 'snmp',
}
SSH_REPORT = {'feed.name': 'SSH Blacklist',
'feed.url': 'http://www.nothink.org/blacklist/blacklist_ssh_day.txt',
'__type': 'Report',
with open(os.path.join(os.path.dirname(__file__),
'misp_event.json')) as handle:
EXAMPLE_MISP_EVENT = handle.read()
with open(os.path.join(os.path.dirname(__file__),
'misp_attribute.json')) as handle:
EXAMPLE_MISP_ATTR = handle.read().strip()
EXAMPLE_REPORT = {
"__type": "Report",
"feed.accuracy": 100.0,
"feed.name": "misp_test",
"feed.url": "http://192.168.56.102/",
"time.observation": "2016-06-15T09:25:26+00:00",
"raw": base64_encode(EXAMPLE_MISP_EVENT)
}
EXAMPLE_EVENT = {
"__type": "Event",
"feed.accuracy": 100.0,
"feed.name": "misp_test",
"feed.url": "http://192.168.56.102/",
"time.observation": "2016-06-15T09:25:26+00:00",
"time.source": "2016-06-11T21:41:44+00:00",
"source.url": "http://fake.example.com/malware/is/here",
"event_description.text": "Payload delivery",
"event_description.url": "http://192.168.56.102/events/view/2",
"classification.type": "infected-system",
"malware.name": "locky",
'misp.attribute_uuid': '575c8598-f1f0-4c16-a94a-0612c0a83866',
# -*- coding: utf-8 -*-
"""
Data is from the HIBP Documentation
"""
import json
import unittest
import pkg_resources
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.hibp.parser_callback import HIBPCallbackParserBot
BREACHREQUEST = json.load(open(pkg_resources.resource_filename('intelmq', 'tests/bots/parsers/hibp/breach_callbacktest.json')))
BREACHRAW = utils.base64_encode(json.dumps(BREACHREQUEST, sort_keys=True))
BR_REP = {"feed.name": "HIBP Enterprise",
"time.observation": "2019-03-01T01:01:01+00:00",
"__type": "Report",
"raw": BREACHRAW
}
BR_EV = {"feed.name": "HIBP Enterprise",
"raw": BREACHRAW,
"time.observation": "2019-03-01T01:01:01+00:00",
"extra.domain_emails": BREACHREQUEST["DomainEmails"],
"extra.breach": BREACHREQUEST["Breach"],
"classification.taxonomy": "information content security",
"classification.type": "leak",
"classification.identifier": "breach",
"source.account": "*****@*****.**",
"source.fqdn": "example.com",
"__type": "Event"
}
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(
os.path.join(os.path.dirname(__file__),
'testdata/outdated_dnssec_key.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {
"feed.name": "ShadowServer Outdated DNSSEC Key",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
"extra.file_name": "2019-01-01-outdated_dnssec_key-test-geo.csv",
}
EVENTS = [{
'__type':
'Event',
'classification.identifier':
'outdated-dnssec-key',
'classification.taxonomy':
'availability',
'classification.type':
'other',
'destination.asn':
65537,
import intelmq.lib.utils as utils
import intelmq.lib.test as test
from intelmq.bots.parsers.cleanmx.parser import \
CleanMXParserBot
with open(os.path.join(os.path.dirname(__file__), 'xmlphishing')) as handle:
PHISHING_FILE = handle.read()
with open(os.path.join(os.path.dirname(__file__), 'xmlviruses')) as handle:
VIRUSES_FILE = handle.read()
PHISHING_REPORT = {
"feed.url":
"http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&format=csv&domain=",
"feed.name": "CleanMX Phishing",
"__type": "Report",
"raw": utils.base64_encode(PHISHING_FILE),
"time.observation": "2015-11-02T13:11:43+00:00"
}
PHISHING_EVENTS = [{
'__type':
'Event',
'classification.type':
'phishing',
'event_description.target':
'DHL',
'extra':
'{"ddescr": "Example Layer", "id": "9377142", "inetnum": '
'"198.18.0.0 - 198.19.255.255", "netname": "EXAMPLE-NETWORK-15", '
'"ns1": "ns2.example.com", "ns2": "ns1.example.com", "phishtank": '
'"4647345", "response": "alive", "review": "198.18.0.1"}',
'feed.name':
