def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.split()
event = Event()
columns = ["source_ip", "source_time"]
for key, value in zip(columns, row):
if key == "source_time":
value = datetime.utcfromtimestamp(int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC"
event.add(key, value.strip())
event.add('feed', 'openbl')
event.add('feed_url', 'http://www.openbl.org/lists/date_all.txt')
event.add('type', 'blacklist')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.replace('\r', '')
values = row.split('\t')
# if special char is in string should not be allowed
if "#" in values[1]:
continue
# if domain name is localhost we are not interested
if values[1].lower().strip() == "localhost":
continue
event = Event(report)
if IPAddress.is_valid(values[1]):
event.add("source.ip", values[1])
else:
event.add("source.fqdn", values[1])
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row.startswith("#") or len(row) == 0:
continue
event = Event(report)
event.add('classification.type', u'c&c')
event.add('source.fqdn', row)
event.add("raw", row)
event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
if row.startswith("#") or len(row) == 0 or row == "Site":
if 'updated' in row:
time_str = row[row.find(': ') + 2:]
time = dateutil.parser.parse(time_str).isoformat()
continue
event = Event(report)
event.add('classification.type', u'malware')
event.add('source.fqdn', row.strip())
event.add('time.source', time)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
report = report.strip()
headers = lib.dcu_headers()
rows = report.split("\n")
for row in rows:
try:
columns = row.strip().split("\t")
fields = dict(zip(headers, columns))
event = Event(lib.convert_dcu_fields(fields))
event.add("feed", "microsoft-dcu")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
except lib.ParsingError as exc:
msg = "Got a parsing problem: %s affected row '%s' IGNORING AND CONTINUING" % (exc.message, row.strip())
self.logger.warning(msg, exc_info=True)
continue
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
if row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
event.add('time.source', timestamp)
event.add('source.ip', ip)
event.add('classification.type', 'brute-force')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val, sanitize=True):
event.add('source.ip', val, sanitize=True)
else:
event.add('source.network', val, sanitize=True)
event.add('classification.type', u'blacklist')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row, sanitize=True):
event.add('source.ip', row, sanitize=True)
else:
event.add('source.fqdn', row, sanitize=True)
event.add('classification.type', u'malware')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
if row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
event.add('time.source', timestamp, sanitize=True)
event.add('source.ip', ip, sanitize=True)
event.add('classification.type', u'brute-force')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
lastgenerated = None
for row in raw_report.splitlines():
event = Event(report)
row = row.strip()
if len(row) == 0:
continue
elif row.startswith("#"):
if 'Generated on' in row:
row = row.strip('# ')[13:]
lastgenerated = dateutil.parser.parse(row).isoformat()
continue
event.add('time.source', lastgenerated)
event.add('classification.type', 'c&c')
event.add('source.fqdn', row)
event.add("raw", row)
event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.split("<tbody>")[1]
raw_report = raw_report.split("</tbody>")[0]
raw_report_splitted = raw_report.split("<tr>")
for row in raw_report_splitted:
row = row.strip()
if row == "":
continue
row_splitted = row.split("<td>")
ip = row_splitted[1].split('">')[1].split("<")[0].strip()
time_source = row_splitted[6].replace("</td></tr>", "").strip()
time_source = time_source + " 00:00:00 UTC"
event = Event(report)
event.add('time.source', time_source)
event.add('classification.type', 'malware')
event.add('source.ip', ip)
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.strip()
url = report.get('feed.url')
path = urlparse(url).path
filename = posixpath.basename(path)
for row in raw_report.split('\n'):
event = Event(report)
event.add('source.ip', row.strip())
if filename in MAPPING:
for key, value in MAPPING[filename].items():
event.add(key, value)
else:
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0:
continue
row = row.split('|')
event = Event()
columns = ["source_url", "source_asn", "source_ip", "source_time", "source_reverse_dns", "source_cc", "__IGNORE__", "additional_information"]
for key, value in zip(columns, row):
value = value.strip()
if key == "source_time":
value += " UTC"
if value != "N/A" and key != "__IGNORE__":
event.add(key, value)
event.add('feed', 'cert-eu')
event.add('type', 'malware')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
report = report.strip()
columns = {
"timestamp": "source_time",
"ip": "source_ip",
"protocol": "transport_protocol",
"port": "source_port",
"hostname": "source_reverse_dns",
"sysdesc": "__TDB__",
"sysname": "__TDB__",
"asn": "source_asn",
"geo": "source_cc",
"region": "source_region",
"city": "source_city",
"version": "__IGNORE__"
}
rows = csv.DictReader(StringIO.StringIO(report))
for row in rows:
event = Event()
for key, value in row.items():
key = columns[key]
if not value:
continue
value = value.strip()
if key is "__IGNORE__" or key is "__TDB__":
continue
# set timezone explicitly to UTC as it is absent in the input
if key == "source_time":
value += " UTC"
event.add(key, value)
event.add('feed', 'shadowserver-snmp')
event.add('type', 'vulnerable service')
event.add('application_protocol', 'snmp')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(
event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
report = encode(report)
columns = ["source_time", "source_url", "source_ip", "source_reverse_dns", "malware", "__IGNORE__", "source_asn"]
for row in unicodecsv.reader(StringIO(report), encoding='utf-8'):
event = Event()
for key, value in zip(columns, row):
if key is "__IGNORE__":
continue
if key is "source_time":
value = value.replace('_',' ')
value += " UTC"
if key is "malware":
value = value.lower()
event.add(key, value)
event.add('feed', 'malwaredomainslist')
event.add('feed_url', 'http://www.malwaredomainlist.com/updatescsv.php')
event.add('type', 'malware') # FIXME
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
lastgenerated = None
for row in raw_report.splitlines():
event = Event(report)
row = row.strip()
if len(row) == 0:
continue
elif row.startswith("#"):
if 'Generated on' in row:
row = row.strip('# ')[13:]
lastgenerated = dateutil.parser.parse(row).isoformat()
continue
event.add('time.source', lastgenerated)
event.add('classification.type', 'c&c')
event.add('source.fqdn', row)
event.add("raw", row)
event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0 or not row.startswith('<td style'):
continue
m = re.search("color: black;\">(\d+.\d+.\d+.\d+)</span></td><td>(.*)</td>", row)
if m:
event = Event()
event.add("source_ip", m.group(1))
event.add('feed', 'netflowhtml')
event.add('feed_url', 'https://tc.edu.tw/net/netflow/lkout/recent/1')
event.add('type', m.group(2))
event = utils.generate_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.replace('\r', '')
values = row.split('\t')
# if special char is in string should not be allowed
if "#" in values[1]:
continue
# if domain name is localhost we are not interested
if values[1].lower().strip() == "localhost":
continue
event = Event(report)
if IPAddress.is_valid(values[1]):
event.add("source.ip", values[1])
else:
event.add("source.fqdn", values[1])
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
self.logger.debug("Will apply regex %s" % self.parameters.regex)
if report:
rowcount = 0
for row in report.split('\n'): # For each line
event = Event()
match = re.search(self.parameters.regex, row)
if match:
for key in match.groupdict():
event.add(key, match.groupdict()[key])
else:
continue # skip lines without matching regex
rowcount += 1
# Get detail from parser parameters, will be nice to have it by
# source parameters.. Avoid adding if parsed
if not 'feed' in match.groupdict():
event.add('feed', self.parameters.feed)
if not 'feed_url' in match.groupdict():
event.add('feed_url', self.parameters.feed_url)
if not 'type' in match.groupdict():
event.add('type', self.parameters.type)
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event,
"observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.logger.info("Processed %d event" % rowcount)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val, sanitize=True):
event.add('source.ip', val, sanitize=True)
else:
event.add('source.network', val, sanitize=True)
event.add('classification.type', u'blacklist')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
if row.startswith("#") or len(row) == 0 or row == "Site":
if 'updated' in row:
time_str = row[row.find(': ') + 2:]
time = dateutil.parser.parse(time_str).isoformat()
continue
event = Event(report)
event.add('classification.type', u'malware')
event.add('source.fqdn', row.strip())
event.add('time.source', time)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('other'):
continue
row = row.split()
event = Event()
columns = ["source_ip"]
for key, value in zip(columns, row):
event.add(key, value)
event.add('feed', 'arbor')
event.add('feed_url', 'http://atlas-public.ec2.arbor.net/public/ssh_attackers')
event.add('type', 'brute-force')
event = utils.generate_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
if row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
event.add('time.source', timestamp, sanitize=True)
event.add('source.ip', ip, sanitize=True)
event.add('classification.type', u'brute-force')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.strip()
url = report.get('feed.url')
path = urlparse(url).path
filename = posixpath.basename(path)
for row in raw_report.split('\n'):
event = Event(report)
event.add('source.ip', row.strip())
if filename in MAPPING:
for key, value in MAPPING[filename].items():
event.add(key, value)
else:
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row):
event.add('source.ip', row)
else:
event.add('source.fqdn', row)
event.add('classification.type', u'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
if not row or row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
if not timestamp:
raise ValueError('No timestamp found.')
event.add('time.source', timestamp)
event.add('source.ip', ip)
event.add('classification.type', 'brute-force')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row.startswith("#") or len(row) == 0:
continue
event = Event(report)
event.add('classification.type', u'c&c')
event.add('source.fqdn', row, sanitize=True)
event.add("raw", row, sanitize=True)
event.add("malware.name", SOURCE_FEEDS[report.value("feed.url")], sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.split("<tbody>")[1]
raw_report = raw_report.split("</tbody>")[0]
raw_report_splitted = raw_report.split("<tr>")
for row in raw_report_splitted:
row = row.strip()
if row == "":
continue
row_splitted = row.split("<td>")
ip = row_splitted[1].split('">')[1].split("<")[0].strip()
time_source = row_splitted[6].replace("</td></tr>", "").strip()
time_source = time_source + " 00:00:00 UTC"
event = Event(report)
event.add('time.source', time_source)
event.add('classification.type', 'malware')
event.add('source.ip', ip)
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
self.columns = {
"# timestamp": None,
"ip": "source.ip",
"asn": "source.asn",
"old_category": None,
"malware": "classification.identifier",
"geoip": None,
"comment": None,
"addtime": None,
"category": None,
"rir": None,
"cc": "source.geolocation.country",
"loadindex": None,
"dstip": "destination.ip",
"srcport": "source.port",
"dstport": "destination.port",
}
self.categories = {
"bots": "botnet drone",
# TODO: more types ?
}
raw_report = utils.base64_decode(report.value("raw"))
for row in csv.DictReader(StringIO(raw_report),
delimiter="\t"):
# Skip unknown categories
if row['old_category'] not in self.categories:
continue
event = Event(report)
# Iterate through mapped fields sanitizing them if needed
for key, field in self.columns.items():
if not field:
continue
event.add(field, row[key], sanitize=True)
event.add('classification.type',
self.categories[row['old_category']])
event.add('time.source', row['# timestamp'] + ' UTC', sanitize=True)
event.add("raw", ",".join(row), sanitize=True)
self.logger.debug('event -> %r', event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.DictReader(io.StringIO(raw_report)):
event = Event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
event.add(key, value)
event.add('classification.type', 'phishing')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
event.add('source.ip', row)
event.add('classification.type', u'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
report = report.strip()
columns = {
"timestamp": "source_time",
"ip": "source_ip",
"protocol" : "transport_protocol",
"port" : "source_port",
"hostname": "source_reverse_dns",
"tag" : "__IGNORE__",
"quote" : "__IGNORE__",
"asn": "source_asn",
"geo": "source_cc",
"region" : "source_region",
"city" : "source_city",
"naics": "__IGNORE__",
"sic": "__IGNORE__"
}
rows = csv.DictReader(StringIO.StringIO(report))
for row in rows:
event = Event()
for key, value in row.items():
key = columns[key]
if not value:
continue
value = value.strip()
if key is "__IGNORE__" or key is "__TDB__":
continue
# set timezone explicitly to UTC as it is absent in the input
if key == "source_time":
value += " UTC"
event.add(key, value)
event.add('feed', 'shadowserver-qotd')
event.add('type', 'vulnerable service')
event.add('application_protocol', 'qotd')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row.startswith("#") or len(row) == 0 or row.startswith('Start'):
continue
values = row.split("\t")
if len(values) < 3:
continue # raise an error
network_ip = values[0]
network_mask = values[2]
network = '%s/%s' % (network_ip, network_mask)
event = Event(report)
event.add('source.network', network, sanitize=True)
event.add('classification.type', u'blacklist')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
columns = [
"source.ip", "__IGNORE__", "event_description.text", "__IGNORE__"
]
headers = True
raw_report = utils.base64_decode(report.get("raw"))
for row in utils.csv_reader(raw_report):
# ignore headers
if headers:
headers = False
continue
event = Event(report)
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value)
event.add('classification.type', u'scanner')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
event = Event(report)
line_contents = line.split('|')
feed_name = line_contents[-1].strip()
file_format = FILE_FORMATS.get(feed_name) or FILE_FORMATS['_default']
if len(line_contents) != len(file_format) + 1:
raise ValueError(f'Incorrect format for feed {event.get("feed.url")}, found line: "{line}"')
if feed_name not in CATEGORY:
raise ValueError(f'Unknown data feed {feed_name}.')
event.update(CATEGORY[feed_name])
for field, (field_name, converter) in zip(line_contents, file_format):
value = converter(field.strip())
if value is not None:
event.add(field_name, value)
event.add('raw', line)
yield event
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
columns = [
"__IGNORE__", "source.url", "event_description.url", "time.source",
"__IGNORE__", "__IGNORE__", "__IGNORE__",
"event_description.target"
]
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.reader(io.StringIO(raw_report)):
# ignore headers
if "phish_id" in row:
continue
event = Event(report)
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value)
event.add('classification.type', u'phishing')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('other'):
continue
event = Event(report)
event.add('classification.type', u'brute-force')
event.add("raw", row, sanitize=True)
columns = ["source.ip"]
row = row.split()
for key, value in zip(columns, row):
event.add(key, value, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
path = urlparse(report['feed.url']).path
filename = posixpath.basename(path)
event = Event(report)
event.add('source.ip', line)
if filename in MAPPING:
for key, value in MAPPING[filename].items():
event.add(key, value)
else:
event.add('classification.type', 'blacklist')
event.add("raw", line)
yield event
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
splitted_row = row.split()
event = Event(report)
columns = ["source.ip", "time.source"]
for key, value in zip(columns, splitted_row):
if key == "time.source":
value = datetime.utcfromtimestamp(
int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC"
event.add(key, value.strip())
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
event = Event(report)
splitted_row = row.split()
columns = ["source.url"]
for key, value in zip(columns, splitted_row):
event.add(key, value)
event.add('classification.type', u'malware')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
event = Event(report)
for key, value in zip(self.parameters.columns, row):
if key in ["__IGNORE__", ""]:
continue
if key in ["time.source", "time.destination"]:
value = parse(value, fuzzy=True).isoformat()
value += " UTC"
# regex from http://stackoverflow.com/a/23483979
# matching ipv4/ipv6 IP within string
elif key in ["source.ip", "destination.ip"]:
value = re.compile(
'(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])'
'\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0'
'-5])|(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])'
'\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])|'
'\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|('
'([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]'
'|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d'
'\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:['
'0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|['
'1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|'
':))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1'
',3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d'
'\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){'
'3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,'
'4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-'
'4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-'
'9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-'
'Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0'
'-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1'
'\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(('
'(:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4'
'}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2['
'0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]'
'{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2'
'[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|'
'[1-9]?\d)){3}))|:)))(%.+)?').match(value).group()
elif key.endswith('.url') and '://' not in value:
value = self.parameters.default_url_protocol + value
elif key in ["classification.type"] and self.type_translation:
if value in self.type_translation:
value = self.type_translation[value]
elif not hasattr(self.parameters, 'type'):
continue
event.add(key, value)
if hasattr(self.parameters, 'type')\
and not event.contains("classification.type"):
event.add('classification.type', self.parameters.type)
event.add("raw", ",".join(row))
yield event
def process(self):
report = self.receive_message()
if report:
report = report.strip()
columns = {
"timestamp": "source_time",
"ip": "source_ip",
"asn": "source_asn",
"geo": "source_cc",
"md5hash": "artifact_hash",
"url" : "reported_destination_url",
"user_agent" : "user_agent",
"host": "reported_destination_reverse_dns",
"method": "comment"
}
rows = csv.DictReader(StringIO.StringIO(report))
for row in rows:
event = Event()
for key, value in row.items():
key = columns[key]
if not value:
continue
value = value.strip()
if key is "__IGNORE__" or key is "__TDB__":
continue
# set timezone explicitly to UTC as it is absent in the input
if key == "source_time":
value += " UTC"
if key== "comment":
value ="HTTP Method ->"+value
event.add(key, value)
event.add('feed', 'shadowserver-Sandbox-Url')
event.add('type', 'malware')
event.add('artifact_hash_type','MD5')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
if self.type == 'whitelist':
new_event = Event()
for key in self.keys:
if key in event:
new_event.add(key, event[key], sanitize=False)
event = new_event
else:
for key in self.keys:
if key in event:
del event[key]
self.send_message(event)
self.acknowledge_message()
def test_multiple_bots(self):
""" Let's simulate multiple IntelMQ instances are pushing the events at once!
Every message must be queued.
Note that if too much clients want connect, connections are refused.
"""
client_count = 5
msg_count = 300
for _ in range(client_count):
bot = TestTCPOutputBot()
bot.setUpClass()
# bot.bot_id = "test-client-{}".format(_)
bot.input_message = []
for i in range(msg_count):
bot.input_message.append(Event(INPUT1, harmonization=self.harmonization))
Process(target=bot._delayed_start).start()
thread = threading.Thread(target=Client().random_client)
thread.start()
self.input_message = None
# can't use standard .bot_run(iteration) or .start() because shutdown() would be called
# and we need to handle multiple connections
self.prepare_bot()
self.bot._Bot__source_pipeline = self.pipe
self.bot._Bot__destination_pipeline = self.pipe
for _ in range(client_count + 1):
# every single calling of process() method will serve to a single connection
with mock.patch('intelmq.lib.utils.load_configuration',
new=self.mocked_config):
with mock.patch('intelmq.lib.utils.log', self.mocked_log):
self.bot.process()
self.bot.stop() # let's call shutdown() and free up bound address
self.assertOutputQueueLen(client_count * msg_count + 2)
def test_intelmq_exchange(self):
""" Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance.
We spawn independent process of the TCPOutput bot that sends a bunch of messages.
"""
bot = TestTCPOutputBot()
bot.setUpClass()
bot.input_message = []
msg_count = 100
for i in range(msg_count):
bot.input_message.append(
Event(INPUT1, harmonization=self.harmonization))
(Process(target=bot._delayed_start)).start()
self.run_bot()
self.assertOutputQueueLen(msg_count)
for i, msg in enumerate(self.get_output_queue()):
report = MessageFactory.unserialize(
msg, harmonization=self.harmonization, default_type='Event')
output = MessageFactory.unserialize(
utils.base64_decode(report["raw"]),
harmonization=self.harmonization,
default_type='Event')
self.assertDictEqual(output, INPUT1)
del report['time.observation']
del report['raw']
self.assertDictEqual(report, REPORT1)
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row.startswith("#") or len(row) == 0 or row.startswith('Start'):
continue
values = row.split("\t")
if len(values) < 3:
continue # raise an error
network_ip = values[0]
network_mask = values[2]
network = '%s/%s' % (network_ip, network_mask)
event = Event(report)
event.add('source.network', network, sanitize=True)
event.add('classification.type', u'blacklist')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
splitted_row = row.split()
event = Event(report)
columns = ["source.ip", "time.source"]
for key, value in zip(columns, splitted_row):
if key == "time.source":
value = datetime.utcfromtimestamp(
int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC"
event.add(key, value.strip(), sanitize=True)
event.add('classification.type', u'blacklist')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
path = urlparse(report['feed.url']).path
filename = posixpath.basename(path)
event = Event(report)
event.add('source.ip', line)
if filename in MAPPING:
for key, value in MAPPING[filename].items():
event.add(key, value)
else:
event.add('classification.type', 'blacklist')
event.add("raw", line)
yield event
def process(self):
report = self.receive_message()
if report:
event = Event()
report = encode(report)
# colums according to https://www.phishtank.com/developer_info.php as of 2015/04/30:
# phish_id,url,phish_detail_url,submission_time,verified,verification_time,online,target
# example:
# 123456,http://www.example.com/,http://www.phishtank.com/phish_detail.php?phish_id=123456,2009-06-19T15:15:47+00:00,yes,2009-06-19T15:37:31+00:00,yes,1st National Example Bank
columns = ["__IGNORE__", "source_url", "description_url", "source_time", "__IGNORE__", "__IGNORE__", "__IGNORE__", "target"]
for row in unicodecsv.reader(StringIO(report), encoding='utf-8'):
if "phish_id" in row:
continue # skip header
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value.strip())
event.add('feed', 'phishtank')
event.add('type', 'phishing')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
columns = [
"source.ip",
"__IGNORE__",
"event_description.text",
"__IGNORE__"
]
headers = True
raw_report = utils.base64_decode(report.get("raw"))
for row in utils.csv_reader(raw_report):
# ignore headers
if headers:
headers = False
continue
event = Event(report)
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value)
event.add('classification.type', u'scanner')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split("\n"):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
event.add("source.ip", row, sanitize=True)
event.add("classification.type", "malware")
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
event = Event(report)
splitted_row = row.split()
columns = ["source.url"]
for key, value in zip(columns, splitted_row):
event.add(key, value, sanitize=True)
event.add('classification.type', u'malware')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('other'):
continue
event = Event(report)
event.add('classification.type', u'brute-force')
event.add("raw", row)
columns = ["source.ip"]
row = row.split()
for key, value in zip(columns, row):
event.add(key, value)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.split()
event = Event()
columns = ["source_url"]
for key, value in zip(columns, row):
event.add(key, value)
event.add('feed', 'malwarepatrol-dansguardian')
event.add('type', 'malware')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
event = Event()
columns = ["__IGNORE__", "source_url", "description_url", "source_time", "__IGNORE__", "__IGNORE__", "__IGNORE__", "target"]
for row in unicodecsv.reader(StringIO(report), encoding='utf-8'):
if "phish_id" in row:
continue
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value.strip())
event.add('feed', 'phishtank')
event.add('type', 'phishing')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
dict_report = json.loads(utils.base64_decode(report.get("raw")))
# add all lists together, only one loop needed
for row in sum(dict_report.values(), []):
event = Event(report)
event.add('classification.type', 'c&c')
try:
event.add('source.ip', row)
except InvalidValue:
event.add('source.fqdn', row)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
"""
Passing through all information from Report to Event.
Also logs a sample line, which will be tested afterwards.
"""
report = self.receive_message()
if not report:
self.acknowledge_message()
return
event = Event()
self.logger.info('Lorem ipsum dolor sit amet')
for key in report.keys():
event.add(key, report[key], sanitize=False)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
columns = self.parameters.columns
if not report or not report.contains("raw"):
self.acknowledge_message()
return
if report:
raw_report = utils.base64_decode(report.get("raw"))
rows = csv.DictReader(StringIO(raw_report))
for row in rows:
event = Event(report)
for key, value in row.items():
key = columns[key]
if not value:
continue
value = value.strip()
if key == u'__IGNORE__' or key == u'__TBD__':
continue
event.add(key, value, sanitize=True)
event.add('classification.type', u'vulnerable service')
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row):
event.add('source.ip', row)
else:
event.add('source.fqdn', row)
event.add('classification.type', 'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.DictReader(io.StringIO(raw_report)):
event = Event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
if key == "source.asn" and value.startswith("ASNA"):
continue
if key == "source.asn":
for asn in value.split(','):
if asn.startswith("AS"):
value = asn.split("AS")[1]
break
event.add(key, value)
event.add('classification.type', u'malware')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
