def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.split()
event = Event()
columns = ["source_ip", "source_time"]
for key, value in zip(columns, row):
if key == "source_time":
value = datetime.utcfromtimestamp(int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC"
event.add(key, value.strip())
event.add('feed', 'openbl')
event.add('feed_url', 'http://www.openbl.org/lists/date_all.txt')
event.add('type', 'blacklist')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.split("<tbody>")[1]
raw_report = raw_report.split("</tbody>")[0]
raw_report_splitted = raw_report.split("<tr>")
for row in raw_report_splitted:
row = row.strip()
if row == "":
continue
row_splitted = row.split("<td>")
fqdn = row_splitted[2].split('">')[1].split("<")[0].strip()
ip = row_splitted[4].split('">')[1].split("<")[0].strip()
time_source = row_splitted[8].replace("</td></tr>", "").strip()
time_source = time_source + " 00:00:00 UTC"
event = Event(report)
event.add('time.source', time_source)
event.add('classification.type', 'malware')
event.add('source.fqdn', fqdn)
event.add('source.ip', ip)
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.split("<tbody>")[1]
raw_report = raw_report.split("</tbody>")[0]
raw_report_splitted = raw_report.split("<tr>")
for row in raw_report_splitted:
row = row.strip()
if row == "":
continue
row_splitted = row.split("<td>")
fqdn = row_splitted[2].split('">')[1].split("<")[0].strip()
ip = row_splitted[4].split('">')[1].split("<")[0].strip()
time_source = row_splitted[8].replace("</td></tr>", "").strip()
time_source = time_source + " 00:00:00 UTC"
event = Event(report)
event.add("time.source", time_source)
event.add("classification.type", "malware")
event.add("source.fqdn", fqdn)
event.add("source.ip", ip)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#'):
self.tempdata.append(line)
else:
splitted_row = line.split('|')
event = Event(report)
columns = ["source.asn", "source.as_name",
"source.ip", "time.source"]
for key, value in zip(columns, splitted_row):
value = value.strip()
if key == "time.source":
value += "+00:00"
if value == "NA":
continue
event.add(key, value)
event.add("classification.type", "brute-force")
event.add("protocol.application", "vnc")
event.add("protocol.transport", "tcp")
event.add("raw", line)
yield event
def process(self):
report = self.receive_message()
if report:
regex_ip = "^[^ \t]+"
regex_timestamp = "# ([^ \t]+ [^ \t]+)"
for row in report.split('\n'):
if row.startswith('#'):
continue
event = Event()
match = re.search(regex_ip, row)
if match:
ip = match.group()
match = re.search(regex_timestamp, row)
if match:
timestamp = match.group(1) + " UTC"
event.add("source_ip", ip)
event.add("source_time", timestamp)
event.add('feed', 'bruteforceblocker')
event.add('feed_url', 'http://danger.rulez.sk/projects/bruteforceblocker/blist.php')
event.add('type', 'brute-force')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw")).strip()
row = raw_report.splitlines()[0]
time_str = row[row.find('(') + 1:row.find(')')]
time = dateutil.parser.parse(time_str).isoformat()
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val):
event.add('source.ip', val)
else:
event.add('source.network', val)
event.add('time.source', time)
event.add('classification.type', u'blacklist')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
splitted_row = row.split('|')
event = Event(report)
columns = ["source.asn", "source.as_name",
"source.ip", "time.source"]
for key, value in zip(columns, splitted_row):
value = value.strip()
if key == "time.source":
value += "+00:00"
if value == "NA":
continue
event.add(key, value)
event.add("classification.type", "brute-force")
event.add("protocol.application", "vnc")
event.add("protocol.transport", "tcp")
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.rstrip()
if row.startswith("#") or len(row) == 0:
continue
values = row.split('\t')[1:]
event = Event(report)
event.add('source.fqdn', values[1])
event.add('event_description.text', values[2])
for i in range(4, len(values)):
if is_valid_date(values[i]):
event.add('time.source', # TODO: verify timezone
values[i] + "T00:00:00+00:00", force=True)
break
event.add('classification.type', u'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
lastgenerated = None
for row in raw_report.splitlines():
event = Event(report)
row = row.strip()
if len(row) == 0:
continue
elif row.startswith("#"):
if 'Generated on' in row:
row = row.strip('# ')[13:]
lastgenerated = dateutil.parser.parse(row).isoformat()
continue
event.add('time.source', lastgenerated)
event.add('source.ip', row)
event.add('classification.type', 'c&c')
event.add("raw", row)
event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
self.event_date = None
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if row.startswith('; Last-Modified:'):
self.event_date = row.split('; Last-Modified: ')[1].strip()
self.event_date = dateparser(self.event_date)
if row == "" or row.startswith(';'):
continue
row_splitted = row.split(';')
network = row_splitted[0].strip()
event = Event(report)
event.add('source.network', network)
event.add('extra', {'blocklist': row_splitted[1].strip()})
if self.event_date:
event.add('time.source', self.event_date.isoformat())
event.add('classification.type', 'spam')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
report = report.strip()
columns = {
"timestamp": "source_time",
"ip": "source_ip",
"protocol" : "transport_protocol",
"port" : "source_port",
"hostname": "source_reverse_dns",
"tag": "__IGNORE__",
"asn": "source_asn",
"geo": "source_cc",
"region" : "source_region",
"city" : "source_city",
"naics" : "__IGNORE__",
"sic": "__IGNORE__",
"programs": "__IGNORE__",
"mountd_port": "__IGNORE__",
"exports": "__IGNORE__"
}
rows = csv.DictReader(StringIO.StringIO(report))
for row in rows:
event = Event()
for key, value in row.items():
if key=='sector':
continue
key = columns[key]
if not value:
continue
value = value.strip()
if key is "__IGNORE__" or key is "__TDB__":
continue
# set timezone explicitly to UTC as it is absent in the input
if key == "source_time":
value += " UTC"
event.add(key, value)
event.add('feed', 'shadowserver-portmapper')
event.add('type', 'vulnerable service')
event.add('application_protocol', 'portmapper')
event.add('description','Open PortMapper')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('<tr>'):
# Get IP and Type
info1 = re.search(
">[\ ]*(\d+\.\d+\.\d+\.\d+)[\ ]*<.*</td><td>([^<]+)</td>", row)
if not info1:
continue
# Get Timestamp
info2 = re.search(
"<td>[\ ]*(\d{4}-\d{2}-\d{2}\ \d{2}:\d{2}:\d{2})[\ ]*</td>",
row)
event = Event(report)
description = info1.group(2)
description = utils.decode(description)
event_type = self.get_type(description)
time_source = info2.group(1) + " UTC-8"
event.add("time.source", time_source)
event.add("source.ip", info1.group(1))
event.add('classification.type', event_type)
event.add('event_description.text', description)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'): # ignore all lines starting with comment mark
continue
row = row.split('|')
event = Event()
columns = ["source_asn", "source_as_name", "source_ip", "source_time"]
for key, value in zip(columns, row):
value = value.strip()
if key == "source_time":
value += " UTC"
event.add(key, value)
event.add('feed', 'dragonresearchgroup')
event.add('feed_url', 'http://dragonresearchgroup.org/insight/vncprobe.txt')
event.add('type', 'brute-force')
event.add('application_protocol', 'vnc')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw")).strip()
row = raw_report.splitlines()[0]
time_str = row[row.find('(')+1:row.find(')')]
time = dateutil.parser.parse(time_str).isoformat()
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val):
event.add('source.ip', val)
else:
event.add('source.network', val)
event.add('time.source', time)
event.add('classification.type', u'blacklist')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
event = Event(report)
line_contents = line.split('|')
feed_name = line_contents[-1].strip()
file_format = FILE_FORMATS.get(feed_name) or FILE_FORMATS['_default']
if len(line_contents) != len(file_format) + 1:
raise ValueError(f'Incorrect format for feed {event.get("feed.url")}, found line: "{line}"')
if feed_name not in CATEGORY:
raise ValueError(f'Unknown data feed {feed_name}.')
event.update(CATEGORY[feed_name])
for field, (field_name, converter) in zip(line_contents, file_format):
value = converter(field.strip())
if value is not None:
event.add(field_name, value)
event.add('raw', line)
yield event
def process(self):
report = self.receive_message()
if report:
report = report.strip()
headers = lib.dcu_headers()
rows = report.split("\n")
for row in rows:
try:
columns = row.strip().split("\t")
fields = dict(zip(headers, columns))
event = Event(lib.convert_dcu_fields(fields))
event.add("feed", "microsoft-dcu")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
except lib.ParsingError as exc:
msg = "Got a parsing problem: %s affected row '%s' IGNORING AND CONTINUING" % (exc.message, row.strip())
self.logger.warning(msg, exc_info=True)
continue
self.acknowledge_message()
def process(self):
report = self.receive_message()
columns = self.parameters.columns
if not report or not report.contains("raw"):
self.acknowledge_message()
return
if report:
raw_report = utils.base64_decode(report.get("raw"))
rows = csv.DictReader(StringIO(raw_report))
for row in rows:
event = Event(report)
for key, value in row.items():
key = columns[key]
if not value:
continue
value = value.strip()
if key == u'__IGNORE__' or key == u'__TBD__':
continue
event.add(key, value, sanitize=True)
event.add('classification.type', u'vulnerable service')
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
self.event_date = None
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if row.startswith('; Last-Modified:'):
self.event_date = row.split('; Last-Modified: ')[1].strip()
self.event_date = dateparser(self.event_date)
if row == "" or row.startswith(';'):
continue
row_splitted = row.split(';')
network = row_splitted[0].strip()
event = Event(report)
event.add('source.network', network)
event.add('extra', {'blocklist': row_splitted[1].strip()})
if self.event_date:
event.add('time.source', self.event_date.isoformat())
event.add('classification.type', 'spam')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0:
continue
row = row.split('|')
event = Event()
columns = ["source_url", "source_asn", "source_ip", "source_time", "source_reverse_dns", "source_cc", "__IGNORE__", "additional_information"]
for key, value in zip(columns, row):
value = value.strip()
if key == "source_time":
value += " UTC"
if value != "N/A" and key != "__IGNORE__":
event.add(key, value)
event.add('feed', 'cert-eu')
event.add('type', 'malware')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.rstrip()
if row.startswith("#") or len(row) == 0:
continue
values = row.split('\t')[1:]
event = Event(report)
event.add('source.fqdn', values[1], sanitize=True)
event.add('event_description.text', values[2], sanitize=True)
for i in range(4, len(values)):
if is_valid_date(values[i]):
event.add('time.source', # TODO: verify timezone
values[i] + "T00:00:00+00:00", sanitize=True)
break
event.add('classification.type', u'malware')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.rstrip()
if row.startswith("#") or len(row) == 0:
continue
values = row.split("\t")[1:]
event = Event(report)
event.add("source.fqdn", values[1])
event.add("event_description.text", values[2])
for i in range(4, len(values)):
if self.is_valid_date(values[i]):
event.add("time.source", values[i] + "T00:00:00+00:00", force=True) # TODO: verify timezone
break
event.add("classification.type", "malware")
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.get("raw"))
lastgenerated = None
for row in raw_report.split('\n'):
event = Event(report)
row = row.strip()
if len(row) == 0:
continue
elif row.startswith("#"):
if 'Generated on' in row:
row = row.strip('# ')[13:]
lastgenerated = dateutil.parser.parse(row).isoformat()
continue
event.add('time.source', lastgenerated)
event.add('classification.type', u'c&c')
event.add('source.fqdn', row)
event.add("raw", row)
event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('<tr>'):
# Get IP and Type
info1 = re.search(
">[\ ]*(\d+\.\d+\.\d+\.\d+)[\ ]*<.*</td><td>([^<]+)</td>", row)
if not info1:
continue
# Get Timestamp
info2 = re.search(
"<td>[\ ]*(\d{4}-\d{2}-\d{2}\ \d{2}:\d{2}:\d{2})[\ ]*</td>",
row)
event = Event(report)
description = info1.group(2)
description = utils.decode(description)
event_type = self.get_type(description)
time_source = info2.group(1) + " UTC-8"
event.add("time.source", time_source)
event.add("source.ip", info1.group(1))
event.add('classification.type', event_type)
event.add('event_description.text', description)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
regex_ip = "^(\d+\.\d+\.\d+\.\d+)"
regex_timestamp = "(\d+\-\d+\-\d+\s\d+\:\d+\:\d+)"
for row in report.split('\n'):
if row.startswith('#'):
continue
event = Event()
match = re.search(regex_ip, row)
if match:
ip = ".".join([octet.lstrip('0') for octet in match.group().split('.')])
match = re.search(regex_timestamp, row)
if match:
timestamp = match.group(1) + " UTC"
event.add("source_ip", ip)
event.add("source_time", timestamp)
event.add('feed', 'dshield')
event.add('feed_url', 'http://dshield.org/asdetailsascii.html')
event.add('type', 'brute-force')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val, sanitize=True):
event.add('source.ip', val, sanitize=True)
else:
event.add('source.network', val, sanitize=True)
event.add('classification.type', u'blacklist')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row.startswith("#") or len(row) == 0:
continue
event = Event(report)
event.add('classification.type', u'c&c')
event.add('source.fqdn', row)
event.add("raw", row)
event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
if row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
event.add('time.source', timestamp, sanitize=True)
event.add('source.ip', ip, sanitize=True)
event.add('classification.type', u'brute-force')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
if row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
event.add('time.source', timestamp, sanitize=True)
event.add('source.ip', ip, sanitize=True)
event.add('classification.type', u'brute-force')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
if row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
event.add('time.source', timestamp)
event.add('source.ip', ip)
event.add('classification.type', 'brute-force')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row, sanitize=True):
event.add('source.ip', row, sanitize=True)
else:
event.add('source.fqdn', row, sanitize=True)
event.add('classification.type', u'malware')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
self.logger.debug("Will apply regex %s" % self.parameters.regex)
if report:
rowcount = 0
for row in report.split('\n'): # For each line
event = Event()
match = re.search(self.parameters.regex, row)
if match:
for key in match.groupdict():
event.add(key, match.groupdict()[key])
else:
continue # skip lines without matching regex
rowcount += 1
# Get detail from parser parameters, will be nice to have it by
# source parameters.. Avoid adding if parsed
if not 'feed' in match.groupdict():
event.add('feed', self.parameters.feed)
if not 'feed_url' in match.groupdict():
event.add('feed_url', self.parameters.feed_url)
if not 'type' in match.groupdict():
event.add('type', self.parameters.type)
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event,
"observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.logger.info("Processed %d event" % rowcount)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
if row.startswith("#") or len(row) == 0 or row == "Site":
if 'updated' in row:
time_str = row[row.find(': ') + 2:]
time = dateutil.parser.parse(time_str).isoformat()
continue
event = Event(report)
event.add('classification.type', u'malware')
event.add('source.fqdn', row.strip())
event.add('time.source', time)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row.startswith("#") or len(row) == 0 or row.startswith('Start'):
if 'updated' in row:
time_str = row[row.find(': ') + 2:]
time = dateutil.parser.parse(time_str).isoformat()
continue
values = row.split("\t")
if len(values) < 3:
continue # raise an error
network_ip = values[0]
network_mask = values[2]
network = '%s/%s' % (network_ip, network_mask)
extra = {}
event = Event(report)
if len(values) > 3:
extra['attacks'] = int(values[3])
if len(values) > 4:
extra['network_name'] = values[4]
if len(values) > 5:
event['source.geolocation.cc'] = values[5]
if len(values) > 6:
event['source.abuse_contact'] = values[6]
if extra:
event.add('extra', extra)
event.add('time.source', time)
event.add('source.network', network)
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val, sanitize=True):
event.add('source.ip', val, sanitize=True)
else:
event.add('source.network', val, sanitize=True)
event.add('classification.type', u'blacklist')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0 or not row.startswith('<td style'):
continue
m = re.search("color: black;\">(\d+.\d+.\d+.\d+)</span></td><td>(.*)</td>", row)
if m:
event = Event()
event.add("source_ip", m.group(1))
event.add('feed', 'netflowhtml')
event.add('feed_url', 'https://tc.edu.tw/net/netflow/lkout/recent/1')
event.add('type', m.group(2))
event = utils.generate_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
raw_report = raw_report.split("<tbody>")[1]
raw_report = raw_report.split("</tbody>")[0]
raw_report_splitted = raw_report.split("<tr>")
for row in raw_report_splitted:
row = row.strip()
if row == "":
continue
row_splitted = row.split("<td>")
ip = row_splitted[1].split('">')[1].split("<")[0].strip()
time_source = row_splitted[6].replace("</td></tr>", "").strip()
time_source = time_source + " 00:00:00 UTC"
event = Event(report)
event.add('time.source', time_source)
event.add('classification.type', u'malware')
event.add('source.ip', ip)
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row):
event.add('source.ip', row)
else:
event.add('source.fqdn', row)
event.add('classification.type', u'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.replace('\r', '')
values = row.split('\t')
# if special char is in string should not be allowed
if "#" in values[1]:
continue
# if domain name is localhost we are not interested
if values[1].lower().strip() == "localhost":
continue
event = Event(report)
if IPAddress.is_valid(values[1]):
event.add("source.ip", values[1])
else:
event.add("source.fqdn", values[1])
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
report = encode(report)
columns = ["source_time", "source_url", "source_ip", "source_reverse_dns", "malware", "__IGNORE__", "source_asn"]
for row in unicodecsv.reader(StringIO(report), encoding='utf-8'):
event = Event()
for key, value in zip(columns, row):
if key is "__IGNORE__":
continue
if key is "source_time":
value = value.replace('_',' ')
value += " UTC"
if key is "malware":
value = value.lower()
event.add(key, value)
event.add('feed', 'malwaredomainslist')
event.add('feed_url', 'http://www.malwaredomainlist.com/updatescsv.php')
event.add('type', 'malware') # FIXME
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
raw_report = raw_report.split("<tbody>")[1]
raw_report = raw_report.split("</tbody>")[0]
raw_report_splitted = raw_report.split("<tr>")
for row in raw_report_splitted:
row = row.strip()
if row == "":
continue
row_splitted = row.split("<td>")
ip = row_splitted[1].split('">')[1].split("<")[0].strip()
time_source = row_splitted[6].replace("</td></tr>", "").strip()
time_source = time_source + " 00:00:00 UTC"
event = Event(report)
event.add('time.source', time_source, sanitize=True)
event.add('classification.type', u'malware')
event.add('source.ip', ip, sanitize=True)
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.replace('\r', '')
values = row.split('\t')
# if special char is in string should not be allowed
if "#" in values[1]:
continue
# if domain name is localhost we are not interested
if values[1].lower().strip() == "localhost":
continue
event = Event(report)
if IPAddress.is_valid(values[1]):
event.add("source.ip", values[1])
else:
event.add("source.fqdn", values[1])
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
if row.startswith("#") or len(row) == 0 or row == "Site":
if 'updated' in row:
time_str = row[row.find(': ') + 2:]
time = dateutil.parser.parse(time_str).isoformat()
continue
event = Event(report)
event.add('classification.type', u'malware')
event.add('source.fqdn', row.strip())
event.add('time.source', time)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row.startswith("#") or len(row) == 0 or row.startswith('Start'):
if 'updated' in row:
time_str = row[row.find(': ')+2:]
time = dateutil.parser.parse(time_str).isoformat()
continue
values = row.split("\t")
if len(values) < 3:
continue # raise an error
network_ip = values[0]
network_mask = values[2]
network = '%s/%s' % (network_ip, network_mask)
extra = {}
event = Event(report)
if len(values) > 3:
extra['attacks'] = int(values[3])
if len(values) > 4:
extra['network_name'] = values[4]
if len(values) > 5:
event['source.geolocation.cc'] = values[5]
if len(values) > 6:
event['source.abuse_contact'] = values[6]
if extra:
event.add('extra', extra, sanitize=True)
event.add('time.source', time, sanitize=True)
event.add('source.network', network, sanitize=True)
event.add('classification.type', 'blacklist')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row.startswith("#") or len(row) == 0:
continue
event = Event(report)
event.add('classification.type', u'c&c')
event.add('source.fqdn', row, sanitize=True)
event.add("raw", row, sanitize=True)
event.add("malware.name", SOURCE_FEEDS[report.value("feed.url")], sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
if not row or row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
if not timestamp:
raise ValueError('No timestamp found.')
event.add('time.source', timestamp)
event.add('source.ip', ip)
event.add('classification.type', 'brute-force')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#'):
self.tempdata.append(line)
if 'Generated on' in line:
row = line.strip('# ')[13:]
self.lastgenerated = dateutil.parser.parse(row).isoformat()
else:
event = Event(report)
event.add('time.source', self.lastgenerated)
event.add('classification.type', 'c&c')
event.add('source.fqdn', line)
event.add("raw", line)
event.add("malware.name", SOURCE_FEEDS[report["feed.url"]])
yield event
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.strip()
url = report.get('feed.url')
path = urlparse(url).path
filename = posixpath.basename(path)
for row in raw_report.split('\n'):
event = Event(report)
event.add('source.ip', row.strip())
if filename in MAPPING:
for key, value in MAPPING[filename].items():
event.add(key, value)
else:
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.strip()
url = report.get('feed.url')
path = urlparse(url).path
filename = posixpath.basename(path)
for row in raw_report.split('\n'):
event = Event(report)
event.add('source.ip', row.strip())
if filename in MAPPING:
for key, value in MAPPING[filename].items():
event.add(key, value)
else:
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#'):
self.tempdata.append(line)
if 'Generated on' in line:
row = line.strip('# ')[13:]
self.lastgenerated = dateutil.parser.parse(row).isoformat()
else:
event = Event(report)
event.add('time.source', self.lastgenerated)
event.add('classification.type', 'c&c')
event.add('source.fqdn', line)
event.add("raw", line)
event.add("malware.name", SOURCE_FEEDS[report["feed.url"]])
yield event
def process(self):
report = self.receive_message()
if report:
for row in report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('other'):
continue
row = row.split()
event = Event()
columns = ["source_ip"]
for key, value in zip(columns, row):
event.add(key, value)
event.add('feed', 'arbor')
event.add('feed_url', 'http://atlas-public.ec2.arbor.net/public/ssh_attackers')
event.add('type', 'brute-force')
event = utils.generate_source_time(event, "source_time")
event = utils.generate_observation_time(event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
if self.type == 'whitelist':
new_event = Event()
for key in self.keys:
if key in event:
new_event.add(key, event[key], sanitize=False)
event = new_event
else:
for key in self.keys:
if key in event:
del event[key]
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report:
report = report.strip()
columns = {
"timestamp": "source_time",
"ip": "source_ip",
"protocol": "transport_protocol",
"port": "source_port",
"hostname": "source_reverse_dns",
"sysdesc": "__TDB__",
"sysname": "__TDB__",
"asn": "source_asn",
"geo": "source_cc",
"region": "source_region",
"city": "source_city",
"version": "__IGNORE__"
}
rows = csv.DictReader(StringIO.StringIO(report))
for row in rows:
event = Event()
for key, value in row.items():
key = columns[key]
if not value:
continue
value = value.strip()
if key is "__IGNORE__" or key is "__TDB__":
continue
# set timezone explicitly to UTC as it is absent in the input
if key == "source_time":
value += " UTC"
event.add(key, value)
event.add('feed', 'shadowserver-snmp')
event.add('type', 'vulnerable service')
event.add('application_protocol', 'snmp')
event = utils.parse_source_time(event, "source_time")
event = utils.generate_observation_time(
event, "observation_time")
event = utils.generate_reported_fields(event)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if row == "" or row[:2] == "//":
continue
event = Event(report)
event.add('classification.type', 'malware')
event.add('source.fqdn', row.split(" ")[1])
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
"""
Passing through all information from Report to Event.
Also logs a sample line, which will be tested afterwards.
"""
report = self.receive_message()
if not report:
self.acknowledge_message()
return
event = Event()
self.logger.info('Lorem ipsum dolor sit amet')
for key in report.keys():
event.add(key, report[key], sanitize=False)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
value = line.split('|')
event = Event(report)
event.add('time.source', value[3].strip() + '+00:00')
if value[0].strip() != 'NA':
event.add('source.asn', value[0].strip())
if value[1].strip() != 'NA':
event.add('source.as_name', value[1].split()[0])
event.add('source.ip', value[2].strip())
if value[4].strip() in DataplaneParserBot.CATEGORY:
event.update(DataplaneParserBot.CATEGORY[value[4].strip()])
else:
raise ValueError('Unknown data feed %r.' % value[4].strip())
event.add('raw', line)
yield event
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
columns = [
"__IGNORE__", "source.url", "event_description.url", "time.source",
"__IGNORE__", "__IGNORE__", "__IGNORE__",
"event_description.target"
]
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.reader(io.StringIO(raw_report)):
# ignore headers
if "phish_id" in row:
continue
event = Event(report)
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value)
event.add('classification.type', u'phishing')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
columns = [
"source.ip", "__IGNORE__", "event_description.text", "__IGNORE__"
]
headers = True
raw_report = utils.base64_decode(report.get("raw"))
for row in utils.csv_reader(raw_report):
# ignore headers
if headers:
headers = False
continue
event = Event(report)
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value)
event.add('classification.type', u'scanner')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
event = Event(report)
splitted_row = row.split()
columns = ["source.url"]
for key, value in zip(columns, splitted_row):
event.add(key, value)
event.add('classification.type', u'malware')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
splitted_row = row.split()
event = Event(report)
columns = ["source.ip", "time.source"]
for key, value in zip(columns, splitted_row):
if key == "time.source":
value = datetime.utcfromtimestamp(
int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC"
event.add(key, value.strip())
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
event.add('source.ip', row)
event.add('classification.type', u'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()