def is_valid(value, sanitize=False):
if sanitize:
value = Base64().sanitize(value)
try:
utils.base64_decode(value)
except TypeError:
return False
if not GenericType().is_valid(value):
return False
return True
def is_valid(value, sanitize=False):
if sanitize:
value = GenericType().sanitize(value)
value = Base64().sanitize(value)
try:
utils.base64_decode(value)
except TypeError:
return False
if not GenericType().is_valid(value):
return False
return True
def parse_azure(self, line, report):
raw = self.recover_line(line)
event = self.new_event(report)
for key, value in line.copy().items():
if key == 'Payload':
if value == 'AA==': # NULL
del line[key]
continue
try:
value = json.loads(utils.base64_decode(value))
# continue unpacking in next loop
except json.decoder.JSONDecodeError:
line[key] = utils.base64_decode(value)
elif key == 'TLP' and value.lower() == 'unknown':
del line[key]
if isinstance(value, dict):
for subkey, subvalue in value.items():
line['%s.%s' % (key, subkey)] = subvalue
del line[key]
for key, value in line.items():
if key == 'ThreatConfidence':
if value == 'None':
continue
value = event.get('feed.accuracy',
100) * CONFIDENCE[value] / 100
elif key == 'DateTimeReceivedUtc':
value = DateTime.from_windows_nt(value)
elif key == 'Payload.ts':
value = DateTime.from_timestamp(value)
elif key == 'Payload.Protocol':
payload_protocol = value[:value.find('/')]
if payload_protocol:
# needs to overwrite a field previously parsed and written
event.add('protocol.application',
payload_protocol,
overwrite=True) # "HTTP/1.1", save additionally
elif not value:
continue
if AZURE[key] != '__IGNORE__':
# feed.accuracy is calculated newly and always needs to be overwritten
event.add(AZURE[key],
value,
overwrite=self.overwrite
or AZURE[key] == "feed.accuracy")
event.add('classification.type', 'infected-system')
event.add('raw', raw)
yield event
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
lastgenerated = None
for row in raw_report.splitlines():
event = Event(report)
row = row.strip()
if len(row) == 0:
continue
elif row.startswith("#"):
if 'Generated on' in row:
row = row.strip('# ')[13:]
lastgenerated = dateutil.parser.parse(row).isoformat()
continue
event.add('time.source', lastgenerated)
event.add('classification.type', 'c&c')
event.add('source.fqdn', row)
event.add("raw", row)
event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
columns = [
"source.ip",
"source.geolocation.cc",
"event_description.text",
"source.asn"
]
headers = True
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.translate({0: None})
for row in csv.reader(io.StringIO(raw_report)):
# ignore headers
if headers:
headers = False
continue
event = self.new_event(report)
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value)
event.add('classification.type', 'scanner')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw = base64_decode(report['raw'])
decoded = json.loads(raw)
event = self.new_event(report)
event['raw'] = raw
if self.minimal_mode:
for intelmqkey, shodankey in MAPPING_MINIMAL.items():
try:
if decoded[shodankey]:
event[intelmqkey] = decoded[shodankey]
except KeyError:
pass
try:
event['source.geolocation.cc'] = decoded["location"]["country_code"]
except KeyError:
pass
event['time.source'] = CONVERSIONS['timestamp'](decoded["timestamp"])
event['extra.shodan'] = decoded
event['classification.type'] = 'other'
else:
event.update(self.apply_mapping(MAPPING, decoded))
event.add('classification.type', 'other')
event.add('classification.identifier', 'shodan-scan')
for protocol in PROTOCOLS:
if protocol in decoded:
event.add('protocol.application', protocol)
self.send_message(event)
self.acknowledge_message()
def parse_json(self, report: dict):
"""
A basic JSON parser
"""
raw_report = utils.base64_decode(report.get("raw"))
for line in json.loads(raw_report):
yield line
def process(self):
report = self.receive_message()
if not report or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in utils.csv_reader(raw_report):
event = Event(report)
self.logger.debug(repr(row))
event.add("time.source", row[0].replace('_', ' ') + " UTC")
if row[1] != '-':
event.add("source.url", self.add_http(row[1]))
try:
event.add("source.ip", row[2])
except InvalidValue:
event.add("source.url", self.add_http(row[2]))
event.add('source.ip', urlparse(row[2]).netloc)
event.add("source.reverse_dns", row[3])
event.add("event_description.text", row[4])
# TODO: ignore abuse contact for now
event.add("source.asn", int(row[6]))
event.add('classification.type', u'malware')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in csv.DictReader(io.StringIO(raw_report), dictreader=True):
event = Event(report)
extra = {}
event.add('time.source', row['timestamp']+' UTC')
event.add('source.ip', row['ip'])
event.add('protocol.transport', row['protocol'])
event.add('source.port', row['port'])
event.add('source.reverse_dns', row['hostname'])
extra['sysdesc'] = row['sysdesc']
extra['sysname'] = row['sysname']
event.add('source.asn', row['asn'])
event.add('source.geolocation.cc', row['geo'])
event.add('source.geolocation.region', row['region'])
event.add('source.geolocation.city', row['city'])
if int(row['naics']):
extra['naics'] = int(row['naics'])
if int(row['sic']):
extra['sic'] = int(row['sic'])
if row['sector']:
extra['sector'] = row['sector']
event.add('extra', extra)
event.add('protocol.application', 'snmp')
event.add('classification.type', 'vulnerable service')
event.add('classification.identifier', 'snmp')
event.add('raw', '"'+','.join(map(str, row.items()))+'"')
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in utils.csv_reader(raw_report):
event = Event(report)
self.logger.debug(repr(row))
event.add("time.source", row[0].replace("_", " ") + " UTC", sanitize=True)
if row[1] != "-":
event.add("source.url", row[1], sanitize=True)
try:
event.add("source.ip", row[2], sanitize=True)
except InvalidValue:
event.add("source.url", row[2], sanitize=True)
event.add("source.ip", urlparse(row[2]).netloc, sanitize=True)
event.add("source.reverse_dns", row[3], sanitize=True)
event.add("event_description.text", row[4], sanitize=True)
# TODO: ignore abuse contact for now
event.add("source.asn", int(row[6]))
event.add("classification.type", "malware")
event.add("raw", ",".join(row), sanitize=True)
self.send_message(event)
self.acknowledge_message()
def parse(self, report: dict):
feed = report['feed.url']
raw_lines = utils.base64_decode(report.get("raw")).splitlines()
self.comments = []
data_lines = []
for r in raw_lines:
if self.__is_comment_line_regex.search(r):
self.comments.append(r)
else:
data_lines.append(self.__sanitize_csv_lines(r))
self.header_line = data_lines.pop(0) # remove CSV header line
fields = [self.__sanitize_csv_lines(f) for f in self.header_line.split(',')] # First line is the CSV header file
if len(fields) != len(FEEDS[feed]['format']):
self.logger.warning("Feed '{}' has not the expected fields: {} != {}".format(feed,
len(fields),
len(FEEDS[feed]['format'])))
raise ValueError("Abusech ip parser is not up to date with the format online")
for line in self.comments:
if 'Last updated' in line:
self.__last_generated_date = dateutil.parser.parse(self.__date_regex.search(line).group(0)).isoformat()
for line in data_lines:
yield line.strip()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
if row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
event.add('time.source', timestamp, sanitize=True)
event.add('source.ip', ip, sanitize=True)
event.add('classification.type', u'brute-force')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
splitted_row = row.split()
event = Event(report)
columns = ["source.ip", "time.source"]
for key, value in zip(columns, splitted_row):
if key == "time.source":
value = datetime.utcfromtimestamp(
int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC"
event.add(key, value.strip(), sanitize=True)
event.add('classification.type', u'blacklist')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
if row.startswith("#") or len(row) == 0 or row == "Site":
if 'updated' in row:
time_str = row[row.find(': ') + 2:]
time = dateutil.parser.parse(time_str).isoformat()
continue
event = Event(report)
event.add('classification.type', u'malware')
event.add('source.fqdn', row.strip())
event.add('time.source', time)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row.startswith("#") or len(row) == 0:
continue
event = Event(report)
event.add('classification.type', u'c&c')
event.add('source.fqdn', row, sanitize=True)
event.add("raw", row, sanitize=True)
event.add("malware.name", SOURCE_FEEDS[report.value("feed.url")], sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
event = MessageFactory.unserialize(base64_decode(report['raw']))
self.send_message(event)
self.acknowledge_message()
def parse(self, report):
raw_report = utils.base64_decode(report.get("raw"))
split = raw_report.split("</tr>")
self.tempdata = ['</tr>'.join(split[:2])]
# TODO: save ending line
for line in split[2:]:
yield line.strip()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.get("raw"))
lastgenerated = None
for row in raw_report.split('\n'):
event = Event(report)
row = row.strip()
if len(row) == 0:
continue
elif row.startswith("#"):
if 'Generated on' in row:
row = row.strip('# ')[13:]
lastgenerated = dateutil.parser.parse(row).isoformat()
continue
event.add('time.source', lastgenerated)
event.add('classification.type', u'c&c')
event.add('source.fqdn', row)
event.add("raw", row)
event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
report_list = [row.strip() for row in raw_report.splitlines()]
index = 0
actual_line = report_list[index]
while parser.lsData != "IPs":
index += 1
actual_line = report_list[index]
parser.feed(actual_line)
count = 0
while actual_line[:8] != "</TBODY>":
index += 1
actual_line = report_list[index]
parser.feed(actual_line)
if parser.lsTag == "input":
if count % 2 == 0:
url = parser.lsValue
url_raw_line = actual_line
else:
ip = parser.lsValue
event = self.new_event(report)
if harmonization.FQDN.is_valid(url, sanitize=True):
event.add("source.fqdn", url)
if harmonization.IPAddress.is_valid(ip, sanitize=True):
event.add("source.ip", ip)
event.add("raw", url_raw_line + actual_line)
event.add("classification.type", "phishing")
self.send_message(event)
count += 1
self.acknowledge_message()
def parse(self, report):
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.translate({0: None})
# ignore lines starting with #. # can have leading spaces/tabs
raw_report = re.sub(r'(?m)^[ \t]*#.*\n?', '', raw_report)
# ignore null bytes
raw_report = re.sub(r'(?m)\0', '', raw_report)
# ignore lines having mix of spaces and tabs only
raw_report = re.sub(r'(?m)^[ \t]*\n?', '', raw_report)
# skip header
if getattr(self.parameters, 'skip_header', False):
self.tempdata.append(raw_report[:raw_report.find('\n')])
raw_report = raw_report[raw_report.find('\n') + 1:]
for row in csv.reader(io.StringIO(raw_report),
delimiter=str(self.parameters.delimiter)):
if self.filter_text and self.filter_type:
text_in_row = self.filter_text in self.parameters.delimiter.join(
row)
if text_in_row and self.filter_type == 'whitelist':
yield row
elif not text_in_row and self.filter_type == 'blacklist':
yield row
else:
continue
else:
yield row
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
if not row or row.startswith('#'):
continue
event = self.new_event(report)
match = re.search(REGEX_IP, row)
ip = None
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
timestamp = None
if match:
timestamp = match.group(1) + " UTC"
if not timestamp:
raise ValueError('No timestamp found.')
elif not ip:
raise ValueError('No ip found.')
event.add('time.source', timestamp)
event.add('source.ip', ip)
event.add('classification.type', 'brute-force')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
splitted_row = row.split()
event = Event(report)
columns = ["source.ip", "time.source"]
for key, value in zip(columns, splitted_row):
if key == "time.source":
value = datetime.utcfromtimestamp(
int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC"
event.add(key, value.strip())
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('other'):
continue
event = Event()
time_observation = DateTime().generate_datetime_now()
event.add('time.observation', time_observation, sanitize=True)
event.add('feed.name', u'arbor')
event.add('feed.url', u'http://atlas-public.ec2.arbor.net/public/ssh_attackers')
event.add('classification.type', u'brute-force')
event.add("raw", row, sanitize=True)
columns = ["source.ip"]
row = row.split()
for key, value in zip(columns, row):
event.add(key, value, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def parse_csv(self, report):
"""
A basic CSV parser.
"""
raw_report = utils.base64_decode(report.get("raw"))
for line in csv.reader(io.StringIO(raw_report)):
yield line
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.replace('\r', '')
values = row.split('\t')
# if special char is in string should not be allowed
if "#" in values[1]:
continue
# if domain name is localhost we are not interested
if values[1].lower().strip() == "localhost":
continue
event = self.new_event(report)
if IPAddress.is_valid(values[1]):
event.add("source.ip", values[1])
else:
event.add("source.fqdn", values[1])
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
if not row or row.startswith('#'):
continue
event = Event(report)
match = re.search(REGEX_IP, row)
if match:
ip = match.group()
match = re.search(REGEX_TIMESTAMP, row)
if match:
timestamp = match.group(1) + " UTC"
if not timestamp:
raise ValueError('No timestamp found.')
event.add('time.source', timestamp)
event.add('source.ip', ip)
event.add('classification.type', 'brute-force')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw")).strip()
if not len(raw_report): # We depend on first line = date
self.acknowledge_message()
return
row = raw_report.splitlines()[0]
time_str = row[row.find('(') + 1:row.find(')')]
time = dateutil.parser.parse(time_str).isoformat()
for row in raw_report.splitlines():
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = self.new_event(report)
if not event.add('source.ip', val, raise_failure=False):
event.add('source.network', val)
event.add('time.source', time)
event.add('classification.type', 'blacklist')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
report_list = [row.strip() for row in raw_report.splitlines()]
index = 0
actual_line = report_list[index]
while parser.lsData != "IPs":
index += 1
actual_line = report_list[index]
parser.feed(actual_line)
count = 0
while actual_line[:8] != "</TBODY>":
index += 1
actual_line = report_list[index]
parser.feed(actual_line)
if parser.lsTag == "input":
if count % 2 == 0:
url = parser.lsValue
url_raw_line = actual_line
else:
ip = parser.lsValue
event = self.new_event(report)
event.add("source.fqdn", url)
event.add("source.ip", ip)
event.add("raw", url_raw_line + actual_line)
event.add("classification.type", "phishing")
self.send_message(event)
count += 1
self.acknowledge_message()
def process(self):
event = self.receive_message()
if self.format_filename:
ev = defaultdict(None)
ev.update(event)
# remove once #671 is done
if 'time.observation' in ev:
ev['time.observation'] = datetime.datetime.strptime(ev['time.observation'],
'%Y-%m-%dT%H:%M:%S+00:00')
if 'time.source' in ev:
ev['time.source'] = datetime.datetime.strptime(ev['time.source'],
'%Y-%m-%dT%H:%M:%S+00:00')
filename = self.parameters.file.format(event=ev)
if not self.file or filename != self.file.name:
self.open_file(filename)
if self.single_key:
event_data = str(event.get(self.single_key))
if self.single_key == 'raw':
event_data = base64_decode(event_data)
else:
event_data = event.to_json(hierarchical=self.parameters.hierarchical_output)
try:
self.file.write(event_data)
self.file.write("\n")
self.file.flush()
except FileNotFoundError:
self.init()
else:
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
report_list = [row.strip() for row in raw_report.splitlines()]
index = 0
while parser.lsData != "Details":
index += 1
parser.feed(report_list[index])
while report_list[index] != "</table>":
index += 1
parser.feed(report_list[index])
if parser.lsCSP:
index += 1
parser.feed(report_list[index])
event = self.new_event(report)
raw_url_line = report_list[index]
event.add("source.url", parser.lsData)
event.add("classification.type", "blacklist")
index += 1
parser.feed(report_list[index])
event.add("classification.identifier", parser.lsData)
event.add("classification.taxonomy", "other")
index += 1
parser.feed(report_list[index])
event.add("time.source", parser.lsData)
event.add("raw", raw_url_line + report_list[index])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
event.add('source.ip', row)
event.add('classification.type', u'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.DictReader(io.StringIO(raw_report)):
event = self.new_event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
event.add(key, value)
event.add('classification.type', 'phishing')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.split("<tbody>")[1]
raw_report = raw_report.split("</tbody>")[0]
raw_report_splitted = raw_report.split("<tr>")
for row in raw_report_splitted:
row = row.strip()
if row == "":
continue
row_splitted = row.split("<td>")
ip = row_splitted[1].split('">')[1].split("<")[0].strip()
time_source = row_splitted[6].replace("</td></tr>", "").strip()
time_source = time_source + " 00:00:00 UTC"
event = Event(report)
event.add('time.source', time_source)
event.add('classification.type', 'malware')
event.add('source.ip', ip)
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('<tr>'):
# Get IP and Type
info1 = re.search(
">[\ ]*(\d+\.\d+\.\d+\.\d+)[\ ]*<.*</td><td>([^<]+)</td>", row)
if not info1:
continue
# Get Timestamp
info2 = re.search(
"<td>[\ ]*(\d{4}-\d{2}-\d{2}\ \d{2}:\d{2}:\d{2})[\ ]*</td>",
row)
event = self.new_event(report)
description = info1.group(2)
description = utils.decode(description)
event_type = self.get_type(description)
time_source = info2.group(1) + " UTC-8"
event.add("time.source", time_source)
event.add("source.ip", info1.group(1))
event.add('classification.type', event_type)
event.add('event_description.text', description)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
columns = [
"__IGNORE__", "source.url", "event_description.url", "time.source",
"__IGNORE__", "__IGNORE__", "__IGNORE__",
"event_description.target"
]
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.reader(io.StringIO(raw_report)):
# ignore headers
if "phish_id" in row:
continue
event = Event(report)
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value)
event.add('classification.type', u'phishing')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw = base64_decode(report['raw'])
decoded = json.loads(raw)
event = self.new_event(report)
event['raw'] = raw
if self.minimal_mode:
for intelmqkey, shodankey in MAPPING_MINIMAL.items():
try:
if decoded[shodankey]:
event[intelmqkey] = decoded[shodankey]
except KeyError:
pass
try:
event['source.geolocation.cc'] = decoded["location"][
"country_code"]
except KeyError:
pass
event['time.source'] = CONVERSIONS['timestamp'](
decoded["timestamp"])
event['extra.shodan'] = decoded
event['classification.type'] = 'other'
else:
event.update(self.apply_mapping(MAPPING, decoded))
event.add('classification.type', 'other')
event.add('classification.identifier', 'shodan-scan')
for protocol in PROTOCOLS:
if protocol in decoded:
event.add('protocol.application', protocol)
self.send_message(event)
self.acknowledge_message()
def parse_json(self, report: dict):
"""
A basic JSON parser
"""
raw_report = utils.base64_decode(report.get("raw"))
for line in json.loads(raw_report):
yield line
def process(self):
report = self.receive_message()
columns = self.parameters.columns
if not report or not report.contains("raw"):
self.acknowledge_message()
return
if report:
raw_report = utils.base64_decode(report.get("raw"))
rows = csv.DictReader(StringIO(raw_report))
for row in rows:
event = Event(report)
for key, value in row.items():
key = columns[key]
if not value:
continue
value = value.strip()
if key == u'__IGNORE__' or key == u'__TBD__':
continue
event.add(key, value, sanitize=True)
event.add('classification.type', u'vulnerable service')
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
event.add('source.ip', row)
event.add('classification.type', u'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in csv.DictReader(io.StringIO(raw_report), dictreader=True):
event = Event(report)
extra = {}
event.add("time.source", row["timestamp"] + " UTC")
event.add("source.ip", row["ip"])
event.add("protocol.transport", row["protocol"])
event.add("source.port", row["port"])
event.add("source.reverse_dns", row["hostname"])
event.add("protocol.application", row["tag"])
extra["quote"] = row["quote"]
event.add("source.asn", row["asn"])
event.add("source.geolocation.cc", row["geo"])
event.add("source.geolocation.region", row["region"])
event.add("source.geolocation.city", row["city"])
if int(row["naics"]):
extra["naics"] = int(row["naics"])
if int(row["sic"]):
extra["sic"] = int(row["sic"])
if row["sector"]:
extra["sector"] = row["sector"]
event.add("extra", extra)
event.add("classification.type", "vulnerable service")
event.add("classification.identifier", "qotd")
event.add("raw", '"' + ",".join(map(str, row.items())) + '"')
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row, sanitize=True):
event.add('source.ip', row, sanitize=True)
else:
event.add('source.fqdn', row, sanitize=True)
event.add('classification.type', u'malware')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.rstrip()
if row.startswith("#") or len(row) == 0:
continue
values = row.split('\t')[1:]
event = Event(report)
event.add('source.fqdn', values[1], sanitize=True)
event.add('event_description.text', values[2], sanitize=True)
for i in range(4, len(values)):
if is_valid_date(values[i]):
event.add('time.source', # TODO: verify timezone
values[i] + "T00:00:00+00:00", sanitize=True)
break
event.add('classification.type', u'malware')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
report_list = [row.strip() for row in raw_report.splitlines()]
index = 0
actual_line = report_list[index]
while actual_line != "Recent domains":
index += 1
actual_line = report_list[index]
parser.set_empty_data
while actual_line != "Recent hosts":
index += 1
actual_line = report_list[index]
count1 = len(parser.lsData)
parser.feed(actual_line)
count2 = len(parser.lsData)
if count1 != count2:
parser.raw_lines.append(actual_line)
parser.process_data
for item in range(len(parser.lsData)):
event = self.new_event(report)
event.add("source.fqdn", parser.lsData[item])
event.add("classification.type", "blacklist")
event.add("raw", parser.raw_lines[item])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.strip()
url = report.get('feed.url')
path = urlparse(url).path
filename = posixpath.basename(path)
for row in raw_report.split('\n'):
event = Event(report)
event.add('source.ip', row.strip())
if filename in MAPPING:
for key, value in MAPPING[filename].items():
event.add(key, value)
else:
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
report_list = [row.strip() for row in raw_report.splitlines()]
index = 0
actual_line = report_list[index]
while actual_line != "Recent domains":
index += 1
actual_line = report_list[index]
parser.set_empty_data
while actual_line != "Recent hosts":
index += 1
actual_line = report_list[index]
count1 = len(parser.lsData)
parser.feed(actual_line)
count2 = len(parser.lsData)
if count1 != count2:
parser.raw_lines.append(actual_line)
parser.process_data
for item in range(len(parser.lsData)):
event = self.new_event(report)
event.add("source.fqdn", parser.lsData[item])
event.add("classification.type", "blacklist")
event.add("raw", parser.raw_lines[item])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
self.event_date = None
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if row.startswith('; Last-Modified:'):
self.event_date = row.split('; Last-Modified: ')[1].strip()
self.event_date = dateparser(self.event_date)
if row == "" or row.startswith(';'):
continue
row_splitted = row.split(';')
network = row_splitted[0].strip()
event = Event(report)
event.add('source.network', network)
event.add('extra', {'blocklist': row_splitted[1].strip()})
if self.event_date:
event.add('time.source', self.event_date.isoformat())
event.add('classification.type', 'spam')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw")).strip()
if not len(raw_report): # We depend on first line = date
self.acknowledge_message()
return
row = raw_report.splitlines()[0]
time_str = row[row.find('(') + 1:row.find(')')]
time = dateutil.parser.parse(time_str).isoformat()
for row in raw_report.splitlines():
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = self.new_event(report)
if not event.add('source.ip', val, raise_failure=False):
event.add('source.network', val)
event.add('time.source', time)
event.add('classification.type', 'blacklist')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.strip()
url = report.get('feed.url')
path = urlparse(url).path
filename = posixpath.basename(path)
for row in raw_report.split('\n'):
event = Event(report)
event.add('source.ip', row.strip())
if filename in MAPPING:
for key, value in MAPPING[filename].items():
event.add(key, value)
else:
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get('raw'))
misp_event = json.loads(raw_report)
# Set the classifier based on the ecsirt tag
classifier = None
if misp_event.get('Tag'):
for tag in misp_event['Tag']:
if tag['name'] in self.MISP_TAXONOMY_MAPPING:
classifier = self.MISP_TAXONOMY_MAPPING[tag['name']]
break
# get the attributes from the event
event_attributes = misp_event['Attribute']
# payload type - get malware variant for the event
malware_variant = None
for attribute in event_attributes:
if attribute['category'] == 'Payload type':
value = attribute['value'].lower()
# TODO: use misp galaxies
if value and harmonization.LowercaseString.is_valid(value):
malware_variant = value
# MISP event URL
url_path = 'event/view/{}'.format(misp_event['id'])
misp_event_url = urljoin(report['feed.url'], url_path)
# Process MISP event attributes as separate IntelMQ events
for attribute in event_attributes:
# get details of attribute
value = attribute['value']
uuid = attribute['uuid']
comment = attribute['comment']
timestamp = attribute['timestamp']
category = attribute['category']
type_ = attribute['type']
# create intelmq events based on the category
if (category in self.SUPPORTED_MISP_CATEGORIES and
type_ in self.MISP_TYPE_MAPPING):
# Create and send the intelmq event
event = Event(report)
event.add('raw', json.dumps(attribute, sort_keys=True))
event.add(self.MISP_TYPE_MAPPING[type_], value)
event.add('misp.event_uuid', misp_event['uuid'])
event.add('misp.attribute_uuid', uuid)
event.add('comment', comment)
event.add('event_description.text', category)
event.add('event_description.url', misp_event_url)
event.add('malware.name', malware_variant)
event.add('classification.type', classifier)
event.add('time.source', '{} UTC'.format(
datetime.utcfromtimestamp(float(timestamp))))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
splitted_row = row.split()
event = self.new_event(report)
columns = ["source.ip", "time.source"]
for key, value in zip(columns, splitted_row):
if key == "time.source":
value = datetime.utcfromtimestamp(
int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC"
event.add(key, value.strip())
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw")).strip()
row = raw_report.splitlines()[0]
time_str = row[row.find('(') + 1:row.find(')')]
time = dateutil.parser.parse(time_str).isoformat()
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val):
event.add('source.ip', val)
else:
event.add('source.network', val)
event.add('time.source', time)
event.add('classification.type', u'blacklist')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
event = Event(report)
splitted_row = row.split()
columns = ["source.url"]
for key, value in zip(columns, splitted_row):
event.add(key, value, sanitize=True)
event.add('classification.type', u'malware')
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split("\n"):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
event.add("source.ip", row, sanitize=True)
event.add("classification.type", "malware")
event.add("raw", row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
columns = [
"source.ip", "__IGNORE__", "event_description.text", "__IGNORE__"
]
headers = True
raw_report = utils.base64_decode(report.get("raw"))
for row in utils.csv_reader(raw_report):
# ignore headers
if headers:
headers = False
continue
event = Event(report)
for key, value in zip(columns, row):
if key == "__IGNORE__":
continue
event.add(key, value)
event.add('classification.type', u'scanner')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def parse_csv(self, report):
"""
A basic CSV parser.
"""
raw_report = utils.base64_decode(report.get("raw"))
for line in csv.reader(io.StringIO(raw_report)):
yield line
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.split('\n'):
if row.startswith("#") or len(row) == 0 or row == "Site":
if 'updated' in row:
time_str = row[row.find(': ') + 2:]
time = dateutil.parser.parse(time_str).isoformat()
continue
event = Event(report)
event.add('classification.type', u'malware')
event.add('source.fqdn', row.strip())
event.add('time.source', time)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
raw_report = raw_report.split("<tbody>")[1]
raw_report = raw_report.split("</tbody>")[0]
raw_report_splitted = raw_report.split("<tr>")
for row in raw_report_splitted:
row = row.strip()
if row == "":
continue
row_splitted = row.split("<td>")
ip = row_splitted[1].split('">')[1].split("<")[0].strip()
time_source = row_splitted[6].replace("</td></tr>", "").strip()
time_source = time_source + " 00:00:00 UTC"
event = Event(report)
event.add('time.source', time_source)
event.add('classification.type', 'malware')
event.add('source.ip', ip)
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()