def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if 'addr' in spec and \
spec.get('source') not in neverignore.get(spec['recontype'], []):
for start, stop in ignorenets.get(spec['recontype'], ()):
if start <= utils.force_ip2int(spec['addr']) <= stop:
return None
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string 'SymantecRandomUserAgent'.
if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \
spec.get('source') == 'USER-AGENT':
if SYMANTEC_UA.match(spec['value']):
spec['value'] = 'SymantecRandomUserAgent'
elif KASPERSKY_UA.match(spec['value']):
spec['value'] = 'KasperskyWeirdUserAgent'
else:
match = SYMANTEC_SEP_UA.match(spec['value'])
if match is not None:
spec['value'] = '%s%s' % match.groups()
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif spec['recontype'] in ['HTTP_CLIENT_HEADER',
'HTTP_CLIENT_HEADER_SERVER'] and \
spec.get('source') in ['AUTHORIZATION',
'PROXY-AUTHORIZATION']:
value = spec['value']
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
spec['value'] = '%s %s' % (authtype, ','.join(
val for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif authtype.lower() in [
'negotiate', 'kerberos', 'oauth', 'ntlm'
]:
spec['value'] = authtype
# p0f in Bro hack: we use the "source" field to provide the
# "distance" and "version" values
elif spec['recontype'] == 'P0F':
distance, version = spec.pop('source').split('-', 1)
try:
spec['distance'] = int(distance)
except ValueError:
pass
if version:
spec['version'] = version
# TCP server banners: try to normalize data
elif spec['recontype'] == 'TCP_SERVER_BANNER':
newvalue = value = utils.nmap_decode_data(spec['value'])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec['value'] = utils.nmap_encode_data(newvalue)
# SSL_{CLIENT,SERVER} JA3
elif ((spec['recontype'] == 'SSL_CLIENT' and spec['source'] == 'ja3')
or (spec['recontype'] == 'SSL_SERVER'
and spec['source'].startswith('ja3-'))):
value = spec['value']
spec.setdefault('infos', {})['raw'] = value
spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
if spec['recontype'] == 'SSL_SERVER':
clientvalue = spec['source'][4:]
spec['infos'].setdefault('client', {})['raw'] = clientvalue
spec['source'] = 'ja3-%s' % hashlib.new(
"md5",
clientvalue.encode(),
).hexdigest()
# Check DNS Blacklist answer
elif spec['recontype'] == 'DNS_ANSWER':
if any((spec.get('value') or "").endswith(dnsbl)
for dnsbl in config.DNS_BLACKLIST_DOMAINS):
dnsbl_val = spec['value']
match = DNSBL_START.search(dnsbl_val)
if match is not None:
spec['recontype'] = 'DNS_BLACKLIST'
spec['value'] = spec.get('addr')
spec.update({
'source':
"%s-%s" % (dnsbl_val[match.end():], spec['source'])
})
addr = match.group()
# IPv4
if addr.count('.') == 4:
spec['addr'] = '.'.join(addr.split('.')[3::-1])
# IPv6
else:
spec['addr'] = utils.int2ip6(
int(addr.replace('.', '')[::-1], 16))
return spec
def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if 'addr' in spec and \
spec.get('source') not in neverignore.get(spec['recontype'], []):
for start, stop in ignorenets.get(spec['recontype'], ()):
if start <= utils.force_ip2int(spec['addr']) <= stop:
return None
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string 'SymantecRandomUserAgent'.
if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \
spec.get('source') == 'USER-AGENT':
if SYMANTEC_UA.match(spec['value']):
spec['value'] = 'SymantecRandomUserAgent'
elif KASPERSKY_UA.match(spec['value']):
spec['value'] = 'KasperskyWeirdUserAgent'
else:
match = SYMANTEC_SEP_UA.match(spec['value'])
if match is not None:
spec['value'] = '%s%s' % match.groups()
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif (spec['recontype']
in {'HTTP_CLIENT_HEADER', 'HTTP_CLIENT_HEADER_SERVER'}
and spec.get('source') in {'AUTHORIZATION', 'PROXY-AUTHORIZATION'}):
value = spec['value']
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
spec['value'] = '%s %s' % (authtype, ','.join(
val for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_NEGOTIATE and NTLM_AUTHENTICATE
try:
auth = utils.decode_b64(value.split(None, 1)[1].encode())
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
pass
spec['value'] = "%s %s" % \
(value.split(None, 1)[0],
ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth)))
elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}:
spec['value'] = authtype
elif (spec['recontype'] == 'HTTP_SERVER_HEADER' and spec.get('source')
in {'WWW-AUTHENTICATE', 'PROXY-AUTHENTICATE'}):
value = spec['value']
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
spec['value'] = '%s %s' % (authtype, ','.join(
val for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_CHALLENGE
try:
auth = utils.decode_b64(value.split(None, 1)[1].encode())
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
pass
spec['value'] = "%s %s" % \
(value.split(None, 1)[0],
ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth)))
elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}:
spec['value'] = authtype
# TCP server banners: try to normalize data
elif spec['recontype'] == 'TCP_SERVER_BANNER':
newvalue = value = utils.nmap_decode_data(spec['value'])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec['value'] = utils.nmap_encode_data(newvalue)
elif spec['recontype'] == 'TCP_CLIENT_BANNER':
probe = utils.get_nmap_probes('tcp').get(
utils.nmap_decode_data(spec['value']))
if probe is not None:
spec.setdefault('infos', {}).update({
'service_name':
'scanner',
'service_product':
'Nmap',
'service_extrainfo':
'TCP probe %s' % probe,
})
elif spec['recontype'] == 'UDP_HONEYPOT_HIT':
data = utils.nmap_decode_data(spec['value'])
probe = utils.get_nmap_probes('udp').get(data)
if probe is not None:
spec.setdefault('infos', {}).update({
'service_name':
'scanner',
'service_product':
'Nmap',
'service_extrainfo':
'UDP probe %s' % probe,
})
else:
payload = utils.get_nmap_udp_payloads().get(data)
if payload is not None:
spec.setdefault('infos', {}).update({
'service_name':
'scanner',
'service_product':
'Nmap',
'service_extrainfo':
'UDP payload %s' % payload,
})
# SSL_{CLIENT,SERVER} JA3
elif ((spec['recontype'] == 'SSL_CLIENT' and spec['source'] == 'ja3')
or (spec['recontype'] == 'SSL_SERVER'
and spec['source'].startswith('ja3-'))):
value = spec['value']
spec.setdefault('infos', {})['raw'] = value
spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
if spec['recontype'] == 'SSL_SERVER':
clientvalue = spec['source'][4:]
spec['infos'].setdefault('client', {})['raw'] = clientvalue
spec['source'] = 'ja3-%s' % hashlib.new(
"md5",
clientvalue.encode(),
).hexdigest()
# SSH_{CLIENT,SERVER}_HASSH
elif spec['recontype'] in ['SSH_CLIENT_HASSH', 'SSH_SERVER_HASSH']:
value = spec['value']
spec.setdefault('infos', {})['raw'] = value
spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
# Check DNS Blacklist answer
elif spec['recontype'] == 'DNS_ANSWER':
if any((spec.get('value') or "").endswith(dnsbl)
for dnsbl in config.DNS_BLACKLIST_DOMAINS):
dnsbl_val = spec['value']
match = DNSBL_START.search(dnsbl_val)
if match is not None:
spec['recontype'] = 'DNS_BLACKLIST'
spec['value'] = spec.get('addr')
spec.update({
'source':
"%s-%s" % (dnsbl_val[match.end():], spec['source'])
})
addr = match.group()
# IPv4
if addr.count('.') == 4:
spec['addr'] = '.'.join(addr.split('.')[3::-1])
# IPv6
else:
spec['addr'] = utils.int2ip6(
int(addr.replace('.', '')[::-1], 16))
return spec
def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if "addr" in spec and spec.get("source") not in neverignore.get(
spec["recontype"], []):
for start, stop in ignorenets.get(spec["recontype"], []):
if start <= utils.force_ip2int(spec["addr"]) <= stop:
return
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string "SymantecRandomUserAgent".
if spec["recontype"] == "HTTP_CLIENT_HEADER" and spec.get(
"source") == "USER-AGENT":
if SYMANTEC_UA.match(spec["value"]):
spec["value"] = "SymantecRandomUserAgent"
elif KASPERSKY_UA.match(spec["value"]):
spec["value"] = "KasperskyWeirdUserAgent"
else:
match = SYMANTEC_SEP_UA.match(spec["value"])
if match is not None:
spec["value"] = "%s%s" % match.groups()
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif spec["recontype"] in {
"HTTP_CLIENT_HEADER",
"HTTP_CLIENT_HEADER_SERVER",
} and spec.get("source") in {"AUTHORIZATION", "PROXY-AUTHORIZATION"}:
value = spec["value"]
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == "digest":
try:
# we only keep relevant info
spec["value"] = "%s %s" % (
authtype,
",".join(
val
for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)),
)
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_NEGOTIATE and NTLM_AUTHENTICATE
yield from _prepare_rec_ntlm(spec, "NTLM_CLIENT_FLAGS")
return
elif authtype.lower() in {"negotiate", "kerberos", "oauth"}:
spec["value"] = authtype
elif spec["recontype"] == "HTTP_SERVER_HEADER" and spec.get("source") in {
"WWW-AUTHENTICATE",
"PROXY-AUTHENTICATE",
}:
value = spec["value"]
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == "digest":
try:
# we only keep relevant info
spec["value"] = "%s %s" % (
authtype,
",".join(
val
for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)),
)
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_CHALLENGE
yield from _prepare_rec_ntlm(spec, "NTLM_SERVER_FLAGS")
return
elif authtype.lower() in {"negotiate", "kerberos", "oauth"}:
spec["value"] = authtype
# TCP server banners: try to normalize data
elif spec["recontype"] == "TCP_SERVER_BANNER":
newvalue = value = utils.nmap_decode_data(spec["value"])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec["value"] = utils.nmap_encode_data(newvalue)
elif spec["recontype"] in {"TCP_CLIENT_BANNER", "TCP_HONEYPOT_HIT"}:
if spec["value"]:
data = utils.nmap_decode_data(spec["value"])
if data in scanners.TCP_PROBES:
scanner, probe = scanners.TCP_PROBES[data]
info = {
"service_name": "scanner",
"service_product": scanner,
}
if probe is not None:
info["service_extrainfo"] = "TCP probe %s" % probe
spec.setdefault("infos", {}).update(info)
else:
probe = utils.get_nmap_probes("tcp").get(data)
if probe is not None:
spec.setdefault("infos", {}).update({
"service_name":
"scanner",
"service_product":
"Nmap",
"service_extrainfo":
"TCP probe %s" % probe,
})
elif spec["recontype"] == "UDP_HONEYPOT_HIT":
data = utils.nmap_decode_data(spec["value"])
if data in scanners.UDP_PROBES:
scanner, probe = scanners.UDP_PROBES[data]
info = {
"service_name": "scanner",
"service_product": scanner,
}
if probe is not None:
info["service_extrainfo"] = "UDP probe %s" % probe
spec.setdefault("infos", {}).update(info)
else:
probe = utils.get_nmap_probes("udp").get(data)
if probe is not None:
spec.setdefault("infos", {}).update({
"service_name":
"scanner",
"service_product":
"Nmap",
"service_extrainfo":
"UDP probe %s" % probe,
})
else:
payload = utils.get_nmap_udp_payloads().get(data)
if payload is not None:
spec.setdefault("infos", {}).update({
"service_name":
"scanner",
"service_product":
"Nmap",
"service_extrainfo":
"UDP payload %s" % payload,
})
elif spec["recontype"] == "STUN_HONEYPOT_REQUEST":
spec["value"] = utils.nmap_decode_data(spec["value"])
# SSL_{CLIENT,SERVER} JA3
elif (spec["recontype"] == "SSL_CLIENT" and spec["source"]
== "ja3") or (spec["recontype"] == "SSL_SERVER"
and spec["source"].startswith("ja3-")):
value = spec["value"]
spec.setdefault("infos", {})["raw"] = value
spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
if spec["recontype"] == "SSL_SERVER":
clientvalue = spec["source"][4:]
spec["infos"].setdefault("client", {})["raw"] = clientvalue
spec["source"] = ("ja3-%s" % hashlib.new(
"md5",
clientvalue.encode(),
).hexdigest())
# SSH_{CLIENT,SERVER}_HASSH
elif spec["recontype"] in ["SSH_CLIENT_HASSH", "SSH_SERVER_HASSH"]:
value = spec["value"]
spec.setdefault("infos", {})["raw"] = value
spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
# Check DNS Blacklist answer
elif spec["recontype"] == "DNS_ANSWER":
if any((spec.get("value") or "").endswith(dnsbl)
for dnsbl in config.DNS_BLACKLIST_DOMAINS):
dnsbl_val = spec["value"]
match = DNSBL_START.search(dnsbl_val)
if match is not None:
spec["recontype"] = "DNS_BLACKLIST"
spec["value"] = spec.get("addr")
spec["source"] = "%s-%s" % (dnsbl_val[match.end():],
spec["source"])
addr = match.group()
# IPv4
if addr.count(".") == 4:
spec["addr"] = ".".join(addr.split(".")[3::-1])
# IPv6
else:
spec["addr"] = utils.int2ip6(
int(addr.replace(".", "")[::-1], 16))
yield spec