def _getinfos_ssh_hostkey(spec):
"""Parse SSH host keys."""
infos = {}
data = utils.nmap_decode_data(spec['value'])
for hashtype in ['md5', 'sha1', 'sha256']:
infos[hashtype] = hashlib.new(hashtype, data).hexdigest()
data = utils.parse_ssh_key(data)
keytype = infos["algo"] = next(data).decode()
if keytype == "ssh-rsa":
try:
infos["exponent"], infos["modulus"] = (
long(utils.encode_hex(elt), 16) for elt in data
)
except Exception:
utils.LOGGER.info("Cannot parse SSH host key for record %r", spec,
exc_info=True)
else:
infos["bits"] = math.ceil(math.log(infos["modulus"], 2))
# convert integer to strings to prevent overflow errors
# (e.g., "MongoDB can only handle up to 8-byte ints")
for val in ["exponent", "modulus"]:
infos[val] = str(infos[val])
elif keytype == 'ecdsa-sha2-nistp256':
infos['bits'] = 256
elif keytype == 'ssh-ed25519':
infos['bits'] = len(next(data)) * 8
return {'infos': infos}
def _extract_passive_SSH_SERVER_HOSTKEY(rec):
"""Handle SSH host keys."""
# TODO: should (probably) be merged, sorted by date/time, keep one
# entry per key type.
#
# (MAYBE) we should add a "lastseen" tag to every intel in view.
value = utils.encode_b64(utils.nmap_decode_data(rec['value'])).decode()
fingerprint = rec['infos']['md5']
key = {
'type': rec['infos']['algo'],
'key': value,
'fingerprint': fingerprint
}
if 'bits' in rec['infos']: # FIXME
key['bits'] = rec['infos']['bits']
fingerprint = utils.decode_hex(fingerprint)
script = {
'id':
'ssh-hostkey',
'ssh-hostkey': [key],
'output':
'\n %s %s (%s)\n%s %s' % (
key.get('bits', '-'), # FIXME
':'.join('%02x' % (ord(i) if isinstance(i, (bytes, str)) else i)
for i in fingerprint),
_KEYS.get(key['type'], (key['type'][4:] if key['type'][:4]
== 'ssh-' else key['type']).upper()),
key['type'],
value),
'key':
key
}
return {
'ports': [{
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'ssh',
'scripts': [script],
}]
}
def _extract_passive_TCP_SERVER_BANNER(rec):
"""Handle banners from tcp servers."""
value = rec['value']
if rec['recontype'] == 'SSH_SERVER':
value += "\r\n"
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'scripts': [{"id": "banner",
"output": value}],
}
port.update(rec.get('infos', {}))
port.update(
utils.match_nmap_svc_fp(output=utils.nmap_decode_data(value),
proto=rec.get('protocol', 'tcp'),
probe="NULL")
)
return {'ports': [port]}
def _extract_passive_TCP_SERVER_BANNER(rec):
"""Handle banners from tcp servers."""
value = rec['value']
if rec['recontype'] == 'SSH_SERVER':
value += "\r\n"
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'scripts': [{"id": "banner",
"output": value}],
}
port.update(rec.get('infos', {}))
port.update(
utils.match_nmap_svc_fp(output=utils.nmap_decode_data(value),
proto=rec.get('protocol', 'tcp'),
probe="NULL")
)
return {'ports': [port]}
def _extract_passive_SSH_SERVER_HOSTKEY(rec):
"""Handle SSH host keys."""
# TODO: should (probably) be merged, sorted by date/time, keep one
# entry per key type.
#
# (MAYBE) we should add a "lastseen" tag to every intel in view.
value = utils.encode_b64(
utils.nmap_decode_data(rec['value'])
).decode()
fingerprint = rec['infos']['md5']
key = {'type': rec['infos']['algo'],
'key': value,
'fingerprint': fingerprint}
if 'bits' in rec['infos']: # FIXME
key['bits'] = rec['infos']['bits']
fingerprint = utils.decode_hex(fingerprint)
script = {
'id': 'ssh-hostkey', 'ssh-hostkey': [key],
'output': '\n %s %s (%s)\n%s %s' % (
key.get('bits', '-'), # FIXME
':'.join('%02x' % (
ord(i) if isinstance(i, (bytes, str)) else i
) for i in fingerprint),
_KEYS.get(
key['type'],
(key['type'][4:] if key['type'][:4] == 'ssh-'
else key['type']).upper()
),
key['type'],
value
),
'key': key
}
return {'ports': [{
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'ssh',
'scripts': [script],
}]}
def _extract_passive_HTTP_SERVER_HEADER(rec):
"""Handle http server headers."""
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'http',
}
# TODO: handle other header values and merge them
if rec.get('source') != 'SERVER':
return {'ports': [port]}
value = rec['value']
script = {'id': 'http-server-header', 'output': value}
port['scripts'] = [script]
banner = (b"HTTP/1.1 200 OK\r\nServer: " + utils.nmap_decode_data(value) +
b"\r\n\r\n")
port.update(
utils.match_nmap_svc_fp(output=banner,
proto=rec.get('protocol', 'tcp'),
probe="GetRequest"))
return {'ports': [port]}
def _extract_passive_HTTP_SERVER_HEADER(rec):
"""Handle http server headers."""
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'http',
}
# TODO: handle other header values and merge them
if rec.get('source') != 'SERVER':
return {'ports': [port]}
value = rec['value']
script = {'id': 'http-server-header', 'output': value}
port['scripts'] = [script]
banner = (b"HTTP/1.1 200 OK\r\nServer: " + utils.nmap_decode_data(value) +
b"\r\n\r\n")
port.update(
utils.match_nmap_svc_fp(output=banner,
proto=rec.get('protocol', 'tcp'),
probe="GetRequest")
)
return {'ports': [port]}
def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if 'addr' in spec and \
spec.get('source') not in neverignore.get(spec['recontype'], []):
for start, stop in ignorenets.get(spec['recontype'], ()):
if start <= utils.force_ip2int(spec['addr']) <= stop:
return None
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string 'SymantecRandomUserAgent'.
if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \
spec.get('source') == 'USER-AGENT':
if SYMANTEC_UA.match(spec['value']):
spec['value'] = 'SymantecRandomUserAgent'
elif KASPERSKY_UA.match(spec['value']):
spec['value'] = 'KasperskyWeirdUserAgent'
else:
match = SYMANTEC_SEP_UA.match(spec['value'])
if match is not None:
spec['value'] = '%s%s' % match.groups()
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif spec['recontype'] in ['HTTP_CLIENT_HEADER',
'HTTP_CLIENT_HEADER_SERVER'] and \
spec.get('source') in ['AUTHORIZATION',
'PROXY-AUTHORIZATION']:
authtype = spec['value'].split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
value = [val for val in
_split_digest_auth(spec['value'][6:].strip())
if DIGEST_AUTH_INFOS.match(val)]
spec['value'] = '%s %s' % (authtype, ','.join(value))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r", spec,
exc_info=True)
elif authtype.lower() in ['negotiate', 'kerberos', 'oauth', 'ntlm']:
spec['value'] = authtype
# p0f in Bro hack: we use the "source" field to provide the
# "distance" and "version" values
elif spec['recontype'] == 'P0F':
distance, version = spec.pop('source').split('-', 1)
try:
spec['distance'] = int(distance)
except ValueError:
pass
if version:
spec['version'] = version
# TCP server banners: try to normalize data
elif spec['recontype'] == 'TCP_SERVER_BANNER':
newvalue = value = utils.nmap_decode_data(spec['value'])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec['value'] = utils.nmap_encode_data(newvalue)
# SSL_{CLIENT,SERVER} JA3
elif ((spec['recontype'] == 'SSL_CLIENT' and spec['source'] == 'ja3') or
(spec['recontype'] == 'SSL_SERVER' and
spec['source'].startswith('ja3-'))):
value = spec['value']
spec.setdefault('infos', {})['raw'] = value
spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
if spec['recontype'] == 'SSL_SERVER':
clientvalue = spec['source'][4:]
spec['infos'].setdefault('client', {})['raw'] = clientvalue
spec['source'] = 'ja3-%s' % hashlib.new(
"md5",
clientvalue.encode(),
).hexdigest()
# Check DNS Blacklist answer
elif spec['recontype'] == 'DNS_ANSWER':
if any(spec['value'].endswith(dnsbl)
for dnsbl in config.DNS_BLACKLIST_DOMAINS):
dnsbl_val = spec['value']
spec['recontype'] = 'DNS_BLACKLIST'
spec['value'] = spec['addr']
spec.update({'source': "%s-%s" %
(dnsbl_val.split('.', 4)[4], spec['source'])})
spec['addr'] = '.'.join(dnsbl_val.split('.')[3::-1])
return spec
def zgrap_parser_http(data, hostrec, port=None):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if 'result' in data:
data.update(data.pop('result'))
if 'response' not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data['response']
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
'Missing field%s %s in zgrab HTTP result',
's' if len(missing_fields) > 1 else '',
', '.join(repr(fld) for fld in missing_fields),
)
return {}
req = resp['request']
url = req.get('url')
res = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp"
}
tls = None
try:
tls = req['tls_handshake']
except KeyError:
# zgrab2
try:
tls = req['tls_log']['handshake_log']
except KeyError:
pass
if tls is not None:
res['service_tunnel'] = 'ssl'
try:
cert = tls['server_certificates']['certificate']['raw']
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault('scripts', []).append({
'id': 'ssl-cert',
'output': output,
'ssl-cert': info,
})
for cert in info:
add_cert_hostnames(cert,
hostrec.setdefault('hostnames', []))
if url:
guessed_port = None
if ':' in url.get('host', ''):
try:
guessed_port = int(url['host'].split(':', 1)[1])
except ValueError:
pass
if port is None:
if guessed_port is None:
if url.get('scheme') == 'https':
port = 443
else:
port = 80
else:
port = guessed_port
elif port != guessed_port:
utils.LOGGER.warning(
'Port %d found from the URL %s differs from the provided port '
'value %d', guessed_port, url.get('path'), port)
port = guessed_port
# Specific paths
if url.get('path').endswith('/.git/index'):
if resp.get('status_code') != 200:
return {}
if not resp.get('body', '').startswith('DIRC'):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = '%s:%d%s' % (hostrec['addr'], port, url['path'][:-5])
res['port'] = port
res.setdefault('scripts', []).append({
'id':
'http-git',
'output':
'\n %s\n Git repository found!\n' % repository,
'http-git': [{
'repository': repository,
'files-found': [".git/index"]
}],
})
return res
if url.get('path').endswith('/owa/auth/logon.aspx'):
if resp.get('status_code') != 200:
return {}
version = set(
m.group(1)
for m in _EXPR_OWA_VERSION.finditer(resp.get('body', '')))
if not version:
return {}
version = sorted(version,
key=lambda v: [int(x) for x in v.split('.')])
res['port'] = port
path = url['path'][:-15]
if len(version) > 1:
output = (
'OWA: path %s, version %s (multiple versions found!)' % (
path,
' / '.join(version),
))
else:
output = 'OWA: path %s, version %s' % (path, version[0])
res.setdefault('scripts', []).append({
'id':
'http-app',
'output':
output,
'http-app': [{
'path': path,
'application': 'OWA',
'version': version[0]
}],
})
return res
if url.get('path').endswith('/centreon/'):
if resp.get('status_code') != 200:
return {}
if not resp.get('body'):
return {}
body = resp['body']
res['port'] = port
path = url['path']
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault('scripts', []).append({
'id':
'http-app',
'output':
'Centreon: path %s%s' % (
path,
'' if version is None else (', version %s' % version),
),
'http-app': [
dict({
'path': path,
'application': 'Centreon'
}, **({} if version is None else {
'version': version
}))
],
})
return res
if url.get('path') != '/':
utils.LOGGER.warning('URL path not supported yet: %s',
url.get('path'))
return {}
elif port is None:
if req.get('tls_handshake') or req.get('tls_log'):
port = 443
else:
port = 80
res['port'] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' +
utils.nmap_decode_data(resp['status_line']) + b"\r\n")
if resp.get('headers'):
headers = resp['headers']
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get('www_authenticate'):
auths = headers.get('www_authenticate')
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode()))
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
pass
keyvals = zip(ntlm_values,
[infos.get(k) for k in ntlm_values])
output = '\n'.join("{}: {}".format(k, v)
for k, v in keyvals if v)
res.setdefault('scripts', []).append({
'id': 'http-ntlm-info',
'output': output,
'ntlm-info': infos
})
if 'DNS_Computer_Name' in infos:
add_hostname(infos['DNS_Computer_Name'], 'ntlm',
hostrec.setdefault('hostnames', []))
if any(val.lower().startswith('ntlm')
for val in req.get('headers', {}).get('authorization', [])):
return res
# the order will be incorrect!
line = '%s %s' % (resp['protocol']['name'], resp['status_line'])
http_hdrs = [{'name': '_status', 'value': line}]
output = [line]
for unk in headers.pop('unknown', []):
headers[unk['key']] = unk['value']
for hdr, values in headers.items():
hdr = hdr.replace('_', '-')
for val in values:
http_hdrs.append({'name': hdr, 'value': val})
output.append('%s: %s' % (hdr, val))
if http_hdrs:
method = req.get('method')
if method:
output.append('')
output.append('(Request type: %s)' % method)
res.setdefault('scripts', []).append({
'id': 'http-headers',
'output': '\n'.join(output),
'http-headers': http_hdrs,
})
handle_http_headers(hostrec, res, http_hdrs, path=url.get('path'))
if headers.get('server'):
banner += (b"Server: " +
utils.nmap_decode_data(headers['server'][0]) +
b"\r\n\r\n")
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
add_cpe_values(hostrec, 'ports.port:%s' % port, info.pop('cpe', []))
res.update(info)
if resp.get('body'):
body = resp['body']
res.setdefault('scripts', []).append({
'id':
'http-content',
'output':
utils.nmap_encode_data(body.encode()),
})
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res['scripts'].append({
'id': 'http-title',
'output': title,
'http-title': {
'title': title
},
})
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault('scripts', []).append(script_http_ls)
service_elasticsearch = create_elasticsearch_service(body)
if service_elasticsearch:
if 'hostname' in service_elasticsearch:
add_hostname(service_elasticsearch.pop('hostname'), 'service',
hostrec.setdefault('hostnames', []))
add_cpe_values(hostrec, 'ports.port:%s' % port,
service_elasticsearch.pop('cpe', []))
res.update(service_elasticsearch)
return res
def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if "addr" in spec and spec.get("source") not in neverignore.get(
spec["recontype"], []):
for start, stop in ignorenets.get(spec["recontype"], []):
if start <= utils.force_ip2int(spec["addr"]) <= stop:
return
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string "SymantecRandomUserAgent".
if spec["recontype"] == "HTTP_CLIENT_HEADER" and spec.get(
"source") == "USER-AGENT":
if SYMANTEC_UA.match(spec["value"]):
spec["value"] = "SymantecRandomUserAgent"
elif KASPERSKY_UA.match(spec["value"]):
spec["value"] = "KasperskyWeirdUserAgent"
else:
match = SYMANTEC_SEP_UA.match(spec["value"])
if match is not None:
spec["value"] = "%s%s" % match.groups()
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif spec["recontype"] in {
"HTTP_CLIENT_HEADER",
"HTTP_CLIENT_HEADER_SERVER",
} and spec.get("source") in {"AUTHORIZATION", "PROXY-AUTHORIZATION"}:
value = spec["value"]
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == "digest":
try:
# we only keep relevant info
spec["value"] = "%s %s" % (
authtype,
",".join(
val
for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)),
)
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_NEGOTIATE and NTLM_AUTHENTICATE
yield from _prepare_rec_ntlm(spec, "NTLM_CLIENT_FLAGS")
return
elif authtype.lower() in {"negotiate", "kerberos", "oauth"}:
spec["value"] = authtype
elif spec["recontype"] == "HTTP_SERVER_HEADER" and spec.get("source") in {
"WWW-AUTHENTICATE",
"PROXY-AUTHENTICATE",
}:
value = spec["value"]
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == "digest":
try:
# we only keep relevant info
spec["value"] = "%s %s" % (
authtype,
",".join(
val
for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)),
)
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_CHALLENGE
yield from _prepare_rec_ntlm(spec, "NTLM_SERVER_FLAGS")
return
elif authtype.lower() in {"negotiate", "kerberos", "oauth"}:
spec["value"] = authtype
# TCP server banners: try to normalize data
elif spec["recontype"] == "TCP_SERVER_BANNER":
newvalue = value = utils.nmap_decode_data(spec["value"])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec["value"] = utils.nmap_encode_data(newvalue)
elif spec["recontype"] in {"TCP_CLIENT_BANNER", "TCP_HONEYPOT_HIT"}:
if spec["value"]:
data = utils.nmap_decode_data(spec["value"])
if data in scanners.TCP_PROBES:
scanner, probe = scanners.TCP_PROBES[data]
info = {
"service_name": "scanner",
"service_product": scanner,
}
if probe is not None:
info["service_extrainfo"] = "TCP probe %s" % probe
spec.setdefault("infos", {}).update(info)
else:
probe = utils.get_nmap_probes("tcp").get(data)
if probe is not None:
spec.setdefault("infos", {}).update({
"service_name":
"scanner",
"service_product":
"Nmap",
"service_extrainfo":
"TCP probe %s" % probe,
})
elif spec["recontype"] == "UDP_HONEYPOT_HIT":
data = utils.nmap_decode_data(spec["value"])
if data in scanners.UDP_PROBES:
scanner, probe = scanners.UDP_PROBES[data]
info = {
"service_name": "scanner",
"service_product": scanner,
}
if probe is not None:
info["service_extrainfo"] = "UDP probe %s" % probe
spec.setdefault("infos", {}).update(info)
else:
probe = utils.get_nmap_probes("udp").get(data)
if probe is not None:
spec.setdefault("infos", {}).update({
"service_name":
"scanner",
"service_product":
"Nmap",
"service_extrainfo":
"UDP probe %s" % probe,
})
else:
payload = utils.get_nmap_udp_payloads().get(data)
if payload is not None:
spec.setdefault("infos", {}).update({
"service_name":
"scanner",
"service_product":
"Nmap",
"service_extrainfo":
"UDP payload %s" % payload,
})
elif spec["recontype"] == "STUN_HONEYPOT_REQUEST":
spec["value"] = utils.nmap_decode_data(spec["value"])
# SSL_{CLIENT,SERVER} JA3
elif (spec["recontype"] == "SSL_CLIENT" and spec["source"]
== "ja3") or (spec["recontype"] == "SSL_SERVER"
and spec["source"].startswith("ja3-")):
value = spec["value"]
spec.setdefault("infos", {})["raw"] = value
spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
if spec["recontype"] == "SSL_SERVER":
clientvalue = spec["source"][4:]
spec["infos"].setdefault("client", {})["raw"] = clientvalue
spec["source"] = ("ja3-%s" % hashlib.new(
"md5",
clientvalue.encode(),
).hexdigest())
# SSH_{CLIENT,SERVER}_HASSH
elif spec["recontype"] in ["SSH_CLIENT_HASSH", "SSH_SERVER_HASSH"]:
value = spec["value"]
spec.setdefault("infos", {})["raw"] = value
spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
# Check DNS Blacklist answer
elif spec["recontype"] == "DNS_ANSWER":
if any((spec.get("value") or "").endswith(dnsbl)
for dnsbl in config.DNS_BLACKLIST_DOMAINS):
dnsbl_val = spec["value"]
match = DNSBL_START.search(dnsbl_val)
if match is not None:
spec["recontype"] = "DNS_BLACKLIST"
spec["value"] = spec.get("addr")
spec["source"] = "%s-%s" % (dnsbl_val[match.end():],
spec["source"])
addr = match.group()
# IPv4
if addr.count(".") == 4:
spec["addr"] = ".".join(addr.split(".")[3::-1])
# IPv6
else:
spec["addr"] = utils.int2ip6(
int(addr.replace(".", "")[::-1], 16))
yield spec
def _getinfos_http_server(spec):
header = utils.nmap_decode_data(spec['value'])
banner = b"HTTP/1.1 200 OK\r\nServer: " + header + b"\r\n\r\n"
res = _getinfos_from_banner(banner, probe="GetRequest")
return res
def _getinfos_tcp_srv_banner(spec):
"""Extract info from a TCP server banner using Nmap database.
"""
return _getinfos_from_banner(utils.nmap_decode_data(spec['value']))
def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if 'addr' in spec and \
spec.get('source') not in neverignore.get(spec['recontype'], []):
for start, stop in ignorenets.get(spec['recontype'], ()):
if start <= utils.force_ip2int(spec['addr']) <= stop:
return None
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string 'SymantecRandomUserAgent'.
if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \
spec.get('source') == 'USER-AGENT':
if SYMANTEC_UA.match(spec['value']):
spec['value'] = 'SymantecRandomUserAgent'
elif KASPERSKY_UA.match(spec['value']):
spec['value'] = 'KasperskyWeirdUserAgent'
else:
match = SYMANTEC_SEP_UA.match(spec['value'])
if match is not None:
spec['value'] = '%s%s' % match.groups()
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif (spec['recontype']
in {'HTTP_CLIENT_HEADER', 'HTTP_CLIENT_HEADER_SERVER'}
and spec.get('source') in {'AUTHORIZATION', 'PROXY-AUTHORIZATION'}):
value = spec['value']
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
spec['value'] = '%s %s' % (authtype, ','.join(
val for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_NEGOTIATE and NTLM_AUTHENTICATE
try:
auth = utils.decode_b64(value.split(None, 1)[1].encode())
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
pass
spec['value'] = "%s %s" % \
(value.split(None, 1)[0],
ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth)))
elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}:
spec['value'] = authtype
elif (spec['recontype'] == 'HTTP_SERVER_HEADER' and spec.get('source')
in {'WWW-AUTHENTICATE', 'PROXY-AUTHENTICATE'}):
value = spec['value']
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
spec['value'] = '%s %s' % (authtype, ','.join(
val for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_CHALLENGE
try:
auth = utils.decode_b64(value.split(None, 1)[1].encode())
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
pass
spec['value'] = "%s %s" % \
(value.split(None, 1)[0],
ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth)))
elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}:
spec['value'] = authtype
# TCP server banners: try to normalize data
elif spec['recontype'] == 'TCP_SERVER_BANNER':
newvalue = value = utils.nmap_decode_data(spec['value'])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec['value'] = utils.nmap_encode_data(newvalue)
elif spec['recontype'] == 'TCP_CLIENT_BANNER':
probe = utils.get_nmap_probes('tcp').get(
utils.nmap_decode_data(spec['value']))
if probe is not None:
spec.setdefault('infos', {}).update({
'service_name':
'scanner',
'service_product':
'Nmap',
'service_extrainfo':
'TCP probe %s' % probe,
})
elif spec['recontype'] == 'UDP_HONEYPOT_HIT':
data = utils.nmap_decode_data(spec['value'])
probe = utils.get_nmap_probes('udp').get(data)
if probe is not None:
spec.setdefault('infos', {}).update({
'service_name':
'scanner',
'service_product':
'Nmap',
'service_extrainfo':
'UDP probe %s' % probe,
})
else:
payload = utils.get_nmap_udp_payloads().get(data)
if payload is not None:
spec.setdefault('infos', {}).update({
'service_name':
'scanner',
'service_product':
'Nmap',
'service_extrainfo':
'UDP payload %s' % payload,
})
# SSL_{CLIENT,SERVER} JA3
elif ((spec['recontype'] == 'SSL_CLIENT' and spec['source'] == 'ja3')
or (spec['recontype'] == 'SSL_SERVER'
and spec['source'].startswith('ja3-'))):
value = spec['value']
spec.setdefault('infos', {})['raw'] = value
spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
if spec['recontype'] == 'SSL_SERVER':
clientvalue = spec['source'][4:]
spec['infos'].setdefault('client', {})['raw'] = clientvalue
spec['source'] = 'ja3-%s' % hashlib.new(
"md5",
clientvalue.encode(),
).hexdigest()
# SSH_{CLIENT,SERVER}_HASSH
elif spec['recontype'] in ['SSH_CLIENT_HASSH', 'SSH_SERVER_HASSH']:
value = spec['value']
spec.setdefault('infos', {})['raw'] = value
spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
# Check DNS Blacklist answer
elif spec['recontype'] == 'DNS_ANSWER':
if any((spec.get('value') or "").endswith(dnsbl)
for dnsbl in config.DNS_BLACKLIST_DOMAINS):
dnsbl_val = spec['value']
match = DNSBL_START.search(dnsbl_val)
if match is not None:
spec['recontype'] = 'DNS_BLACKLIST'
spec['value'] = spec.get('addr')
spec.update({
'source':
"%s-%s" % (dnsbl_val[match.end():], spec['source'])
})
addr = match.group()
# IPv4
if addr.count('.') == 4:
spec['addr'] = '.'.join(addr.split('.')[3::-1])
# IPv6
else:
spec['addr'] = utils.int2ip6(
int(addr.replace('.', '')[::-1], 16))
return spec
def zgrap_parser_http(data, hostrec):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if 'result' in data:
data.update(data.pop('result'))
if 'response' not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data['response']
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
'Missing field%s %s in zgrab HTTP result',
's' if len(missing_fields) > 1 else '',
', '.join(repr(fld) for fld in missing_fields),
)
return {}
req = resp['request']
url = req.get('url')
res = {"service_name": "http", "service_method": "probed",
"state_state": "open", "state_reason": "response",
"protocol": "tcp"}
tls = None
try:
tls = req['tls_handshake']
except KeyError:
# zgrab2
try:
tls = req['tls_log']['handshake_log']
except KeyError:
pass
if tls is not None:
res['service_tunnel'] = 'ssl'
try:
cert = tls['server_certificates']['certificate']['raw']
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault('scripts', []).append({
'id': 'ssl-cert',
'output': output,
'ssl-cert': info,
})
for cert in info:
add_cert_hostnames(cert,
hostrec.setdefault('hostnames', []))
if url:
port = None
if ':' in url.get('host', ''):
try:
port = int(url['host'].split(':', 1)[1])
except ValueError:
pass
if port is None:
if url.get('scheme') == 'https':
port = 443
else:
port = 80
# Specific paths
if url.get('path').endswith('/.git/index'):
if resp.get('status_code') != 200:
return {}
if not resp.get('body', '').startswith('DIRC'):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = '%s:%d%s' % (hostrec['addr'], port, url['path'][:-5])
res['port'] = port
res.setdefault('scripts', []).append({
'id': 'http-git',
'output': '\n %s\n Git repository found!\n' % repository,
'http-git': [{'repository': repository,
'files-found': [".git/index"]}],
})
return res
if url.get('path').endswith('/owa/auth/logon.aspx'):
if resp.get('status_code') != 200:
return {}
version = set(
m.group(1)
for m in _EXPR_OWA_VERSION.finditer(resp.get('body', ''))
)
if not version:
return {}
version = sorted(version,
key=lambda v: [int(x) for x in v.split('.')])
res['port'] = port
path = url['path'][:-15]
if len(version) > 1:
output = (
'OWA: path %s, version %s (multiple versions found!)' % (
path,
' / '.join(version),
)
)
else:
output = 'OWA: path %s, version %s' % (path, version[0])
res.setdefault('scripts', []).append({
'id': 'http-app',
'output': output,
'http-app': [{'path': path,
'application': 'OWA',
'version': version[0]}],
})
return res
elif req.get('tls_handshake') or req.get('tls_log'):
# zgrab / zgrab2
port = 443
else:
port = 80
res['port'] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' +
utils.nmap_decode_data(resp['status_line']) + b"\r\n")
if resp.get('headers'):
headers = resp['headers']
# the order will be incorrect!
line = '%s %s' % (resp['protocol']['name'], resp['status_line'])
http_hdrs = [{'name': '_status', 'value': line}]
output = [line]
if 'unknown' in headers:
for unk in headers.pop('unknown'):
headers[unk['key']] = unk['value']
for hdr, values in viewitems(headers):
hdr = hdr.replace('_', '-')
for val in values:
http_hdrs.append({'name': hdr, 'value': val})
output.append('%s: %s' % (hdr, val))
if http_hdrs:
method = req.get('method')
if method:
output.append('')
output.append('(Request type: %s)' % method)
res.setdefault('scripts', []).append({
'id': 'http-headers', 'output': '\n'.join(output),
'http-headers': http_hdrs,
})
if headers.get('server'):
server = resp['headers']['server']
res.setdefault('scripts', []).append({
'id': 'http-server-header', 'output': server[0],
'http-server-header': server,
})
banner += (b"Server: " + utils.nmap_decode_data(server[0]) +
b"\r\n\r\n")
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
res.update(info)
if resp.get('body'):
body = resp['body']
res.setdefault('scripts', []).append({
'id': 'http-content',
'output': utils.nmap_encode_data(body.encode()),
})
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res['scripts'].append({
'id': 'http-title', 'output': title,
'http-title': {'title': title},
})
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault('scripts', []).append(script_http_ls)
return res
def zgrap_parser_http(data):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if 'result' in data:
data.update(data.pop('result'))
if 'response' not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data['response']
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
'Missing field%s %s in zgrab HTTP result',
's' if len(missing_fields) > 1 else '',
', '.join(repr(fld) for fld in missing_fields),
)
return {}
req = resp['request']
res = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "syn-ack",
"protocol": "tcp"
}
url = req.get('url')
if url:
port = None
if ':' in url.get('host', ''):
try:
port = int(url['host'].split(':', 1)[1])
except ValueError:
pass
if port is None:
if url.get('scheme') == 'https':
port = 443
else:
port = 80
elif req.get('tls_handshake') or req.get('tls_log'):
# zgrab / zgrab2
port = 443
else:
port = 80
res['port'] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' +
utils.nmap_decode_data(resp['status_line']) + b"\r\n")
if resp.get('headers'):
headers = resp['headers']
# the order will be incorrect!
http_hdrs = [{'name': '_status', 'value': resp['status_line']}]
output = [resp['status_line']]
if 'unknown' in headers:
for unk in headers.pop('unknown'):
headers[unk['key']] = unk['value']
for hdr, values in viewitems(headers):
hdr = hdr.replace('_', '-')
for val in values:
http_hdrs.append({'name': hdr, 'value': val})
output.append('%s: %s' % (hdr, val))
if http_hdrs:
method = req.get('method')
if method:
output.append('')
output.append('(Request type: %s)' % method)
res.setdefault('scripts', []).append({
'id': 'http-headers',
'output': '\n'.join(output),
'http-headers': http_hdrs,
})
if headers.get('server'):
server = resp['headers']['server']
res.setdefault('scripts', []).append({
'id': 'http-server-header',
'output': server[0],
'http-server-header': server,
})
banner += (b"Server: " + utils.nmap_decode_data(server[0]) +
b"\r\n\r\n")
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
res.update(info)
if resp.get('body'):
body = resp['body']
res.setdefault('scripts', []).append({
'id':
'http-content',
'output':
utils.nmap_encode_data(body.encode()),
})
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res['scripts'].append({
'id': 'http-title',
'output': title,
'http-title': {
'title': title
},
})
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault('scripts', []).append(script_http_ls)
tls = None
try:
tls = req['tls_handshake']
except KeyError:
# zgrab2
try:
tls = req['tls_log']['handshake_log']
except KeyError:
pass
if tls is not None:
res['service_tunnel'] = 'ssl'
try:
cert = tls['server_certificates']['certificate']['raw']
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault('scripts', []).append({
'id':
'ssl-cert',
'output':
"\n".join(output),
'ssl-cert':
info,
})
return res
def handle_http_headers(
host: NmapHost,
port: NmapPort,
headers: List[HttpHeader],
path: str = "/",
handle_server: bool = True,
) -> None:
"""This function enriches scan results based on HTTP headers reported
by the Nmap script http-headers or any similar report, such as
Masscan or Zgrab(2).
"""
# 1. add a script "http-server-header" if it does not exist
if handle_server:
srv_headers = [
nmap_decode_data(h["value"])
for h in headers
if h["name"] == "server" and h["value"]
]
if srv_headers and not any(
s["id"] == "http-server-header" for s in port.get("scripts", [])
):
port.setdefault("scripts", []).append(
{
"id": "http-server-header",
"output": "\n".join(nmap_encode_data(hdr) for hdr in srv_headers),
"http-server-header": [
nmap_encode_data(hdr) for hdr in srv_headers
],
}
)
# 2. add a script "http-app" for MS SharePoint, and merge it if
# necessary
try:
header = next(
nmap_decode_data(h["value"])
for h in headers
if h["name"] == "microsoftsharepointteamservices" and h["value"]
)
except StopIteration:
pass
else:
version = nmap_encode_data(header.split(b":", 1)[0])
add_cpe_values(
host,
"ports.port:%s" % port.get("port", -1),
["cpe:/a:microsoft:sharepoint_server:%s" % version],
)
script = {
"id": "http-app",
"output": "SharePoint: path %s, version %s" % (path, version),
"http-app": [
{"path": path, "application": "SharePoint", "version": version}
],
}
try:
cur_script = next(
s for s in port.get("scripts", []) if s["id"] == "http-app"
)
except StopIteration:
port.setdefault("scripts", []).append(script)
else:
merge_http_app_scripts(cur_script, script, "http-app")
def _getinfos_ssh_server(spec):
"""Convert an SSH server banner to a TCP banner and use
_getinfos_tcp_srv_banner()"""
return _getinfos_from_banner(utils.nmap_decode_data(
spec['value']
) + b'\r\n')
def _getinfos_tcp_srv_banner(spec):
"""Extract info from a TCP server banner using Nmap database.
"""
return _getinfos_from_banner(utils.nmap_decode_data(spec['value']))
def zgrap_parser_http(data: Dict[str, Any],
hostrec: NmapHost,
port: Optional[int] = None) -> NmapPort:
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if "result" in data:
data.update(data.pop("result"))
if "response" not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data["response"]
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
"Missing field%s %s in zgrab HTTP result",
"s" if len(missing_fields) > 1 else "",
", ".join(repr(fld) for fld in missing_fields),
)
return {}
req = resp["request"]
url = req.get("url")
res: NmapPort = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp",
}
tls = None
try:
tls = req["tls_handshake"]
except KeyError:
# zgrab2
try:
tls = req["tls_log"]["handshake_log"]
except KeyError:
pass
if tls is not None:
res["service_tunnel"] = "ssl"
try:
cert = tls["server_certificates"]["certificate"]["raw"]
except KeyError:
pass
else:
output, info_cert = create_ssl_cert(cert.encode(), b64encoded=True)
if info_cert:
res.setdefault("scripts", []).append({
"id": "ssl-cert",
"output": output,
"ssl-cert": info_cert,
})
for cert in info_cert:
add_cert_hostnames(cert,
hostrec.setdefault("hostnames", []))
if url:
try:
_, guessed_port = utils.url2hostport("%(scheme)s://%(host)s" % url)
except ValueError:
utils.LOGGER.warning("Cannot guess port from url %r", url)
guessed_port = 80 # because reasons
else:
if port is not None and port != guessed_port:
utils.LOGGER.warning(
"Port %d found from the URL %s differs from the provided port "
"value %d",
guessed_port,
url.get("path"),
port,
)
port = guessed_port
if port is None:
port = guessed_port
# Specific paths
if url.get("path").endswith("/.git/index"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body", "").startswith("DIRC"):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5])
res["port"] = port
res.setdefault("scripts", []).append({
"id":
"http-git",
"output":
"\n %s\n Git repository found!\n" % repository,
"http-git": [
{
"repository": repository,
"files-found": [".git/index"]
},
],
})
return res
if url.get("path").endswith("/owa/auth/logon.aspx"):
if resp.get("status_code") != 200:
return {}
version_set = set(
m.group(1)
for m in _EXPR_OWA_VERSION.finditer(resp.get("body", "")))
if not version_set:
return {}
version_list = sorted(version_set,
key=lambda v: [int(x) for x in v.split(".")])
res["port"] = port
path = url["path"][:-15]
if version_list:
parsed_version = EXCHANGE_BUILDS.get(version_list[0],
"unknown build number")
if len(version_list) > 1:
version_list = [
"%s (%s)" %
(vers, EXCHANGE_BUILDS.get(vers,
"unknown build number"))
for vers in version_list
]
output = "OWA: path %s, version %s (multiple versions found!)" % (
path,
" / ".join(version_list),
)
else:
output = "OWA: path %s, version %s (%s)" % (
path,
version_list[0],
parsed_version,
)
res.setdefault("scripts", []).append({
"id":
"http-app",
"output":
output,
"http-app": [{
"path": path,
"application": "OWA",
"version": version_list[0],
"parsed_version": parsed_version,
}],
})
return res
if url.get("path").endswith("/centreon/"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
path = url["path"]
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
version: Optional[str]
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault("scripts", []).append({
"id":
"http-app",
"output":
"Centreon: path %s%s" % (
path,
"" if version is None else (", version %s" % version),
),
"http-app": [
dict(
{
"path": path,
"application": "Centreon"
},
**({} if version is None else {
"version": version
}),
)
],
})
return res
if url.get("path").endswith("/.well-known/security.txt"):
if resp.get("status_code") != 200:
return {}
if not resp.get("headers"):
return {}
if not any(
ctype.split(";", 1)[0].lower() == "text/plain"
for ctype in resp["headers"].get("content_type", [])):
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
parsed: Dict[str, List[str]] = {}
for line in body.splitlines():
line = line.strip().split("#", 1)[0]
if not line:
continue
if ":" not in line:
utils.LOGGER.warning(
"Invalid line in security.txt file [%r]", line)
continue
key, value = line.split(":", 1)
parsed.setdefault(key.strip().lower(),
[]).append(value.strip())
res.setdefault("scripts", []).append({
"id": "http-securitytxt",
"output": body,
"http-securitytxt":
{key: " / ".join(value)
for key, value in parsed.items()},
})
return res
if url.get("path") != "/":
utils.LOGGER.warning("URL path not supported yet: %s",
url.get("path"))
return {}
elif port is None:
if req.get("tls_handshake") or req.get("tls_log"):
port = 443
else:
port = 80
res["port"] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
if resp.get("headers"):
headers = resp["headers"]
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get("www_authenticate"):
auths = headers.get("www_authenticate")
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode()))
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
continue
if not infos:
continue
keyvals = zip(ntlm_values,
[infos.get(k) for k in ntlm_values])
output = "\n".join("{}: {}".format(k, v)
for k, v in keyvals if v)
res.setdefault("scripts", []).append({
"id":
"ntlm-info",
"output":
output,
"ntlm-info":
dict(infos, protocol="http"),
})
if "DNS_Computer_Name" in infos:
add_hostname(
infos["DNS_Computer_Name"],
"ntlm",
hostrec.setdefault("hostnames", []),
)
if any(val.lower().startswith("ntlm")
for val in req.get("headers", {}).get("authorization", [])):
return res
# If we have headers_raw value, let's use it. Else, let's fake it as well as we can.
http_hdrs: List[HttpHeader] = []
output_list: List[str] = []
has_raw_value = False
if resp.get("headers_raw"):
try:
banner = utils.decode_b64(resp.get("headers_raw").encode())
except Exception:
utils.LOGGER.warning(
"Cannot decode raw headers, using parsed result")
else:
output_list = [
utils.nmap_encode_data(line)
for line in re.split(b"\r?\n", banner)
]
banner_split = banner.split(b"\n")
http_hdrs = [{
"name":
"_status",
"value":
utils.nmap_encode_data(banner_split[0].strip()),
}]
http_hdrs.extend(
{
"name": utils.nmap_encode_data(hdrname).lower(),
"value": utils.nmap_encode_data(hdrval),
} for hdrname, hdrval in (m.groups() for m in (
utils.RAW_HTTP_HEADER.search(part.strip())
for part in banner_split) if m))
has_raw_value = True
if not has_raw_value: # no headers_raw or decoding failed
# The order will be incorrect!
banner = (utils.nmap_decode_data(resp["protocol"]["name"]) + b" " +
utils.nmap_decode_data(resp["status_line"]) + b"\r\n")
line = "%s %s" % (resp["protocol"]["name"], resp["status_line"])
http_hdrs = [{"name": "_status", "value": line}]
output_list = [line]
for unk in headers.pop("unknown", []):
headers[unk["key"]] = unk["value"]
for hdr, values in headers.items():
hdr = hdr.replace("_", "-")
for val in values:
http_hdrs.append({"name": hdr, "value": val})
output_list.append("%s: %s" % (hdr, val))
if headers.get("server"):
banner += (b"Server: " +
utils.nmap_decode_data(headers["server"][0]) +
b"\r\n\r\n")
if http_hdrs:
method = req.get("method")
if method:
output_list.append("")
output_list.append("(Request type: %s)" % method)
script: NmapScript = {
"id": "http-headers",
"output": "\n".join(output_list),
"http-headers": http_hdrs,
}
if has_raw_value:
script["masscan"] = {"raw": utils.encode_b64(banner).decode()}
res.setdefault("scripts", []).append(script)
handle_http_headers(hostrec, res, http_hdrs, path=url.get("path"))
info: NmapServiceMatch = utils.match_nmap_svc_fp(banner,
proto="tcp",
probe="GetRequest")
if info:
add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", []))
res.update(cast(NmapPort, info))
add_service_hostname(info, hostrec.setdefault("hostnames", []))
if resp.get("body"):
body = resp["body"]
res.setdefault("scripts", []).append({
"id":
"http-content",
"output":
utils.nmap_encode_data(body.encode()),
})
handle_http_content(hostrec, res, body.encode())
return res
def _getinfos_ssh_server(spec):
"""Convert an SSH server banner to a TCP banner and use
_getinfos_tcp_srv_banner()"""
return _getinfos_from_banner(utils.nmap_decode_data(
spec['value']
) + b'\r\n')
def _getinfos_http_server(spec):
header = utils.nmap_decode_data(spec['value'])
banner = b"HTTP/1.1 200 OK\r\nServer: " + header + b"\r\n\r\n"
res = _getinfos_from_banner(banner, probe="GetRequest")
return res
def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if 'addr' in spec and \
spec.get('source') not in neverignore.get(spec['recontype'], []):
for n in ignorenets.get(spec['recontype'], ()):
if n[0] <= spec['addr'] <= n[1]:
return None
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string 'SymantecRandomUserAgent'.
if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \
spec.get('source') == 'USER-AGENT':
if SYMANTEC_UA.match(spec['value']):
spec['value'] = 'SymantecRandomUserAgent'
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif spec['recontype'] in ['HTTP_CLIENT_HEADER',
'HTTP_CLIENT_HEADER_SERVER'] and \
spec.get('source') in ['AUTHORIZATION',
'PROXY-AUTHORIZATION']:
authtype = spec['value'].split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
value = [val for val in
_split_digest_auth(spec['value'][6:].strip())
if DIGEST_AUTH_INFOS.match(val)]
spec['value'] = '%s %s' % (authtype, ','.join(value))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r", spec,
exc_info=True)
elif authtype.lower() in ['negotiate', 'kerberos', 'oauth', 'ntlm']:
spec['value'] = authtype
# p0f in Bro hack: we use the "source" field to provide the
# "distance" and "version" values
elif spec['recontype'] == 'P0F':
distance, version = spec.pop('source').split('-', 1)
try:
spec['distance'] = int(distance)
except ValueError:
pass
if version:
spec['version'] = version
# TCP server banners: try to normalize data
elif spec['recontype'] == 'TCP_SERVER_BANNER':
newvalue = value = utils.nmap_decode_data(spec['value'])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec['value'] = utils.nmap_encode_data(newvalue)
# Finally we prepare the record to be stored. For that, we make
# sure that no indexed value has a size greater than MAXVALLEN. If
# so, we replace the value with its SHA1 hash and store the
# original value in full[original column name].
if len(spec['value']) > utils.MAXVALLEN:
spec['fullvalue'] = spec['value']
spec['value'] = hashlib.sha1(spec['fullvalue'].encode()).hexdigest()
if 'targetval' in spec and len(spec['targetval']) > utils.MAXVALLEN:
spec['fulltargetval'] = spec['targetval']
spec['targetval'] = hashlib.sha1(
spec['fulltargetval'].encode()
).hexdigest()
return spec
def zgrap_parser_http(data, hostrec, port=None):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if "result" in data:
data.update(data.pop("result"))
if "response" not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data["response"]
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
"Missing field%s %s in zgrab HTTP result",
"s" if len(missing_fields) > 1 else "",
", ".join(repr(fld) for fld in missing_fields),
)
return {}
req = resp["request"]
url = req.get("url")
res = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp",
}
tls = None
try:
tls = req["tls_handshake"]
except KeyError:
# zgrab2
try:
tls = req["tls_log"]["handshake_log"]
except KeyError:
pass
if tls is not None:
res["service_tunnel"] = "ssl"
try:
cert = tls["server_certificates"]["certificate"]["raw"]
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault("scripts", []).append(
{
"id": "ssl-cert",
"output": output,
"ssl-cert": info,
}
)
for cert in info:
add_cert_hostnames(cert, hostrec.setdefault("hostnames", []))
if url:
guessed_port = None
if ":" in url.get("host", ""):
try:
guessed_port = int(url["host"].split(":", 1)[1])
except ValueError:
pass
if port is None:
if guessed_port is None:
if url.get("scheme") == "https":
port = 443
else:
port = 80
else:
port = guessed_port
elif port != guessed_port:
utils.LOGGER.warning(
"Port %d found from the URL %s differs from the provided port "
"value %d",
guessed_port,
url.get("path"),
port,
)
port = guessed_port
# Specific paths
if url.get("path").endswith("/.git/index"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body", "").startswith("DIRC"):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5])
res["port"] = port
res.setdefault("scripts", []).append(
{
"id": "http-git",
"output": "\n %s\n Git repository found!\n" % repository,
"http-git": [
{"repository": repository, "files-found": [".git/index"]}
],
}
)
return res
if url.get("path").endswith("/owa/auth/logon.aspx"):
if resp.get("status_code") != 200:
return {}
version = set(
m.group(1) for m in _EXPR_OWA_VERSION.finditer(resp.get("body", ""))
)
if not version:
return {}
version = sorted(version, key=lambda v: [int(x) for x in v.split(".")])
res["port"] = port
path = url["path"][:-15]
if len(version) > 1:
output = "OWA: path %s, version %s (multiple versions found!)" % (
path,
" / ".join(version),
)
else:
output = "OWA: path %s, version %s" % (path, version[0])
res.setdefault("scripts", []).append(
{
"id": "http-app",
"output": output,
"http-app": [
{"path": path, "application": "OWA", "version": version[0]}
],
}
)
return res
if url.get("path").endswith("/centreon/"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
path = url["path"]
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault("scripts", []).append(
{
"id": "http-app",
"output": "Centreon: path %s%s"
% (
path,
"" if version is None else (", version %s" % version),
),
"http-app": [
dict(
{"path": path, "application": "Centreon"},
**({} if version is None else {"version": version}),
)
],
}
)
return res
if url.get("path") != "/":
utils.LOGGER.warning("URL path not supported yet: %s", url.get("path"))
return {}
elif port is None:
if req.get("tls_handshake") or req.get("tls_log"):
port = 443
else:
port = 80
res["port"] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (
utils.nmap_decode_data(resp["protocol"]["name"])
+ b" "
+ utils.nmap_decode_data(resp["status_line"])
+ b"\r\n"
)
if resp.get("headers"):
headers = resp["headers"]
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get("www_authenticate"):
auths = headers.get("www_authenticate")
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode())
)
except (UnicodeDecodeError, TypeError, ValueError, binascii.Error):
pass
keyvals = zip(ntlm_values, [infos.get(k) for k in ntlm_values])
output = "\n".join("{}: {}".format(k, v) for k, v in keyvals if v)
res.setdefault("scripts", []).append(
{"id": "http-ntlm-info", "output": output, "ntlm-info": infos}
)
if "DNS_Computer_Name" in infos:
add_hostname(
infos["DNS_Computer_Name"],
"ntlm",
hostrec.setdefault("hostnames", []),
)
if any(
val.lower().startswith("ntlm")
for val in req.get("headers", {}).get("authorization", [])
):
return res
# the order will be incorrect!
line = "%s %s" % (resp["protocol"]["name"], resp["status_line"])
http_hdrs = [{"name": "_status", "value": line}]
output = [line]
for unk in headers.pop("unknown", []):
headers[unk["key"]] = unk["value"]
for hdr, values in headers.items():
hdr = hdr.replace("_", "-")
for val in values:
http_hdrs.append({"name": hdr, "value": val})
output.append("%s: %s" % (hdr, val))
if http_hdrs:
method = req.get("method")
if method:
output.append("")
output.append("(Request type: %s)" % method)
res.setdefault("scripts", []).append(
{
"id": "http-headers",
"output": "\n".join(output),
"http-headers": http_hdrs,
}
)
handle_http_headers(hostrec, res, http_hdrs, path=url.get("path"))
if headers.get("server"):
banner += (
b"Server: " + utils.nmap_decode_data(headers["server"][0]) + b"\r\n\r\n"
)
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", []))
res.update(info)
if resp.get("body"):
body = resp["body"]
res.setdefault("scripts", []).append(
{
"id": "http-content",
"output": utils.nmap_encode_data(body.encode()),
}
)
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res["scripts"].append(
{
"id": "http-title",
"output": title,
"http-title": {"title": title},
}
)
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault("scripts", []).append(script_http_ls)
service_elasticsearch = create_elasticsearch_service(body)
if service_elasticsearch:
if "hostname" in service_elasticsearch:
add_hostname(
service_elasticsearch.pop("hostname"),
"service",
hostrec.setdefault("hostnames", []),
)
add_cpe_values(
hostrec, "ports.port:%s" % port, service_elasticsearch.pop("cpe", [])
)
res.update(service_elasticsearch)
return res
def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if 'addr' in spec and \
spec.get('source') not in neverignore.get(spec['recontype'], []):
for start, stop in ignorenets.get(spec['recontype'], ()):
if start <= utils.force_ip2int(spec['addr']) <= stop:
return None
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string 'SymantecRandomUserAgent'.
if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \
spec.get('source') == 'USER-AGENT':
if SYMANTEC_UA.match(spec['value']):
spec['value'] = 'SymantecRandomUserAgent'
elif KASPERSKY_UA.match(spec['value']):
spec['value'] = 'KasperskyWeirdUserAgent'
else:
match = SYMANTEC_SEP_UA.match(spec['value'])
if match is not None:
spec['value'] = '%s%s' % match.groups()
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif spec['recontype'] in ['HTTP_CLIENT_HEADER',
'HTTP_CLIENT_HEADER_SERVER'] and \
spec.get('source') in ['AUTHORIZATION',
'PROXY-AUTHORIZATION']:
authtype = spec['value'].split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
value = [val for val in
_split_digest_auth(spec['value'][6:].strip())
if DIGEST_AUTH_INFOS.match(val)]
spec['value'] = '%s %s' % (authtype, ','.join(value))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r", spec,
exc_info=True)
elif authtype.lower() in ['negotiate', 'kerberos', 'oauth', 'ntlm']:
spec['value'] = authtype
# p0f in Bro hack: we use the "source" field to provide the
# "distance" and "version" values
elif spec['recontype'] == 'P0F':
distance, version = spec.pop('source').split('-', 1)
try:
spec['distance'] = int(distance)
except ValueError:
pass
if version:
spec['version'] = version
# TCP server banners: try to normalize data
elif spec['recontype'] == 'TCP_SERVER_BANNER':
newvalue = value = utils.nmap_decode_data(spec['value'])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec['value'] = utils.nmap_encode_data(newvalue)
# SSL_{CLIENT,SERVER} JA3
elif ((spec['recontype'] == 'SSL_CLIENT' and spec['source'] == 'ja3') or
(spec['recontype'] == 'SSL_SERVER' and
spec['source'].startswith('ja3-'))):
value = spec['value']
spec.setdefault('infos', {})['raw'] = value
spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
if spec['recontype'] == 'SSL_SERVER':
clientvalue = spec['source'][4:]
spec['infos'].setdefault('client', {})['raw'] = clientvalue
spec['source'] = 'ja3-%s' % hashlib.new(
"md5",
clientvalue.encode(),
).hexdigest()
# Check DNS Blacklist answer
elif spec['recontype'] == 'DNS_ANSWER':
if any(spec['value'].endswith(dnsbl)
for dnsbl in config.DNS_BLACKLIST_DOMAINS):
dnsbl_val = spec['value']
spec['recontype'] = 'DNS_BLACKLIST'
spec['value'] = spec['addr']
spec.update({'source': "%s-%s" %
(dnsbl_val.split('.', 4)[4], spec['source'])})
spec['addr'] = '.'.join(dnsbl_val.split('.')[3::-1])
return spec