def _extract_passive_HTTP_SERVER_HEADER(rec):
"""Handle http server headers."""
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'http',
}
host = {'ports': [port]}
# TODO: handle other header values and merge them
if rec.get('source') != 'SERVER':
return {'ports': [port]}
value = rec['value']
script = {'id': 'http-server-header', 'output': value}
port['scripts'] = [script]
banner = (b"HTTP/1.1 200 OK\r\nServer: " + utils.nmap_decode_data(value) +
b"\r\n\r\n")
nmap_info = utils.match_nmap_svc_fp(output=banner,
proto=rec.get('protocol', 'tcp'),
probe="GetRequest")
add_cpe_values(host, 'ports.port:%s' % port, nmap_info.pop('cpe', []))
host['cpes'] = list(viewvalues(host['cpes']))
for cpe in host['cpes']:
cpe['origins'] = sorted(cpe['origins'])
if not host['cpes']:
del host['cpes']
port.update(nmap_info)
return host
def _extract_passive_HTTP_SERVER_HEADER(rec):
"""Handle http server headers."""
if (
rec["value"]
and rec.get("source") in ["WWW-AUTHENTICATE", "PROXY-AUTHENTICATE"]
and rec["value"].split(None, 1)[0].lower() in {"ntlm", "negotiate"}
and len(rec["value"].split(None, 1)) > 1
):
return _extract_passive_NTLM(rec, service="http")
port = {
"state_state": "open",
"state_reason": "passive",
"port": rec["port"],
"protocol": rec.get("protocol", "tcp"),
"service_name": "http",
}
host = {"ports": [port]}
if rec.get("source") == "MICROSOFTSHAREPOINTTEAMSERVICES":
version = rec["value"].split(":", 1)[0]
add_cpe_values(
host,
"ports.port:%s" % port["port"],
["cpe:/a:microsoft:sharepoint_server:%s" % version],
)
host["cpes"] = list(host["cpes"].values())
# Let's pretend the application is on '/UNKNOWN/'
port["scripts"] = [
{
"id": "http-app",
"output": "SharePoint: path /UNKNOWN/, version %s" % (version),
"http-app": [
{
"path": "/UNKNOWN/",
"application": "SharePoint",
"version": version,
}
],
}
]
return host
# TODO: handle other header values and merge them
if rec.get("source") != "SERVER":
return host
value = rec["value"]
script = {"id": "http-server-header", "output": value}
port["scripts"] = [script]
banner = (
b"HTTP/1.1 200 OK\r\nServer: " + utils.nmap_decode_data(value) + b"\r\n\r\n"
)
nmap_info = utils.match_nmap_svc_fp(
output=banner, proto=rec.get("protocol", "tcp"), probe="GetRequest"
)
add_cpe_values(host, "ports.port:%s" % port, nmap_info.pop("cpe", []))
host["cpes"] = list(host["cpes"].values())
for cpe in host["cpes"]:
cpe["origins"] = sorted(cpe["origins"])
if not host["cpes"]:
del host["cpes"]
port.update(nmap_info)
return host
def _getinfos_from_banner(banner, proto="tcp", probe="NULL"):
infos = utils.match_nmap_svc_fp(banner, proto=proto, probe=probe)
if not infos:
return {}
res = {'infos': infos}
_fix_infos_size(res)
return res
def _extract_passive_TCP_SERVER_BANNER(rec):
"""Handle banners from tcp servers."""
value = rec['value']
if rec['recontype'] == 'SSH_SERVER':
value += "\r\n"
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'scripts': [{
"id": "banner",
"output": value
}],
}
host = {'ports': [port]}
port.update(rec.get('infos', {}))
nmap_info = utils.match_nmap_svc_fp(output=utils.nmap_decode_data(value),
proto=rec.get('protocol', 'tcp'),
probe="NULL")
add_cpe_values(host, 'ports.port:%s' % port, nmap_info.pop('cpe', []))
host['cpes'] = list(viewvalues(host['cpes']))
for cpe in host['cpes']:
cpe['origins'] = sorted(cpe['origins'])
if not host['cpes']:
del host['cpes']
port.update(nmap_info)
return host
def _extract_passive_TCP_SERVER_BANNER(rec):
"""Handle banners from tcp servers."""
value = rec["value"]
if rec["recontype"] == "SSH_SERVER":
value += "\r\n"
port = {
"state_state": "open",
"state_reason": "passive",
"port": rec["port"],
"protocol": rec.get("protocol", "tcp"),
"scripts": [{"id": "banner", "output": value}],
}
host = {"ports": [port]}
port.update(rec.get("infos", {}))
nmap_info = utils.match_nmap_svc_fp(
output=utils.nmap_decode_data(value),
proto=rec.get("protocol", "tcp"),
probe="NULL",
)
add_cpe_values(host, "ports.port:%s" % port, nmap_info.pop("cpe", []))
host["cpes"] = list(host["cpes"].values())
for cpe in host["cpes"]:
cpe["origins"] = sorted(cpe["origins"])
if not host["cpes"]:
del host["cpes"]
port.update(nmap_info)
return host
def _getinfos_from_banner(banner, proto="tcp", probe="NULL"):
infos = utils.match_nmap_svc_fp(banner, proto=proto, probe=probe) or {}
try:
del infos['cpe']
except KeyError:
pass
if not infos:
return {}
return {'infos': infos}
def _extract_passive_HTTP_SERVER_HEADER(rec):
"""Handle http server headers."""
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'http',
}
host = {'ports': [port]}
if rec.get('source') == 'MICROSOFTSHAREPOINTTEAMSERVICES':
version = rec['value'].split(':', 1)[0]
add_cpe_values(host, 'ports.port:%s' % port['port'],
["cpe:/a:microsoft:sharepoint_server:%s" % version])
host['cpes'] = list(host['cpes'].values())
# Let's pretend the application is on '/UNKNOWN/'
port['scripts'] = [{
'id':
'http-app',
'output':
'SharePoint: path /UNKNOWN/, version %s' % (version),
'http-app': [{
'path': '/UNKNOWN/',
'application': 'SharePoint',
'version': version
}],
}]
return host
# TODO: handle other header values and merge them
if rec.get('source') != 'SERVER':
return host
value = rec['value']
script = {'id': 'http-server-header', 'output': value}
port['scripts'] = [script]
banner = (b"HTTP/1.1 200 OK\r\nServer: " + utils.nmap_decode_data(value) +
b"\r\n\r\n")
nmap_info = utils.match_nmap_svc_fp(output=banner,
proto=rec.get('protocol', 'tcp'),
probe="GetRequest")
add_cpe_values(host, 'ports.port:%s' % port, nmap_info.pop('cpe', []))
host['cpes'] = list(host['cpes'].values())
for cpe in host['cpes']:
cpe['origins'] = sorted(cpe['origins'])
if not host['cpes']:
del host['cpes']
port.update(nmap_info)
return host
def _extract_passive_TCP_SERVER_BANNER(rec):
"""Handle banners from tcp servers."""
value = rec['value']
if rec['recontype'] == 'SSH_SERVER':
value += "\r\n"
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'scripts': [{"id": "banner",
"output": value}],
}
port.update(rec.get('infos', {}))
port.update(
utils.match_nmap_svc_fp(output=utils.nmap_decode_data(value),
proto=rec.get('protocol', 'tcp'),
probe="NULL")
)
return {'ports': [port]}
def _extract_passive_TCP_SERVER_BANNER(rec):
"""Handle banners from tcp servers."""
value = rec['value']
if rec['recontype'] == 'SSH_SERVER':
value += "\r\n"
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'scripts': [{"id": "banner",
"output": value}],
}
port.update(rec.get('infos', {}))
port.update(
utils.match_nmap_svc_fp(output=utils.nmap_decode_data(value),
proto=rec.get('protocol', 'tcp'),
probe="NULL")
)
return {'ports': [port]}
def _extract_passive_HTTP_SERVER_HEADER(rec):
"""Handle http server headers."""
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'http',
}
# TODO: handle other header values and merge them
if rec.get('source') != 'SERVER':
return {'ports': [port]}
value = rec['value']
script = {'id': 'http-server-header', 'output': value}
port['scripts'] = [script]
banner = (b"HTTP/1.1 200 OK\r\nServer: " + utils.nmap_decode_data(value) +
b"\r\n\r\n")
port.update(
utils.match_nmap_svc_fp(output=banner,
proto=rec.get('protocol', 'tcp'),
probe="GetRequest"))
return {'ports': [port]}
def _extract_passive_HTTP_SERVER_HEADER(rec):
"""Handle http server headers."""
port = {
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'http',
}
# TODO: handle other header values and merge them
if rec.get('source') != 'SERVER':
return {'ports': [port]}
value = rec['value']
script = {'id': 'http-server-header', 'output': value}
port['scripts'] = [script]
banner = (b"HTTP/1.1 200 OK\r\nServer: " + utils.nmap_decode_data(value) +
b"\r\n\r\n")
port.update(
utils.match_nmap_svc_fp(output=banner,
proto=rec.get('protocol', 'tcp'),
probe="GetRequest")
)
return {'ports': [port]}
def zgrap_parser_http(data, hostrec, port=None):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if "result" in data:
data.update(data.pop("result"))
if "response" not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data["response"]
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
"Missing field%s %s in zgrab HTTP result",
"s" if len(missing_fields) > 1 else "",
", ".join(repr(fld) for fld in missing_fields),
)
return {}
req = resp["request"]
url = req.get("url")
res = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp",
}
tls = None
try:
tls = req["tls_handshake"]
except KeyError:
# zgrab2
try:
tls = req["tls_log"]["handshake_log"]
except KeyError:
pass
if tls is not None:
res["service_tunnel"] = "ssl"
try:
cert = tls["server_certificates"]["certificate"]["raw"]
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault("scripts", []).append(
{
"id": "ssl-cert",
"output": output,
"ssl-cert": info,
}
)
for cert in info:
add_cert_hostnames(cert, hostrec.setdefault("hostnames", []))
if url:
guessed_port = None
if ":" in url.get("host", ""):
try:
guessed_port = int(url["host"].split(":", 1)[1])
except ValueError:
pass
if port is None:
if guessed_port is None:
if url.get("scheme") == "https":
port = 443
else:
port = 80
else:
port = guessed_port
elif port != guessed_port:
utils.LOGGER.warning(
"Port %d found from the URL %s differs from the provided port "
"value %d",
guessed_port,
url.get("path"),
port,
)
port = guessed_port
# Specific paths
if url.get("path").endswith("/.git/index"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body", "").startswith("DIRC"):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5])
res["port"] = port
res.setdefault("scripts", []).append(
{
"id": "http-git",
"output": "\n %s\n Git repository found!\n" % repository,
"http-git": [
{"repository": repository, "files-found": [".git/index"]}
],
}
)
return res
if url.get("path").endswith("/owa/auth/logon.aspx"):
if resp.get("status_code") != 200:
return {}
version = set(
m.group(1) for m in _EXPR_OWA_VERSION.finditer(resp.get("body", ""))
)
if not version:
return {}
version = sorted(version, key=lambda v: [int(x) for x in v.split(".")])
res["port"] = port
path = url["path"][:-15]
if len(version) > 1:
output = "OWA: path %s, version %s (multiple versions found!)" % (
path,
" / ".join(version),
)
else:
output = "OWA: path %s, version %s" % (path, version[0])
res.setdefault("scripts", []).append(
{
"id": "http-app",
"output": output,
"http-app": [
{"path": path, "application": "OWA", "version": version[0]}
],
}
)
return res
if url.get("path").endswith("/centreon/"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
path = url["path"]
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault("scripts", []).append(
{
"id": "http-app",
"output": "Centreon: path %s%s"
% (
path,
"" if version is None else (", version %s" % version),
),
"http-app": [
dict(
{"path": path, "application": "Centreon"},
**({} if version is None else {"version": version}),
)
],
}
)
return res
if url.get("path") != "/":
utils.LOGGER.warning("URL path not supported yet: %s", url.get("path"))
return {}
elif port is None:
if req.get("tls_handshake") or req.get("tls_log"):
port = 443
else:
port = 80
res["port"] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (
utils.nmap_decode_data(resp["protocol"]["name"])
+ b" "
+ utils.nmap_decode_data(resp["status_line"])
+ b"\r\n"
)
if resp.get("headers"):
headers = resp["headers"]
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get("www_authenticate"):
auths = headers.get("www_authenticate")
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode())
)
except (UnicodeDecodeError, TypeError, ValueError, binascii.Error):
pass
keyvals = zip(ntlm_values, [infos.get(k) for k in ntlm_values])
output = "\n".join("{}: {}".format(k, v) for k, v in keyvals if v)
res.setdefault("scripts", []).append(
{"id": "http-ntlm-info", "output": output, "ntlm-info": infos}
)
if "DNS_Computer_Name" in infos:
add_hostname(
infos["DNS_Computer_Name"],
"ntlm",
hostrec.setdefault("hostnames", []),
)
if any(
val.lower().startswith("ntlm")
for val in req.get("headers", {}).get("authorization", [])
):
return res
# the order will be incorrect!
line = "%s %s" % (resp["protocol"]["name"], resp["status_line"])
http_hdrs = [{"name": "_status", "value": line}]
output = [line]
for unk in headers.pop("unknown", []):
headers[unk["key"]] = unk["value"]
for hdr, values in headers.items():
hdr = hdr.replace("_", "-")
for val in values:
http_hdrs.append({"name": hdr, "value": val})
output.append("%s: %s" % (hdr, val))
if http_hdrs:
method = req.get("method")
if method:
output.append("")
output.append("(Request type: %s)" % method)
res.setdefault("scripts", []).append(
{
"id": "http-headers",
"output": "\n".join(output),
"http-headers": http_hdrs,
}
)
handle_http_headers(hostrec, res, http_hdrs, path=url.get("path"))
if headers.get("server"):
banner += (
b"Server: " + utils.nmap_decode_data(headers["server"][0]) + b"\r\n\r\n"
)
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", []))
res.update(info)
if resp.get("body"):
body = resp["body"]
res.setdefault("scripts", []).append(
{
"id": "http-content",
"output": utils.nmap_encode_data(body.encode()),
}
)
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res["scripts"].append(
{
"id": "http-title",
"output": title,
"http-title": {"title": title},
}
)
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault("scripts", []).append(script_http_ls)
service_elasticsearch = create_elasticsearch_service(body)
if service_elasticsearch:
if "hostname" in service_elasticsearch:
add_hostname(
service_elasticsearch.pop("hostname"),
"service",
hostrec.setdefault("hostnames", []),
)
add_cpe_values(
hostrec, "ports.port:%s" % port, service_elasticsearch.pop("cpe", [])
)
res.update(service_elasticsearch)
return res
def zgrap_parser_http(data):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if 'result' in data:
data.update(data.pop('result'))
if 'response' not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data['response']
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
'Missing field%s %s in zgrab HTTP result',
's' if len(missing_fields) > 1 else '',
', '.join(repr(fld) for fld in missing_fields),
)
return {}
req = resp['request']
res = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "syn-ack",
"protocol": "tcp"
}
url = req.get('url')
if url:
port = None
if ':' in url.get('host', ''):
try:
port = int(url['host'].split(':', 1)[1])
except ValueError:
pass
if port is None:
if url.get('scheme') == 'https':
port = 443
else:
port = 80
elif req.get('tls_handshake') or req.get('tls_log'):
# zgrab / zgrab2
port = 443
else:
port = 80
res['port'] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' +
utils.nmap_decode_data(resp['status_line']) + b"\r\n")
if resp.get('headers'):
headers = resp['headers']
# the order will be incorrect!
http_hdrs = [{'name': '_status', 'value': resp['status_line']}]
output = [resp['status_line']]
if 'unknown' in headers:
for unk in headers.pop('unknown'):
headers[unk['key']] = unk['value']
for hdr, values in viewitems(headers):
hdr = hdr.replace('_', '-')
for val in values:
http_hdrs.append({'name': hdr, 'value': val})
output.append('%s: %s' % (hdr, val))
if http_hdrs:
method = req.get('method')
if method:
output.append('')
output.append('(Request type: %s)' % method)
res.setdefault('scripts', []).append({
'id': 'http-headers',
'output': '\n'.join(output),
'http-headers': http_hdrs,
})
if headers.get('server'):
server = resp['headers']['server']
res.setdefault('scripts', []).append({
'id': 'http-server-header',
'output': server[0],
'http-server-header': server,
})
banner += (b"Server: " + utils.nmap_decode_data(server[0]) +
b"\r\n\r\n")
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
res.update(info)
if resp.get('body'):
body = resp['body']
res.setdefault('scripts', []).append({
'id':
'http-content',
'output':
utils.nmap_encode_data(body.encode()),
})
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res['scripts'].append({
'id': 'http-title',
'output': title,
'http-title': {
'title': title
},
})
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault('scripts', []).append(script_http_ls)
tls = None
try:
tls = req['tls_handshake']
except KeyError:
# zgrab2
try:
tls = req['tls_log']['handshake_log']
except KeyError:
pass
if tls is not None:
res['service_tunnel'] = 'ssl'
try:
cert = tls['server_certificates']['certificate']['raw']
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault('scripts', []).append({
'id':
'ssl-cert',
'output':
"\n".join(output),
'ssl-cert':
info,
})
return res
def zgrap_parser_http(data: Dict[str, Any],
hostrec: NmapHost,
port: Optional[int] = None) -> NmapPort:
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if "result" in data:
data.update(data.pop("result"))
if "response" not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data["response"]
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
"Missing field%s %s in zgrab HTTP result",
"s" if len(missing_fields) > 1 else "",
", ".join(repr(fld) for fld in missing_fields),
)
return {}
req = resp["request"]
url = req.get("url")
res: NmapPort = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp",
}
tls = None
try:
tls = req["tls_handshake"]
except KeyError:
# zgrab2
try:
tls = req["tls_log"]["handshake_log"]
except KeyError:
pass
if tls is not None:
res["service_tunnel"] = "ssl"
try:
cert = tls["server_certificates"]["certificate"]["raw"]
except KeyError:
pass
else:
output, info_cert = create_ssl_cert(cert.encode(), b64encoded=True)
if info_cert:
res.setdefault("scripts", []).append({
"id": "ssl-cert",
"output": output,
"ssl-cert": info_cert,
})
for cert in info_cert:
add_cert_hostnames(cert,
hostrec.setdefault("hostnames", []))
if url:
try:
_, guessed_port = utils.url2hostport("%(scheme)s://%(host)s" % url)
except ValueError:
utils.LOGGER.warning("Cannot guess port from url %r", url)
guessed_port = 80 # because reasons
else:
if port is not None and port != guessed_port:
utils.LOGGER.warning(
"Port %d found from the URL %s differs from the provided port "
"value %d",
guessed_port,
url.get("path"),
port,
)
port = guessed_port
if port is None:
port = guessed_port
# Specific paths
if url.get("path").endswith("/.git/index"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body", "").startswith("DIRC"):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5])
res["port"] = port
res.setdefault("scripts", []).append({
"id":
"http-git",
"output":
"\n %s\n Git repository found!\n" % repository,
"http-git": [
{
"repository": repository,
"files-found": [".git/index"]
},
],
})
return res
if url.get("path").endswith("/owa/auth/logon.aspx"):
if resp.get("status_code") != 200:
return {}
version_set = set(
m.group(1)
for m in _EXPR_OWA_VERSION.finditer(resp.get("body", "")))
if not version_set:
return {}
version_list = sorted(version_set,
key=lambda v: [int(x) for x in v.split(".")])
res["port"] = port
path = url["path"][:-15]
if version_list:
parsed_version = EXCHANGE_BUILDS.get(version_list[0],
"unknown build number")
if len(version_list) > 1:
version_list = [
"%s (%s)" %
(vers, EXCHANGE_BUILDS.get(vers,
"unknown build number"))
for vers in version_list
]
output = "OWA: path %s, version %s (multiple versions found!)" % (
path,
" / ".join(version_list),
)
else:
output = "OWA: path %s, version %s (%s)" % (
path,
version_list[0],
parsed_version,
)
res.setdefault("scripts", []).append({
"id":
"http-app",
"output":
output,
"http-app": [{
"path": path,
"application": "OWA",
"version": version_list[0],
"parsed_version": parsed_version,
}],
})
return res
if url.get("path").endswith("/centreon/"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
path = url["path"]
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
version: Optional[str]
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault("scripts", []).append({
"id":
"http-app",
"output":
"Centreon: path %s%s" % (
path,
"" if version is None else (", version %s" % version),
),
"http-app": [
dict(
{
"path": path,
"application": "Centreon"
},
**({} if version is None else {
"version": version
}),
)
],
})
return res
if url.get("path").endswith("/.well-known/security.txt"):
if resp.get("status_code") != 200:
return {}
if not resp.get("headers"):
return {}
if not any(
ctype.split(";", 1)[0].lower() == "text/plain"
for ctype in resp["headers"].get("content_type", [])):
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
parsed: Dict[str, List[str]] = {}
for line in body.splitlines():
line = line.strip().split("#", 1)[0]
if not line:
continue
if ":" not in line:
utils.LOGGER.warning(
"Invalid line in security.txt file [%r]", line)
continue
key, value = line.split(":", 1)
parsed.setdefault(key.strip().lower(),
[]).append(value.strip())
res.setdefault("scripts", []).append({
"id": "http-securitytxt",
"output": body,
"http-securitytxt":
{key: " / ".join(value)
for key, value in parsed.items()},
})
return res
if url.get("path") != "/":
utils.LOGGER.warning("URL path not supported yet: %s",
url.get("path"))
return {}
elif port is None:
if req.get("tls_handshake") or req.get("tls_log"):
port = 443
else:
port = 80
res["port"] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
if resp.get("headers"):
headers = resp["headers"]
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get("www_authenticate"):
auths = headers.get("www_authenticate")
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode()))
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
continue
if not infos:
continue
keyvals = zip(ntlm_values,
[infos.get(k) for k in ntlm_values])
output = "\n".join("{}: {}".format(k, v)
for k, v in keyvals if v)
res.setdefault("scripts", []).append({
"id":
"ntlm-info",
"output":
output,
"ntlm-info":
dict(infos, protocol="http"),
})
if "DNS_Computer_Name" in infos:
add_hostname(
infos["DNS_Computer_Name"],
"ntlm",
hostrec.setdefault("hostnames", []),
)
if any(val.lower().startswith("ntlm")
for val in req.get("headers", {}).get("authorization", [])):
return res
# If we have headers_raw value, let's use it. Else, let's fake it as well as we can.
http_hdrs: List[HttpHeader] = []
output_list: List[str] = []
has_raw_value = False
if resp.get("headers_raw"):
try:
banner = utils.decode_b64(resp.get("headers_raw").encode())
except Exception:
utils.LOGGER.warning(
"Cannot decode raw headers, using parsed result")
else:
output_list = [
utils.nmap_encode_data(line)
for line in re.split(b"\r?\n", banner)
]
banner_split = banner.split(b"\n")
http_hdrs = [{
"name":
"_status",
"value":
utils.nmap_encode_data(banner_split[0].strip()),
}]
http_hdrs.extend(
{
"name": utils.nmap_encode_data(hdrname).lower(),
"value": utils.nmap_encode_data(hdrval),
} for hdrname, hdrval in (m.groups() for m in (
utils.RAW_HTTP_HEADER.search(part.strip())
for part in banner_split) if m))
has_raw_value = True
if not has_raw_value: # no headers_raw or decoding failed
# The order will be incorrect!
banner = (utils.nmap_decode_data(resp["protocol"]["name"]) + b" " +
utils.nmap_decode_data(resp["status_line"]) + b"\r\n")
line = "%s %s" % (resp["protocol"]["name"], resp["status_line"])
http_hdrs = [{"name": "_status", "value": line}]
output_list = [line]
for unk in headers.pop("unknown", []):
headers[unk["key"]] = unk["value"]
for hdr, values in headers.items():
hdr = hdr.replace("_", "-")
for val in values:
http_hdrs.append({"name": hdr, "value": val})
output_list.append("%s: %s" % (hdr, val))
if headers.get("server"):
banner += (b"Server: " +
utils.nmap_decode_data(headers["server"][0]) +
b"\r\n\r\n")
if http_hdrs:
method = req.get("method")
if method:
output_list.append("")
output_list.append("(Request type: %s)" % method)
script: NmapScript = {
"id": "http-headers",
"output": "\n".join(output_list),
"http-headers": http_hdrs,
}
if has_raw_value:
script["masscan"] = {"raw": utils.encode_b64(banner).decode()}
res.setdefault("scripts", []).append(script)
handle_http_headers(hostrec, res, http_hdrs, path=url.get("path"))
info: NmapServiceMatch = utils.match_nmap_svc_fp(banner,
proto="tcp",
probe="GetRequest")
if info:
add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", []))
res.update(cast(NmapPort, info))
add_service_hostname(info, hostrec.setdefault("hostnames", []))
if resp.get("body"):
body = resp["body"]
res.setdefault("scripts", []).append({
"id":
"http-content",
"output":
utils.nmap_encode_data(body.encode()),
})
handle_http_content(hostrec, res, body.encode())
return res
def zgrap_parser_http(data, hostrec, port=None):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if 'result' in data:
data.update(data.pop('result'))
if 'response' not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data['response']
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
'Missing field%s %s in zgrab HTTP result',
's' if len(missing_fields) > 1 else '',
', '.join(repr(fld) for fld in missing_fields),
)
return {}
req = resp['request']
url = req.get('url')
res = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp"
}
tls = None
try:
tls = req['tls_handshake']
except KeyError:
# zgrab2
try:
tls = req['tls_log']['handshake_log']
except KeyError:
pass
if tls is not None:
res['service_tunnel'] = 'ssl'
try:
cert = tls['server_certificates']['certificate']['raw']
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault('scripts', []).append({
'id': 'ssl-cert',
'output': output,
'ssl-cert': info,
})
for cert in info:
add_cert_hostnames(cert,
hostrec.setdefault('hostnames', []))
if url:
guessed_port = None
if ':' in url.get('host', ''):
try:
guessed_port = int(url['host'].split(':', 1)[1])
except ValueError:
pass
if port is None:
if guessed_port is None:
if url.get('scheme') == 'https':
port = 443
else:
port = 80
else:
port = guessed_port
elif port != guessed_port:
utils.LOGGER.warning(
'Port %d found from the URL %s differs from the provided port '
'value %d', guessed_port, url.get('path'), port)
port = guessed_port
# Specific paths
if url.get('path').endswith('/.git/index'):
if resp.get('status_code') != 200:
return {}
if not resp.get('body', '').startswith('DIRC'):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = '%s:%d%s' % (hostrec['addr'], port, url['path'][:-5])
res['port'] = port
res.setdefault('scripts', []).append({
'id':
'http-git',
'output':
'\n %s\n Git repository found!\n' % repository,
'http-git': [{
'repository': repository,
'files-found': [".git/index"]
}],
})
return res
if url.get('path').endswith('/owa/auth/logon.aspx'):
if resp.get('status_code') != 200:
return {}
version = set(
m.group(1)
for m in _EXPR_OWA_VERSION.finditer(resp.get('body', '')))
if not version:
return {}
version = sorted(version,
key=lambda v: [int(x) for x in v.split('.')])
res['port'] = port
path = url['path'][:-15]
if len(version) > 1:
output = (
'OWA: path %s, version %s (multiple versions found!)' % (
path,
' / '.join(version),
))
else:
output = 'OWA: path %s, version %s' % (path, version[0])
res.setdefault('scripts', []).append({
'id':
'http-app',
'output':
output,
'http-app': [{
'path': path,
'application': 'OWA',
'version': version[0]
}],
})
return res
if url.get('path').endswith('/centreon/'):
if resp.get('status_code') != 200:
return {}
if not resp.get('body'):
return {}
body = resp['body']
res['port'] = port
path = url['path']
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault('scripts', []).append({
'id':
'http-app',
'output':
'Centreon: path %s%s' % (
path,
'' if version is None else (', version %s' % version),
),
'http-app': [
dict({
'path': path,
'application': 'Centreon'
}, **({} if version is None else {
'version': version
}))
],
})
return res
if url.get('path') != '/':
utils.LOGGER.warning('URL path not supported yet: %s',
url.get('path'))
return {}
elif port is None:
if req.get('tls_handshake') or req.get('tls_log'):
port = 443
else:
port = 80
res['port'] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' +
utils.nmap_decode_data(resp['status_line']) + b"\r\n")
if resp.get('headers'):
headers = resp['headers']
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get('www_authenticate'):
auths = headers.get('www_authenticate')
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode()))
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
pass
keyvals = zip(ntlm_values,
[infos.get(k) for k in ntlm_values])
output = '\n'.join("{}: {}".format(k, v)
for k, v in keyvals if v)
res.setdefault('scripts', []).append({
'id': 'http-ntlm-info',
'output': output,
'ntlm-info': infos
})
if 'DNS_Computer_Name' in infos:
add_hostname(infos['DNS_Computer_Name'], 'ntlm',
hostrec.setdefault('hostnames', []))
if any(val.lower().startswith('ntlm')
for val in req.get('headers', {}).get('authorization', [])):
return res
# the order will be incorrect!
line = '%s %s' % (resp['protocol']['name'], resp['status_line'])
http_hdrs = [{'name': '_status', 'value': line}]
output = [line]
for unk in headers.pop('unknown', []):
headers[unk['key']] = unk['value']
for hdr, values in headers.items():
hdr = hdr.replace('_', '-')
for val in values:
http_hdrs.append({'name': hdr, 'value': val})
output.append('%s: %s' % (hdr, val))
if http_hdrs:
method = req.get('method')
if method:
output.append('')
output.append('(Request type: %s)' % method)
res.setdefault('scripts', []).append({
'id': 'http-headers',
'output': '\n'.join(output),
'http-headers': http_hdrs,
})
handle_http_headers(hostrec, res, http_hdrs, path=url.get('path'))
if headers.get('server'):
banner += (b"Server: " +
utils.nmap_decode_data(headers['server'][0]) +
b"\r\n\r\n")
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
add_cpe_values(hostrec, 'ports.port:%s' % port, info.pop('cpe', []))
res.update(info)
if resp.get('body'):
body = resp['body']
res.setdefault('scripts', []).append({
'id':
'http-content',
'output':
utils.nmap_encode_data(body.encode()),
})
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res['scripts'].append({
'id': 'http-title',
'output': title,
'http-title': {
'title': title
},
})
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault('scripts', []).append(script_http_ls)
service_elasticsearch = create_elasticsearch_service(body)
if service_elasticsearch:
if 'hostname' in service_elasticsearch:
add_hostname(service_elasticsearch.pop('hostname'), 'service',
hostrec.setdefault('hostnames', []))
add_cpe_values(hostrec, 'ports.port:%s' % port,
service_elasticsearch.pop('cpe', []))
res.update(service_elasticsearch)
return res
def zgrap_parser_http(data, hostrec):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if 'result' in data:
data.update(data.pop('result'))
if 'response' not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data['response']
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
'Missing field%s %s in zgrab HTTP result',
's' if len(missing_fields) > 1 else '',
', '.join(repr(fld) for fld in missing_fields),
)
return {}
req = resp['request']
url = req.get('url')
res = {"service_name": "http", "service_method": "probed",
"state_state": "open", "state_reason": "response",
"protocol": "tcp"}
tls = None
try:
tls = req['tls_handshake']
except KeyError:
# zgrab2
try:
tls = req['tls_log']['handshake_log']
except KeyError:
pass
if tls is not None:
res['service_tunnel'] = 'ssl'
try:
cert = tls['server_certificates']['certificate']['raw']
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault('scripts', []).append({
'id': 'ssl-cert',
'output': output,
'ssl-cert': info,
})
for cert in info:
add_cert_hostnames(cert,
hostrec.setdefault('hostnames', []))
if url:
port = None
if ':' in url.get('host', ''):
try:
port = int(url['host'].split(':', 1)[1])
except ValueError:
pass
if port is None:
if url.get('scheme') == 'https':
port = 443
else:
port = 80
# Specific paths
if url.get('path').endswith('/.git/index'):
if resp.get('status_code') != 200:
return {}
if not resp.get('body', '').startswith('DIRC'):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = '%s:%d%s' % (hostrec['addr'], port, url['path'][:-5])
res['port'] = port
res.setdefault('scripts', []).append({
'id': 'http-git',
'output': '\n %s\n Git repository found!\n' % repository,
'http-git': [{'repository': repository,
'files-found': [".git/index"]}],
})
return res
if url.get('path').endswith('/owa/auth/logon.aspx'):
if resp.get('status_code') != 200:
return {}
version = set(
m.group(1)
for m in _EXPR_OWA_VERSION.finditer(resp.get('body', ''))
)
if not version:
return {}
version = sorted(version,
key=lambda v: [int(x) for x in v.split('.')])
res['port'] = port
path = url['path'][:-15]
if len(version) > 1:
output = (
'OWA: path %s, version %s (multiple versions found!)' % (
path,
' / '.join(version),
)
)
else:
output = 'OWA: path %s, version %s' % (path, version[0])
res.setdefault('scripts', []).append({
'id': 'http-app',
'output': output,
'http-app': [{'path': path,
'application': 'OWA',
'version': version[0]}],
})
return res
elif req.get('tls_handshake') or req.get('tls_log'):
# zgrab / zgrab2
port = 443
else:
port = 80
res['port'] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' +
utils.nmap_decode_data(resp['status_line']) + b"\r\n")
if resp.get('headers'):
headers = resp['headers']
# the order will be incorrect!
line = '%s %s' % (resp['protocol']['name'], resp['status_line'])
http_hdrs = [{'name': '_status', 'value': line}]
output = [line]
if 'unknown' in headers:
for unk in headers.pop('unknown'):
headers[unk['key']] = unk['value']
for hdr, values in viewitems(headers):
hdr = hdr.replace('_', '-')
for val in values:
http_hdrs.append({'name': hdr, 'value': val})
output.append('%s: %s' % (hdr, val))
if http_hdrs:
method = req.get('method')
if method:
output.append('')
output.append('(Request type: %s)' % method)
res.setdefault('scripts', []).append({
'id': 'http-headers', 'output': '\n'.join(output),
'http-headers': http_hdrs,
})
if headers.get('server'):
server = resp['headers']['server']
res.setdefault('scripts', []).append({
'id': 'http-server-header', 'output': server[0],
'http-server-header': server,
})
banner += (b"Server: " + utils.nmap_decode_data(server[0]) +
b"\r\n\r\n")
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
res.update(info)
if resp.get('body'):
body = resp['body']
res.setdefault('scripts', []).append({
'id': 'http-content',
'output': utils.nmap_encode_data(body.encode()),
})
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res['scripts'].append({
'id': 'http-title', 'output': title,
'http-title': {'title': title},
})
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault('scripts', []).append(script_http_ls)
return res
def _getinfos_from_banner(banner, proto="tcp", probe="NULL"):
infos = utils.match_nmap_svc_fp(banner, proto=proto, probe=probe)
if not infos:
return {}
return {'infos': infos}
