def _display_gnmap_host(host: NmapHost, out: TextIO = sys.stdout) -> None:
addr = host["addr"]
hostname = None
for name in host.get("hostnames", []):
if name.get("type") == "PTR":
hostname = name.get("name")
if hostname is not None:
break
if hostname is None:
name = addr
else:
name = "%s (%s)" % (addr, hostname)
if host.get("state"):
out.write("Host: %s Status: %s\n" % (name, host["state"].capitalize()))
ports = []
info = []
for port in host.get("ports", []):
if port.get("port") == -1:
continue
if "service_product" in port:
version = port["service_product"]
for key in ["version", "extrainfo"]:
key = "service_%s" % key
if key in port:
version += " %s" % port[key]
version = version.replace("/", "|")
else:
version = ""
ports.append("%d/%s/%s//%s//%s/" % (
port["port"],
port["state_state"],
port["protocol"],
port.get("service_name", ""),
version,
))
if ports:
info.append("Ports: %s" % ", ".join(ports))
extraports = []
for state, counts in host.get("extraports", {}).items():
extraports.append("%s (%d)" % (state, counts["total"]))
if extraports:
info.append("Ignored State: %s" % ", ".join(extraports))
for osmatch in host.get("os", {}).get("osmatch", []):
info.append("OS: %s" % osmatch["name"])
break
# TODO: data from tcpsequence and ipidsequence is currently
# missing
if info:
out.write("Host: %s %s\n" % (name, "\t".join(info)))
def _display_xml_table_elem(
doc: NmapHost,
first: bool = False,
name: Optional[str] = None,
out: TextIO = sys.stdout,
) -> None:
if first:
assert name is None
name = "" if name is None else " key=%s" % saxutils.quoteattr(name)
if isinstance(doc, list):
if not first:
out.write("<table%s>\n" % name)
for subdoc in doc:
_display_xml_table_elem(subdoc, out=out)
if not first:
out.write("</table>\n")
elif isinstance(doc, dict):
if not first:
out.write("<table%s>\n" % name)
for key, subdoc in doc.items():
_display_xml_table_elem(subdoc, name=key, out=out)
if not first:
out.write("</table>\n")
else:
out.write("<elem%s>%s</elem>\n" % (
name,
saxutils.escape(
str(doc),
entities={"\n": " "},
),
))
def set_openports_attribute(host: NmapHost) -> None:
"""This function sets the "openports" value in the `host` record,
based on the elements of the "ports" list. This is used in MongoDB to
speed up queries based on open ports.
"""
openports = host["openports"] = {"count": 0}
for port in host.get("ports", []):
if port.get("state_state") != "open":
continue
cur = openports.setdefault(port["protocol"], {"count": 0, "ports": []})
if port["port"] not in cur["ports"]:
openports["count"] += 1
cur["count"] += 1
cur["ports"].append(port["port"])
def cleanup_synack_honeypot_host(host: NmapHost, update_openports: bool = True) -> None:
"""This function will clean the `host` record if it has too many (at
least `VIEW_SYNACK_HONEYPOT_COUNT`) open ports that may be "syn-ack"
honeypots (which means, ports for which is_real_service_port() returns
False).
"""
if VIEW_SYNACK_HONEYPOT_COUNT is None:
return
n_ports = len(host.get("ports", []))
if n_ports < VIEW_SYNACK_HONEYPOT_COUNT:
return
# check if we have too many open ports that could be "syn-ack
# honeypots"...
newports = [port for port in host["ports"] if is_real_service_port(port)]
if n_ports - len(newports) > VIEW_SYNACK_HONEYPOT_COUNT:
# ... if so, keep only the ports that cannot be "syn-ack
# honeypots"
host["ports"] = newports
host["synack_honeypot"] = True
if update_openports:
set_openports_attribute(host)
def zgrap_parser_http(data: Dict[str, Any],
hostrec: NmapHost,
port: Optional[int] = None) -> NmapPort:
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if "result" in data:
data.update(data.pop("result"))
if "response" not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data["response"]
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
"Missing field%s %s in zgrab HTTP result",
"s" if len(missing_fields) > 1 else "",
", ".join(repr(fld) for fld in missing_fields),
)
return {}
req = resp["request"]
url = req.get("url")
res: NmapPort = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp",
}
tls = None
try:
tls = req["tls_handshake"]
except KeyError:
# zgrab2
try:
tls = req["tls_log"]["handshake_log"]
except KeyError:
pass
if tls is not None:
res["service_tunnel"] = "ssl"
try:
cert = tls["server_certificates"]["certificate"]["raw"]
except KeyError:
pass
else:
output, info_cert = create_ssl_cert(cert.encode(), b64encoded=True)
if info_cert:
res.setdefault("scripts", []).append({
"id": "ssl-cert",
"output": output,
"ssl-cert": info_cert,
})
for cert in info_cert:
add_cert_hostnames(cert,
hostrec.setdefault("hostnames", []))
if url:
try:
_, guessed_port = utils.url2hostport("%(scheme)s://%(host)s" % url)
except ValueError:
utils.LOGGER.warning("Cannot guess port from url %r", url)
guessed_port = 80 # because reasons
else:
if port is not None and port != guessed_port:
utils.LOGGER.warning(
"Port %d found from the URL %s differs from the provided port "
"value %d",
guessed_port,
url.get("path"),
port,
)
port = guessed_port
if port is None:
port = guessed_port
# Specific paths
if url.get("path").endswith("/.git/index"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body", "").startswith("DIRC"):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5])
res["port"] = port
res.setdefault("scripts", []).append({
"id":
"http-git",
"output":
"\n %s\n Git repository found!\n" % repository,
"http-git": [
{
"repository": repository,
"files-found": [".git/index"]
},
],
})
return res
if url.get("path").endswith("/owa/auth/logon.aspx"):
if resp.get("status_code") != 200:
return {}
version_set = set(
m.group(1)
for m in _EXPR_OWA_VERSION.finditer(resp.get("body", "")))
if not version_set:
return {}
version_list = sorted(version_set,
key=lambda v: [int(x) for x in v.split(".")])
res["port"] = port
path = url["path"][:-15]
if len(version_set) > 1:
output = "OWA: path %s, version %s (multiple versions found!)" % (
path,
" / ".join(version_list),
)
else:
output = "OWA: path %s, version %s" % (path, version_list[0])
res.setdefault("scripts", []).append({
"id":
"http-app",
"output":
output,
"http-app": [{
"path": path,
"application": "OWA",
"version": version_list[0]
}],
})
return res
if url.get("path").endswith("/centreon/"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
path = url["path"]
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
version: Optional[str]
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault("scripts", []).append({
"id":
"http-app",
"output":
"Centreon: path %s%s" % (
path,
"" if version is None else (", version %s" % version),
),
"http-app": [
dict(
{
"path": path,
"application": "Centreon"
},
**({} if version is None else {
"version": version
}),
)
],
})
return res
if url.get("path") != "/":
utils.LOGGER.warning("URL path not supported yet: %s",
url.get("path"))
return {}
elif port is None:
if req.get("tls_handshake") or req.get("tls_log"):
port = 443
else:
port = 80
res["port"] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (utils.nmap_decode_data(resp["protocol"]["name"]) + b" " +
utils.nmap_decode_data(resp["status_line"]) + b"\r\n")
if resp.get("headers"):
headers = resp["headers"]
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get("www_authenticate"):
auths = headers.get("www_authenticate")
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode()))
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
continue
if not infos:
continue
keyvals = zip(ntlm_values,
[infos.get(k) for k in ntlm_values])
output = "\n".join("{}: {}".format(k, v)
for k, v in keyvals if v)
res.setdefault("scripts", []).append({
"id":
"ntlm-info",
"output":
output,
"ntlm-info":
dict(infos, protocol="http"),
})
if "DNS_Computer_Name" in infos:
add_hostname(
infos["DNS_Computer_Name"],
"ntlm",
hostrec.setdefault("hostnames", []),
)
if any(val.lower().startswith("ntlm")
for val in req.get("headers", {}).get("authorization", [])):
return res
# the order will be incorrect!
line = "%s %s" % (resp["protocol"]["name"], resp["status_line"])
http_hdrs: List[HttpHeader] = [{"name": "_status", "value": line}]
output_list = [line]
for unk in headers.pop("unknown", []):
headers[unk["key"]] = unk["value"]
for hdr, values in headers.items():
hdr = hdr.replace("_", "-")
for val in values:
http_hdrs.append({"name": hdr, "value": val})
output_list.append("%s: %s" % (hdr, val))
if http_hdrs:
method = req.get("method")
if method:
output_list.append("")
output_list.append("(Request type: %s)" % method)
res.setdefault("scripts", []).append({
"id":
"http-headers",
"output":
"\n".join(output_list),
"http-headers":
http_hdrs,
})
handle_http_headers(hostrec, res, http_hdrs, path=url.get("path"))
if headers.get("server"):
banner += (b"Server: " +
utils.nmap_decode_data(headers["server"][0]) +
b"\r\n\r\n")
info: NmapServiceMatch = utils.match_nmap_svc_fp(banner,
proto="tcp",
probe="GetRequest")
if info:
add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", []))
res.update(cast(NmapPort, info))
if resp.get("body"):
body = resp["body"]
res.setdefault("scripts", []).append({
"id":
"http-content",
"output":
utils.nmap_encode_data(body.encode()),
})
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res["scripts"].append({
"id": "http-title",
"output": title,
"http-title": {
"title": title
},
})
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault("scripts", []).append(script_http_ls)
service_elasticsearch = create_elasticsearch_service(body)
if service_elasticsearch:
if "service_hostname" in service_elasticsearch:
add_hostname(
service_elasticsearch["service_hostname"],
"service",
hostrec.setdefault("hostnames", []),
)
add_cpe_values(hostrec, "ports.port:%s" % port,
service_elasticsearch.pop("cpe", []))
res.update(cast(NmapPort, service_elasticsearch))
return res
def _display_xml_host(host: NmapHost, out: TextIO = sys.stdout) -> None:
out.write("<host")
for k in ["timedout", "timeoutcounter"]:
if k in host:
out.write(" %s=%s" % (k, saxutils.quoteattr(host[k])))
for k in ["starttime", "endtime"]:
if k in host:
out.write(" %s=%s" %
(k, saxutils.quoteattr(host[k].strftime("%s"))))
out.write(">")
if "state" in host:
out.write('<status state="%s"' % host["state"])
for k in ["reason", "reason_ttl"]:
kk = "state_%s" % k
if kk in host:
out.write(' %s="%s"' % (k, host[kk]))
out.write("/>")
out.write("\n")
if "addr" in host:
out.write('<address addr="%s" addrtype="ipv%d"/>\n' % (
host["addr"],
6 if ":" in host["addr"] else 4,
))
for atype, addrs in host.get("addresses", {}).items():
for addr in addrs:
extra = ""
if atype == "mac":
manuf = utils.mac2manuf(addr)
# if manuf:
# if len(manuf) > 1 and manuf[1]:
# manuf = manuf[1]
# else:
# manuf = manuf[0]
# extra = ' vendor=%s' % saxutils.quoteattr(manuf[0])
if manuf and manuf[0]:
extra = " vendor=%s" % saxutils.quoteattr(manuf[0])
out.write('<address addr="%s" addrtype="%s"%s/>\n' %
(addr, atype, extra))
if "hostnames" in host:
out.write("<hostnames>\n")
for hostname in host["hostnames"]:
out.write("<hostname")
for k in ["name", "type"]:
if k in hostname:
out.write(' %s="%s"' % (k, hostname[k]))
out.write("/>\n")
out.write("</hostnames>\n")
out.write("<ports>")
for state, counts in host.get("extraports", {}).items():
out.write('<extraports state="%s" count="%d">\n' %
(state, counts["total"]))
for reason, count in counts["reasons"].items():
out.write('<extrareasons reason="%s" count="%d"/>\n' %
(reason, count))
out.write("</extraports>\n")
hostscripts: List[NmapScript] = []
for p in host.get("ports", []):
if p.get("port") == -1:
hostscripts = p["scripts"]
continue
out.write("<port")
if "protocol" in p:
out.write(' protocol="%s"' % p["protocol"])
if "port" in p:
out.write(' portid="%s"' % p["port"])
out.write("><state")
for k in ["state", "reason", "reason_ttl"]:
kk = "state_%s" % k
if kk in p:
out.write(" %s=%s" % (k, saxutils.quoteattr(str(p[kk]))))
out.write("/>")
if "service_name" in p:
out.write('<service name="%s"' % p["service_name"])
for k in [
"servicefp",
"product",
"version",
"extrainfo",
"ostype",
"method",
"conf",
]:
kk = "service_%s" % k
if kk in p:
if isinstance(p[kk], str):
out.write(" %s=%s" % (k, saxutils.quoteattr(p[kk])))
else:
out.write(' %s="%s"' % (k, p[kk]))
# TODO: CPE
out.write("></service>")
for s in p.get("scripts", []):
_display_xml_script(s, out=out)
out.write("</port>\n")
out.write("</ports>\n")
if hostscripts:
out.write("<hostscript>")
for s in hostscripts:
_display_xml_script(s, out=out)
out.write("</hostscript>")
for trace in host.get("traces", []):
out.write("<trace")
if "port" in trace:
out.write(" port=%s" % (saxutils.quoteattr(str(trace["port"]))))
if "protocol" in trace:
out.write(" proto=%s" % (saxutils.quoteattr(trace["protocol"])))
out.write(">\n")
for hop in sorted(trace.get("hops", []),
key=lambda hop: cast(int, hop["ttl"])):
out.write("<hop")
if "ttl" in hop:
out.write(" ttl=%s" % (saxutils.quoteattr(str(hop["ttl"]))))
if "ipaddr" in hop:
out.write(" ipaddr=%s" % (saxutils.quoteattr(hop["ipaddr"])))
if "rtt" in hop:
out.write(
" rtt=%s" %
(saxutils.quoteattr("%.2f" % hop["rtt"] if isinstance(
hop["rtt"], float) else hop["rtt"])))
if "host" in hop:
out.write(" host=%s" % (saxutils.quoteattr(hop["host"])))
out.write("/>\n")
out.write("</trace>\n")
out.write("</host>\n")
def _display_honeyd_conf(
host: NmapHost,
honeyd_routes: HoneydRoutes,
honeyd_entries: HoneydNodes,
out: TextIO = sys.stdout,
) -> Tuple[HoneydRoutes, HoneydNodes]:
addr = host["addr"]
hname = "host_%s" % addr.replace(".", "_").replace(":", "_")
out.write("create %s\n" % hname)
defaction = HONEYD_DEFAULT_ACTION
if "extraports" in host:
extra = host["extraports"]
defaction = max(
max(extra.values(),
key=lambda state: cast(int,
cast(dict, state)["total"]))
["reasons"].items(),
key=lambda reason: cast(Tuple[str, int], reason)[1],
)[0]
try:
defaction = HONEYD_ACTION_FROM_NMAP_STATE[defaction]
except KeyError:
pass
out.write("set %s default tcp action %s\n" % (hname, defaction))
for p in host.get("ports", []):
try:
out.write("add %s %s port %d %s\n" % (
hname,
p["protocol"],
p["port"],
_nmap_port2honeyd_action(p),
))
except KeyError:
# let's skip pseudo-port records that are only containers for host
# scripts.
pass
if host.get("traces"):
trace = max(host["traces"], key=lambda x: len(x["hops"]))["hops"]
if trace:
trace.sort(key=lambda x: x["ttl"])
curhop = trace[0]
honeyd_entries.add(curhop["ipaddr"])
for t in trace[1:]:
key = (curhop["ipaddr"], t["ipaddr"])
latency = max(t["rtt"] - curhop["rtt"], 0)
route = honeyd_routes.get(key)
if route is None:
honeyd_routes[key] = {
"count": 1,
"high": latency,
"low": latency,
"mean": latency,
"targets": set([host["addr"]]),
}
else:
route["targets"].add(host["addr"])
honeyd_routes[key] = {
"count":
route["count"] + 1,
"high":
max(route["high"], latency),
"low":
min(route["low"], latency),
"mean": (route["mean"] * route["count"] + latency) /
float(route["count"] + 1),
"targets":
route["targets"],
}
curhop = t
out.write("bind %s %s\n\n" % (addr, hname))
return honeyd_routes, honeyd_entries
def merge_host_docs(rec1: NmapHost, rec2: NmapHost) -> NmapHost:
"""Merge two host records and return the result. Unmergeable /
hard-to-merge fields are lost (e.g., extraports).
"""
if rec1.get("schema_version") != rec2.get("schema_version"):
raise ValueError(
"Cannot merge host documents. "
"Schema versions differ (%r != %r)"
% (rec1.get("schema_version"), rec2.get("schema_version"))
)
rec = {}
if "schema_version" in rec1:
rec["schema_version"] = rec1["schema_version"]
# When we have different values, we will use the one from the
# most recent scan, rec2. If one result has no "endtime", we
# consider it as older.
if (rec1.get("endtime") or datetime.fromtimestamp(0)) > (
rec2.get("endtime") or datetime.fromtimestamp(0)
):
rec1, rec2 = rec2, rec1
for fname, function in [("starttime", min), ("endtime", max)]:
try:
rec[fname] = function(
record[fname] for record in [rec1, rec2] if fname in record
)
except ValueError:
pass
sa_honeypot = rec1.get("synack_honeypot") or rec2.get("synack_honeypot")
rec["state"] = "up" if rec1.get("state") == "up" else rec2.get("state")
if rec["state"] is None:
del rec["state"]
rec["state_reason"] = rec2.get("state_reason", rec1.get("state_reason"))
if rec["state_reason"] is None:
del rec["state_reason"]
rec["categories"] = list(
set(rec1.get("categories", [])).union(rec2.get("categories", []))
)
for field in ["addr", "os"]:
rec[field] = rec2[field] if rec2.get(field) else rec1.get(field)
if not rec[field]:
del rec[field]
rec["source"] = list(set(rec1.get("source", [])).union(set(rec2.get("source", []))))
rec["traces"] = rec2.get("traces", [])
for trace in rec1.get("traces", []):
# Skip this result (from rec1) if a more recent traceroute
# result exists using the same protocol and port in the
# most recent scan (rec2).
if any(
other["protocol"] == trace["protocol"]
and other.get("port") == trace.get("port")
for other in rec["traces"]
):
continue
rec["traces"].append(trace)
rec["cpes"] = rec2.get("cpes", [])
for cpe in rec1.get("cpes", []):
origins = set(cpe.pop("origins", []))
cpe["origins"] = None
try:
other = next(
ocpe for ocpe in rec["cpes"] if dict(ocpe, origins=None) == cpe
)
except StopIteration:
rec["cpes"].append(dict(cpe, origins=origins))
else:
other["origins"] = set(other.get("origins", [])).union(origins)
for cpe in rec["cpes"]:
cpe["origins"] = list(cpe.get("origins", []))
rec["infos"] = {}
for record in [rec1, rec2]:
rec["infos"].update(record.get("infos", {}))
# We want to make sure of (type, name) unicity
hostnames = dict(
((h["type"], h["name"]), h.get("domains"))
for h in (rec1.get("hostnames", []) + rec2.get("hostnames", []))
)
rec["hostnames"] = [
{"type": h[0], "name": h[1], "domains": d} for h, d in hostnames.items()
]
addresses: NmapAddress = {}
for record in [rec1, rec2]:
for atype, addrs in record.get("addresses", {}).items():
cur_addrs = addresses.setdefault(atype, [])
for addr in addrs:
addr = addr.lower()
if addr not in cur_addrs:
cur_addrs.append(addr)
if addresses:
rec["addresses"] = addresses
sa_honeypot_check = False
if sa_honeypot:
rec["synack_honeypot"] = True
for record in [rec1, rec2]:
if not record.get("synack_honeypot"):
sa_honeypot_check = True
record["ports"] = [
port
for port in record.get("ports", [])
if is_real_service_port(port)
]
ports = dict(
((port.get("protocol"), port["port"]), port.copy())
for port in rec2.get("ports", [])
)
for port in rec1.get("ports", []):
if (port.get("protocol"), port["port"]) in ports:
curport = ports[(port.get("protocol"), port["port"])]
if "scripts" in curport:
curport["scripts"] = curport["scripts"][:]
else:
curport["scripts"] = []
present_scripts = set(script["id"] for script in curport["scripts"])
for script in port.get("scripts", []):
if script["id"] not in present_scripts:
curport["scripts"].append(script)
elif script["id"] in _SCRIPT_MERGE:
# Merge scripts
curscript = next(
x for x in curport["scripts"] if x["id"] == script["id"]
)
merge_scripts(curscript, script, script["id"])
if not curport["scripts"]:
del curport["scripts"]
if "service_name" in port:
if "service_name" not in curport:
for key in port:
if key.startswith("service_"):
curport[key] = port[key]
elif port["service_name"] == curport["service_name"]:
# if the "old" record has information missing
# from the "new" record and information from
# both records is consistent, let's keep the
# "old" data.
for key in port:
if key.startswith("service_") and key not in curport:
curport[key] = port[key]
if "screenshot" in port and "screenshot" not in curport:
for key in ["screenshot", "screendata", "screenwords"]:
if key in port:
curport[key] = port[key]
else:
ports[(port.get("protocol"), port["port"])] = port
if sa_honeypot and sa_honeypot_check:
rec["ports"] = sorted(
(port for port in ports.values() if is_real_service_port(port)),
key=lambda port: (
port.get("protocol") or "~",
port.get("port"),
),
)
else:
rec["ports"] = sorted(
ports.values(),
key=lambda port: (
port.get("protocol") or "~",
port.get("port"),
),
)
if not sa_honeypot:
cleanup_synack_honeypot_host(rec, update_openports=False)
set_openports_attribute(rec)
for field in ["traces", "infos", "ports", "cpes"]:
if not rec[field]:
del rec[field]
return rec