def zgrap_parser_http(data, hostrec, port=None):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if "result" in data:
data.update(data.pop("result"))
if "response" not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data["response"]
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
"Missing field%s %s in zgrab HTTP result",
"s" if len(missing_fields) > 1 else "",
", ".join(repr(fld) for fld in missing_fields),
)
return {}
req = resp["request"]
url = req.get("url")
res = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp",
}
tls = None
try:
tls = req["tls_handshake"]
except KeyError:
# zgrab2
try:
tls = req["tls_log"]["handshake_log"]
except KeyError:
pass
if tls is not None:
res["service_tunnel"] = "ssl"
try:
cert = tls["server_certificates"]["certificate"]["raw"]
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault("scripts", []).append(
{
"id": "ssl-cert",
"output": output,
"ssl-cert": info,
}
)
for cert in info:
add_cert_hostnames(cert, hostrec.setdefault("hostnames", []))
if url:
guessed_port = None
if ":" in url.get("host", ""):
try:
guessed_port = int(url["host"].split(":", 1)[1])
except ValueError:
pass
if port is None:
if guessed_port is None:
if url.get("scheme") == "https":
port = 443
else:
port = 80
else:
port = guessed_port
elif port != guessed_port:
utils.LOGGER.warning(
"Port %d found from the URL %s differs from the provided port "
"value %d",
guessed_port,
url.get("path"),
port,
)
port = guessed_port
# Specific paths
if url.get("path").endswith("/.git/index"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body", "").startswith("DIRC"):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5])
res["port"] = port
res.setdefault("scripts", []).append(
{
"id": "http-git",
"output": "\n %s\n Git repository found!\n" % repository,
"http-git": [
{"repository": repository, "files-found": [".git/index"]}
],
}
)
return res
if url.get("path").endswith("/owa/auth/logon.aspx"):
if resp.get("status_code") != 200:
return {}
version = set(
m.group(1) for m in _EXPR_OWA_VERSION.finditer(resp.get("body", ""))
)
if not version:
return {}
version = sorted(version, key=lambda v: [int(x) for x in v.split(".")])
res["port"] = port
path = url["path"][:-15]
if len(version) > 1:
output = "OWA: path %s, version %s (multiple versions found!)" % (
path,
" / ".join(version),
)
else:
output = "OWA: path %s, version %s" % (path, version[0])
res.setdefault("scripts", []).append(
{
"id": "http-app",
"output": output,
"http-app": [
{"path": path, "application": "OWA", "version": version[0]}
],
}
)
return res
if url.get("path").endswith("/centreon/"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
path = url["path"]
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault("scripts", []).append(
{
"id": "http-app",
"output": "Centreon: path %s%s"
% (
path,
"" if version is None else (", version %s" % version),
),
"http-app": [
dict(
{"path": path, "application": "Centreon"},
**({} if version is None else {"version": version}),
)
],
}
)
return res
if url.get("path") != "/":
utils.LOGGER.warning("URL path not supported yet: %s", url.get("path"))
return {}
elif port is None:
if req.get("tls_handshake") or req.get("tls_log"):
port = 443
else:
port = 80
res["port"] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (
utils.nmap_decode_data(resp["protocol"]["name"])
+ b" "
+ utils.nmap_decode_data(resp["status_line"])
+ b"\r\n"
)
if resp.get("headers"):
headers = resp["headers"]
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get("www_authenticate"):
auths = headers.get("www_authenticate")
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode())
)
except (UnicodeDecodeError, TypeError, ValueError, binascii.Error):
pass
keyvals = zip(ntlm_values, [infos.get(k) for k in ntlm_values])
output = "\n".join("{}: {}".format(k, v) for k, v in keyvals if v)
res.setdefault("scripts", []).append(
{"id": "http-ntlm-info", "output": output, "ntlm-info": infos}
)
if "DNS_Computer_Name" in infos:
add_hostname(
infos["DNS_Computer_Name"],
"ntlm",
hostrec.setdefault("hostnames", []),
)
if any(
val.lower().startswith("ntlm")
for val in req.get("headers", {}).get("authorization", [])
):
return res
# the order will be incorrect!
line = "%s %s" % (resp["protocol"]["name"], resp["status_line"])
http_hdrs = [{"name": "_status", "value": line}]
output = [line]
for unk in headers.pop("unknown", []):
headers[unk["key"]] = unk["value"]
for hdr, values in headers.items():
hdr = hdr.replace("_", "-")
for val in values:
http_hdrs.append({"name": hdr, "value": val})
output.append("%s: %s" % (hdr, val))
if http_hdrs:
method = req.get("method")
if method:
output.append("")
output.append("(Request type: %s)" % method)
res.setdefault("scripts", []).append(
{
"id": "http-headers",
"output": "\n".join(output),
"http-headers": http_hdrs,
}
)
handle_http_headers(hostrec, res, http_hdrs, path=url.get("path"))
if headers.get("server"):
banner += (
b"Server: " + utils.nmap_decode_data(headers["server"][0]) + b"\r\n\r\n"
)
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", []))
res.update(info)
if resp.get("body"):
body = resp["body"]
res.setdefault("scripts", []).append(
{
"id": "http-content",
"output": utils.nmap_encode_data(body.encode()),
}
)
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res["scripts"].append(
{
"id": "http-title",
"output": title,
"http-title": {"title": title},
}
)
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault("scripts", []).append(script_http_ls)
service_elasticsearch = create_elasticsearch_service(body)
if service_elasticsearch:
if "hostname" in service_elasticsearch:
add_hostname(
service_elasticsearch.pop("hostname"),
"service",
hostrec.setdefault("hostnames", []),
)
add_cpe_values(
hostrec, "ports.port:%s" % port, service_elasticsearch.pop("cpe", [])
)
res.update(service_elasticsearch)
return res
def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if 'addr' in spec and \
spec.get('source') not in neverignore.get(spec['recontype'], []):
for start, stop in ignorenets.get(spec['recontype'], ()):
if start <= utils.force_ip2int(spec['addr']) <= stop:
return None
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string 'SymantecRandomUserAgent'.
if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \
spec.get('source') == 'USER-AGENT':
if SYMANTEC_UA.match(spec['value']):
spec['value'] = 'SymantecRandomUserAgent'
elif KASPERSKY_UA.match(spec['value']):
spec['value'] = 'KasperskyWeirdUserAgent'
else:
match = SYMANTEC_SEP_UA.match(spec['value'])
if match is not None:
spec['value'] = '%s%s' % match.groups()
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif (spec['recontype']
in {'HTTP_CLIENT_HEADER', 'HTTP_CLIENT_HEADER_SERVER'}
and spec.get('source') in {'AUTHORIZATION', 'PROXY-AUTHORIZATION'}):
value = spec['value']
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
spec['value'] = '%s %s' % (authtype, ','.join(
val for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_NEGOTIATE and NTLM_AUTHENTICATE
try:
auth = utils.decode_b64(value.split(None, 1)[1].encode())
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
pass
spec['value'] = "%s %s" % \
(value.split(None, 1)[0],
ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth)))
elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}:
spec['value'] = authtype
elif (spec['recontype'] == 'HTTP_SERVER_HEADER' and spec.get('source')
in {'WWW-AUTHENTICATE', 'PROXY-AUTHENTICATE'}):
value = spec['value']
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == 'digest':
try:
# we only keep relevant info
spec['value'] = '%s %s' % (authtype, ','.join(
val for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)))
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_CHALLENGE
try:
auth = utils.decode_b64(value.split(None, 1)[1].encode())
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
pass
spec['value'] = "%s %s" % \
(value.split(None, 1)[0],
ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth)))
elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}:
spec['value'] = authtype
# TCP server banners: try to normalize data
elif spec['recontype'] == 'TCP_SERVER_BANNER':
newvalue = value = utils.nmap_decode_data(spec['value'])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec['value'] = utils.nmap_encode_data(newvalue)
elif spec['recontype'] == 'TCP_CLIENT_BANNER':
probe = utils.get_nmap_probes('tcp').get(
utils.nmap_decode_data(spec['value']))
if probe is not None:
spec.setdefault('infos', {}).update({
'service_name':
'scanner',
'service_product':
'Nmap',
'service_extrainfo':
'TCP probe %s' % probe,
})
elif spec['recontype'] == 'UDP_HONEYPOT_HIT':
data = utils.nmap_decode_data(spec['value'])
probe = utils.get_nmap_probes('udp').get(data)
if probe is not None:
spec.setdefault('infos', {}).update({
'service_name':
'scanner',
'service_product':
'Nmap',
'service_extrainfo':
'UDP probe %s' % probe,
})
else:
payload = utils.get_nmap_udp_payloads().get(data)
if payload is not None:
spec.setdefault('infos', {}).update({
'service_name':
'scanner',
'service_product':
'Nmap',
'service_extrainfo':
'UDP payload %s' % payload,
})
# SSL_{CLIENT,SERVER} JA3
elif ((spec['recontype'] == 'SSL_CLIENT' and spec['source'] == 'ja3')
or (spec['recontype'] == 'SSL_SERVER'
and spec['source'].startswith('ja3-'))):
value = spec['value']
spec.setdefault('infos', {})['raw'] = value
spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
if spec['recontype'] == 'SSL_SERVER':
clientvalue = spec['source'][4:]
spec['infos'].setdefault('client', {})['raw'] = clientvalue
spec['source'] = 'ja3-%s' % hashlib.new(
"md5",
clientvalue.encode(),
).hexdigest()
# SSH_{CLIENT,SERVER}_HASSH
elif spec['recontype'] in ['SSH_CLIENT_HASSH', 'SSH_SERVER_HASSH']:
value = spec['value']
spec.setdefault('infos', {})['raw'] = value
spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
# Check DNS Blacklist answer
elif spec['recontype'] == 'DNS_ANSWER':
if any((spec.get('value') or "").endswith(dnsbl)
for dnsbl in config.DNS_BLACKLIST_DOMAINS):
dnsbl_val = spec['value']
match = DNSBL_START.search(dnsbl_val)
if match is not None:
spec['recontype'] = 'DNS_BLACKLIST'
spec['value'] = spec.get('addr')
spec.update({
'source':
"%s-%s" % (dnsbl_val[match.end():], spec['source'])
})
addr = match.group()
# IPv4
if addr.count('.') == 4:
spec['addr'] = '.'.join(addr.split('.')[3::-1])
# IPv6
else:
spec['addr'] = utils.int2ip6(
int(addr.replace('.', '')[::-1], 16))
return spec
def zgrap_parser_http(data, hostrec, port=None):
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if 'result' in data:
data.update(data.pop('result'))
if 'response' not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data['response']
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
'Missing field%s %s in zgrab HTTP result',
's' if len(missing_fields) > 1 else '',
', '.join(repr(fld) for fld in missing_fields),
)
return {}
req = resp['request']
url = req.get('url')
res = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp"
}
tls = None
try:
tls = req['tls_handshake']
except KeyError:
# zgrab2
try:
tls = req['tls_log']['handshake_log']
except KeyError:
pass
if tls is not None:
res['service_tunnel'] = 'ssl'
try:
cert = tls['server_certificates']['certificate']['raw']
except KeyError:
pass
else:
output, info = create_ssl_cert(cert.encode(), b64encoded=True)
if info:
res.setdefault('scripts', []).append({
'id': 'ssl-cert',
'output': output,
'ssl-cert': info,
})
for cert in info:
add_cert_hostnames(cert,
hostrec.setdefault('hostnames', []))
if url:
guessed_port = None
if ':' in url.get('host', ''):
try:
guessed_port = int(url['host'].split(':', 1)[1])
except ValueError:
pass
if port is None:
if guessed_port is None:
if url.get('scheme') == 'https':
port = 443
else:
port = 80
else:
port = guessed_port
elif port != guessed_port:
utils.LOGGER.warning(
'Port %d found from the URL %s differs from the provided port '
'value %d', guessed_port, url.get('path'), port)
port = guessed_port
# Specific paths
if url.get('path').endswith('/.git/index'):
if resp.get('status_code') != 200:
return {}
if not resp.get('body', '').startswith('DIRC'):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = '%s:%d%s' % (hostrec['addr'], port, url['path'][:-5])
res['port'] = port
res.setdefault('scripts', []).append({
'id':
'http-git',
'output':
'\n %s\n Git repository found!\n' % repository,
'http-git': [{
'repository': repository,
'files-found': [".git/index"]
}],
})
return res
if url.get('path').endswith('/owa/auth/logon.aspx'):
if resp.get('status_code') != 200:
return {}
version = set(
m.group(1)
for m in _EXPR_OWA_VERSION.finditer(resp.get('body', '')))
if not version:
return {}
version = sorted(version,
key=lambda v: [int(x) for x in v.split('.')])
res['port'] = port
path = url['path'][:-15]
if len(version) > 1:
output = (
'OWA: path %s, version %s (multiple versions found!)' % (
path,
' / '.join(version),
))
else:
output = 'OWA: path %s, version %s' % (path, version[0])
res.setdefault('scripts', []).append({
'id':
'http-app',
'output':
output,
'http-app': [{
'path': path,
'application': 'OWA',
'version': version[0]
}],
})
return res
if url.get('path').endswith('/centreon/'):
if resp.get('status_code') != 200:
return {}
if not resp.get('body'):
return {}
body = resp['body']
res['port'] = port
path = url['path']
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault('scripts', []).append({
'id':
'http-app',
'output':
'Centreon: path %s%s' % (
path,
'' if version is None else (', version %s' % version),
),
'http-app': [
dict({
'path': path,
'application': 'Centreon'
}, **({} if version is None else {
'version': version
}))
],
})
return res
if url.get('path') != '/':
utils.LOGGER.warning('URL path not supported yet: %s',
url.get('path'))
return {}
elif port is None:
if req.get('tls_handshake') or req.get('tls_log'):
port = 443
else:
port = 80
res['port'] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' +
utils.nmap_decode_data(resp['status_line']) + b"\r\n")
if resp.get('headers'):
headers = resp['headers']
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get('www_authenticate'):
auths = headers.get('www_authenticate')
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode()))
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
pass
keyvals = zip(ntlm_values,
[infos.get(k) for k in ntlm_values])
output = '\n'.join("{}: {}".format(k, v)
for k, v in keyvals if v)
res.setdefault('scripts', []).append({
'id': 'http-ntlm-info',
'output': output,
'ntlm-info': infos
})
if 'DNS_Computer_Name' in infos:
add_hostname(infos['DNS_Computer_Name'], 'ntlm',
hostrec.setdefault('hostnames', []))
if any(val.lower().startswith('ntlm')
for val in req.get('headers', {}).get('authorization', [])):
return res
# the order will be incorrect!
line = '%s %s' % (resp['protocol']['name'], resp['status_line'])
http_hdrs = [{'name': '_status', 'value': line}]
output = [line]
for unk in headers.pop('unknown', []):
headers[unk['key']] = unk['value']
for hdr, values in headers.items():
hdr = hdr.replace('_', '-')
for val in values:
http_hdrs.append({'name': hdr, 'value': val})
output.append('%s: %s' % (hdr, val))
if http_hdrs:
method = req.get('method')
if method:
output.append('')
output.append('(Request type: %s)' % method)
res.setdefault('scripts', []).append({
'id': 'http-headers',
'output': '\n'.join(output),
'http-headers': http_hdrs,
})
handle_http_headers(hostrec, res, http_hdrs, path=url.get('path'))
if headers.get('server'):
banner += (b"Server: " +
utils.nmap_decode_data(headers['server'][0]) +
b"\r\n\r\n")
info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest")
if info:
add_cpe_values(hostrec, 'ports.port:%s' % port, info.pop('cpe', []))
res.update(info)
if resp.get('body'):
body = resp['body']
res.setdefault('scripts', []).append({
'id':
'http-content',
'output':
utils.nmap_encode_data(body.encode()),
})
match = _EXPR_TITLE.search(body)
if match is not None:
title = match.groups()[0]
res['scripts'].append({
'id': 'http-title',
'output': title,
'http-title': {
'title': title
},
})
script_http_ls = create_http_ls(body, url=url)
if script_http_ls is not None:
res.setdefault('scripts', []).append(script_http_ls)
service_elasticsearch = create_elasticsearch_service(body)
if service_elasticsearch:
if 'hostname' in service_elasticsearch:
add_hostname(service_elasticsearch.pop('hostname'), 'service',
hostrec.setdefault('hostnames', []))
add_cpe_values(hostrec, 'ports.port:%s' % port,
service_elasticsearch.pop('cpe', []))
res.update(service_elasticsearch)
return res
def zgrap_parser_http(data: Dict[str, Any],
hostrec: NmapHost,
port: Optional[int] = None) -> NmapPort:
"""This function handles data from `{"data": {"http": [...]}}`
records. `data` should be the content, i.e. the `[...]`. It should
consist of simple dictionary, that may contain a `"response"` key
and/or a `"redirect_response_chain"` key.
The output is a port dict (i.e., the content of the "ports" key of an
`nmap` of `view` record in IVRE), that may be empty.
"""
if not data:
return {}
# for zgrab2 results
if "result" in data:
data.update(data.pop("result"))
if "response" not in data:
utils.LOGGER.warning('Missing "response" field in zgrab HTTP result')
return {}
resp = data["response"]
needed_fields = set(["request", "status_code", "status_line"])
missing_fields = needed_fields.difference(resp)
if missing_fields:
utils.LOGGER.warning(
"Missing field%s %s in zgrab HTTP result",
"s" if len(missing_fields) > 1 else "",
", ".join(repr(fld) for fld in missing_fields),
)
return {}
req = resp["request"]
url = req.get("url")
res: NmapPort = {
"service_name": "http",
"service_method": "probed",
"state_state": "open",
"state_reason": "response",
"protocol": "tcp",
}
tls = None
try:
tls = req["tls_handshake"]
except KeyError:
# zgrab2
try:
tls = req["tls_log"]["handshake_log"]
except KeyError:
pass
if tls is not None:
res["service_tunnel"] = "ssl"
try:
cert = tls["server_certificates"]["certificate"]["raw"]
except KeyError:
pass
else:
output, info_cert = create_ssl_cert(cert.encode(), b64encoded=True)
if info_cert:
res.setdefault("scripts", []).append({
"id": "ssl-cert",
"output": output,
"ssl-cert": info_cert,
})
for cert in info_cert:
add_cert_hostnames(cert,
hostrec.setdefault("hostnames", []))
if url:
try:
_, guessed_port = utils.url2hostport("%(scheme)s://%(host)s" % url)
except ValueError:
utils.LOGGER.warning("Cannot guess port from url %r", url)
guessed_port = 80 # because reasons
else:
if port is not None and port != guessed_port:
utils.LOGGER.warning(
"Port %d found from the URL %s differs from the provided port "
"value %d",
guessed_port,
url.get("path"),
port,
)
port = guessed_port
if port is None:
port = guessed_port
# Specific paths
if url.get("path").endswith("/.git/index"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body", "").startswith("DIRC"):
return {}
# Due to an issue with ZGrab2 output, we cannot, for now,
# process the content of the file. See
# <https://github.com/zmap/zgrab2/issues/263>.
repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5])
res["port"] = port
res.setdefault("scripts", []).append({
"id":
"http-git",
"output":
"\n %s\n Git repository found!\n" % repository,
"http-git": [
{
"repository": repository,
"files-found": [".git/index"]
},
],
})
return res
if url.get("path").endswith("/owa/auth/logon.aspx"):
if resp.get("status_code") != 200:
return {}
version_set = set(
m.group(1)
for m in _EXPR_OWA_VERSION.finditer(resp.get("body", "")))
if not version_set:
return {}
version_list = sorted(version_set,
key=lambda v: [int(x) for x in v.split(".")])
res["port"] = port
path = url["path"][:-15]
if version_list:
parsed_version = EXCHANGE_BUILDS.get(version_list[0],
"unknown build number")
if len(version_list) > 1:
version_list = [
"%s (%s)" %
(vers, EXCHANGE_BUILDS.get(vers,
"unknown build number"))
for vers in version_list
]
output = "OWA: path %s, version %s (multiple versions found!)" % (
path,
" / ".join(version_list),
)
else:
output = "OWA: path %s, version %s (%s)" % (
path,
version_list[0],
parsed_version,
)
res.setdefault("scripts", []).append({
"id":
"http-app",
"output":
output,
"http-app": [{
"path": path,
"application": "OWA",
"version": version_list[0],
"parsed_version": parsed_version,
}],
})
return res
if url.get("path").endswith("/centreon/"):
if resp.get("status_code") != 200:
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
path = url["path"]
match = _EXPR_TITLE.search(body)
if match is None:
return {}
if match.groups()[0] != "Centreon - IT & Network Monitoring":
return {}
match = _EXPR_CENTREON_VERSION.search(body)
version: Optional[str]
if match is None:
version = None
else:
version = match.group(1) or match.group(2)
res.setdefault("scripts", []).append({
"id":
"http-app",
"output":
"Centreon: path %s%s" % (
path,
"" if version is None else (", version %s" % version),
),
"http-app": [
dict(
{
"path": path,
"application": "Centreon"
},
**({} if version is None else {
"version": version
}),
)
],
})
return res
if url.get("path").endswith("/.well-known/security.txt"):
if resp.get("status_code") != 200:
return {}
if not resp.get("headers"):
return {}
if not any(
ctype.split(";", 1)[0].lower() == "text/plain"
for ctype in resp["headers"].get("content_type", [])):
return {}
if not resp.get("body"):
return {}
body = resp["body"]
res["port"] = port
parsed: Dict[str, List[str]] = {}
for line in body.splitlines():
line = line.strip().split("#", 1)[0]
if not line:
continue
if ":" not in line:
utils.LOGGER.warning(
"Invalid line in security.txt file [%r]", line)
continue
key, value = line.split(":", 1)
parsed.setdefault(key.strip().lower(),
[]).append(value.strip())
res.setdefault("scripts", []).append({
"id": "http-securitytxt",
"output": body,
"http-securitytxt":
{key: " / ".join(value)
for key, value in parsed.items()},
})
return res
if url.get("path") != "/":
utils.LOGGER.warning("URL path not supported yet: %s",
url.get("path"))
return {}
elif port is None:
if req.get("tls_handshake") or req.get("tls_log"):
port = 443
else:
port = 80
res["port"] = port
# Since Zgrab does not preserve the order of the headers, we need
# to reconstruct a banner to use Nmap fingerprints
if resp.get("headers"):
headers = resp["headers"]
# Check the Authenticate header first: if we requested it with
# an Authorization header, we don't want to gather other information
if headers.get("www_authenticate"):
auths = headers.get("www_authenticate")
for auth in auths:
if ntlm._is_ntlm_message(auth):
try:
infos = ntlm.ntlm_extract_info(
utils.decode_b64(auth.split(None, 1)[1].encode()))
except (UnicodeDecodeError, TypeError, ValueError,
binascii.Error):
continue
if not infos:
continue
keyvals = zip(ntlm_values,
[infos.get(k) for k in ntlm_values])
output = "\n".join("{}: {}".format(k, v)
for k, v in keyvals if v)
res.setdefault("scripts", []).append({
"id":
"ntlm-info",
"output":
output,
"ntlm-info":
dict(infos, protocol="http"),
})
if "DNS_Computer_Name" in infos:
add_hostname(
infos["DNS_Computer_Name"],
"ntlm",
hostrec.setdefault("hostnames", []),
)
if any(val.lower().startswith("ntlm")
for val in req.get("headers", {}).get("authorization", [])):
return res
# If we have headers_raw value, let's use it. Else, let's fake it as well as we can.
http_hdrs: List[HttpHeader] = []
output_list: List[str] = []
has_raw_value = False
if resp.get("headers_raw"):
try:
banner = utils.decode_b64(resp.get("headers_raw").encode())
except Exception:
utils.LOGGER.warning(
"Cannot decode raw headers, using parsed result")
else:
output_list = [
utils.nmap_encode_data(line)
for line in re.split(b"\r?\n", banner)
]
banner_split = banner.split(b"\n")
http_hdrs = [{
"name":
"_status",
"value":
utils.nmap_encode_data(banner_split[0].strip()),
}]
http_hdrs.extend(
{
"name": utils.nmap_encode_data(hdrname).lower(),
"value": utils.nmap_encode_data(hdrval),
} for hdrname, hdrval in (m.groups() for m in (
utils.RAW_HTTP_HEADER.search(part.strip())
for part in banner_split) if m))
has_raw_value = True
if not has_raw_value: # no headers_raw or decoding failed
# The order will be incorrect!
banner = (utils.nmap_decode_data(resp["protocol"]["name"]) + b" " +
utils.nmap_decode_data(resp["status_line"]) + b"\r\n")
line = "%s %s" % (resp["protocol"]["name"], resp["status_line"])
http_hdrs = [{"name": "_status", "value": line}]
output_list = [line]
for unk in headers.pop("unknown", []):
headers[unk["key"]] = unk["value"]
for hdr, values in headers.items():
hdr = hdr.replace("_", "-")
for val in values:
http_hdrs.append({"name": hdr, "value": val})
output_list.append("%s: %s" % (hdr, val))
if headers.get("server"):
banner += (b"Server: " +
utils.nmap_decode_data(headers["server"][0]) +
b"\r\n\r\n")
if http_hdrs:
method = req.get("method")
if method:
output_list.append("")
output_list.append("(Request type: %s)" % method)
script: NmapScript = {
"id": "http-headers",
"output": "\n".join(output_list),
"http-headers": http_hdrs,
}
if has_raw_value:
script["masscan"] = {"raw": utils.encode_b64(banner).decode()}
res.setdefault("scripts", []).append(script)
handle_http_headers(hostrec, res, http_hdrs, path=url.get("path"))
info: NmapServiceMatch = utils.match_nmap_svc_fp(banner,
proto="tcp",
probe="GetRequest")
if info:
add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", []))
res.update(cast(NmapPort, info))
add_service_hostname(info, hostrec.setdefault("hostnames", []))
if resp.get("body"):
body = resp["body"]
res.setdefault("scripts", []).append({
"id":
"http-content",
"output":
utils.nmap_encode_data(body.encode()),
})
handle_http_content(hostrec, res, body.encode())
return res
def _prepare_rec(spec, ignorenets, neverignore):
# First of all, let's see if we are supposed to ignore this spec,
# and if so, do so.
if "addr" in spec and spec.get("source") not in neverignore.get(
spec["recontype"], []):
for start, stop in ignorenets.get(spec["recontype"], []):
if start <= utils.force_ip2int(spec["addr"]) <= stop:
return
# Then, let's clean up the records.
# Change Symantec's random user agents (matching SYMANTEC_UA) to
# the constant string "SymantecRandomUserAgent".
if spec["recontype"] == "HTTP_CLIENT_HEADER" and spec.get(
"source") == "USER-AGENT":
if SYMANTEC_UA.match(spec["value"]):
spec["value"] = "SymantecRandomUserAgent"
elif KASPERSKY_UA.match(spec["value"]):
spec["value"] = "KasperskyWeirdUserAgent"
else:
match = SYMANTEC_SEP_UA.match(spec["value"])
if match is not None:
spec["value"] = "%s%s" % match.groups()
# Change any Digest authorization header to remove non-constant
# information. On one hand we loose the necessary information to
# try to recover the passwords, but on the other hand we store
# specs with different challenges but the same username, realm,
# host and sensor in the same records.
elif spec["recontype"] in {
"HTTP_CLIENT_HEADER",
"HTTP_CLIENT_HEADER_SERVER",
} and spec.get("source") in {"AUTHORIZATION", "PROXY-AUTHORIZATION"}:
value = spec["value"]
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == "digest":
try:
# we only keep relevant info
spec["value"] = "%s %s" % (
authtype,
",".join(
val
for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)),
)
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_NEGOTIATE and NTLM_AUTHENTICATE
yield from _prepare_rec_ntlm(spec, "NTLM_CLIENT_FLAGS")
return
elif authtype.lower() in {"negotiate", "kerberos", "oauth"}:
spec["value"] = authtype
elif spec["recontype"] == "HTTP_SERVER_HEADER" and spec.get("source") in {
"WWW-AUTHENTICATE",
"PROXY-AUTHENTICATE",
}:
value = spec["value"]
if value:
authtype = value.split(None, 1)[0]
if authtype.lower() == "digest":
try:
# we only keep relevant info
spec["value"] = "%s %s" % (
authtype,
",".join(
val
for val in _split_digest_auth(value[6:].strip())
if DIGEST_AUTH_INFOS.match(val)),
)
except Exception:
utils.LOGGER.warning("Cannot parse digest error for %r",
spec,
exc_info=True)
elif ntlm._is_ntlm_message(value):
# NTLM_CHALLENGE
yield from _prepare_rec_ntlm(spec, "NTLM_SERVER_FLAGS")
return
elif authtype.lower() in {"negotiate", "kerberos", "oauth"}:
spec["value"] = authtype
# TCP server banners: try to normalize data
elif spec["recontype"] == "TCP_SERVER_BANNER":
newvalue = value = utils.nmap_decode_data(spec["value"])
for pattern, replace in TCP_SERVER_PATTERNS:
if pattern.search(newvalue):
newvalue = pattern.sub(replace, newvalue)
if newvalue != value:
spec["value"] = utils.nmap_encode_data(newvalue)
elif spec["recontype"] in {"TCP_CLIENT_BANNER", "TCP_HONEYPOT_HIT"}:
if spec["value"]:
data = utils.nmap_decode_data(spec["value"])
if data in scanners.TCP_PROBES:
scanner, probe = scanners.TCP_PROBES[data]
info = {
"service_name": "scanner",
"service_product": scanner,
}
if probe is not None:
info["service_extrainfo"] = "TCP probe %s" % probe
spec.setdefault("infos", {}).update(info)
else:
probe = utils.get_nmap_probes("tcp").get(data)
if probe is not None:
spec.setdefault("infos", {}).update({
"service_name":
"scanner",
"service_product":
"Nmap",
"service_extrainfo":
"TCP probe %s" % probe,
})
elif spec["recontype"] == "UDP_HONEYPOT_HIT":
data = utils.nmap_decode_data(spec["value"])
if data in scanners.UDP_PROBES:
scanner, probe = scanners.UDP_PROBES[data]
info = {
"service_name": "scanner",
"service_product": scanner,
}
if probe is not None:
info["service_extrainfo"] = "UDP probe %s" % probe
spec.setdefault("infos", {}).update(info)
else:
probe = utils.get_nmap_probes("udp").get(data)
if probe is not None:
spec.setdefault("infos", {}).update({
"service_name":
"scanner",
"service_product":
"Nmap",
"service_extrainfo":
"UDP probe %s" % probe,
})
else:
payload = utils.get_nmap_udp_payloads().get(data)
if payload is not None:
spec.setdefault("infos", {}).update({
"service_name":
"scanner",
"service_product":
"Nmap",
"service_extrainfo":
"UDP payload %s" % payload,
})
elif spec["recontype"] == "STUN_HONEYPOT_REQUEST":
spec["value"] = utils.nmap_decode_data(spec["value"])
# SSL_{CLIENT,SERVER} JA3
elif (spec["recontype"] == "SSL_CLIENT" and spec["source"]
== "ja3") or (spec["recontype"] == "SSL_SERVER"
and spec["source"].startswith("ja3-")):
value = spec["value"]
spec.setdefault("infos", {})["raw"] = value
spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
if spec["recontype"] == "SSL_SERVER":
clientvalue = spec["source"][4:]
spec["infos"].setdefault("client", {})["raw"] = clientvalue
spec["source"] = ("ja3-%s" % hashlib.new(
"md5",
clientvalue.encode(),
).hexdigest())
# SSH_{CLIENT,SERVER}_HASSH
elif spec["recontype"] in ["SSH_CLIENT_HASSH", "SSH_SERVER_HASSH"]:
value = spec["value"]
spec.setdefault("infos", {})["raw"] = value
spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
# Check DNS Blacklist answer
elif spec["recontype"] == "DNS_ANSWER":
if any((spec.get("value") or "").endswith(dnsbl)
for dnsbl in config.DNS_BLACKLIST_DOMAINS):
dnsbl_val = spec["value"]
match = DNSBL_START.search(dnsbl_val)
if match is not None:
spec["recontype"] = "DNS_BLACKLIST"
spec["value"] = spec.get("addr")
spec["source"] = "%s-%s" % (dnsbl_val[match.end():],
spec["source"])
addr = match.group()
# IPv4
if addr.count(".") == 4:
spec["addr"] = ".".join(addr.split(".")[3::-1])
# IPv6
else:
spec["addr"] = utils.int2ip6(
int(addr.replace(".", "")[::-1], 16))
yield spec