def test(self, v4=True, v6=True):
start = datetime.now()
for srvname, addr, res in self.do_test(v4=v4, v6=v6):
if not res:
continue
if (len(res) == 1 and res[0].rtype == 'SOA'):
# SOA only: transfer failed
continue
LOGGER.info('AXFR success for %r on %r', self.domain, addr)
line_fmt = "| %%-%ds %%-%ds %%s" % (
max(len(r.name) for r in res),
max(len(r.rtype) for r in res),
)
yield {
"addr":
addr,
"hostnames": [{
"name": srvname,
"type": "user",
"domains": list(get_domains(srvname))
}],
"schema_version":
SCHEMA_VERSION,
"starttime":
start,
"endtime":
datetime.now(),
"ports": [{
"port":
53,
"protocol":
"tcp",
"service_name":
"domain",
"state_state":
"open",
"scripts": [{
"id":
"dns-zone-transfer",
"output":
'\nDomain: %s\n%s\n\\\n' % (
self.domain,
'\n'.join(line_fmt % (r.name, r.rtype, r.data)
for r in res),
),
"dns-zone-transfer": [{
"domain":
self.domain,
"records": [{
"name": r.name,
"ttl": r.ttl,
"class": r.rclass,
"type": r.rtype,
"data": r.data
} for r in res]
}]
}]
}],
}
start = datetime.now()
def _getinfos_dns(spec):
"""Extract domain names in an handy-to-index-and-query form."""
infos = {}
fullinfos = {}
fields = {'domain': 'value', 'domaintarget': 'targetval'}
for field in fields:
try:
if fields[field] not in spec:
continue
infos[field] = []
fullinfos[field] = []
for domain in utils.get_domains(
spec.get('full' + fields[field], spec[fields[field]])):
infos[field].append(domain[:utils.MAXVALLEN])
if len(domain) > utils.MAXVALLEN:
fullinfos[field].append(domain)
if not infos[field]:
del infos[field]
if not fullinfos[field]:
del fullinfos[field]
except Exception:
pass
res = {}
if infos:
res['infos'] = infos
if fullinfos:
res['fullinfos'] = fullinfos
return res
def _getinfos_dns(spec):
"""Extract domain names in an handy-to-index-and-query form."""
infos = {}
fullinfos = {}
fields = {'domain': 'value', 'domaintarget': 'targetval'}
for field in fields:
try:
if fields[field] not in spec:
continue
infos[field] = []
fullinfos[field] = []
for domain in utils.get_domains(
spec.get('full' + fields[field],
spec[fields[field]])):
infos[field].append(domain[:utils.MAXVALLEN])
if len(domain) > utils.MAXVALLEN:
fullinfos[field].append(domain)
if not infos[field]:
del infos[field]
if not fullinfos[field]:
del fullinfos[field]
except Exception:
pass
res = {}
if infos:
res['infos'] = infos
if fullinfos:
res['fullinfos'] = fullinfos
return res
def test(self,
v4: bool = True,
v6: bool = True) -> Generator[NmapHost, None, None]:
yield from super().test(v4=v4, v6=v6)
for srvname, addr, _ in self.results:
srvname = srvname.rstrip(".")
yield {
"addr":
addr,
"hostnames": [{
"name": srvname,
"type": "user",
"domains": list(get_domains(srvname)),
}],
"schema_version":
SCHEMA_VERSION,
"starttime":
self.start,
"endtime":
self.stop,
"ports": [{
"port":
53,
"protocol":
"udp",
"service_name":
"domain",
"state_state":
"open",
"scripts": [
{
"id":
"dns-domains",
"output":
"Server is authoritative for %s" % self.domain,
"dns-domains": [{
"domain":
self.domain,
"parents":
list(get_domains(self.domain)),
}],
},
],
}],
}
def _getinfos_dns_blacklist(spec):
"""Extract and properly format DNSBL records."""
infos = {}
try:
if "source" in spec:
infos["domain"] = []
for domain in utils.get_domains(spec["source"].split("-")[-4]):
infos["domain"].append(domain)
if not infos["domain"]:
del infos["domain"]
except Exception:
pass
res = {}
if infos:
res["infos"] = infos
return res
def _getinfos_dns_blacklist(spec):
"""Extract and properly format DNSBL records."""
infos = {}
try:
if 'source' in spec:
infos['domain'] = []
for domain in utils.get_domains(spec['source'].split('-')[-4]):
infos['domain'].append(domain)
if not infos['domain']:
del infos['domain']
except Exception:
pass
res = {}
if infos:
res['infos'] = infos
return res
def _getinfos_dns_blacklist(spec):
"""Extract and properly format DNSBL records."""
infos = {}
try:
if 'source' in spec:
infos['domain'] = []
for domain in utils.get_domains(spec['source'].split('-')[-4]):
infos['domain'].append(domain)
if not infos['domain']:
del infos['domain']
except Exception:
pass
res = {}
if infos:
res['infos'] = infos
return res
def _getinfos_dns(spec):
"""Extract domain names in an handy-to-index-and-query form."""
infos = {}
fields = {"domain": "value", "domaintarget": "targetval"}
for field, value in fields.items():
try:
if value not in spec:
continue
infos[field] = []
for domain in utils.get_domains(spec[value]):
infos[field].append(domain)
if not infos[field]:
del infos[field]
except Exception:
pass
res = {}
if infos:
res["infos"] = infos
return res
def merge_dns_domains_scripts(curscript, script, script_id):
domains = {
res["domain"]: (res["parents"] if "parents" in res else list(
get_domains(res["domain"])))
for scr in [script, curscript] for res in scr.get(script_id, [])
if "domain" in res
}
domains_order = sorted(domains, key=lambda v: v.strip().split(".")[::-1])
if len(domains_order) == 1:
output = "Server is authoritative for %s" % domains_order[0]
else:
output = "Server is authoritative for:\n%s" % "\n".join(
" %s" % dom for dom in domains_order)
curscript["output"] = output
curscript[script_id] = [{
"domain": dom,
"parents": domains[dom]
} for dom in domains_order]
return curscript
def startElement(self, name, attrs):
if name == "nmaprun":
if self._curscan is not None:
sys.stderr.write("WARNING, self._curscan should be None at " "this point(got %r)\n" % self._curscan)
self._curscan = dict(attrs)
self._curscan["_id"] = self._filehash
elif name == "scaninfo" and self._curscan is not None:
self._addscaninfo(dict(attrs))
elif name == "host":
if self._curhost is not None:
sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost)
self._curhost = {"schema_version": 1}
if self._curscan:
self._curhost["scanid"] = self._curscan["_id"]
for attr in attrs.keys():
self._curhost[attr] = attrs[attr]
for field in ["starttime", "endtime"]:
if field in self._curhost:
self._curhost[field] = datetime.datetime.utcfromtimestamp(int(self._curhost[field]))
elif name == "address" and self._curhost is not None:
if attrs["addrtype"] != "ipv4":
if "addresses" not in self._curhost:
self._curhost["addresses"] = {attrs["addrtype"]: [attrs["addr"]]}
elif attrs["addrtype"] not in self._curhost:
self._curhost["addresses"].update({attrs["addrtype"]: [attrs["addr"]]})
else:
addresses = self._curhost["addresses"][attrs["addrtype"]]
addresses.append(attrs["addr"])
self._curhost["addresses"][attrs["addrtype"]] = addresses
else:
try:
self._curhost["addr"] = utils.ip2int(attrs["addr"])
except utils.socket.error:
self._curhost["addr"] = attrs["addr"]
elif name == "hostnames":
if self._curhostnames is not None:
sys.stderr.write(
"WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames
)
self._curhostnames = []
elif name == "hostname":
if self._curhostnames is None:
sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n")
self._curhostnames = []
hostname = dict(attrs)
if "name" in attrs:
hostname["domains"] = list(utils.get_domains(attrs["name"]))
self._curhostnames.append(hostname)
elif name == "status" and self._curhost is not None:
self._curhost["state"] = attrs["state"]
if "reason" in attrs:
self._curhost["state_reason"] = attrs["reason"]
if "reason_ttl" in attrs:
self._curhost["state_reason_ttl"] = int(attrs["reason_ttl"])
elif name == "extraports":
if self._curextraports is not None:
sys.stderr.write(
"WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports
)
self._curextraports = {attrs["state"]: [int(attrs["count"]), {}]}
elif name == "extrareasons" and self._curextraports is not None:
self._curextraports[self._curextraports.keys()[0]][1][attrs["reason"]] = int(attrs["count"])
elif name == "port":
if self._curport is not None:
sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport)
self._curport = {"protocol": attrs["protocol"], "port": int(attrs["portid"])}
elif name == "state" and self._curport is not None:
for attr in attrs.keys():
self._curport["state_%s" % attr] = attrs[attr]
for field in ["state_reason_ttl"]:
if field in self._curport:
self._curport[field] = int(self._curport[field])
for field in ["state_reason_ip"]:
if field in self._curport:
try:
self._curport[field] = utils.ip2int(self._curport[field])
except utils.socket.error:
pass
elif name == "service" and self._curport is not None:
if attrs.get("method") == "table":
# discard information from nmap-services
return
for attr in attrs.keys():
self._curport["service_%s" % attr] = attrs[attr]
for field in ["service_conf", "service_rpcnum", "service_lowver", "service_highver"]:
if field in self._curport:
self._curport[field] = int(self._curport[field])
elif name == "script":
if self._curscript is not None:
sys.stderr.write(
"WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript
)
self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys())
elif name in ["table", "elem"]:
if self._curscript.get("id") in IGNORE_TABLE_ELEMS:
return
if name == "elem":
# start recording characters
if self._curdata is not None:
sys.stderr.write(
"WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata
)
self._curdata = ""
if "key" in attrs:
key = attrs["key"].replace(".", "_")
obj = {key: {}}
else:
key = None
obj = []
if not self._curtablepath:
if not self._curtable:
self._curtable = obj
elif key is not None:
self._curtable.update(obj)
if key is None:
key = len(self._curtable)
self._curtablepath.append(key)
return
lastlevel = self._curtable
for k in self._curtablepath[:-1]:
lastlevel = lastlevel[k]
k = self._curtablepath[-1]
if type(k) is int:
if k < len(lastlevel):
if key is not None:
lastlevel[k].update(obj)
else:
lastlevel.append(obj)
if key is None:
key = len(lastlevel[k])
else:
if key is None:
if lastlevel[k]:
key = len(lastlevel[k])
else:
key = 0
lastlevel[k] = obj
else:
lastlevel[k].update(obj)
self._curtablepath.append(key)
elif name == "os":
self._curhost["os"] = {}
elif name == "portused" and "os" in self._curhost:
self._curhost["os"]["portused"] = {
"port": "%s_%s" % (attrs["proto"], attrs["portid"]),
"state": attrs["state"],
}
elif name in ["osclass", "osmatch"] and "os" in self._curhost:
if name not in self._curhost["os"]:
self._curhost["os"][name] = [dict(attrs)]
else:
self._curhost["os"][name].append(dict(attrs))
elif name == "osfingerprint" and "os" in self._curhost:
self._curhost["os"]["fingerprint"] = attrs["fingerprint"]
elif name == "trace":
if self._curtrace is not None:
sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace)
if "proto" not in attrs:
self._curtrace = {"protocol": None}
elif attrs["proto"] in ["tcp", "udp"]:
self._curtrace = {"protocol": attrs["proto"], "port": int(attrs["port"])}
else:
self._curtrace = {"protocol": attrs["proto"]}
self._curtrace["hops"] = []
elif name == "hop" and self._curtrace is not None:
attrsdict = dict(attrs)
try:
attrsdict["ipaddr"] = utils.ip2int(attrs["ipaddr"])
except utils.socket.error:
pass
try:
attrsdict["rtt"] = float(attrs["rtt"])
except ValueError:
pass
try:
attrsdict["ttl"] = int(attrs["ttl"])
except ValueError:
pass
if "host" in attrsdict:
attrsdict["domains"] = list(utils.get_domains(attrsdict["host"]))
self._curtrace["hops"].append(attrsdict)
elif name == "cpe":
# start recording
self._curdata = ""
def startElement(self, name, attrs):
if name == 'nmaprun':
if self._curscan is not None:
sys.stderr.write("WARNING, self._curscan should be None at "
"this point(got %r)\n" % self._curscan)
self._curscan = dict(attrs)
self._curscan['_id'] = self._filehash
elif name == 'scaninfo' and self._curscan is not None:
self._addscaninfo(dict(attrs))
elif name == 'host':
if self._curhost is not None:
sys.stderr.write("WARNING, self._curhost should be None at "
"this point (got %r)\n" % self._curhost)
self._curhost = {"schema_version": 1}
if self._curscan:
self._curhost['scanid'] = self._curscan['_id']
for attr in attrs.keys():
self._curhost[attr] = attrs[attr]
for field in ['starttime', 'endtime']:
if field in self._curhost:
self._curhost[field] = datetime.datetime.utcfromtimestamp(
int(self._curhost[field])
)
elif name == 'address' and self._curhost is not None:
if attrs['addrtype'] != 'ipv4':
if 'addresses' not in self._curhost:
self._curhost['addresses'] = {
attrs['addrtype']: [attrs['addr']]
}
elif attrs['addrtype'] not in self._curhost:
self._curhost['addresses'].update({
attrs['addrtype']: [attrs['addr']]
})
else:
addresses = self._curhost['addresses'][attrs['addrtype']]
addresses.append(attrs['addr'])
self._curhost['addresses'][attrs['addrtype']] = addresses
else:
try:
self._curhost['addr'] = utils.ip2int(attrs['addr'])
except utils.socket.error:
self._curhost['addr'] = attrs['addr']
elif name == 'hostnames':
if self._curhostnames is not None:
sys.stderr.write("WARNING, self._curhostnames should be None "
"at this point "
"(got %r)\n" % self._curhostnames)
self._curhostnames = []
elif name == 'hostname':
if self._curhostnames is None:
sys.stderr.write("WARNING, self._curhostnames should NOT be "
"None at this point\n")
self._curhostnames = []
hostname = dict(attrs)
if 'name' in attrs:
hostname['domains'] = list(utils.get_domains(attrs['name']))
self._curhostnames.append(hostname)
elif name == 'status' and self._curhost is not None:
self._curhost['state'] = attrs['state']
if 'reason' in attrs:
self._curhost['state_reason'] = attrs['reason']
if 'reason_ttl' in attrs:
self._curhost['state_reason_ttl'] = int(attrs['reason_ttl'])
elif name == 'extraports':
if self._curextraports is not None:
sys.stderr.write("WARNING, self._curextraports should be None"
" at this point "
"(got %r)\n" % self._curextraports)
self._curextraports = {attrs['state']: [int(attrs['count']), {}]}
elif name == 'extrareasons' and self._curextraports is not None:
self._curextraports[self._curextraports.keys()[0]][1][
attrs['reason']] = int(attrs['count'])
elif name == 'port':
if self._curport is not None:
sys.stderr.write("WARNING, self._curport should be None at "
"this point (got %r)\n" % self._curport)
self._curport = {'protocol': attrs['protocol'],
'port': int(attrs['portid'])}
elif name == 'state' and self._curport is not None:
for attr in attrs.keys():
self._curport['state_%s' % attr] = attrs[attr]
for field in ['state_reason_ttl']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
for field in ['state_reason_ip']:
if field in self._curport:
try:
self._curport[field] = utils.ip2int(
self._curport[field])
except utils.socket.error:
pass
elif name == 'service' and self._curport is not None:
for attr in attrs.keys():
self._curport['service_%s' % attr] = attrs[attr]
for field in ['service_conf', 'service_rpcnum',
'service_lowver', 'service_highver']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
elif name == 'script':
if self._curscript is not None:
sys.stderr.write("WARNING, self._curscript should be None "
"at this point (got %r)\n" % self._curscript)
self._curscript = dict([attr, attrs[attr]]
for attr in attrs.keys())
elif name in ['table', 'elem']:
if self._curscript.get('id') in IGNORE_TABLE_ELEMS:
return
if name == 'elem':
# start recording characters
if self._curdata is not None:
sys.stderr.write("WARNING, self._curdata should be None"
" at this point "
"(got %r)\n" % self._curdata)
self._curdata = ''
if 'key' in attrs:
key = attrs['key'].replace('.', '_')
obj = {key: {}}
else:
key = None
obj = []
if not self._curtablepath:
if not self._curtable:
self._curtable = obj
elif key is not None:
self._curtable.update(obj)
if key is None:
key = len(self._curtable)
self._curtablepath.append(key)
return
lastlevel = self._curtable
for k in self._curtablepath[:-1]:
lastlevel = lastlevel[k]
k = self._curtablepath[-1]
if type(k) is int:
if k < len(lastlevel):
if key is not None:
lastlevel[k].update(obj)
else:
lastlevel.append(obj)
if key is None:
key = len(lastlevel[k])
else:
if key is None:
if lastlevel[k]:
key = len(lastlevel[k])
else:
key = 0
lastlevel[k] = obj
else:
lastlevel[k].update(obj)
self._curtablepath.append(key)
elif name == 'os':
self._curhost['os'] = {}
elif name == 'portused' and 'os' in self._curhost:
self._curhost['os']['portused'] = {
'port': '%s_%s' % (attrs['proto'], attrs['portid']),
'state': attrs['state'],
}
elif name in ['osclass', 'osmatch'] and 'os' in self._curhost:
if name not in self._curhost['os']:
self._curhost['os'][name] = [dict(attrs)]
else:
self._curhost['os'][name].append(dict(attrs))
elif name == 'osfingerprint' and 'os' in self._curhost:
self._curhost['os']['fingerprint'] = attrs['fingerprint']
elif name == 'trace':
if self._curtrace is not None:
sys.stderr.write("WARNING, self._curtrace should be None "
"at this point (got %r)\n" % self._curtrace)
if 'proto' not in attrs:
self._curtrace = {'protocol': None}
elif attrs['proto'] in ['tcp', 'udp']:
self._curtrace = {'protocol': attrs['proto'],
'port': int(attrs['port'])}
else:
self._curtrace = {'protocol': attrs['proto']}
self._curtrace['hops'] = []
elif name == 'hop' and self._curtrace is not None:
attrsdict = dict(attrs)
try:
attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr'])
except utils.socket.error:
pass
try:
attrsdict['rtt'] = float(attrs['rtt'])
except ValueError:
pass
try:
attrsdict['ttl'] = int(attrs['ttl'])
except ValueError:
pass
if 'host' in attrsdict:
attrsdict['domains'] = list(
utils.get_domains(attrsdict['host']))
self._curtrace['hops'].append(attrsdict)
elif name == 'cpe':
# start recording
self._curdata = ''
def startElement(self, name, attrs):
if name == 'nmaprun':
if self._curscan is not None:
sys.stderr.write("WARNING, self._curscan should be None at "
"this point (got %r)\n" % self._curscan)
self._curscan = dict(attrs)
self.scanner = self._curscan.get("scanner", self.scanner)
if self.scanner == "masscan":
# We need to force "merge" mode due to the nature of
# Masscan results
self.merge = True
self._curscan['_id'] = self._filehash
elif name == 'scaninfo' and self._curscan is not None:
self._addscaninfo(dict(attrs))
elif name == 'host':
if self._curhost is not None:
sys.stderr.write("WARNING, self._curhost should be None at "
"this point (got %r)\n" % self._curhost)
self._curhost = {"schema_version": SCHEMA_VERSION}
if self._curscan:
self._curhost['scanid'] = self._curscan['_id']
for attr in attrs.keys():
self._curhost[attr] = attrs[attr]
for field in ['starttime', 'endtime']:
if field in self._curhost:
self._curhost[field] = datetime.datetime.utcfromtimestamp(
int(self._curhost[field])
)
if 'starttime' not in self._curhost and 'endtime' in self._curhost:
# Masscan
self._curhost['starttime'] = self._curhost['endtime']
elif name == 'address' and self._curhost is not None:
if attrs['addrtype'] != 'ipv4':
self._curhost.setdefault(
'addresses', {}).setdefault(
attrs['addrtype'], []).append(attrs['addr'])
else:
try:
self._curhost['addr'] = utils.ip2int(attrs['addr'])
except utils.socket.error:
self._curhost['addr'] = attrs['addr']
elif name == 'hostnames':
if self._curhostnames is not None:
sys.stderr.write("WARNING, self._curhostnames should be None "
"at this point "
"(got %r)\n" % self._curhostnames)
self._curhostnames = []
elif name == 'hostname':
if self._curhostnames is None:
sys.stderr.write("WARNING, self._curhostnames should NOT be "
"None at this point\n")
self._curhostnames = []
hostname = dict(attrs)
if 'name' in attrs:
hostname['domains'] = list(utils.get_domains(attrs['name']))
self._curhostnames.append(hostname)
elif name == 'status' and self._curhost is not None:
self._curhost['state'] = attrs['state']
if 'reason' in attrs:
self._curhost['state_reason'] = attrs['reason']
if 'reason_ttl' in attrs:
self._curhost['state_reason_ttl'] = int(attrs['reason_ttl'])
elif name == 'extraports':
if self._curextraports is not None:
sys.stderr.write("WARNING, self._curextraports should be None"
" at this point "
"(got %r)\n" % self._curextraports)
self._curextraports = {
attrs['state']: {"total": int(attrs['count']), "reasons": {}},
}
elif name == 'extrareasons' and self._curextraports is not None:
self._curextraports[next(iter(self._curextraports))]["reasons"][
attrs['reason']] = int(attrs['count'])
elif name == 'port':
if self._curport is not None:
sys.stderr.write("WARNING, self._curport should be None at "
"this point (got %r)\n" % self._curport)
self._curport = {'protocol': attrs['protocol'],
'port': int(attrs['portid'])}
elif name == 'state' and self._curport is not None:
for attr in attrs.keys():
self._curport['state_%s' % attr] = attrs[attr]
for field in ['state_reason_ttl']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
for field in ['state_reason_ip']:
if field in self._curport:
try:
self._curport[field] = utils.ip2int(
self._curport[field])
except utils.socket.error:
pass
elif name == 'service' and self._curport is not None:
if attrs.get("method") == "table":
# discard information from nmap-services
return
if self.scanner == "masscan":
# create fake scripts from masscan "service" tags
self._curport.setdefault('scripts', []).append({
"id": MASSCAN_SERVICES_NMAP_SCRIPTS.get(attrs['name'],
attrs['name']),
"output": MASSCAN_ENCODING.sub(
_masscan_decode,
attrs["banner"],
)
})
# get service name
service = MASSCAN_SERVICES_NMAP_SERVICES.get(attrs['name'])
if service is not None:
self._curport['service_name'] = service
return
for attr in attrs.keys():
self._curport['service_%s' % attr] = attrs[attr]
for field in ['service_conf', 'service_rpcnum',
'service_lowver', 'service_highver']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
elif name == 'script':
if self._curscript is not None:
sys.stderr.write("WARNING, self._curscript should be None "
"at this point (got %r)\n" % self._curscript)
self._curscript = dict([attr, attrs[attr]]
for attr in attrs.keys())
elif name in ['table', 'elem']:
if self._curscript.get('id') in IGNORE_TABLE_ELEMS:
return
if name == 'elem':
# start recording characters
if self._curdata is not None:
sys.stderr.write("WARNING, self._curdata should be None"
" at this point "
"(got %r)\n" % self._curdata)
self._curdata = ''
if 'key' in attrs:
key = attrs['key'].replace('.', '_')
obj = {key: {}}
else:
key = None
obj = []
if not self._curtablepath:
if not self._curtable:
self._curtable = obj
elif key is not None:
self._curtable.update(obj)
if key is None:
key = len(self._curtable)
self._curtablepath.append(key)
return
lastlevel = self._curtable
for k in self._curtablepath[:-1]:
lastlevel = lastlevel[k]
k = self._curtablepath[-1]
if type(k) is int:
if k < len(lastlevel):
if key is not None:
lastlevel[k].update(obj)
else:
lastlevel.append(obj)
if key is None:
key = len(lastlevel[k])
else:
if key is None:
if lastlevel[k]:
key = len(lastlevel[k])
else:
key = 0
lastlevel[k] = obj
else:
lastlevel[k].update(obj)
self._curtablepath.append(key)
elif name == 'os':
self._curhost['os'] = {}
elif name == 'portused' and 'os' in self._curhost:
self._curhost['os']['portused'] = {
'port': '%s_%s' % (attrs['proto'], attrs['portid']),
'state': attrs['state'],
}
elif name in ['osclass', 'osmatch'] and 'os' in self._curhost:
self._curhost['os'].setdefault(name, []).append(dict(attrs))
elif name == 'osfingerprint' and 'os' in self._curhost:
self._curhost['os']['fingerprint'] = attrs['fingerprint']
elif name == 'trace':
if self._curtrace is not None:
sys.stderr.write("WARNING, self._curtrace should be None "
"at this point (got %r)\n" % self._curtrace)
if 'proto' not in attrs:
self._curtrace = {'protocol': None}
elif attrs['proto'] in ['tcp', 'udp']:
self._curtrace = {'protocol': attrs['proto'],
'port': int(attrs['port'])}
else:
self._curtrace = {'protocol': attrs['proto']}
self._curtrace['hops'] = []
elif name == 'hop' and self._curtrace is not None:
attrsdict = dict(attrs)
try:
attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr'])
except utils.socket.error:
pass
try:
attrsdict['rtt'] = float(attrs['rtt'])
except ValueError:
pass
try:
attrsdict['ttl'] = int(attrs['ttl'])
except ValueError:
pass
if 'host' in attrsdict:
attrsdict['domains'] = list(
utils.get_domains(attrsdict['host']))
self._curtrace['hops'].append(attrsdict)
elif name == 'cpe':
# start recording
self._curdata = ''
def test(self, v4=True, v6=True):
yield from super().test(v4=v4, v6=v6)
for srvname, addr, res in self.results:
srvname = srvname.rstrip(".")
res = [literal_eval(r) for r in sorted(res)]
if not res:
output = "Domain %s has no TLS-RPT configuration" % self.domain
structured = {
"domain": self.domain,
"warnings": ["Domain has no TLS-RPT configuration"],
}
elif len(res) > 1:
output = ("Domain %s has more than one TLS-RPT configuration" %
self.domain)
structured = {
"domain": self.domain,
"value": " / ".join(res),
"warnings":
["Domain has more than one TLS-RPT configuration"],
}
else:
value = res[0]
structured = {
"domain": self.domain,
"value": value,
}
warnings = []
if value.startswith("v=TLSRPTv1;"):
if not value[11:].startswith("rua="):
warnings.append(
"TLS-RPT configuration should contain 'rua=' after 'v=TLSRPTv1;'"
)
else:
warnings.append(
"TLS-RPT configuration should start with 'v=TLSRPTv1;'"
)
if not (value.startswith("rua=") or ";rua=" in value):
warnings.append(
"TLS-RPT configuration should contain 'rua=' after 'v=TLSRPTv1;'"
)
if "rua=" in value:
ruas = value.split("rua=", 1)[1]
for rua_val in ruas.split(","):
if rua_val.startswith("https://"):
if HTTPS_REGEXP.search(rua_val[8:]) is None:
warnings.append(
"TLS-RPT contains an invalid HTTPS URL: %r"
% rua_val)
elif rua_val.startswith("mailto:"):
if MAIL_REGEXP.search(rua_val[7:]) is None:
warnings.append(
"TLS-RPT contains an invalid e-mail URL: %r"
% rua_val)
else:
warnings.append(
"TLS-RPT contains an invalid URL: %r" %
rua_val)
else:
warnings.append(
"TLS-RPT does not contain an rua entry: %r" % value)
if warnings:
structured["warnings"] = warnings
output = (
"Domain %s has a TLS-RPT configuration with warnings:\n%s"
% (self.domain, "\n".join(warnings)))
else:
output = "Domain %s has a valid TLS-RPT configuration" % self.domain
yield {
"addr":
addr,
"hostnames": [{
"name": srvname,
"type": "user",
"domains": list(get_domains(srvname)),
}],
"schema_version":
SCHEMA_VERSION,
"starttime":
self.start,
"endtime":
self.stop,
"ports": [{
"port":
53,
"protocol":
"udp",
"service_name":
"domain",
"state_state":
"open",
"scripts": [
{
"id": "dns-tls-rpt",
"output": output,
"dns-tls-rpt": [structured],
},
],
}],
}
def test(self, v4=True, v6=True):
self.start = datetime.now()
results = {}
self.results = list(self.do_test(v4=v4, v6=v6))
for srvname, addr, res in self.results:
srvname = srvname.rstrip(".")
results.setdefault(res, {}).setdefault(addr, []).append(srvname)
if len(results) < 1:
return
self.stop = datetime.now()
good_value = max(results, key=lambda val: len(results[val]))
good_value_repr = "\n".join(" %r" % r for r in sorted(good_value))
good_value_sorted = sorted(good_value)
for val, servers in results.items():
if val == good_value:
continue
for addr, names in servers.items():
yield {
"addr":
addr,
"hostnames": [{
"name": name,
"type": "user",
"domains": list(get_domains(name)),
} for name in names],
"schema_version":
SCHEMA_VERSION,
"starttime":
self.start,
"endtime":
self.stop,
"ports": [{
"port":
53,
"protocol":
"udp",
"service_name":
"domain",
"state_state":
"open",
"scripts": [
{
"id":
"dns-check-consistency",
"output":
"DNS inconsistency\n\n%s (%s)\nThis server:\n%s\nMost common answer:\n%s"
% (
self.name,
self.rtype,
"\n".join(" %r" % r for r in sorted(val)),
good_value_repr,
),
"dns-check-consistency": [{
"domain":
self.domain,
"name":
self.name,
"rtype":
self.rtype,
"value":
sorted(val),
"reference_value":
good_value_sorted,
}],
},
],
}],
}
def test(self, v4=True, v6=True):
start = datetime.now()
for srvname, addr, res in self.do_test(v4=v4, v6=v6):
srvname = srvname.rstrip(".")
if not res:
continue
if len(res) == 1 and res[0].rtype == "SOA":
# SOA only: transfer failed
continue
LOGGER.info("AXFR success for %r on %r", self.domain, addr)
line_fmt = "| %%-%ds %%-%ds %%s" % (
max(len(r.name) for r in res),
max(len(r.rtype) for r in res),
)
yield {
"addr":
addr,
"hostnames": [{
"name": srvname,
"type": "user",
"domains": list(get_domains(srvname)),
}],
"schema_version":
SCHEMA_VERSION,
"starttime":
start,
"endtime":
datetime.now(),
"ports": [
{
"port":
53,
"protocol":
"tcp",
"service_name":
"domain",
"state_state":
"open",
"scripts": [
{
"id":
"dns-zone-transfer",
"output":
"\nDomain: %s\n%s\n\\\n" % (
self.domain,
"\n".join(line_fmt %
(r.name, r.rtype, r.data)
for r in res),
),
"dns-zone-transfer": [{
"domain":
self.domain,
"records": [{
"name": r.name,
"ttl": r.ttl,
"class": r.rclass,
"type": r.rtype,
"data": r.data,
} for r in res],
}],
},
],
},
],
}
hosts = {}
for r in res:
if r.rclass != "IN":
continue
if r.rtype in ["A", "AAAA"]:
name = r.name.rstrip(".")
hosts.setdefault(r.data, set()).add((r.rtype, name))
for host, records in hosts.items():
yield {
"addr":
host,
"hostnames": [{
"name": rec[1],
"type": rec[0],
"domains": list(get_domains(rec[1])),
} for rec in records],
"schema_version":
SCHEMA_VERSION,
"starttime":
start,
"endtime":
datetime.now(),
}
start = datetime.now()
def startElement(self, name, attrs):
if name == 'nmaprun':
if self._curscan is not None:
sys.stderr.write("WARNING, self._curscan should be None at "
"this point (got %r)\n" % self._curscan)
self._curscan = dict(attrs)
self.scanner = self._curscan.get("scanner", self.scanner)
if self.scanner == "masscan":
# We need to force "merge" mode due to the nature of
# Masscan results
self.merge = True
self._curscan['_id'] = self._filehash
elif name == 'scaninfo' and self._curscan is not None:
self._addscaninfo(dict(attrs))
elif name == 'host':
if self._curhost is not None:
sys.stderr.write("WARNING, self._curhost should be None at "
"this point (got %r)\n" % self._curhost)
self._curhost = {"schema_version": SCHEMA_VERSION}
if self._curscan:
self._curhost['scanid'] = self._curscan['_id']
for attr in attrs.keys():
self._curhost[attr] = attrs[attr]
for field in ['starttime', 'endtime']:
if field in self._curhost:
self._curhost[field] = datetime.datetime.utcfromtimestamp(
int(self._curhost[field]))
if 'starttime' not in self._curhost and 'endtime' in self._curhost:
# Masscan
self._curhost['starttime'] = self._curhost['endtime']
elif name == 'address' and self._curhost is not None:
if attrs['addrtype'] != 'ipv4':
self._curhost.setdefault('addresses', {}).setdefault(
attrs['addrtype'], []).append(attrs['addr'])
else:
try:
self._curhost['addr'] = utils.ip2int(attrs['addr'])
except utils.socket.error:
self._curhost['addr'] = attrs['addr']
elif name == 'hostnames':
if self._curhostnames is not None:
sys.stderr.write("WARNING, self._curhostnames should be None "
"at this point "
"(got %r)\n" % self._curhostnames)
self._curhostnames = []
elif name == 'hostname':
if self._curhostnames is None:
sys.stderr.write("WARNING, self._curhostnames should NOT be "
"None at this point\n")
self._curhostnames = []
hostname = dict(attrs)
if 'name' in attrs:
hostname['domains'] = list(utils.get_domains(attrs['name']))
self._curhostnames.append(hostname)
elif name == 'status' and self._curhost is not None:
self._curhost['state'] = attrs['state']
if 'reason' in attrs:
self._curhost['state_reason'] = attrs['reason']
if 'reason_ttl' in attrs:
self._curhost['state_reason_ttl'] = int(attrs['reason_ttl'])
elif name == 'extraports':
if self._curextraports is not None:
sys.stderr.write("WARNING, self._curextraports should be None"
" at this point "
"(got %r)\n" % self._curextraports)
self._curextraports = {
attrs['state']: {
"total": int(attrs['count']),
"reasons": {}
},
}
elif name == 'extrareasons' and self._curextraports is not None:
self._curextraports[next(iter(
self._curextraports))]["reasons"][attrs['reason']] = int(
attrs['count'])
elif name == 'port':
if self._curport is not None:
sys.stderr.write("WARNING, self._curport should be None at "
"this point (got %r)\n" % self._curport)
self._curport = {
'protocol': attrs['protocol'],
'port': int(attrs['portid'])
}
elif name == 'state' and self._curport is not None:
for attr in attrs.keys():
self._curport['state_%s' % attr] = attrs[attr]
for field in ['state_reason_ttl']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
for field in ['state_reason_ip']:
if field in self._curport:
try:
self._curport[field] = utils.ip2int(
self._curport[field])
except utils.socket.error:
pass
elif name == 'service' and self._curport is not None:
if attrs.get("method") == "table":
# discard information from nmap-services
return
if self.scanner == "masscan":
# create fake scripts from masscan "service" tags
self._curport.setdefault('scripts', []).append({
"id":
MASSCAN_SERVICES_NMAP_SCRIPTS.get(attrs['name'],
attrs['name']),
"output":
MASSCAN_ENCODING.sub(
_masscan_decode,
attrs["banner"],
)
})
# get service name
service = MASSCAN_SERVICES_NMAP_SERVICES.get(attrs['name'])
if service is not None:
self._curport['service_name'] = service
return
for attr in attrs.keys():
self._curport['service_%s' % attr] = attrs[attr]
for field in [
'service_conf', 'service_rpcnum', 'service_lowver',
'service_highver'
]:
if field in self._curport:
self._curport[field] = int(self._curport[field])
elif name == 'script':
if self._curscript is not None:
sys.stderr.write("WARNING, self._curscript should be None "
"at this point (got %r)\n" % self._curscript)
self._curscript = dict([attr, attrs[attr]]
for attr in attrs.keys())
elif name in ['table', 'elem']:
if self._curscript.get('id') in IGNORE_TABLE_ELEMS:
return
if name == 'elem':
# start recording characters
if self._curdata is not None:
sys.stderr.write("WARNING, self._curdata should be None"
" at this point "
"(got %r)\n" % self._curdata)
self._curdata = ''
if 'key' in attrs:
key = attrs['key'].replace('.', '_')
obj = {key: {}}
else:
key = None
obj = []
if not self._curtablepath:
if not self._curtable:
self._curtable = obj
elif key is not None:
self._curtable.update(obj)
if key is None:
key = len(self._curtable)
self._curtablepath.append(key)
return
lastlevel = self._curtable
for k in self._curtablepath[:-1]:
lastlevel = lastlevel[k]
k = self._curtablepath[-1]
if type(k) is int:
if k < len(lastlevel):
if key is not None:
lastlevel[k].update(obj)
else:
lastlevel.append(obj)
if key is None:
key = len(lastlevel[k])
else:
if key is None:
if lastlevel[k]:
key = len(lastlevel[k])
else:
key = 0
lastlevel[k] = obj
else:
lastlevel[k].update(obj)
self._curtablepath.append(key)
elif name == 'os':
self._curhost['os'] = {}
elif name == 'portused' and 'os' in self._curhost:
self._curhost['os']['portused'] = {
'port': '%s_%s' % (attrs['proto'], attrs['portid']),
'state': attrs['state'],
}
elif name in ['osclass', 'osmatch'] and 'os' in self._curhost:
self._curhost['os'].setdefault(name, []).append(dict(attrs))
elif name == 'osfingerprint' and 'os' in self._curhost:
self._curhost['os']['fingerprint'] = attrs['fingerprint']
elif name == 'trace':
if self._curtrace is not None:
sys.stderr.write("WARNING, self._curtrace should be None "
"at this point (got %r)\n" % self._curtrace)
if 'proto' not in attrs:
self._curtrace = {'protocol': None}
elif attrs['proto'] in ['tcp', 'udp']:
self._curtrace = {
'protocol': attrs['proto'],
'port': int(attrs['port'])
}
else:
self._curtrace = {'protocol': attrs['proto']}
self._curtrace['hops'] = []
elif name == 'hop' and self._curtrace is not None:
attrsdict = dict(attrs)
try:
attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr'])
except utils.socket.error:
pass
try:
attrsdict['rtt'] = float(attrs['rtt'])
except ValueError:
pass
try:
attrsdict['ttl'] = int(attrs['ttl'])
except ValueError:
pass
if 'host' in attrsdict:
attrsdict['domains'] = list(
utils.get_domains(attrsdict['host']))
self._curtrace['hops'].append(attrsdict)
elif name == 'cpe':
# start recording
self._curdata = ''
def startElement(self, name, attrs):
if name == 'nmaprun':
if self._curscan is not None:
sys.stderr.write("WARNING, self._curscan should be None at "
"this point(got %r)\n" % self._curscan)
self._curscan = dict(attrs)
self._curscan['_id'] = self._filehash
elif name == 'scaninfo' and self._curscan is not None:
self._addscaninfo(dict(attrs))
elif name == 'host':
if self._curhost is not None:
sys.stderr.write("WARNING, self._curhost should be None at "
"this point (got %r)\n" % self._curhost)
self._curhost = {"schema_version": SCHEMA_VERSION}
if self._curscan:
self._curhost['scanid'] = self._curscan['_id']
for attr in attrs.keys():
self._curhost[attr] = attrs[attr]
for field in ['starttime', 'endtime']:
if field in self._curhost:
self._curhost[field] = datetime.datetime.utcfromtimestamp(
int(self._curhost[field]))
elif name == 'address' and self._curhost is not None:
if attrs['addrtype'] != 'ipv4':
if 'addresses' not in self._curhost:
self._curhost['addresses'] = {
attrs['addrtype']: [attrs['addr']]
}
elif attrs['addrtype'] not in self._curhost:
self._curhost['addresses'].update(
{attrs['addrtype']: [attrs['addr']]})
else:
addresses = self._curhost['addresses'][attrs['addrtype']]
addresses.append(attrs['addr'])
self._curhost['addresses'][attrs['addrtype']] = addresses
else:
try:
self._curhost['addr'] = utils.ip2int(attrs['addr'])
except utils.socket.error:
self._curhost['addr'] = attrs['addr']
elif name == 'hostnames':
if self._curhostnames is not None:
sys.stderr.write("WARNING, self._curhostnames should be None "
"at this point "
"(got %r)\n" % self._curhostnames)
self._curhostnames = []
elif name == 'hostname':
if self._curhostnames is None:
sys.stderr.write("WARNING, self._curhostnames should NOT be "
"None at this point\n")
self._curhostnames = []
hostname = dict(attrs)
if 'name' in attrs:
hostname['domains'] = list(utils.get_domains(attrs['name']))
self._curhostnames.append(hostname)
elif name == 'status' and self._curhost is not None:
self._curhost['state'] = attrs['state']
if 'reason' in attrs:
self._curhost['state_reason'] = attrs['reason']
if 'reason_ttl' in attrs:
self._curhost['state_reason_ttl'] = int(attrs['reason_ttl'])
elif name == 'extraports':
if self._curextraports is not None:
sys.stderr.write("WARNING, self._curextraports should be None"
" at this point "
"(got %r)\n" % self._curextraports)
self._curextraports = {attrs['state']: [int(attrs['count']), {}]}
elif name == 'extrareasons' and self._curextraports is not None:
self._curextraports[self._curextraports.keys()[0]][1][
attrs['reason']] = int(attrs['count'])
elif name == 'port':
if self._curport is not None:
sys.stderr.write("WARNING, self._curport should be None at "
"this point (got %r)\n" % self._curport)
self._curport = {
'protocol': attrs['protocol'],
'port': int(attrs['portid'])
}
elif name == 'state' and self._curport is not None:
for attr in attrs.keys():
self._curport['state_%s' % attr] = attrs[attr]
for field in ['state_reason_ttl']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
for field in ['state_reason_ip']:
if field in self._curport:
try:
self._curport[field] = utils.ip2int(
self._curport[field])
except utils.socket.error:
pass
elif name == 'service' and self._curport is not None:
if attrs.get("method") == "table":
# discard information from nmap-services
return
for attr in attrs.keys():
self._curport['service_%s' % attr] = attrs[attr]
for field in [
'service_conf', 'service_rpcnum', 'service_lowver',
'service_highver'
]:
if field in self._curport:
self._curport[field] = int(self._curport[field])
elif name == 'script':
if self._curscript is not None:
sys.stderr.write("WARNING, self._curscript should be None "
"at this point (got %r)\n" % self._curscript)
self._curscript = dict([attr, attrs[attr]]
for attr in attrs.keys())
elif name in ['table', 'elem']:
if self._curscript.get('id') in IGNORE_TABLE_ELEMS:
return
if name == 'elem':
# start recording characters
if self._curdata is not None:
sys.stderr.write("WARNING, self._curdata should be None"
" at this point "
"(got %r)\n" % self._curdata)
self._curdata = ''
if 'key' in attrs:
key = attrs['key'].replace('.', '_')
obj = {key: {}}
else:
key = None
obj = []
if not self._curtablepath:
if not self._curtable:
self._curtable = obj
elif key is not None:
self._curtable.update(obj)
if key is None:
key = len(self._curtable)
self._curtablepath.append(key)
return
lastlevel = self._curtable
for k in self._curtablepath[:-1]:
lastlevel = lastlevel[k]
k = self._curtablepath[-1]
if type(k) is int:
if k < len(lastlevel):
if key is not None:
lastlevel[k].update(obj)
else:
lastlevel.append(obj)
if key is None:
key = len(lastlevel[k])
else:
if key is None:
if lastlevel[k]:
key = len(lastlevel[k])
else:
key = 0
lastlevel[k] = obj
else:
lastlevel[k].update(obj)
self._curtablepath.append(key)
elif name == 'os':
self._curhost['os'] = {}
elif name == 'portused' and 'os' in self._curhost:
self._curhost['os']['portused'] = {
'port': '%s_%s' % (attrs['proto'], attrs['portid']),
'state': attrs['state'],
}
elif name in ['osclass', 'osmatch'] and 'os' in self._curhost:
if name not in self._curhost['os']:
self._curhost['os'][name] = [dict(attrs)]
else:
self._curhost['os'][name].append(dict(attrs))
elif name == 'osfingerprint' and 'os' in self._curhost:
self._curhost['os']['fingerprint'] = attrs['fingerprint']
elif name == 'trace':
if self._curtrace is not None:
sys.stderr.write("WARNING, self._curtrace should be None "
"at this point (got %r)\n" % self._curtrace)
if 'proto' not in attrs:
self._curtrace = {'protocol': None}
elif attrs['proto'] in ['tcp', 'udp']:
self._curtrace = {
'protocol': attrs['proto'],
'port': int(attrs['port'])
}
else:
self._curtrace = {'protocol': attrs['proto']}
self._curtrace['hops'] = []
elif name == 'hop' and self._curtrace is not None:
attrsdict = dict(attrs)
try:
attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr'])
except utils.socket.error:
pass
try:
attrsdict['rtt'] = float(attrs['rtt'])
except ValueError:
pass
try:
attrsdict['ttl'] = int(attrs['ttl'])
except ValueError:
pass
if 'host' in attrsdict:
attrsdict['domains'] = list(
utils.get_domains(attrsdict['host']))
self._curtrace['hops'].append(attrsdict)
elif name == 'cpe':
# start recording
self._curdata = ''
