def __init__(self, start, stop, categories=None, rand=True, maxnbr=None,
state=None, name=None):
super().__init__(
geoiputils.IPRanges(ranges=[(utils.ip2int(start),
utils.ip2int(stop))]),
rand=rand, maxnbr=maxnbr, state=state,
name=name or 'RANGE-%s-%s' % (start, stop), categories=categories,
)
def __init__(self, start, stop, categories=None, rand=True, maxnbr=None,
state=None):
Target.__init__(
self,
geoiputils.IPRanges(ranges=[(utils.ip2int(start),
utils.ip2int(stop))]),
rand=rand, maxnbr=maxnbr, state=state
)
if categories is None:
categories = ['RANGE-%s-%s' % (start, stop)]
self.infos = {'categories': categories}
def __init__(self, start, stop, categories=None, rand=True, maxnbr=None,
state=None):
Target.__init__(
self,
geoiputils.IPRanges(ranges=[(utils.ip2int(start),
utils.ip2int(stop))]),
rand=rand, maxnbr=maxnbr, state=state
)
if categories is None:
categories = ['RANGE-%s-%s' % (start, stop)]
self.infos = {'categories': categories}
def r2res(r):
return [
utils.ip2int(r['addr']),
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
def parse_p0f_line(line, include_port=False, sensor=None, recontype=None):
line = [line.split(b' - ')[0]] + line.split(b' - ')[1].split(b' -> ')
if line[1].startswith(b'UNKNOWN '):
sig = line[1][line[1].index(b'UNKNOWN ') + 8:][1:-1].split(b':')[:6]
osname, version, dist = b'?', b'?', -1
else:
sig = line[1][line[1].index(b' Signature: ') + 12:][
1:-1].split(b':')[:6]
if b' (up: ' in line[1]:
osname = line[1][:line[1].index(b' (up: ')]
else:
osname = line[1][:line[1].index(b' Signature: ')]
osname, version = osname.split(b' ')[0], b' '.join(osname.split(b' ')[1:])
dist = int(P0F_DIST.search(line[2]).groups()[0])
# We wildcard any window size which is not Sxxx or Tyyy
if sig[0][0] not in b'ST':
sig[0] = b'*'
spec = {
'addr': utils.ip2int(line[0][line[0].index(b'> ')
+ 2:line[0].index(b':')]),
'distance': dist,
'value': osname.decode(),
'version': version.decode(),
'signature': ':'.join(str(elt) for elt in sig),
}
if include_port:
spec.update({'port': int(line[0][line[0].index(b':') + 1:])})
if sensor is not None:
spec['sensor'] = sensor
if recontype is not None:
spec['recontype'] = recontype
return float(line[0][1:line[0].index(b'>')]), spec
def parse_p0f_line(line, include_port=False, sensor=None, recontype=None):
line = [line.split(b' - ')[0]] + line.split(b' - ')[1].split(b' -> ')
if line[1].startswith(b'UNKNOWN '):
sig = line[1][line[1].index(b'UNKNOWN ') + 8:][1:-1].split(b':')[:6]
osname, version, dist = b'?', b'?', -1
else:
sig = line[1][line[1].index(b' Signature: ') + 12:][
1:-1].split(b':')[:6]
if b' (up: ' in line[1]:
osname = line[1][:line[1].index(b' (up: ')]
else:
osname = line[1][:line[1].index(b' Signature: ')]
osname = osname.split(b' ')
osname, version = osname[0], b' '.join(osname[1:])
dist = int(P0F_DIST.search(line[2]).groups()[0])
# We wildcard any window size which is not Sxxx or Tyyy
if sig[0][0] not in b'ST':
sig[0] = b'*'
spec = {
'addr': utils.ip2int(line[0][line[0].index(b'> ') + 2:
line[0].index(b':')]),
'distance': dist,
'value': osname.decode(),
'version': version.decode(),
'signature': ':'.join(str(elt) for elt in sig),
}
if include_port:
spec.update({'port': int(line[0][line[0].index(b':') + 1:])})
if sensor is not None:
spec['sensor'] = sensor
if recontype is not None:
spec['recontype'] = recontype
return float(line[0][1:line[0].index(b'>')]), spec
def bgp_raw_to_csv(fname, out):
out = open(os.path.join(config.GEOIP_PATH, out), 'w')
cur = None
with open(os.path.join(config.GEOIP_PATH, fname), 'rb') as fdesc:
for line in fdesc:
start, stop = (utils.ip2int(elt) for elt in
utils.net2range(line[:-1].split(None, 1)[0]))
if cur:
if start >= cur[0] and stop <= cur[1]:
continue
if start >= cur[0] and start <= cur[1]:
cur = (cur[0], stop)
continue
if stop >= cur[0] and stop <= cur[1]:
cur = (start, cur[1])
continue
if start <= cur[0] and stop >= cur[1]:
cur = (start, stop)
continue
if start == cur[1] + 1:
cur = (cur[0], stop)
continue
if stop == cur[0] + 1:
cur = (start, cur[1])
continue
out.write('%d,%d\n' % cur)
cur = (start, stop)
if cur:
out.write('%d,%d\n' % cur)
def bgp_raw_to_csv(fname, out):
out = open(os.path.join(config.GEOIP_PATH, out), 'wb')
cur = []
with open(os.path.join(config.GEOIP_PATH, fname), 'rb') as fdesc:
for line in fdesc:
start, stop = (utils.ip2int(elt) for elt in
utils.net2range(line[:-1].split()[0]))
if cur:
if start >= cur[0] and stop <= cur[1]:
continue
if start >= cur[0] and start <= cur[1]:
cur = [cur[0], stop]
continue
if stop >= cur[0] and stop <= cur[1]:
cur = [start, cur[1]]
continue
if start <= cur[0] and stop >= cur[1]:
cur = [start, stop]
continue
if start == cur[1] + 1:
cur = [cur[0], stop]
continue
if stop == cur[0] + 1:
cur = [start, cur[1]]
continue
out.write(('"%s","%s","%d","%d"\n' % (
utils.int2ip(cur[0]), utils.int2ip(cur[1]), cur[0], cur[1],
)).encode())
cur = [start, stop]
if cur:
out.write(('"%s","%s","%d","%d"\n' % (
utils.int2ip(cur[0]), utils.int2ip(cur[1]), cur[0], cur[1],
)).encode())
def __getaddr__(self, line):
addr = self.match_addr.match(line)
if addr is not None:
try:
return utils.ip2int(addr.groups()[0])
except utils.socket.error:
pass
def __getaddr__(self, line):
addr = self.match_addr.match(line)
if addr is not None:
try:
return utils.ip2int(addr.groups()[0])
except utils.socket.error:
pass
def get_scan_results(self) -> Generator[NmapHost, None, None]:
starttime = datetime.now()
results = sorted(self.parse().items(),
key=lambda a_p: utils.ip2int(a_p[0]))
endtime = datetime.now()
for addr, ports in results:
rec = {"addr": addr, "starttime": starttime, "endtime": endtime}
for (proto, portnum), progs in sorted(ports.items()):
port = {"protocol": proto, "port": portnum}
if progs:
port["scripts"] = [
{
"id":
"localscan",
"output":
"Program%s: %s" % ("s" if len(progs) > 1 else "",
", ".join(sorted(progs))),
"localscan": {
"programs": sorted(progs)
},
},
]
# TODO: remove when NmapHost is implemented
rec.setdefault("ports", []).append(port) # type: ignore
yield rec
def handle_rec(
sensor,
ignorenets,
neverignore,
# these argmuments are provided by *<line.split()>
timestamp,
host,
port,
recon_type,
source,
value,
targetval):
recon_type = recon_type[14:] # skip PassiveRecon::
if host == '-':
spec = {
'targetval': targetval,
'recontype': recon_type,
'value': value
}
else:
try:
host = utils.ip2int(host)
except:
pass
spec = {'addr': host, 'recontype': recon_type, 'value': value}
if sensor is not None:
spec.update({'sensor': sensor})
if port != '-':
spec.update({'port': int(port)})
if source != '-':
spec.update({'source': source})
spec = _prepare_rec(spec, ignorenets, neverignore)
return float(timestamp), spec
def handle_rec(sensor, ignorenets, neverignore,
# these argmuments are provided by **bro_line
timestamp, host, srvport, recon_type, source, value, targetval):
if host is None:
spec = {
'targetval': targetval,
'recontype': recon_type,
'value': value
}
else:
try:
host = utils.ip2int(host)
except:
pass
spec = {
'addr': host,
'recontype': recon_type,
'value': value
}
if sensor is not None:
spec.update({'sensor': sensor})
if srvport is not None:
spec.update({'port': srvport})
if source is not None:
spec.update({'source': source})
spec = _prepare_rec(spec, ignorenets, neverignore)
# Python 2/3 compat: python 3 has datetime.timestamp()
try:
float_ts = timestamp.timestamp()
except AttributeError:
float_ts = time.mktime(timestamp.timetuple()) \
+ timestamp.microsecond / (1000.*1000.)
return float_ts, spec
def parse_p0f_line(line, include_port=False, sensor=None, recontype=None):
line = [line.split(' - ')[0]] + line.split(' - ')[1].split(' -> ')
if line[1].startswith('UNKNOWN '):
sig = line[1][line[1].index('UNKNOWN ') + 8:][1:-1].split(':')[:6]
OS, version, dist = '?', '?', -1
else:
sig = line[1][line[1].index(' Signature: ') + 12:][1:-1].split(':')[:6]
if ' (up: ' in line[1]:
OS = line[1][:line[1].index(' (up: ')]
else:
OS = line[1][:line[1].index(' Signature: ')]
OS, version = OS.split(' ')[0], ' '.join(OS.split(' ')[1:])
dist = int(P0F_DIST.search(line[2]).groups()[0])
# We wildcard any window size which is not Sxxx or Tyyy
if sig[0][0] not in ['S', 'T']:
sig[0] = '*'
spec = {
'addr':
utils.ip2int(line[0][line[0].index('> ') + 2:line[0].index(':')]),
'distance': dist,
'value': OS,
'version': version,
'signature': ":".join(map(str, sig)),
}
if include_port:
spec.update({'port': int(line[0][line[0].index(':') + 1:])})
if sensor is not None:
spec['sensor'] = sensor
if recontype is not None:
spec['recontype'] = recontype
return float(line[0][1:line[0].index('>')]), spec
def parse_p0f_line(line, include_port=False, sensor=None, recontype=None):
line = [line.split(' - ')[0]] + line.split(' - ')[1].split(' -> ')
if line[1].startswith('UNKNOWN '):
sig = line[1][line[1].index('UNKNOWN ') + 8:][1:-1].split(':')[:6]
OS, version, dist = '?', '?', -1
else:
sig = line[1][line[1].index(' Signature: ') + 12:][
1:-1].split(':')[:6]
if ' (up: ' in line[1]:
OS = line[1][:line[1].index(' (up: ')]
else:
OS = line[1][:line[1].index(' Signature: ')]
OS, version = OS.split(' ')[0], ' '.join(OS.split(' ')[1:])
dist = int(P0F_DIST.search(line[2]).groups()[0])
# We wildcard any window size which is not Sxxx or Tyyy
if sig[0][0] not in ['S', 'T']:
sig[0] = '*'
spec = {
'addr': utils.ip2int(line[0][line[0].index('> ')
+ 2:line[0].index(':')]),
'distance': dist,
'value': OS,
'version': version,
'signature': ":".join(map(str, sig)),
}
if include_port:
spec.update({'port': int(line[0][line[0].index(':') + 1:])})
if sensor is not None:
spec['sensor'] = sensor
if recontype is not None:
spec['recontype'] = recontype
return float(line[0][1:line[0].index('>')]), spec
def handle_rec(sensor, ignorenets, neverignore,
# these argmuments are provided by *<line.split()>
timestamp, host, port, recon_type, source, value,
targetval):
recon_type = recon_type[14:] # skip PassiveRecon::
if host == '-':
spec = {
'targetval': targetval,
'recontype': recon_type,
'value': value
}
else:
try:
host = utils.ip2int(host)
except:
pass
spec = {
'addr': host,
'recontype': recon_type,
'value': value
}
if sensor is not None:
spec.update({'sensor': sensor})
if port != '-':
spec.update({'port': int(port)})
if source != '-':
spec.update({'source': source})
spec = _prepare_rec(spec, ignorenets, neverignore)
return float(timestamp), spec
def bgp_raw_to_csv(fname: str, outname: str) -> None:
cur = None
assert config.GEOIP_PATH is not None
with open(os.path.join(config.GEOIP_PATH, fname),
"rb") as fdesc, open(os.path.join(config.GEOIP_PATH, outname),
"w",
encoding="utf8") as out:
for line in fdesc:
start, stop = (
utils.ip2int(elt)
for elt in utils.net2range(line[:-1].split(None, 1)[0]))
if cur:
if start >= cur[0] and stop <= cur[1]:
continue
if cur[0] <= start <= cur[1]:
cur = (cur[0], stop)
continue
if cur[0] <= stop <= cur[1]:
cur = (start, cur[1])
continue
if start <= cur[0] and stop >= cur[1]:
cur = (start, stop)
continue
if start == cur[1] + 1:
cur = (cur[0], stop)
continue
if stop == cur[0] + 1:
cur = (start, cur[1])
continue
out.write("%d,%d\n" % cur)
cur = (start, stop)
if cur:
out.write("%d,%d\n" % cur)
def r2res(r):
return [
utils.ip2int(r['addr']),
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
def _getaddr(cls, line):
addr = cls.match_addr.match(line)
if addr is not None:
try:
return utils.ip2int(addr.groups()[0])
except utils.socket.error:
pass
return None
def graphhost(host: Record) -> Tuple[List[int], List[int]]:
if "ports" not in host:
return [], []
hh, pp = [], []
addr = utils.ip2int(host["addr"])
for p in host["ports"]:
if p.get("state_state") == "open":
hh.append(addr)
pp.append(p["port"])
return hh, pp
def bgp_raw_to_csv(fname, out):
out = open(os.path.join(config.GEOIP_PATH, out), 'wb')
cur = []
with open(os.path.join(config.GEOIP_PATH, fname), 'rb') as fdesc:
for line in fdesc:
start, stop = (utils.ip2int(elt)
for elt in utils.net2range(line[:-1].split()[0]))
if cur:
if start >= cur[0] and stop <= cur[1]:
continue
if start >= cur[0] and start <= cur[1]:
cur = [cur[0], stop]
continue
if stop >= cur[0] and stop <= cur[1]:
cur = [start, cur[1]]
continue
if start <= cur[0] and stop >= cur[1]:
cur = [start, stop]
continue
if start == cur[1] + 1:
cur = [cur[0], stop]
continue
if stop == cur[0] + 1:
cur = [start, cur[1]]
continue
out.write(('"%s","%s","%d","%d"\n' % (
utils.int2ip(cur[0]),
utils.int2ip(cur[1]),
cur[0],
cur[1],
)).encode())
cur = [start, stop]
if cur:
out.write(('"%s","%s","%d","%d"\n' % (
utils.int2ip(cur[0]),
utils.int2ip(cur[1]),
cur[0],
cur[1],
)).encode())
def get_scan_results(self):
starttime = datetime.now()
results = sorted(self.parse().items(),
key=lambda a_p: utils.ip2int(a_p[0]))
endtime = datetime.now()
for addr, ports in results:
rec = {"addr": addr, "starttime": starttime, "endtime": endtime}
for (proto, port), progs in sorted(ports.items()):
port = {"protocol": proto, "port": port}
if progs:
port["scripts"] = [
{
"id":
"localscan",
"output":
"Program%s: %s" % ("s" if len(progs) > 1 else "",
", ".join(sorted(progs))),
"localscan": {
"programs": sorted(progs)
},
},
]
rec.setdefault("ports", []).append(port)
yield rec
def r2res(r):
return utils.ip2int(r['addr'])
def r2res(r):
return [utils.ip2int(r['addr']), r['openports']['count']]
def get_nmap_action(action):
flt_params = get_nmap_base()
preamble = "[\n"
postamble = "]\n"
r2res = lambda x: x
if action == "timeline":
result, count = db.view.get_open_port_count(flt_params.flt)
if request.params.get("modulo") is None:
r2time = lambda r: int(utils.datetime2timestamp(r['starttime']))
else:
r2time = lambda r: (int(utils.datetime2timestamp(r['starttime'])) %
int(request.params.get("modulo")))
if flt_params.ipsasnumbers:
r2res = lambda r: [
r2time(r),
utils.ip2int(r['addr']), r['openports']['count']
]
else:
r2res = lambda r: [r2time(r), r['addr'], r['openports']['count']]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": [\n'
postamble = ']}\n'
result = list(db.view.getlocations(flt_params.flt))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {
"count": r['count']
},
}
elif action == "countopenports":
result, count = db.view.get_open_port_count(flt_params.flt)
if flt_params.ipsasnumbers:
r2res = lambda r: [
utils.ip2int(r['addr']), r['openports']['count']
]
else:
r2res = lambda r: [r['addr'], r['openports']['count']]
elif action == "ipsports":
result, count = db.view.get_ips_ports(flt_params.flt)
if flt_params.ipsasnumbers:
r2res = lambda r: [
utils.ip2int(r['addr']),
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
else:
r2res = lambda r: [
r['addr'],
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
elif action == "onlyips":
result, count = db.view.get_ips(flt_params.flt)
if flt_params.ipsasnumbers:
r2res = lambda r: utils.ip2int(r['addr'])
else:
r2res = lambda r: r['addr']
elif action == "diffcats":
if request.params.get("onlydiff"):
output = db.view.diff_categories(request.params.get("cat1"),
request.params.get("cat2"),
flt=flt_params.flt,
include_both_open=False)
else:
output = db.view.diff_categories(request.params.get("cat1"),
request.params.get("cat2"),
flt=flt_params.flt)
count = 0
result = {}
if flt_params.ipsasnumbers:
for res in output:
result.setdefault(res["addr"],
[]).append([res['port'], res['value']])
count += 1
else:
for res in output:
result.setdefault(utils.int2ip(res["addr"]),
[]).append([res['port'], res['value']])
count += 1
result = viewitems(result)
if flt_params.callback is not None:
if count >= config.WEB_WARN_DOTS_COUNT:
yield (
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count)
yield '%s(\n' % flt_params.callback
yield preamble
# hack to avoid a trailing comma
result = iter(result)
try:
rec = next(result)
except StopIteration:
pass
else:
yield json.dumps(r2res(rec))
for rec in result:
yield ",\n" + json.dumps(r2res(rec))
yield "\n"
yield postamble
if flt_params.callback is not None:
yield ");"
if count >= config.WEB_WARN_DOTS_COUNT:
yield '}\n'
else:
yield "\n"
def startElement(self, name, attrs):
if name == "nmaprun":
if self._curscan is not None:
sys.stderr.write("WARNING, self._curscan should be None at " "this point(got %r)\n" % self._curscan)
self._curscan = dict(attrs)
self._curscan["_id"] = self._filehash
elif name == "scaninfo" and self._curscan is not None:
self._addscaninfo(dict(attrs))
elif name == "host":
if self._curhost is not None:
sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost)
self._curhost = {"schema_version": 1}
if self._curscan:
self._curhost["scanid"] = self._curscan["_id"]
for attr in attrs.keys():
self._curhost[attr] = attrs[attr]
for field in ["starttime", "endtime"]:
if field in self._curhost:
self._curhost[field] = datetime.datetime.utcfromtimestamp(int(self._curhost[field]))
elif name == "address" and self._curhost is not None:
if attrs["addrtype"] != "ipv4":
if "addresses" not in self._curhost:
self._curhost["addresses"] = {attrs["addrtype"]: [attrs["addr"]]}
elif attrs["addrtype"] not in self._curhost:
self._curhost["addresses"].update({attrs["addrtype"]: [attrs["addr"]]})
else:
addresses = self._curhost["addresses"][attrs["addrtype"]]
addresses.append(attrs["addr"])
self._curhost["addresses"][attrs["addrtype"]] = addresses
else:
try:
self._curhost["addr"] = utils.ip2int(attrs["addr"])
except utils.socket.error:
self._curhost["addr"] = attrs["addr"]
elif name == "hostnames":
if self._curhostnames is not None:
sys.stderr.write(
"WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames
)
self._curhostnames = []
elif name == "hostname":
if self._curhostnames is None:
sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n")
self._curhostnames = []
hostname = dict(attrs)
if "name" in attrs:
hostname["domains"] = list(utils.get_domains(attrs["name"]))
self._curhostnames.append(hostname)
elif name == "status" and self._curhost is not None:
self._curhost["state"] = attrs["state"]
if "reason" in attrs:
self._curhost["state_reason"] = attrs["reason"]
if "reason_ttl" in attrs:
self._curhost["state_reason_ttl"] = int(attrs["reason_ttl"])
elif name == "extraports":
if self._curextraports is not None:
sys.stderr.write(
"WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports
)
self._curextraports = {attrs["state"]: [int(attrs["count"]), {}]}
elif name == "extrareasons" and self._curextraports is not None:
self._curextraports[self._curextraports.keys()[0]][1][attrs["reason"]] = int(attrs["count"])
elif name == "port":
if self._curport is not None:
sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport)
self._curport = {"protocol": attrs["protocol"], "port": int(attrs["portid"])}
elif name == "state" and self._curport is not None:
for attr in attrs.keys():
self._curport["state_%s" % attr] = attrs[attr]
for field in ["state_reason_ttl"]:
if field in self._curport:
self._curport[field] = int(self._curport[field])
for field in ["state_reason_ip"]:
if field in self._curport:
try:
self._curport[field] = utils.ip2int(self._curport[field])
except utils.socket.error:
pass
elif name == "service" and self._curport is not None:
if attrs.get("method") == "table":
# discard information from nmap-services
return
for attr in attrs.keys():
self._curport["service_%s" % attr] = attrs[attr]
for field in ["service_conf", "service_rpcnum", "service_lowver", "service_highver"]:
if field in self._curport:
self._curport[field] = int(self._curport[field])
elif name == "script":
if self._curscript is not None:
sys.stderr.write(
"WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript
)
self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys())
elif name in ["table", "elem"]:
if self._curscript.get("id") in IGNORE_TABLE_ELEMS:
return
if name == "elem":
# start recording characters
if self._curdata is not None:
sys.stderr.write(
"WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata
)
self._curdata = ""
if "key" in attrs:
key = attrs["key"].replace(".", "_")
obj = {key: {}}
else:
key = None
obj = []
if not self._curtablepath:
if not self._curtable:
self._curtable = obj
elif key is not None:
self._curtable.update(obj)
if key is None:
key = len(self._curtable)
self._curtablepath.append(key)
return
lastlevel = self._curtable
for k in self._curtablepath[:-1]:
lastlevel = lastlevel[k]
k = self._curtablepath[-1]
if type(k) is int:
if k < len(lastlevel):
if key is not None:
lastlevel[k].update(obj)
else:
lastlevel.append(obj)
if key is None:
key = len(lastlevel[k])
else:
if key is None:
if lastlevel[k]:
key = len(lastlevel[k])
else:
key = 0
lastlevel[k] = obj
else:
lastlevel[k].update(obj)
self._curtablepath.append(key)
elif name == "os":
self._curhost["os"] = {}
elif name == "portused" and "os" in self._curhost:
self._curhost["os"]["portused"] = {
"port": "%s_%s" % (attrs["proto"], attrs["portid"]),
"state": attrs["state"],
}
elif name in ["osclass", "osmatch"] and "os" in self._curhost:
if name not in self._curhost["os"]:
self._curhost["os"][name] = [dict(attrs)]
else:
self._curhost["os"][name].append(dict(attrs))
elif name == "osfingerprint" and "os" in self._curhost:
self._curhost["os"]["fingerprint"] = attrs["fingerprint"]
elif name == "trace":
if self._curtrace is not None:
sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace)
if "proto" not in attrs:
self._curtrace = {"protocol": None}
elif attrs["proto"] in ["tcp", "udp"]:
self._curtrace = {"protocol": attrs["proto"], "port": int(attrs["port"])}
else:
self._curtrace = {"protocol": attrs["proto"]}
self._curtrace["hops"] = []
elif name == "hop" and self._curtrace is not None:
attrsdict = dict(attrs)
try:
attrsdict["ipaddr"] = utils.ip2int(attrs["ipaddr"])
except utils.socket.error:
pass
try:
attrsdict["rtt"] = float(attrs["rtt"])
except ValueError:
pass
try:
attrsdict["ttl"] = int(attrs["ttl"])
except ValueError:
pass
if "host" in attrsdict:
attrsdict["domains"] = list(utils.get_domains(attrsdict["host"]))
self._curtrace["hops"].append(attrsdict)
elif name == "cpe":
# start recording
self._curdata = ""
def r2res(r):
return [
utils.ip2int(r['addr']),
r.get('openports', {}).get('count', 0)
]
def _getaddr(line):
try:
return utils.ip2int(line.split("#", 1)[0].strip())
except utils.socket.error:
return None
def r2res(r):
return [utils.ip2int(r['addr']), r['openports']['count']]
def force_ip_int(addr):
try:
return utils.ip2int(addr)
except (TypeError, struct.error):
return addr
def __getaddr__(self, line):
try:
return utils.ip2int(line.split('#', 1)[0].strip())
except utils.socket.error:
pass
def r2res(r):
return utils.ip2int(r['addr'])
def startElement(self, name, attrs):
if name == 'nmaprun':
if self._curscan is not None:
sys.stderr.write("WARNING, self._curscan should be None at "
"this point(got %r)\n" % self._curscan)
self._curscan = dict(attrs)
self._curscan['_id'] = self._filehash
elif name == 'scaninfo' and self._curscan is not None:
self._addscaninfo(dict(attrs))
elif name == 'host':
if self._curhost is not None:
sys.stderr.write("WARNING, self._curhost should be None at "
"this point (got %r)\n" % self._curhost)
self._curhost = {"schema_version": 1}
if self._curscan:
self._curhost['scanid'] = self._curscan['_id']
for attr in attrs.keys():
self._curhost[attr] = attrs[attr]
for field in ['starttime', 'endtime']:
if field in self._curhost:
self._curhost[field] = datetime.datetime.utcfromtimestamp(
int(self._curhost[field])
)
elif name == 'address' and self._curhost is not None:
if attrs['addrtype'] != 'ipv4':
if 'addresses' not in self._curhost:
self._curhost['addresses'] = {
attrs['addrtype']: [attrs['addr']]
}
elif attrs['addrtype'] not in self._curhost:
self._curhost['addresses'].update({
attrs['addrtype']: [attrs['addr']]
})
else:
addresses = self._curhost['addresses'][attrs['addrtype']]
addresses.append(attrs['addr'])
self._curhost['addresses'][attrs['addrtype']] = addresses
else:
try:
self._curhost['addr'] = utils.ip2int(attrs['addr'])
except utils.socket.error:
self._curhost['addr'] = attrs['addr']
elif name == 'hostnames':
if self._curhostnames is not None:
sys.stderr.write("WARNING, self._curhostnames should be None "
"at this point "
"(got %r)\n" % self._curhostnames)
self._curhostnames = []
elif name == 'hostname':
if self._curhostnames is None:
sys.stderr.write("WARNING, self._curhostnames should NOT be "
"None at this point\n")
self._curhostnames = []
hostname = dict(attrs)
if 'name' in attrs:
hostname['domains'] = list(utils.get_domains(attrs['name']))
self._curhostnames.append(hostname)
elif name == 'status' and self._curhost is not None:
self._curhost['state'] = attrs['state']
if 'reason' in attrs:
self._curhost['state_reason'] = attrs['reason']
if 'reason_ttl' in attrs:
self._curhost['state_reason_ttl'] = int(attrs['reason_ttl'])
elif name == 'extraports':
if self._curextraports is not None:
sys.stderr.write("WARNING, self._curextraports should be None"
" at this point "
"(got %r)\n" % self._curextraports)
self._curextraports = {attrs['state']: [int(attrs['count']), {}]}
elif name == 'extrareasons' and self._curextraports is not None:
self._curextraports[self._curextraports.keys()[0]][1][
attrs['reason']] = int(attrs['count'])
elif name == 'port':
if self._curport is not None:
sys.stderr.write("WARNING, self._curport should be None at "
"this point (got %r)\n" % self._curport)
self._curport = {'protocol': attrs['protocol'],
'port': int(attrs['portid'])}
elif name == 'state' and self._curport is not None:
for attr in attrs.keys():
self._curport['state_%s' % attr] = attrs[attr]
for field in ['state_reason_ttl']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
for field in ['state_reason_ip']:
if field in self._curport:
try:
self._curport[field] = utils.ip2int(
self._curport[field])
except utils.socket.error:
pass
elif name == 'service' and self._curport is not None:
for attr in attrs.keys():
self._curport['service_%s' % attr] = attrs[attr]
for field in ['service_conf', 'service_rpcnum',
'service_lowver', 'service_highver']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
elif name == 'script':
if self._curscript is not None:
sys.stderr.write("WARNING, self._curscript should be None "
"at this point (got %r)\n" % self._curscript)
self._curscript = dict([attr, attrs[attr]]
for attr in attrs.keys())
elif name in ['table', 'elem']:
if self._curscript.get('id') in IGNORE_TABLE_ELEMS:
return
if name == 'elem':
# start recording characters
if self._curdata is not None:
sys.stderr.write("WARNING, self._curdata should be None"
" at this point "
"(got %r)\n" % self._curdata)
self._curdata = ''
if 'key' in attrs:
key = attrs['key'].replace('.', '_')
obj = {key: {}}
else:
key = None
obj = []
if not self._curtablepath:
if not self._curtable:
self._curtable = obj
elif key is not None:
self._curtable.update(obj)
if key is None:
key = len(self._curtable)
self._curtablepath.append(key)
return
lastlevel = self._curtable
for k in self._curtablepath[:-1]:
lastlevel = lastlevel[k]
k = self._curtablepath[-1]
if type(k) is int:
if k < len(lastlevel):
if key is not None:
lastlevel[k].update(obj)
else:
lastlevel.append(obj)
if key is None:
key = len(lastlevel[k])
else:
if key is None:
if lastlevel[k]:
key = len(lastlevel[k])
else:
key = 0
lastlevel[k] = obj
else:
lastlevel[k].update(obj)
self._curtablepath.append(key)
elif name == 'os':
self._curhost['os'] = {}
elif name == 'portused' and 'os' in self._curhost:
self._curhost['os']['portused'] = {
'port': '%s_%s' % (attrs['proto'], attrs['portid']),
'state': attrs['state'],
}
elif name in ['osclass', 'osmatch'] and 'os' in self._curhost:
if name not in self._curhost['os']:
self._curhost['os'][name] = [dict(attrs)]
else:
self._curhost['os'][name].append(dict(attrs))
elif name == 'osfingerprint' and 'os' in self._curhost:
self._curhost['os']['fingerprint'] = attrs['fingerprint']
elif name == 'trace':
if self._curtrace is not None:
sys.stderr.write("WARNING, self._curtrace should be None "
"at this point (got %r)\n" % self._curtrace)
if 'proto' not in attrs:
self._curtrace = {'protocol': None}
elif attrs['proto'] in ['tcp', 'udp']:
self._curtrace = {'protocol': attrs['proto'],
'port': int(attrs['port'])}
else:
self._curtrace = {'protocol': attrs['proto']}
self._curtrace['hops'] = []
elif name == 'hop' and self._curtrace is not None:
attrsdict = dict(attrs)
try:
attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr'])
except utils.socket.error:
pass
try:
attrsdict['rtt'] = float(attrs['rtt'])
except ValueError:
pass
try:
attrsdict['ttl'] = int(attrs['ttl'])
except ValueError:
pass
if 'host' in attrsdict:
attrsdict['domains'] = list(
utils.get_domains(attrsdict['host']))
self._curtrace['hops'].append(attrsdict)
elif name == 'cpe':
# start recording
self._curdata = ''
def startElement(self, name, attrs):
if name == 'nmaprun':
if self._curscan is not None:
sys.stderr.write("WARNING, self._curscan should be None at "
"this point (got %r)\n" % self._curscan)
self._curscan = dict(attrs)
self.scanner = self._curscan.get("scanner", self.scanner)
if self.scanner == "masscan":
# We need to force "merge" mode due to the nature of
# Masscan results
self.merge = True
self._curscan['_id'] = self._filehash
elif name == 'scaninfo' and self._curscan is not None:
self._addscaninfo(dict(attrs))
elif name == 'host':
if self._curhost is not None:
sys.stderr.write("WARNING, self._curhost should be None at "
"this point (got %r)\n" % self._curhost)
self._curhost = {"schema_version": SCHEMA_VERSION}
if self._curscan:
self._curhost['scanid'] = self._curscan['_id']
for attr in attrs.keys():
self._curhost[attr] = attrs[attr]
for field in ['starttime', 'endtime']:
if field in self._curhost:
self._curhost[field] = datetime.datetime.utcfromtimestamp(
int(self._curhost[field])
)
if 'starttime' not in self._curhost and 'endtime' in self._curhost:
# Masscan
self._curhost['starttime'] = self._curhost['endtime']
elif name == 'address' and self._curhost is not None:
if attrs['addrtype'] != 'ipv4':
self._curhost.setdefault(
'addresses', {}).setdefault(
attrs['addrtype'], []).append(attrs['addr'])
else:
try:
self._curhost['addr'] = utils.ip2int(attrs['addr'])
except utils.socket.error:
self._curhost['addr'] = attrs['addr']
elif name == 'hostnames':
if self._curhostnames is not None:
sys.stderr.write("WARNING, self._curhostnames should be None "
"at this point "
"(got %r)\n" % self._curhostnames)
self._curhostnames = []
elif name == 'hostname':
if self._curhostnames is None:
sys.stderr.write("WARNING, self._curhostnames should NOT be "
"None at this point\n")
self._curhostnames = []
hostname = dict(attrs)
if 'name' in attrs:
hostname['domains'] = list(utils.get_domains(attrs['name']))
self._curhostnames.append(hostname)
elif name == 'status' and self._curhost is not None:
self._curhost['state'] = attrs['state']
if 'reason' in attrs:
self._curhost['state_reason'] = attrs['reason']
if 'reason_ttl' in attrs:
self._curhost['state_reason_ttl'] = int(attrs['reason_ttl'])
elif name == 'extraports':
if self._curextraports is not None:
sys.stderr.write("WARNING, self._curextraports should be None"
" at this point "
"(got %r)\n" % self._curextraports)
self._curextraports = {
attrs['state']: {"total": int(attrs['count']), "reasons": {}},
}
elif name == 'extrareasons' and self._curextraports is not None:
self._curextraports[next(iter(self._curextraports))]["reasons"][
attrs['reason']] = int(attrs['count'])
elif name == 'port':
if self._curport is not None:
sys.stderr.write("WARNING, self._curport should be None at "
"this point (got %r)\n" % self._curport)
self._curport = {'protocol': attrs['protocol'],
'port': int(attrs['portid'])}
elif name == 'state' and self._curport is not None:
for attr in attrs.keys():
self._curport['state_%s' % attr] = attrs[attr]
for field in ['state_reason_ttl']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
for field in ['state_reason_ip']:
if field in self._curport:
try:
self._curport[field] = utils.ip2int(
self._curport[field])
except utils.socket.error:
pass
elif name == 'service' and self._curport is not None:
if attrs.get("method") == "table":
# discard information from nmap-services
return
if self.scanner == "masscan":
# create fake scripts from masscan "service" tags
self._curport.setdefault('scripts', []).append({
"id": MASSCAN_SERVICES_NMAP_SCRIPTS.get(attrs['name'],
attrs['name']),
"output": MASSCAN_ENCODING.sub(
_masscan_decode,
attrs["banner"],
)
})
# get service name
service = MASSCAN_SERVICES_NMAP_SERVICES.get(attrs['name'])
if service is not None:
self._curport['service_name'] = service
return
for attr in attrs.keys():
self._curport['service_%s' % attr] = attrs[attr]
for field in ['service_conf', 'service_rpcnum',
'service_lowver', 'service_highver']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
elif name == 'script':
if self._curscript is not None:
sys.stderr.write("WARNING, self._curscript should be None "
"at this point (got %r)\n" % self._curscript)
self._curscript = dict([attr, attrs[attr]]
for attr in attrs.keys())
elif name in ['table', 'elem']:
if self._curscript.get('id') in IGNORE_TABLE_ELEMS:
return
if name == 'elem':
# start recording characters
if self._curdata is not None:
sys.stderr.write("WARNING, self._curdata should be None"
" at this point "
"(got %r)\n" % self._curdata)
self._curdata = ''
if 'key' in attrs:
key = attrs['key'].replace('.', '_')
obj = {key: {}}
else:
key = None
obj = []
if not self._curtablepath:
if not self._curtable:
self._curtable = obj
elif key is not None:
self._curtable.update(obj)
if key is None:
key = len(self._curtable)
self._curtablepath.append(key)
return
lastlevel = self._curtable
for k in self._curtablepath[:-1]:
lastlevel = lastlevel[k]
k = self._curtablepath[-1]
if type(k) is int:
if k < len(lastlevel):
if key is not None:
lastlevel[k].update(obj)
else:
lastlevel.append(obj)
if key is None:
key = len(lastlevel[k])
else:
if key is None:
if lastlevel[k]:
key = len(lastlevel[k])
else:
key = 0
lastlevel[k] = obj
else:
lastlevel[k].update(obj)
self._curtablepath.append(key)
elif name == 'os':
self._curhost['os'] = {}
elif name == 'portused' and 'os' in self._curhost:
self._curhost['os']['portused'] = {
'port': '%s_%s' % (attrs['proto'], attrs['portid']),
'state': attrs['state'],
}
elif name in ['osclass', 'osmatch'] and 'os' in self._curhost:
self._curhost['os'].setdefault(name, []).append(dict(attrs))
elif name == 'osfingerprint' and 'os' in self._curhost:
self._curhost['os']['fingerprint'] = attrs['fingerprint']
elif name == 'trace':
if self._curtrace is not None:
sys.stderr.write("WARNING, self._curtrace should be None "
"at this point (got %r)\n" % self._curtrace)
if 'proto' not in attrs:
self._curtrace = {'protocol': None}
elif attrs['proto'] in ['tcp', 'udp']:
self._curtrace = {'protocol': attrs['proto'],
'port': int(attrs['port'])}
else:
self._curtrace = {'protocol': attrs['proto']}
self._curtrace['hops'] = []
elif name == 'hop' and self._curtrace is not None:
attrsdict = dict(attrs)
try:
attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr'])
except utils.socket.error:
pass
try:
attrsdict['rtt'] = float(attrs['rtt'])
except ValueError:
pass
try:
attrsdict['ttl'] = int(attrs['ttl'])
except ValueError:
pass
if 'host' in attrsdict:
attrsdict['domains'] = list(
utils.get_domains(attrsdict['host']))
self._curtrace['hops'].append(attrsdict)
elif name == 'cpe':
# start recording
self._curdata = ''
def startElement(self, name, attrs):
if name == 'nmaprun':
if self._curscan is not None:
sys.stderr.write("WARNING, self._curscan should be None at "
"this point(got %r)\n" % self._curscan)
self._curscan = dict(attrs)
self._curscan['_id'] = self._filehash
elif name == 'scaninfo' and self._curscan is not None:
self._addscaninfo(dict(attrs))
elif name == 'host':
if self._curhost is not None:
sys.stderr.write("WARNING, self._curhost should be None at "
"this point (got %r)\n" % self._curhost)
self._curhost = {"schema_version": SCHEMA_VERSION}
if self._curscan:
self._curhost['scanid'] = self._curscan['_id']
for attr in attrs.keys():
self._curhost[attr] = attrs[attr]
for field in ['starttime', 'endtime']:
if field in self._curhost:
self._curhost[field] = datetime.datetime.utcfromtimestamp(
int(self._curhost[field]))
elif name == 'address' and self._curhost is not None:
if attrs['addrtype'] != 'ipv4':
if 'addresses' not in self._curhost:
self._curhost['addresses'] = {
attrs['addrtype']: [attrs['addr']]
}
elif attrs['addrtype'] not in self._curhost:
self._curhost['addresses'].update(
{attrs['addrtype']: [attrs['addr']]})
else:
addresses = self._curhost['addresses'][attrs['addrtype']]
addresses.append(attrs['addr'])
self._curhost['addresses'][attrs['addrtype']] = addresses
else:
try:
self._curhost['addr'] = utils.ip2int(attrs['addr'])
except utils.socket.error:
self._curhost['addr'] = attrs['addr']
elif name == 'hostnames':
if self._curhostnames is not None:
sys.stderr.write("WARNING, self._curhostnames should be None "
"at this point "
"(got %r)\n" % self._curhostnames)
self._curhostnames = []
elif name == 'hostname':
if self._curhostnames is None:
sys.stderr.write("WARNING, self._curhostnames should NOT be "
"None at this point\n")
self._curhostnames = []
hostname = dict(attrs)
if 'name' in attrs:
hostname['domains'] = list(utils.get_domains(attrs['name']))
self._curhostnames.append(hostname)
elif name == 'status' and self._curhost is not None:
self._curhost['state'] = attrs['state']
if 'reason' in attrs:
self._curhost['state_reason'] = attrs['reason']
if 'reason_ttl' in attrs:
self._curhost['state_reason_ttl'] = int(attrs['reason_ttl'])
elif name == 'extraports':
if self._curextraports is not None:
sys.stderr.write("WARNING, self._curextraports should be None"
" at this point "
"(got %r)\n" % self._curextraports)
self._curextraports = {attrs['state']: [int(attrs['count']), {}]}
elif name == 'extrareasons' and self._curextraports is not None:
self._curextraports[self._curextraports.keys()[0]][1][
attrs['reason']] = int(attrs['count'])
elif name == 'port':
if self._curport is not None:
sys.stderr.write("WARNING, self._curport should be None at "
"this point (got %r)\n" % self._curport)
self._curport = {
'protocol': attrs['protocol'],
'port': int(attrs['portid'])
}
elif name == 'state' and self._curport is not None:
for attr in attrs.keys():
self._curport['state_%s' % attr] = attrs[attr]
for field in ['state_reason_ttl']:
if field in self._curport:
self._curport[field] = int(self._curport[field])
for field in ['state_reason_ip']:
if field in self._curport:
try:
self._curport[field] = utils.ip2int(
self._curport[field])
except utils.socket.error:
pass
elif name == 'service' and self._curport is not None:
if attrs.get("method") == "table":
# discard information from nmap-services
return
for attr in attrs.keys():
self._curport['service_%s' % attr] = attrs[attr]
for field in [
'service_conf', 'service_rpcnum', 'service_lowver',
'service_highver'
]:
if field in self._curport:
self._curport[field] = int(self._curport[field])
elif name == 'script':
if self._curscript is not None:
sys.stderr.write("WARNING, self._curscript should be None "
"at this point (got %r)\n" % self._curscript)
self._curscript = dict([attr, attrs[attr]]
for attr in attrs.keys())
elif name in ['table', 'elem']:
if self._curscript.get('id') in IGNORE_TABLE_ELEMS:
return
if name == 'elem':
# start recording characters
if self._curdata is not None:
sys.stderr.write("WARNING, self._curdata should be None"
" at this point "
"(got %r)\n" % self._curdata)
self._curdata = ''
if 'key' in attrs:
key = attrs['key'].replace('.', '_')
obj = {key: {}}
else:
key = None
obj = []
if not self._curtablepath:
if not self._curtable:
self._curtable = obj
elif key is not None:
self._curtable.update(obj)
if key is None:
key = len(self._curtable)
self._curtablepath.append(key)
return
lastlevel = self._curtable
for k in self._curtablepath[:-1]:
lastlevel = lastlevel[k]
k = self._curtablepath[-1]
if type(k) is int:
if k < len(lastlevel):
if key is not None:
lastlevel[k].update(obj)
else:
lastlevel.append(obj)
if key is None:
key = len(lastlevel[k])
else:
if key is None:
if lastlevel[k]:
key = len(lastlevel[k])
else:
key = 0
lastlevel[k] = obj
else:
lastlevel[k].update(obj)
self._curtablepath.append(key)
elif name == 'os':
self._curhost['os'] = {}
elif name == 'portused' and 'os' in self._curhost:
self._curhost['os']['portused'] = {
'port': '%s_%s' % (attrs['proto'], attrs['portid']),
'state': attrs['state'],
}
elif name in ['osclass', 'osmatch'] and 'os' in self._curhost:
if name not in self._curhost['os']:
self._curhost['os'][name] = [dict(attrs)]
else:
self._curhost['os'][name].append(dict(attrs))
elif name == 'osfingerprint' and 'os' in self._curhost:
self._curhost['os']['fingerprint'] = attrs['fingerprint']
elif name == 'trace':
if self._curtrace is not None:
sys.stderr.write("WARNING, self._curtrace should be None "
"at this point (got %r)\n" % self._curtrace)
if 'proto' not in attrs:
self._curtrace = {'protocol': None}
elif attrs['proto'] in ['tcp', 'udp']:
self._curtrace = {
'protocol': attrs['proto'],
'port': int(attrs['port'])
}
else:
self._curtrace = {'protocol': attrs['proto']}
self._curtrace['hops'] = []
elif name == 'hop' and self._curtrace is not None:
attrsdict = dict(attrs)
try:
attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr'])
except utils.socket.error:
pass
try:
attrsdict['rtt'] = float(attrs['rtt'])
except ValueError:
pass
try:
attrsdict['ttl'] = int(attrs['ttl'])
except ValueError:
pass
if 'host' in attrsdict:
attrsdict['domains'] = list(
utils.get_domains(attrsdict['host']))
self._curtrace['hops'].append(attrsdict)
elif name == 'cpe':
# start recording
self._curdata = ''
def __getaddr__(self, line):
try:
return utils.ip2int(line.split('#', 1)[0].strip())
except utils.socket.error:
pass
def force_ip_int(addr):
try:
return utils.ip2int(addr)
except (TypeError, struct.error):
return addr
