def __init__(self, start, stop, categories=None, rand=True, maxnbr=None, state=None, name=None): super().__init__( geoiputils.IPRanges(ranges=[(utils.ip2int(start), utils.ip2int(stop))]), rand=rand, maxnbr=maxnbr, state=state, name=name or 'RANGE-%s-%s' % (start, stop), categories=categories, )
def __init__(self, start, stop, categories=None, rand=True, maxnbr=None, state=None): Target.__init__( self, geoiputils.IPRanges(ranges=[(utils.ip2int(start), utils.ip2int(stop))]), rand=rand, maxnbr=maxnbr, state=state ) if categories is None: categories = ['RANGE-%s-%s' % (start, stop)] self.infos = {'categories': categories}
def r2res(r): return [ utils.ip2int(r['addr']), [[p['port'], p['state_state']] for p in r.get('ports', []) if 'state_state' in p] ]
def parse_p0f_line(line, include_port=False, sensor=None, recontype=None): line = [line.split(b' - ')[0]] + line.split(b' - ')[1].split(b' -> ') if line[1].startswith(b'UNKNOWN '): sig = line[1][line[1].index(b'UNKNOWN ') + 8:][1:-1].split(b':')[:6] osname, version, dist = b'?', b'?', -1 else: sig = line[1][line[1].index(b' Signature: ') + 12:][ 1:-1].split(b':')[:6] if b' (up: ' in line[1]: osname = line[1][:line[1].index(b' (up: ')] else: osname = line[1][:line[1].index(b' Signature: ')] osname, version = osname.split(b' ')[0], b' '.join(osname.split(b' ')[1:]) dist = int(P0F_DIST.search(line[2]).groups()[0]) # We wildcard any window size which is not Sxxx or Tyyy if sig[0][0] not in b'ST': sig[0] = b'*' spec = { 'addr': utils.ip2int(line[0][line[0].index(b'> ') + 2:line[0].index(b':')]), 'distance': dist, 'value': osname.decode(), 'version': version.decode(), 'signature': ':'.join(str(elt) for elt in sig), } if include_port: spec.update({'port': int(line[0][line[0].index(b':') + 1:])}) if sensor is not None: spec['sensor'] = sensor if recontype is not None: spec['recontype'] = recontype return float(line[0][1:line[0].index(b'>')]), spec
def parse_p0f_line(line, include_port=False, sensor=None, recontype=None): line = [line.split(b' - ')[0]] + line.split(b' - ')[1].split(b' -> ') if line[1].startswith(b'UNKNOWN '): sig = line[1][line[1].index(b'UNKNOWN ') + 8:][1:-1].split(b':')[:6] osname, version, dist = b'?', b'?', -1 else: sig = line[1][line[1].index(b' Signature: ') + 12:][ 1:-1].split(b':')[:6] if b' (up: ' in line[1]: osname = line[1][:line[1].index(b' (up: ')] else: osname = line[1][:line[1].index(b' Signature: ')] osname = osname.split(b' ') osname, version = osname[0], b' '.join(osname[1:]) dist = int(P0F_DIST.search(line[2]).groups()[0]) # We wildcard any window size which is not Sxxx or Tyyy if sig[0][0] not in b'ST': sig[0] = b'*' spec = { 'addr': utils.ip2int(line[0][line[0].index(b'> ') + 2: line[0].index(b':')]), 'distance': dist, 'value': osname.decode(), 'version': version.decode(), 'signature': ':'.join(str(elt) for elt in sig), } if include_port: spec.update({'port': int(line[0][line[0].index(b':') + 1:])}) if sensor is not None: spec['sensor'] = sensor if recontype is not None: spec['recontype'] = recontype return float(line[0][1:line[0].index(b'>')]), spec
def bgp_raw_to_csv(fname, out): out = open(os.path.join(config.GEOIP_PATH, out), 'w') cur = None with open(os.path.join(config.GEOIP_PATH, fname), 'rb') as fdesc: for line in fdesc: start, stop = (utils.ip2int(elt) for elt in utils.net2range(line[:-1].split(None, 1)[0])) if cur: if start >= cur[0] and stop <= cur[1]: continue if start >= cur[0] and start <= cur[1]: cur = (cur[0], stop) continue if stop >= cur[0] and stop <= cur[1]: cur = (start, cur[1]) continue if start <= cur[0] and stop >= cur[1]: cur = (start, stop) continue if start == cur[1] + 1: cur = (cur[0], stop) continue if stop == cur[0] + 1: cur = (start, cur[1]) continue out.write('%d,%d\n' % cur) cur = (start, stop) if cur: out.write('%d,%d\n' % cur)
def bgp_raw_to_csv(fname, out): out = open(os.path.join(config.GEOIP_PATH, out), 'wb') cur = [] with open(os.path.join(config.GEOIP_PATH, fname), 'rb') as fdesc: for line in fdesc: start, stop = (utils.ip2int(elt) for elt in utils.net2range(line[:-1].split()[0])) if cur: if start >= cur[0] and stop <= cur[1]: continue if start >= cur[0] and start <= cur[1]: cur = [cur[0], stop] continue if stop >= cur[0] and stop <= cur[1]: cur = [start, cur[1]] continue if start <= cur[0] and stop >= cur[1]: cur = [start, stop] continue if start == cur[1] + 1: cur = [cur[0], stop] continue if stop == cur[0] + 1: cur = [start, cur[1]] continue out.write(('"%s","%s","%d","%d"\n' % ( utils.int2ip(cur[0]), utils.int2ip(cur[1]), cur[0], cur[1], )).encode()) cur = [start, stop] if cur: out.write(('"%s","%s","%d","%d"\n' % ( utils.int2ip(cur[0]), utils.int2ip(cur[1]), cur[0], cur[1], )).encode())
def __getaddr__(self, line): addr = self.match_addr.match(line) if addr is not None: try: return utils.ip2int(addr.groups()[0]) except utils.socket.error: pass
def get_scan_results(self) -> Generator[NmapHost, None, None]: starttime = datetime.now() results = sorted(self.parse().items(), key=lambda a_p: utils.ip2int(a_p[0])) endtime = datetime.now() for addr, ports in results: rec = {"addr": addr, "starttime": starttime, "endtime": endtime} for (proto, portnum), progs in sorted(ports.items()): port = {"protocol": proto, "port": portnum} if progs: port["scripts"] = [ { "id": "localscan", "output": "Program%s: %s" % ("s" if len(progs) > 1 else "", ", ".join(sorted(progs))), "localscan": { "programs": sorted(progs) }, }, ] # TODO: remove when NmapHost is implemented rec.setdefault("ports", []).append(port) # type: ignore yield rec
def handle_rec( sensor, ignorenets, neverignore, # these argmuments are provided by *<line.split()> timestamp, host, port, recon_type, source, value, targetval): recon_type = recon_type[14:] # skip PassiveRecon:: if host == '-': spec = { 'targetval': targetval, 'recontype': recon_type, 'value': value } else: try: host = utils.ip2int(host) except: pass spec = {'addr': host, 'recontype': recon_type, 'value': value} if sensor is not None: spec.update({'sensor': sensor}) if port != '-': spec.update({'port': int(port)}) if source != '-': spec.update({'source': source}) spec = _prepare_rec(spec, ignorenets, neverignore) return float(timestamp), spec
def handle_rec(sensor, ignorenets, neverignore, # these argmuments are provided by **bro_line timestamp, host, srvport, recon_type, source, value, targetval): if host is None: spec = { 'targetval': targetval, 'recontype': recon_type, 'value': value } else: try: host = utils.ip2int(host) except: pass spec = { 'addr': host, 'recontype': recon_type, 'value': value } if sensor is not None: spec.update({'sensor': sensor}) if srvport is not None: spec.update({'port': srvport}) if source is not None: spec.update({'source': source}) spec = _prepare_rec(spec, ignorenets, neverignore) # Python 2/3 compat: python 3 has datetime.timestamp() try: float_ts = timestamp.timestamp() except AttributeError: float_ts = time.mktime(timestamp.timetuple()) \ + timestamp.microsecond / (1000.*1000.) return float_ts, spec
def parse_p0f_line(line, include_port=False, sensor=None, recontype=None): line = [line.split(' - ')[0]] + line.split(' - ')[1].split(' -> ') if line[1].startswith('UNKNOWN '): sig = line[1][line[1].index('UNKNOWN ') + 8:][1:-1].split(':')[:6] OS, version, dist = '?', '?', -1 else: sig = line[1][line[1].index(' Signature: ') + 12:][1:-1].split(':')[:6] if ' (up: ' in line[1]: OS = line[1][:line[1].index(' (up: ')] else: OS = line[1][:line[1].index(' Signature: ')] OS, version = OS.split(' ')[0], ' '.join(OS.split(' ')[1:]) dist = int(P0F_DIST.search(line[2]).groups()[0]) # We wildcard any window size which is not Sxxx or Tyyy if sig[0][0] not in ['S', 'T']: sig[0] = '*' spec = { 'addr': utils.ip2int(line[0][line[0].index('> ') + 2:line[0].index(':')]), 'distance': dist, 'value': OS, 'version': version, 'signature': ":".join(map(str, sig)), } if include_port: spec.update({'port': int(line[0][line[0].index(':') + 1:])}) if sensor is not None: spec['sensor'] = sensor if recontype is not None: spec['recontype'] = recontype return float(line[0][1:line[0].index('>')]), spec
def parse_p0f_line(line, include_port=False, sensor=None, recontype=None): line = [line.split(' - ')[0]] + line.split(' - ')[1].split(' -> ') if line[1].startswith('UNKNOWN '): sig = line[1][line[1].index('UNKNOWN ') + 8:][1:-1].split(':')[:6] OS, version, dist = '?', '?', -1 else: sig = line[1][line[1].index(' Signature: ') + 12:][ 1:-1].split(':')[:6] if ' (up: ' in line[1]: OS = line[1][:line[1].index(' (up: ')] else: OS = line[1][:line[1].index(' Signature: ')] OS, version = OS.split(' ')[0], ' '.join(OS.split(' ')[1:]) dist = int(P0F_DIST.search(line[2]).groups()[0]) # We wildcard any window size which is not Sxxx or Tyyy if sig[0][0] not in ['S', 'T']: sig[0] = '*' spec = { 'addr': utils.ip2int(line[0][line[0].index('> ') + 2:line[0].index(':')]), 'distance': dist, 'value': OS, 'version': version, 'signature': ":".join(map(str, sig)), } if include_port: spec.update({'port': int(line[0][line[0].index(':') + 1:])}) if sensor is not None: spec['sensor'] = sensor if recontype is not None: spec['recontype'] = recontype return float(line[0][1:line[0].index('>')]), spec
def handle_rec(sensor, ignorenets, neverignore, # these argmuments are provided by *<line.split()> timestamp, host, port, recon_type, source, value, targetval): recon_type = recon_type[14:] # skip PassiveRecon:: if host == '-': spec = { 'targetval': targetval, 'recontype': recon_type, 'value': value } else: try: host = utils.ip2int(host) except: pass spec = { 'addr': host, 'recontype': recon_type, 'value': value } if sensor is not None: spec.update({'sensor': sensor}) if port != '-': spec.update({'port': int(port)}) if source != '-': spec.update({'source': source}) spec = _prepare_rec(spec, ignorenets, neverignore) return float(timestamp), spec
def bgp_raw_to_csv(fname: str, outname: str) -> None: cur = None assert config.GEOIP_PATH is not None with open(os.path.join(config.GEOIP_PATH, fname), "rb") as fdesc, open(os.path.join(config.GEOIP_PATH, outname), "w", encoding="utf8") as out: for line in fdesc: start, stop = ( utils.ip2int(elt) for elt in utils.net2range(line[:-1].split(None, 1)[0])) if cur: if start >= cur[0] and stop <= cur[1]: continue if cur[0] <= start <= cur[1]: cur = (cur[0], stop) continue if cur[0] <= stop <= cur[1]: cur = (start, cur[1]) continue if start <= cur[0] and stop >= cur[1]: cur = (start, stop) continue if start == cur[1] + 1: cur = (cur[0], stop) continue if stop == cur[0] + 1: cur = (start, cur[1]) continue out.write("%d,%d\n" % cur) cur = (start, stop) if cur: out.write("%d,%d\n" % cur)
def _getaddr(cls, line): addr = cls.match_addr.match(line) if addr is not None: try: return utils.ip2int(addr.groups()[0]) except utils.socket.error: pass return None
def graphhost(host: Record) -> Tuple[List[int], List[int]]: if "ports" not in host: return [], [] hh, pp = [], [] addr = utils.ip2int(host["addr"]) for p in host["ports"]: if p.get("state_state") == "open": hh.append(addr) pp.append(p["port"]) return hh, pp
def get_scan_results(self): starttime = datetime.now() results = sorted(self.parse().items(), key=lambda a_p: utils.ip2int(a_p[0])) endtime = datetime.now() for addr, ports in results: rec = {"addr": addr, "starttime": starttime, "endtime": endtime} for (proto, port), progs in sorted(ports.items()): port = {"protocol": proto, "port": port} if progs: port["scripts"] = [ { "id": "localscan", "output": "Program%s: %s" % ("s" if len(progs) > 1 else "", ", ".join(sorted(progs))), "localscan": { "programs": sorted(progs) }, }, ] rec.setdefault("ports", []).append(port) yield rec
def r2res(r): return utils.ip2int(r['addr'])
def r2res(r): return [utils.ip2int(r['addr']), r['openports']['count']]
def get_nmap_action(action): flt_params = get_nmap_base() preamble = "[\n" postamble = "]\n" r2res = lambda x: x if action == "timeline": result, count = db.view.get_open_port_count(flt_params.flt) if request.params.get("modulo") is None: r2time = lambda r: int(utils.datetime2timestamp(r['starttime'])) else: r2time = lambda r: (int(utils.datetime2timestamp(r['starttime'])) % int(request.params.get("modulo"))) if flt_params.ipsasnumbers: r2res = lambda r: [ r2time(r), utils.ip2int(r['addr']), r['openports']['count'] ] else: r2res = lambda r: [r2time(r), r['addr'], r['openports']['count']] elif action == "coordinates": preamble = '{"type": "GeometryCollection", "geometries": [\n' postamble = ']}\n' result = list(db.view.getlocations(flt_params.flt)) count = len(result) r2res = lambda r: { "type": "Point", "coordinates": r['_id'], "properties": { "count": r['count'] }, } elif action == "countopenports": result, count = db.view.get_open_port_count(flt_params.flt) if flt_params.ipsasnumbers: r2res = lambda r: [ utils.ip2int(r['addr']), r['openports']['count'] ] else: r2res = lambda r: [r['addr'], r['openports']['count']] elif action == "ipsports": result, count = db.view.get_ips_ports(flt_params.flt) if flt_params.ipsasnumbers: r2res = lambda r: [ utils.ip2int(r['addr']), [[p['port'], p['state_state']] for p in r.get('ports', []) if 'state_state' in p] ] else: r2res = lambda r: [ r['addr'], [[p['port'], p['state_state']] for p in r.get('ports', []) if 'state_state' in p] ] elif action == "onlyips": result, count = db.view.get_ips(flt_params.flt) if flt_params.ipsasnumbers: r2res = lambda r: utils.ip2int(r['addr']) else: r2res = lambda r: r['addr'] elif action == "diffcats": if request.params.get("onlydiff"): output = db.view.diff_categories(request.params.get("cat1"), request.params.get("cat2"), flt=flt_params.flt, include_both_open=False) else: output = db.view.diff_categories(request.params.get("cat1"), request.params.get("cat2"), flt=flt_params.flt) count = 0 result = {} if flt_params.ipsasnumbers: for res in output: result.setdefault(res["addr"], []).append([res['port'], res['value']]) count += 1 else: for res in output: result.setdefault(utils.int2ip(res["addr"]), []).append([res['port'], res['value']]) count += 1 result = viewitems(result) if flt_params.callback is not None: if count >= config.WEB_WARN_DOTS_COUNT: yield ( 'if(confirm("You are about to ask your browser to display %d ' 'dots, which is a lot and might slow down, freeze or crash ' 'your browser. Do you want to continue?")) {\n' % count) yield '%s(\n' % flt_params.callback yield preamble # hack to avoid a trailing comma result = iter(result) try: rec = next(result) except StopIteration: pass else: yield json.dumps(r2res(rec)) for rec in result: yield ",\n" + json.dumps(r2res(rec)) yield "\n" yield postamble if flt_params.callback is not None: yield ");" if count >= config.WEB_WARN_DOTS_COUNT: yield '}\n' else: yield "\n"
def startElement(self, name, attrs): if name == "nmaprun": if self._curscan is not None: sys.stderr.write("WARNING, self._curscan should be None at " "this point(got %r)\n" % self._curscan) self._curscan = dict(attrs) self._curscan["_id"] = self._filehash elif name == "scaninfo" and self._curscan is not None: self._addscaninfo(dict(attrs)) elif name == "host": if self._curhost is not None: sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost) self._curhost = {"schema_version": 1} if self._curscan: self._curhost["scanid"] = self._curscan["_id"] for attr in attrs.keys(): self._curhost[attr] = attrs[attr] for field in ["starttime", "endtime"]: if field in self._curhost: self._curhost[field] = datetime.datetime.utcfromtimestamp(int(self._curhost[field])) elif name == "address" and self._curhost is not None: if attrs["addrtype"] != "ipv4": if "addresses" not in self._curhost: self._curhost["addresses"] = {attrs["addrtype"]: [attrs["addr"]]} elif attrs["addrtype"] not in self._curhost: self._curhost["addresses"].update({attrs["addrtype"]: [attrs["addr"]]}) else: addresses = self._curhost["addresses"][attrs["addrtype"]] addresses.append(attrs["addr"]) self._curhost["addresses"][attrs["addrtype"]] = addresses else: try: self._curhost["addr"] = utils.ip2int(attrs["addr"]) except utils.socket.error: self._curhost["addr"] = attrs["addr"] elif name == "hostnames": if self._curhostnames is not None: sys.stderr.write( "WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames ) self._curhostnames = [] elif name == "hostname": if self._curhostnames is None: sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n") self._curhostnames = [] hostname = dict(attrs) if "name" in attrs: hostname["domains"] = list(utils.get_domains(attrs["name"])) self._curhostnames.append(hostname) elif name == "status" and self._curhost is not None: self._curhost["state"] = attrs["state"] if "reason" in attrs: self._curhost["state_reason"] = attrs["reason"] if "reason_ttl" in attrs: self._curhost["state_reason_ttl"] = int(attrs["reason_ttl"]) elif name == "extraports": if self._curextraports is not None: sys.stderr.write( "WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports ) self._curextraports = {attrs["state"]: [int(attrs["count"]), {}]} elif name == "extrareasons" and self._curextraports is not None: self._curextraports[self._curextraports.keys()[0]][1][attrs["reason"]] = int(attrs["count"]) elif name == "port": if self._curport is not None: sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport) self._curport = {"protocol": attrs["protocol"], "port": int(attrs["portid"])} elif name == "state" and self._curport is not None: for attr in attrs.keys(): self._curport["state_%s" % attr] = attrs[attr] for field in ["state_reason_ttl"]: if field in self._curport: self._curport[field] = int(self._curport[field]) for field in ["state_reason_ip"]: if field in self._curport: try: self._curport[field] = utils.ip2int(self._curport[field]) except utils.socket.error: pass elif name == "service" and self._curport is not None: if attrs.get("method") == "table": # discard information from nmap-services return for attr in attrs.keys(): self._curport["service_%s" % attr] = attrs[attr] for field in ["service_conf", "service_rpcnum", "service_lowver", "service_highver"]: if field in self._curport: self._curport[field] = int(self._curport[field]) elif name == "script": if self._curscript is not None: sys.stderr.write( "WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript ) self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys()) elif name in ["table", "elem"]: if self._curscript.get("id") in IGNORE_TABLE_ELEMS: return if name == "elem": # start recording characters if self._curdata is not None: sys.stderr.write( "WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata ) self._curdata = "" if "key" in attrs: key = attrs["key"].replace(".", "_") obj = {key: {}} else: key = None obj = [] if not self._curtablepath: if not self._curtable: self._curtable = obj elif key is not None: self._curtable.update(obj) if key is None: key = len(self._curtable) self._curtablepath.append(key) return lastlevel = self._curtable for k in self._curtablepath[:-1]: lastlevel = lastlevel[k] k = self._curtablepath[-1] if type(k) is int: if k < len(lastlevel): if key is not None: lastlevel[k].update(obj) else: lastlevel.append(obj) if key is None: key = len(lastlevel[k]) else: if key is None: if lastlevel[k]: key = len(lastlevel[k]) else: key = 0 lastlevel[k] = obj else: lastlevel[k].update(obj) self._curtablepath.append(key) elif name == "os": self._curhost["os"] = {} elif name == "portused" and "os" in self._curhost: self._curhost["os"]["portused"] = { "port": "%s_%s" % (attrs["proto"], attrs["portid"]), "state": attrs["state"], } elif name in ["osclass", "osmatch"] and "os" in self._curhost: if name not in self._curhost["os"]: self._curhost["os"][name] = [dict(attrs)] else: self._curhost["os"][name].append(dict(attrs)) elif name == "osfingerprint" and "os" in self._curhost: self._curhost["os"]["fingerprint"] = attrs["fingerprint"] elif name == "trace": if self._curtrace is not None: sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace) if "proto" not in attrs: self._curtrace = {"protocol": None} elif attrs["proto"] in ["tcp", "udp"]: self._curtrace = {"protocol": attrs["proto"], "port": int(attrs["port"])} else: self._curtrace = {"protocol": attrs["proto"]} self._curtrace["hops"] = [] elif name == "hop" and self._curtrace is not None: attrsdict = dict(attrs) try: attrsdict["ipaddr"] = utils.ip2int(attrs["ipaddr"]) except utils.socket.error: pass try: attrsdict["rtt"] = float(attrs["rtt"]) except ValueError: pass try: attrsdict["ttl"] = int(attrs["ttl"]) except ValueError: pass if "host" in attrsdict: attrsdict["domains"] = list(utils.get_domains(attrsdict["host"])) self._curtrace["hops"].append(attrsdict) elif name == "cpe": # start recording self._curdata = ""
def r2res(r): return [ utils.ip2int(r['addr']), r.get('openports', {}).get('count', 0) ]
def _getaddr(line): try: return utils.ip2int(line.split("#", 1)[0].strip()) except utils.socket.error: return None
def force_ip_int(addr): try: return utils.ip2int(addr) except (TypeError, struct.error): return addr
def __getaddr__(self, line): try: return utils.ip2int(line.split('#', 1)[0].strip()) except utils.socket.error: pass
def startElement(self, name, attrs): if name == 'nmaprun': if self._curscan is not None: sys.stderr.write("WARNING, self._curscan should be None at " "this point(got %r)\n" % self._curscan) self._curscan = dict(attrs) self._curscan['_id'] = self._filehash elif name == 'scaninfo' and self._curscan is not None: self._addscaninfo(dict(attrs)) elif name == 'host': if self._curhost is not None: sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost) self._curhost = {"schema_version": 1} if self._curscan: self._curhost['scanid'] = self._curscan['_id'] for attr in attrs.keys(): self._curhost[attr] = attrs[attr] for field in ['starttime', 'endtime']: if field in self._curhost: self._curhost[field] = datetime.datetime.utcfromtimestamp( int(self._curhost[field]) ) elif name == 'address' and self._curhost is not None: if attrs['addrtype'] != 'ipv4': if 'addresses' not in self._curhost: self._curhost['addresses'] = { attrs['addrtype']: [attrs['addr']] } elif attrs['addrtype'] not in self._curhost: self._curhost['addresses'].update({ attrs['addrtype']: [attrs['addr']] }) else: addresses = self._curhost['addresses'][attrs['addrtype']] addresses.append(attrs['addr']) self._curhost['addresses'][attrs['addrtype']] = addresses else: try: self._curhost['addr'] = utils.ip2int(attrs['addr']) except utils.socket.error: self._curhost['addr'] = attrs['addr'] elif name == 'hostnames': if self._curhostnames is not None: sys.stderr.write("WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames) self._curhostnames = [] elif name == 'hostname': if self._curhostnames is None: sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n") self._curhostnames = [] hostname = dict(attrs) if 'name' in attrs: hostname['domains'] = list(utils.get_domains(attrs['name'])) self._curhostnames.append(hostname) elif name == 'status' and self._curhost is not None: self._curhost['state'] = attrs['state'] if 'reason' in attrs: self._curhost['state_reason'] = attrs['reason'] if 'reason_ttl' in attrs: self._curhost['state_reason_ttl'] = int(attrs['reason_ttl']) elif name == 'extraports': if self._curextraports is not None: sys.stderr.write("WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports) self._curextraports = {attrs['state']: [int(attrs['count']), {}]} elif name == 'extrareasons' and self._curextraports is not None: self._curextraports[self._curextraports.keys()[0]][1][ attrs['reason']] = int(attrs['count']) elif name == 'port': if self._curport is not None: sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport) self._curport = {'protocol': attrs['protocol'], 'port': int(attrs['portid'])} elif name == 'state' and self._curport is not None: for attr in attrs.keys(): self._curport['state_%s' % attr] = attrs[attr] for field in ['state_reason_ttl']: if field in self._curport: self._curport[field] = int(self._curport[field]) for field in ['state_reason_ip']: if field in self._curport: try: self._curport[field] = utils.ip2int( self._curport[field]) except utils.socket.error: pass elif name == 'service' and self._curport is not None: for attr in attrs.keys(): self._curport['service_%s' % attr] = attrs[attr] for field in ['service_conf', 'service_rpcnum', 'service_lowver', 'service_highver']: if field in self._curport: self._curport[field] = int(self._curport[field]) elif name == 'script': if self._curscript is not None: sys.stderr.write("WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript) self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys()) elif name in ['table', 'elem']: if self._curscript.get('id') in IGNORE_TABLE_ELEMS: return if name == 'elem': # start recording characters if self._curdata is not None: sys.stderr.write("WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata) self._curdata = '' if 'key' in attrs: key = attrs['key'].replace('.', '_') obj = {key: {}} else: key = None obj = [] if not self._curtablepath: if not self._curtable: self._curtable = obj elif key is not None: self._curtable.update(obj) if key is None: key = len(self._curtable) self._curtablepath.append(key) return lastlevel = self._curtable for k in self._curtablepath[:-1]: lastlevel = lastlevel[k] k = self._curtablepath[-1] if type(k) is int: if k < len(lastlevel): if key is not None: lastlevel[k].update(obj) else: lastlevel.append(obj) if key is None: key = len(lastlevel[k]) else: if key is None: if lastlevel[k]: key = len(lastlevel[k]) else: key = 0 lastlevel[k] = obj else: lastlevel[k].update(obj) self._curtablepath.append(key) elif name == 'os': self._curhost['os'] = {} elif name == 'portused' and 'os' in self._curhost: self._curhost['os']['portused'] = { 'port': '%s_%s' % (attrs['proto'], attrs['portid']), 'state': attrs['state'], } elif name in ['osclass', 'osmatch'] and 'os' in self._curhost: if name not in self._curhost['os']: self._curhost['os'][name] = [dict(attrs)] else: self._curhost['os'][name].append(dict(attrs)) elif name == 'osfingerprint' and 'os' in self._curhost: self._curhost['os']['fingerprint'] = attrs['fingerprint'] elif name == 'trace': if self._curtrace is not None: sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace) if 'proto' not in attrs: self._curtrace = {'protocol': None} elif attrs['proto'] in ['tcp', 'udp']: self._curtrace = {'protocol': attrs['proto'], 'port': int(attrs['port'])} else: self._curtrace = {'protocol': attrs['proto']} self._curtrace['hops'] = [] elif name == 'hop' and self._curtrace is not None: attrsdict = dict(attrs) try: attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr']) except utils.socket.error: pass try: attrsdict['rtt'] = float(attrs['rtt']) except ValueError: pass try: attrsdict['ttl'] = int(attrs['ttl']) except ValueError: pass if 'host' in attrsdict: attrsdict['domains'] = list( utils.get_domains(attrsdict['host'])) self._curtrace['hops'].append(attrsdict) elif name == 'cpe': # start recording self._curdata = ''
def startElement(self, name, attrs): if name == 'nmaprun': if self._curscan is not None: sys.stderr.write("WARNING, self._curscan should be None at " "this point (got %r)\n" % self._curscan) self._curscan = dict(attrs) self.scanner = self._curscan.get("scanner", self.scanner) if self.scanner == "masscan": # We need to force "merge" mode due to the nature of # Masscan results self.merge = True self._curscan['_id'] = self._filehash elif name == 'scaninfo' and self._curscan is not None: self._addscaninfo(dict(attrs)) elif name == 'host': if self._curhost is not None: sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost) self._curhost = {"schema_version": SCHEMA_VERSION} if self._curscan: self._curhost['scanid'] = self._curscan['_id'] for attr in attrs.keys(): self._curhost[attr] = attrs[attr] for field in ['starttime', 'endtime']: if field in self._curhost: self._curhost[field] = datetime.datetime.utcfromtimestamp( int(self._curhost[field]) ) if 'starttime' not in self._curhost and 'endtime' in self._curhost: # Masscan self._curhost['starttime'] = self._curhost['endtime'] elif name == 'address' and self._curhost is not None: if attrs['addrtype'] != 'ipv4': self._curhost.setdefault( 'addresses', {}).setdefault( attrs['addrtype'], []).append(attrs['addr']) else: try: self._curhost['addr'] = utils.ip2int(attrs['addr']) except utils.socket.error: self._curhost['addr'] = attrs['addr'] elif name == 'hostnames': if self._curhostnames is not None: sys.stderr.write("WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames) self._curhostnames = [] elif name == 'hostname': if self._curhostnames is None: sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n") self._curhostnames = [] hostname = dict(attrs) if 'name' in attrs: hostname['domains'] = list(utils.get_domains(attrs['name'])) self._curhostnames.append(hostname) elif name == 'status' and self._curhost is not None: self._curhost['state'] = attrs['state'] if 'reason' in attrs: self._curhost['state_reason'] = attrs['reason'] if 'reason_ttl' in attrs: self._curhost['state_reason_ttl'] = int(attrs['reason_ttl']) elif name == 'extraports': if self._curextraports is not None: sys.stderr.write("WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports) self._curextraports = { attrs['state']: {"total": int(attrs['count']), "reasons": {}}, } elif name == 'extrareasons' and self._curextraports is not None: self._curextraports[next(iter(self._curextraports))]["reasons"][ attrs['reason']] = int(attrs['count']) elif name == 'port': if self._curport is not None: sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport) self._curport = {'protocol': attrs['protocol'], 'port': int(attrs['portid'])} elif name == 'state' and self._curport is not None: for attr in attrs.keys(): self._curport['state_%s' % attr] = attrs[attr] for field in ['state_reason_ttl']: if field in self._curport: self._curport[field] = int(self._curport[field]) for field in ['state_reason_ip']: if field in self._curport: try: self._curport[field] = utils.ip2int( self._curport[field]) except utils.socket.error: pass elif name == 'service' and self._curport is not None: if attrs.get("method") == "table": # discard information from nmap-services return if self.scanner == "masscan": # create fake scripts from masscan "service" tags self._curport.setdefault('scripts', []).append({ "id": MASSCAN_SERVICES_NMAP_SCRIPTS.get(attrs['name'], attrs['name']), "output": MASSCAN_ENCODING.sub( _masscan_decode, attrs["banner"], ) }) # get service name service = MASSCAN_SERVICES_NMAP_SERVICES.get(attrs['name']) if service is not None: self._curport['service_name'] = service return for attr in attrs.keys(): self._curport['service_%s' % attr] = attrs[attr] for field in ['service_conf', 'service_rpcnum', 'service_lowver', 'service_highver']: if field in self._curport: self._curport[field] = int(self._curport[field]) elif name == 'script': if self._curscript is not None: sys.stderr.write("WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript) self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys()) elif name in ['table', 'elem']: if self._curscript.get('id') in IGNORE_TABLE_ELEMS: return if name == 'elem': # start recording characters if self._curdata is not None: sys.stderr.write("WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata) self._curdata = '' if 'key' in attrs: key = attrs['key'].replace('.', '_') obj = {key: {}} else: key = None obj = [] if not self._curtablepath: if not self._curtable: self._curtable = obj elif key is not None: self._curtable.update(obj) if key is None: key = len(self._curtable) self._curtablepath.append(key) return lastlevel = self._curtable for k in self._curtablepath[:-1]: lastlevel = lastlevel[k] k = self._curtablepath[-1] if type(k) is int: if k < len(lastlevel): if key is not None: lastlevel[k].update(obj) else: lastlevel.append(obj) if key is None: key = len(lastlevel[k]) else: if key is None: if lastlevel[k]: key = len(lastlevel[k]) else: key = 0 lastlevel[k] = obj else: lastlevel[k].update(obj) self._curtablepath.append(key) elif name == 'os': self._curhost['os'] = {} elif name == 'portused' and 'os' in self._curhost: self._curhost['os']['portused'] = { 'port': '%s_%s' % (attrs['proto'], attrs['portid']), 'state': attrs['state'], } elif name in ['osclass', 'osmatch'] and 'os' in self._curhost: self._curhost['os'].setdefault(name, []).append(dict(attrs)) elif name == 'osfingerprint' and 'os' in self._curhost: self._curhost['os']['fingerprint'] = attrs['fingerprint'] elif name == 'trace': if self._curtrace is not None: sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace) if 'proto' not in attrs: self._curtrace = {'protocol': None} elif attrs['proto'] in ['tcp', 'udp']: self._curtrace = {'protocol': attrs['proto'], 'port': int(attrs['port'])} else: self._curtrace = {'protocol': attrs['proto']} self._curtrace['hops'] = [] elif name == 'hop' and self._curtrace is not None: attrsdict = dict(attrs) try: attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr']) except utils.socket.error: pass try: attrsdict['rtt'] = float(attrs['rtt']) except ValueError: pass try: attrsdict['ttl'] = int(attrs['ttl']) except ValueError: pass if 'host' in attrsdict: attrsdict['domains'] = list( utils.get_domains(attrsdict['host'])) self._curtrace['hops'].append(attrsdict) elif name == 'cpe': # start recording self._curdata = ''
def startElement(self, name, attrs): if name == 'nmaprun': if self._curscan is not None: sys.stderr.write("WARNING, self._curscan should be None at " "this point(got %r)\n" % self._curscan) self._curscan = dict(attrs) self._curscan['_id'] = self._filehash elif name == 'scaninfo' and self._curscan is not None: self._addscaninfo(dict(attrs)) elif name == 'host': if self._curhost is not None: sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost) self._curhost = {"schema_version": SCHEMA_VERSION} if self._curscan: self._curhost['scanid'] = self._curscan['_id'] for attr in attrs.keys(): self._curhost[attr] = attrs[attr] for field in ['starttime', 'endtime']: if field in self._curhost: self._curhost[field] = datetime.datetime.utcfromtimestamp( int(self._curhost[field])) elif name == 'address' and self._curhost is not None: if attrs['addrtype'] != 'ipv4': if 'addresses' not in self._curhost: self._curhost['addresses'] = { attrs['addrtype']: [attrs['addr']] } elif attrs['addrtype'] not in self._curhost: self._curhost['addresses'].update( {attrs['addrtype']: [attrs['addr']]}) else: addresses = self._curhost['addresses'][attrs['addrtype']] addresses.append(attrs['addr']) self._curhost['addresses'][attrs['addrtype']] = addresses else: try: self._curhost['addr'] = utils.ip2int(attrs['addr']) except utils.socket.error: self._curhost['addr'] = attrs['addr'] elif name == 'hostnames': if self._curhostnames is not None: sys.stderr.write("WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames) self._curhostnames = [] elif name == 'hostname': if self._curhostnames is None: sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n") self._curhostnames = [] hostname = dict(attrs) if 'name' in attrs: hostname['domains'] = list(utils.get_domains(attrs['name'])) self._curhostnames.append(hostname) elif name == 'status' and self._curhost is not None: self._curhost['state'] = attrs['state'] if 'reason' in attrs: self._curhost['state_reason'] = attrs['reason'] if 'reason_ttl' in attrs: self._curhost['state_reason_ttl'] = int(attrs['reason_ttl']) elif name == 'extraports': if self._curextraports is not None: sys.stderr.write("WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports) self._curextraports = {attrs['state']: [int(attrs['count']), {}]} elif name == 'extrareasons' and self._curextraports is not None: self._curextraports[self._curextraports.keys()[0]][1][ attrs['reason']] = int(attrs['count']) elif name == 'port': if self._curport is not None: sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport) self._curport = { 'protocol': attrs['protocol'], 'port': int(attrs['portid']) } elif name == 'state' and self._curport is not None: for attr in attrs.keys(): self._curport['state_%s' % attr] = attrs[attr] for field in ['state_reason_ttl']: if field in self._curport: self._curport[field] = int(self._curport[field]) for field in ['state_reason_ip']: if field in self._curport: try: self._curport[field] = utils.ip2int( self._curport[field]) except utils.socket.error: pass elif name == 'service' and self._curport is not None: if attrs.get("method") == "table": # discard information from nmap-services return for attr in attrs.keys(): self._curport['service_%s' % attr] = attrs[attr] for field in [ 'service_conf', 'service_rpcnum', 'service_lowver', 'service_highver' ]: if field in self._curport: self._curport[field] = int(self._curport[field]) elif name == 'script': if self._curscript is not None: sys.stderr.write("WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript) self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys()) elif name in ['table', 'elem']: if self._curscript.get('id') in IGNORE_TABLE_ELEMS: return if name == 'elem': # start recording characters if self._curdata is not None: sys.stderr.write("WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata) self._curdata = '' if 'key' in attrs: key = attrs['key'].replace('.', '_') obj = {key: {}} else: key = None obj = [] if not self._curtablepath: if not self._curtable: self._curtable = obj elif key is not None: self._curtable.update(obj) if key is None: key = len(self._curtable) self._curtablepath.append(key) return lastlevel = self._curtable for k in self._curtablepath[:-1]: lastlevel = lastlevel[k] k = self._curtablepath[-1] if type(k) is int: if k < len(lastlevel): if key is not None: lastlevel[k].update(obj) else: lastlevel.append(obj) if key is None: key = len(lastlevel[k]) else: if key is None: if lastlevel[k]: key = len(lastlevel[k]) else: key = 0 lastlevel[k] = obj else: lastlevel[k].update(obj) self._curtablepath.append(key) elif name == 'os': self._curhost['os'] = {} elif name == 'portused' and 'os' in self._curhost: self._curhost['os']['portused'] = { 'port': '%s_%s' % (attrs['proto'], attrs['portid']), 'state': attrs['state'], } elif name in ['osclass', 'osmatch'] and 'os' in self._curhost: if name not in self._curhost['os']: self._curhost['os'][name] = [dict(attrs)] else: self._curhost['os'][name].append(dict(attrs)) elif name == 'osfingerprint' and 'os' in self._curhost: self._curhost['os']['fingerprint'] = attrs['fingerprint'] elif name == 'trace': if self._curtrace is not None: sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace) if 'proto' not in attrs: self._curtrace = {'protocol': None} elif attrs['proto'] in ['tcp', 'udp']: self._curtrace = { 'protocol': attrs['proto'], 'port': int(attrs['port']) } else: self._curtrace = {'protocol': attrs['proto']} self._curtrace['hops'] = [] elif name == 'hop' and self._curtrace is not None: attrsdict = dict(attrs) try: attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr']) except utils.socket.error: pass try: attrsdict['rtt'] = float(attrs['rtt']) except ValueError: pass try: attrsdict['ttl'] = int(attrs['ttl']) except ValueError: pass if 'host' in attrsdict: attrsdict['domains'] = list( utils.get_domains(attrsdict['host'])) self._curtrace['hops'].append(attrsdict) elif name == 'cpe': # start recording self._curdata = ''