def test(self, v4=True, v6=True): start = datetime.now() for srvname, addr, res in self.do_test(v4=v4, v6=v6): if not res: continue if (len(res) == 1 and res[0].rtype == 'SOA'): # SOA only: transfer failed continue LOGGER.info('AXFR success for %r on %r', self.domain, addr) line_fmt = "| %%-%ds %%-%ds %%s" % ( max(len(r.name) for r in res), max(len(r.rtype) for r in res), ) yield { "addr": addr, "hostnames": [{ "name": srvname, "type": "user", "domains": list(get_domains(srvname)) }], "schema_version": SCHEMA_VERSION, "starttime": start, "endtime": datetime.now(), "ports": [{ "port": 53, "protocol": "tcp", "service_name": "domain", "state_state": "open", "scripts": [{ "id": "dns-zone-transfer", "output": '\nDomain: %s\n%s\n\\\n' % ( self.domain, '\n'.join(line_fmt % (r.name, r.rtype, r.data) for r in res), ), "dns-zone-transfer": [{ "domain": self.domain, "records": [{ "name": r.name, "ttl": r.ttl, "class": r.rclass, "type": r.rtype, "data": r.data } for r in res] }] }] }], } start = datetime.now()
def _getinfos_dns(spec): """Extract domain names in an handy-to-index-and-query form.""" infos = {} fullinfos = {} fields = {'domain': 'value', 'domaintarget': 'targetval'} for field in fields: try: if fields[field] not in spec: continue infos[field] = [] fullinfos[field] = [] for domain in utils.get_domains( spec.get('full' + fields[field], spec[fields[field]])): infos[field].append(domain[:utils.MAXVALLEN]) if len(domain) > utils.MAXVALLEN: fullinfos[field].append(domain) if not infos[field]: del infos[field] if not fullinfos[field]: del fullinfos[field] except Exception: pass res = {} if infos: res['infos'] = infos if fullinfos: res['fullinfos'] = fullinfos return res
def test(self, v4: bool = True, v6: bool = True) -> Generator[NmapHost, None, None]: yield from super().test(v4=v4, v6=v6) for srvname, addr, _ in self.results: srvname = srvname.rstrip(".") yield { "addr": addr, "hostnames": [{ "name": srvname, "type": "user", "domains": list(get_domains(srvname)), }], "schema_version": SCHEMA_VERSION, "starttime": self.start, "endtime": self.stop, "ports": [{ "port": 53, "protocol": "udp", "service_name": "domain", "state_state": "open", "scripts": [ { "id": "dns-domains", "output": "Server is authoritative for %s" % self.domain, "dns-domains": [{ "domain": self.domain, "parents": list(get_domains(self.domain)), }], }, ], }], }
def _getinfos_dns_blacklist(spec): """Extract and properly format DNSBL records.""" infos = {} try: if "source" in spec: infos["domain"] = [] for domain in utils.get_domains(spec["source"].split("-")[-4]): infos["domain"].append(domain) if not infos["domain"]: del infos["domain"] except Exception: pass res = {} if infos: res["infos"] = infos return res
def _getinfos_dns_blacklist(spec): """Extract and properly format DNSBL records.""" infos = {} try: if 'source' in spec: infos['domain'] = [] for domain in utils.get_domains(spec['source'].split('-')[-4]): infos['domain'].append(domain) if not infos['domain']: del infos['domain'] except Exception: pass res = {} if infos: res['infos'] = infos return res
def _getinfos_dns(spec): """Extract domain names in an handy-to-index-and-query form.""" infos = {} fields = {"domain": "value", "domaintarget": "targetval"} for field, value in fields.items(): try: if value not in spec: continue infos[field] = [] for domain in utils.get_domains(spec[value]): infos[field].append(domain) if not infos[field]: del infos[field] except Exception: pass res = {} if infos: res["infos"] = infos return res
def merge_dns_domains_scripts(curscript, script, script_id): domains = { res["domain"]: (res["parents"] if "parents" in res else list( get_domains(res["domain"]))) for scr in [script, curscript] for res in scr.get(script_id, []) if "domain" in res } domains_order = sorted(domains, key=lambda v: v.strip().split(".")[::-1]) if len(domains_order) == 1: output = "Server is authoritative for %s" % domains_order[0] else: output = "Server is authoritative for:\n%s" % "\n".join( " %s" % dom for dom in domains_order) curscript["output"] = output curscript[script_id] = [{ "domain": dom, "parents": domains[dom] } for dom in domains_order] return curscript
def startElement(self, name, attrs): if name == "nmaprun": if self._curscan is not None: sys.stderr.write("WARNING, self._curscan should be None at " "this point(got %r)\n" % self._curscan) self._curscan = dict(attrs) self._curscan["_id"] = self._filehash elif name == "scaninfo" and self._curscan is not None: self._addscaninfo(dict(attrs)) elif name == "host": if self._curhost is not None: sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost) self._curhost = {"schema_version": 1} if self._curscan: self._curhost["scanid"] = self._curscan["_id"] for attr in attrs.keys(): self._curhost[attr] = attrs[attr] for field in ["starttime", "endtime"]: if field in self._curhost: self._curhost[field] = datetime.datetime.utcfromtimestamp(int(self._curhost[field])) elif name == "address" and self._curhost is not None: if attrs["addrtype"] != "ipv4": if "addresses" not in self._curhost: self._curhost["addresses"] = {attrs["addrtype"]: [attrs["addr"]]} elif attrs["addrtype"] not in self._curhost: self._curhost["addresses"].update({attrs["addrtype"]: [attrs["addr"]]}) else: addresses = self._curhost["addresses"][attrs["addrtype"]] addresses.append(attrs["addr"]) self._curhost["addresses"][attrs["addrtype"]] = addresses else: try: self._curhost["addr"] = utils.ip2int(attrs["addr"]) except utils.socket.error: self._curhost["addr"] = attrs["addr"] elif name == "hostnames": if self._curhostnames is not None: sys.stderr.write( "WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames ) self._curhostnames = [] elif name == "hostname": if self._curhostnames is None: sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n") self._curhostnames = [] hostname = dict(attrs) if "name" in attrs: hostname["domains"] = list(utils.get_domains(attrs["name"])) self._curhostnames.append(hostname) elif name == "status" and self._curhost is not None: self._curhost["state"] = attrs["state"] if "reason" in attrs: self._curhost["state_reason"] = attrs["reason"] if "reason_ttl" in attrs: self._curhost["state_reason_ttl"] = int(attrs["reason_ttl"]) elif name == "extraports": if self._curextraports is not None: sys.stderr.write( "WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports ) self._curextraports = {attrs["state"]: [int(attrs["count"]), {}]} elif name == "extrareasons" and self._curextraports is not None: self._curextraports[self._curextraports.keys()[0]][1][attrs["reason"]] = int(attrs["count"]) elif name == "port": if self._curport is not None: sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport) self._curport = {"protocol": attrs["protocol"], "port": int(attrs["portid"])} elif name == "state" and self._curport is not None: for attr in attrs.keys(): self._curport["state_%s" % attr] = attrs[attr] for field in ["state_reason_ttl"]: if field in self._curport: self._curport[field] = int(self._curport[field]) for field in ["state_reason_ip"]: if field in self._curport: try: self._curport[field] = utils.ip2int(self._curport[field]) except utils.socket.error: pass elif name == "service" and self._curport is not None: if attrs.get("method") == "table": # discard information from nmap-services return for attr in attrs.keys(): self._curport["service_%s" % attr] = attrs[attr] for field in ["service_conf", "service_rpcnum", "service_lowver", "service_highver"]: if field in self._curport: self._curport[field] = int(self._curport[field]) elif name == "script": if self._curscript is not None: sys.stderr.write( "WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript ) self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys()) elif name in ["table", "elem"]: if self._curscript.get("id") in IGNORE_TABLE_ELEMS: return if name == "elem": # start recording characters if self._curdata is not None: sys.stderr.write( "WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata ) self._curdata = "" if "key" in attrs: key = attrs["key"].replace(".", "_") obj = {key: {}} else: key = None obj = [] if not self._curtablepath: if not self._curtable: self._curtable = obj elif key is not None: self._curtable.update(obj) if key is None: key = len(self._curtable) self._curtablepath.append(key) return lastlevel = self._curtable for k in self._curtablepath[:-1]: lastlevel = lastlevel[k] k = self._curtablepath[-1] if type(k) is int: if k < len(lastlevel): if key is not None: lastlevel[k].update(obj) else: lastlevel.append(obj) if key is None: key = len(lastlevel[k]) else: if key is None: if lastlevel[k]: key = len(lastlevel[k]) else: key = 0 lastlevel[k] = obj else: lastlevel[k].update(obj) self._curtablepath.append(key) elif name == "os": self._curhost["os"] = {} elif name == "portused" and "os" in self._curhost: self._curhost["os"]["portused"] = { "port": "%s_%s" % (attrs["proto"], attrs["portid"]), "state": attrs["state"], } elif name in ["osclass", "osmatch"] and "os" in self._curhost: if name not in self._curhost["os"]: self._curhost["os"][name] = [dict(attrs)] else: self._curhost["os"][name].append(dict(attrs)) elif name == "osfingerprint" and "os" in self._curhost: self._curhost["os"]["fingerprint"] = attrs["fingerprint"] elif name == "trace": if self._curtrace is not None: sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace) if "proto" not in attrs: self._curtrace = {"protocol": None} elif attrs["proto"] in ["tcp", "udp"]: self._curtrace = {"protocol": attrs["proto"], "port": int(attrs["port"])} else: self._curtrace = {"protocol": attrs["proto"]} self._curtrace["hops"] = [] elif name == "hop" and self._curtrace is not None: attrsdict = dict(attrs) try: attrsdict["ipaddr"] = utils.ip2int(attrs["ipaddr"]) except utils.socket.error: pass try: attrsdict["rtt"] = float(attrs["rtt"]) except ValueError: pass try: attrsdict["ttl"] = int(attrs["ttl"]) except ValueError: pass if "host" in attrsdict: attrsdict["domains"] = list(utils.get_domains(attrsdict["host"])) self._curtrace["hops"].append(attrsdict) elif name == "cpe": # start recording self._curdata = ""
def startElement(self, name, attrs): if name == 'nmaprun': if self._curscan is not None: sys.stderr.write("WARNING, self._curscan should be None at " "this point(got %r)\n" % self._curscan) self._curscan = dict(attrs) self._curscan['_id'] = self._filehash elif name == 'scaninfo' and self._curscan is not None: self._addscaninfo(dict(attrs)) elif name == 'host': if self._curhost is not None: sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost) self._curhost = {"schema_version": 1} if self._curscan: self._curhost['scanid'] = self._curscan['_id'] for attr in attrs.keys(): self._curhost[attr] = attrs[attr] for field in ['starttime', 'endtime']: if field in self._curhost: self._curhost[field] = datetime.datetime.utcfromtimestamp( int(self._curhost[field]) ) elif name == 'address' and self._curhost is not None: if attrs['addrtype'] != 'ipv4': if 'addresses' not in self._curhost: self._curhost['addresses'] = { attrs['addrtype']: [attrs['addr']] } elif attrs['addrtype'] not in self._curhost: self._curhost['addresses'].update({ attrs['addrtype']: [attrs['addr']] }) else: addresses = self._curhost['addresses'][attrs['addrtype']] addresses.append(attrs['addr']) self._curhost['addresses'][attrs['addrtype']] = addresses else: try: self._curhost['addr'] = utils.ip2int(attrs['addr']) except utils.socket.error: self._curhost['addr'] = attrs['addr'] elif name == 'hostnames': if self._curhostnames is not None: sys.stderr.write("WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames) self._curhostnames = [] elif name == 'hostname': if self._curhostnames is None: sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n") self._curhostnames = [] hostname = dict(attrs) if 'name' in attrs: hostname['domains'] = list(utils.get_domains(attrs['name'])) self._curhostnames.append(hostname) elif name == 'status' and self._curhost is not None: self._curhost['state'] = attrs['state'] if 'reason' in attrs: self._curhost['state_reason'] = attrs['reason'] if 'reason_ttl' in attrs: self._curhost['state_reason_ttl'] = int(attrs['reason_ttl']) elif name == 'extraports': if self._curextraports is not None: sys.stderr.write("WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports) self._curextraports = {attrs['state']: [int(attrs['count']), {}]} elif name == 'extrareasons' and self._curextraports is not None: self._curextraports[self._curextraports.keys()[0]][1][ attrs['reason']] = int(attrs['count']) elif name == 'port': if self._curport is not None: sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport) self._curport = {'protocol': attrs['protocol'], 'port': int(attrs['portid'])} elif name == 'state' and self._curport is not None: for attr in attrs.keys(): self._curport['state_%s' % attr] = attrs[attr] for field in ['state_reason_ttl']: if field in self._curport: self._curport[field] = int(self._curport[field]) for field in ['state_reason_ip']: if field in self._curport: try: self._curport[field] = utils.ip2int( self._curport[field]) except utils.socket.error: pass elif name == 'service' and self._curport is not None: for attr in attrs.keys(): self._curport['service_%s' % attr] = attrs[attr] for field in ['service_conf', 'service_rpcnum', 'service_lowver', 'service_highver']: if field in self._curport: self._curport[field] = int(self._curport[field]) elif name == 'script': if self._curscript is not None: sys.stderr.write("WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript) self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys()) elif name in ['table', 'elem']: if self._curscript.get('id') in IGNORE_TABLE_ELEMS: return if name == 'elem': # start recording characters if self._curdata is not None: sys.stderr.write("WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata) self._curdata = '' if 'key' in attrs: key = attrs['key'].replace('.', '_') obj = {key: {}} else: key = None obj = [] if not self._curtablepath: if not self._curtable: self._curtable = obj elif key is not None: self._curtable.update(obj) if key is None: key = len(self._curtable) self._curtablepath.append(key) return lastlevel = self._curtable for k in self._curtablepath[:-1]: lastlevel = lastlevel[k] k = self._curtablepath[-1] if type(k) is int: if k < len(lastlevel): if key is not None: lastlevel[k].update(obj) else: lastlevel.append(obj) if key is None: key = len(lastlevel[k]) else: if key is None: if lastlevel[k]: key = len(lastlevel[k]) else: key = 0 lastlevel[k] = obj else: lastlevel[k].update(obj) self._curtablepath.append(key) elif name == 'os': self._curhost['os'] = {} elif name == 'portused' and 'os' in self._curhost: self._curhost['os']['portused'] = { 'port': '%s_%s' % (attrs['proto'], attrs['portid']), 'state': attrs['state'], } elif name in ['osclass', 'osmatch'] and 'os' in self._curhost: if name not in self._curhost['os']: self._curhost['os'][name] = [dict(attrs)] else: self._curhost['os'][name].append(dict(attrs)) elif name == 'osfingerprint' and 'os' in self._curhost: self._curhost['os']['fingerprint'] = attrs['fingerprint'] elif name == 'trace': if self._curtrace is not None: sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace) if 'proto' not in attrs: self._curtrace = {'protocol': None} elif attrs['proto'] in ['tcp', 'udp']: self._curtrace = {'protocol': attrs['proto'], 'port': int(attrs['port'])} else: self._curtrace = {'protocol': attrs['proto']} self._curtrace['hops'] = [] elif name == 'hop' and self._curtrace is not None: attrsdict = dict(attrs) try: attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr']) except utils.socket.error: pass try: attrsdict['rtt'] = float(attrs['rtt']) except ValueError: pass try: attrsdict['ttl'] = int(attrs['ttl']) except ValueError: pass if 'host' in attrsdict: attrsdict['domains'] = list( utils.get_domains(attrsdict['host'])) self._curtrace['hops'].append(attrsdict) elif name == 'cpe': # start recording self._curdata = ''
def startElement(self, name, attrs): if name == 'nmaprun': if self._curscan is not None: sys.stderr.write("WARNING, self._curscan should be None at " "this point (got %r)\n" % self._curscan) self._curscan = dict(attrs) self.scanner = self._curscan.get("scanner", self.scanner) if self.scanner == "masscan": # We need to force "merge" mode due to the nature of # Masscan results self.merge = True self._curscan['_id'] = self._filehash elif name == 'scaninfo' and self._curscan is not None: self._addscaninfo(dict(attrs)) elif name == 'host': if self._curhost is not None: sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost) self._curhost = {"schema_version": SCHEMA_VERSION} if self._curscan: self._curhost['scanid'] = self._curscan['_id'] for attr in attrs.keys(): self._curhost[attr] = attrs[attr] for field in ['starttime', 'endtime']: if field in self._curhost: self._curhost[field] = datetime.datetime.utcfromtimestamp( int(self._curhost[field]) ) if 'starttime' not in self._curhost and 'endtime' in self._curhost: # Masscan self._curhost['starttime'] = self._curhost['endtime'] elif name == 'address' and self._curhost is not None: if attrs['addrtype'] != 'ipv4': self._curhost.setdefault( 'addresses', {}).setdefault( attrs['addrtype'], []).append(attrs['addr']) else: try: self._curhost['addr'] = utils.ip2int(attrs['addr']) except utils.socket.error: self._curhost['addr'] = attrs['addr'] elif name == 'hostnames': if self._curhostnames is not None: sys.stderr.write("WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames) self._curhostnames = [] elif name == 'hostname': if self._curhostnames is None: sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n") self._curhostnames = [] hostname = dict(attrs) if 'name' in attrs: hostname['domains'] = list(utils.get_domains(attrs['name'])) self._curhostnames.append(hostname) elif name == 'status' and self._curhost is not None: self._curhost['state'] = attrs['state'] if 'reason' in attrs: self._curhost['state_reason'] = attrs['reason'] if 'reason_ttl' in attrs: self._curhost['state_reason_ttl'] = int(attrs['reason_ttl']) elif name == 'extraports': if self._curextraports is not None: sys.stderr.write("WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports) self._curextraports = { attrs['state']: {"total": int(attrs['count']), "reasons": {}}, } elif name == 'extrareasons' and self._curextraports is not None: self._curextraports[next(iter(self._curextraports))]["reasons"][ attrs['reason']] = int(attrs['count']) elif name == 'port': if self._curport is not None: sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport) self._curport = {'protocol': attrs['protocol'], 'port': int(attrs['portid'])} elif name == 'state' and self._curport is not None: for attr in attrs.keys(): self._curport['state_%s' % attr] = attrs[attr] for field in ['state_reason_ttl']: if field in self._curport: self._curport[field] = int(self._curport[field]) for field in ['state_reason_ip']: if field in self._curport: try: self._curport[field] = utils.ip2int( self._curport[field]) except utils.socket.error: pass elif name == 'service' and self._curport is not None: if attrs.get("method") == "table": # discard information from nmap-services return if self.scanner == "masscan": # create fake scripts from masscan "service" tags self._curport.setdefault('scripts', []).append({ "id": MASSCAN_SERVICES_NMAP_SCRIPTS.get(attrs['name'], attrs['name']), "output": MASSCAN_ENCODING.sub( _masscan_decode, attrs["banner"], ) }) # get service name service = MASSCAN_SERVICES_NMAP_SERVICES.get(attrs['name']) if service is not None: self._curport['service_name'] = service return for attr in attrs.keys(): self._curport['service_%s' % attr] = attrs[attr] for field in ['service_conf', 'service_rpcnum', 'service_lowver', 'service_highver']: if field in self._curport: self._curport[field] = int(self._curport[field]) elif name == 'script': if self._curscript is not None: sys.stderr.write("WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript) self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys()) elif name in ['table', 'elem']: if self._curscript.get('id') in IGNORE_TABLE_ELEMS: return if name == 'elem': # start recording characters if self._curdata is not None: sys.stderr.write("WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata) self._curdata = '' if 'key' in attrs: key = attrs['key'].replace('.', '_') obj = {key: {}} else: key = None obj = [] if not self._curtablepath: if not self._curtable: self._curtable = obj elif key is not None: self._curtable.update(obj) if key is None: key = len(self._curtable) self._curtablepath.append(key) return lastlevel = self._curtable for k in self._curtablepath[:-1]: lastlevel = lastlevel[k] k = self._curtablepath[-1] if type(k) is int: if k < len(lastlevel): if key is not None: lastlevel[k].update(obj) else: lastlevel.append(obj) if key is None: key = len(lastlevel[k]) else: if key is None: if lastlevel[k]: key = len(lastlevel[k]) else: key = 0 lastlevel[k] = obj else: lastlevel[k].update(obj) self._curtablepath.append(key) elif name == 'os': self._curhost['os'] = {} elif name == 'portused' and 'os' in self._curhost: self._curhost['os']['portused'] = { 'port': '%s_%s' % (attrs['proto'], attrs['portid']), 'state': attrs['state'], } elif name in ['osclass', 'osmatch'] and 'os' in self._curhost: self._curhost['os'].setdefault(name, []).append(dict(attrs)) elif name == 'osfingerprint' and 'os' in self._curhost: self._curhost['os']['fingerprint'] = attrs['fingerprint'] elif name == 'trace': if self._curtrace is not None: sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace) if 'proto' not in attrs: self._curtrace = {'protocol': None} elif attrs['proto'] in ['tcp', 'udp']: self._curtrace = {'protocol': attrs['proto'], 'port': int(attrs['port'])} else: self._curtrace = {'protocol': attrs['proto']} self._curtrace['hops'] = [] elif name == 'hop' and self._curtrace is not None: attrsdict = dict(attrs) try: attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr']) except utils.socket.error: pass try: attrsdict['rtt'] = float(attrs['rtt']) except ValueError: pass try: attrsdict['ttl'] = int(attrs['ttl']) except ValueError: pass if 'host' in attrsdict: attrsdict['domains'] = list( utils.get_domains(attrsdict['host'])) self._curtrace['hops'].append(attrsdict) elif name == 'cpe': # start recording self._curdata = ''
def test(self, v4=True, v6=True): yield from super().test(v4=v4, v6=v6) for srvname, addr, res in self.results: srvname = srvname.rstrip(".") res = [literal_eval(r) for r in sorted(res)] if not res: output = "Domain %s has no TLS-RPT configuration" % self.domain structured = { "domain": self.domain, "warnings": ["Domain has no TLS-RPT configuration"], } elif len(res) > 1: output = ("Domain %s has more than one TLS-RPT configuration" % self.domain) structured = { "domain": self.domain, "value": " / ".join(res), "warnings": ["Domain has more than one TLS-RPT configuration"], } else: value = res[0] structured = { "domain": self.domain, "value": value, } warnings = [] if value.startswith("v=TLSRPTv1;"): if not value[11:].startswith("rua="): warnings.append( "TLS-RPT configuration should contain 'rua=' after 'v=TLSRPTv1;'" ) else: warnings.append( "TLS-RPT configuration should start with 'v=TLSRPTv1;'" ) if not (value.startswith("rua=") or ";rua=" in value): warnings.append( "TLS-RPT configuration should contain 'rua=' after 'v=TLSRPTv1;'" ) if "rua=" in value: ruas = value.split("rua=", 1)[1] for rua_val in ruas.split(","): if rua_val.startswith("https://"): if HTTPS_REGEXP.search(rua_val[8:]) is None: warnings.append( "TLS-RPT contains an invalid HTTPS URL: %r" % rua_val) elif rua_val.startswith("mailto:"): if MAIL_REGEXP.search(rua_val[7:]) is None: warnings.append( "TLS-RPT contains an invalid e-mail URL: %r" % rua_val) else: warnings.append( "TLS-RPT contains an invalid URL: %r" % rua_val) else: warnings.append( "TLS-RPT does not contain an rua entry: %r" % value) if warnings: structured["warnings"] = warnings output = ( "Domain %s has a TLS-RPT configuration with warnings:\n%s" % (self.domain, "\n".join(warnings))) else: output = "Domain %s has a valid TLS-RPT configuration" % self.domain yield { "addr": addr, "hostnames": [{ "name": srvname, "type": "user", "domains": list(get_domains(srvname)), }], "schema_version": SCHEMA_VERSION, "starttime": self.start, "endtime": self.stop, "ports": [{ "port": 53, "protocol": "udp", "service_name": "domain", "state_state": "open", "scripts": [ { "id": "dns-tls-rpt", "output": output, "dns-tls-rpt": [structured], }, ], }], }
def test(self, v4=True, v6=True): self.start = datetime.now() results = {} self.results = list(self.do_test(v4=v4, v6=v6)) for srvname, addr, res in self.results: srvname = srvname.rstrip(".") results.setdefault(res, {}).setdefault(addr, []).append(srvname) if len(results) < 1: return self.stop = datetime.now() good_value = max(results, key=lambda val: len(results[val])) good_value_repr = "\n".join(" %r" % r for r in sorted(good_value)) good_value_sorted = sorted(good_value) for val, servers in results.items(): if val == good_value: continue for addr, names in servers.items(): yield { "addr": addr, "hostnames": [{ "name": name, "type": "user", "domains": list(get_domains(name)), } for name in names], "schema_version": SCHEMA_VERSION, "starttime": self.start, "endtime": self.stop, "ports": [{ "port": 53, "protocol": "udp", "service_name": "domain", "state_state": "open", "scripts": [ { "id": "dns-check-consistency", "output": "DNS inconsistency\n\n%s (%s)\nThis server:\n%s\nMost common answer:\n%s" % ( self.name, self.rtype, "\n".join(" %r" % r for r in sorted(val)), good_value_repr, ), "dns-check-consistency": [{ "domain": self.domain, "name": self.name, "rtype": self.rtype, "value": sorted(val), "reference_value": good_value_sorted, }], }, ], }], }
def test(self, v4=True, v6=True): start = datetime.now() for srvname, addr, res in self.do_test(v4=v4, v6=v6): srvname = srvname.rstrip(".") if not res: continue if len(res) == 1 and res[0].rtype == "SOA": # SOA only: transfer failed continue LOGGER.info("AXFR success for %r on %r", self.domain, addr) line_fmt = "| %%-%ds %%-%ds %%s" % ( max(len(r.name) for r in res), max(len(r.rtype) for r in res), ) yield { "addr": addr, "hostnames": [{ "name": srvname, "type": "user", "domains": list(get_domains(srvname)), }], "schema_version": SCHEMA_VERSION, "starttime": start, "endtime": datetime.now(), "ports": [ { "port": 53, "protocol": "tcp", "service_name": "domain", "state_state": "open", "scripts": [ { "id": "dns-zone-transfer", "output": "\nDomain: %s\n%s\n\\\n" % ( self.domain, "\n".join(line_fmt % (r.name, r.rtype, r.data) for r in res), ), "dns-zone-transfer": [{ "domain": self.domain, "records": [{ "name": r.name, "ttl": r.ttl, "class": r.rclass, "type": r.rtype, "data": r.data, } for r in res], }], }, ], }, ], } hosts = {} for r in res: if r.rclass != "IN": continue if r.rtype in ["A", "AAAA"]: name = r.name.rstrip(".") hosts.setdefault(r.data, set()).add((r.rtype, name)) for host, records in hosts.items(): yield { "addr": host, "hostnames": [{ "name": rec[1], "type": rec[0], "domains": list(get_domains(rec[1])), } for rec in records], "schema_version": SCHEMA_VERSION, "starttime": start, "endtime": datetime.now(), } start = datetime.now()
def startElement(self, name, attrs): if name == 'nmaprun': if self._curscan is not None: sys.stderr.write("WARNING, self._curscan should be None at " "this point (got %r)\n" % self._curscan) self._curscan = dict(attrs) self.scanner = self._curscan.get("scanner", self.scanner) if self.scanner == "masscan": # We need to force "merge" mode due to the nature of # Masscan results self.merge = True self._curscan['_id'] = self._filehash elif name == 'scaninfo' and self._curscan is not None: self._addscaninfo(dict(attrs)) elif name == 'host': if self._curhost is not None: sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost) self._curhost = {"schema_version": SCHEMA_VERSION} if self._curscan: self._curhost['scanid'] = self._curscan['_id'] for attr in attrs.keys(): self._curhost[attr] = attrs[attr] for field in ['starttime', 'endtime']: if field in self._curhost: self._curhost[field] = datetime.datetime.utcfromtimestamp( int(self._curhost[field])) if 'starttime' not in self._curhost and 'endtime' in self._curhost: # Masscan self._curhost['starttime'] = self._curhost['endtime'] elif name == 'address' and self._curhost is not None: if attrs['addrtype'] != 'ipv4': self._curhost.setdefault('addresses', {}).setdefault( attrs['addrtype'], []).append(attrs['addr']) else: try: self._curhost['addr'] = utils.ip2int(attrs['addr']) except utils.socket.error: self._curhost['addr'] = attrs['addr'] elif name == 'hostnames': if self._curhostnames is not None: sys.stderr.write("WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames) self._curhostnames = [] elif name == 'hostname': if self._curhostnames is None: sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n") self._curhostnames = [] hostname = dict(attrs) if 'name' in attrs: hostname['domains'] = list(utils.get_domains(attrs['name'])) self._curhostnames.append(hostname) elif name == 'status' and self._curhost is not None: self._curhost['state'] = attrs['state'] if 'reason' in attrs: self._curhost['state_reason'] = attrs['reason'] if 'reason_ttl' in attrs: self._curhost['state_reason_ttl'] = int(attrs['reason_ttl']) elif name == 'extraports': if self._curextraports is not None: sys.stderr.write("WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports) self._curextraports = { attrs['state']: { "total": int(attrs['count']), "reasons": {} }, } elif name == 'extrareasons' and self._curextraports is not None: self._curextraports[next(iter( self._curextraports))]["reasons"][attrs['reason']] = int( attrs['count']) elif name == 'port': if self._curport is not None: sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport) self._curport = { 'protocol': attrs['protocol'], 'port': int(attrs['portid']) } elif name == 'state' and self._curport is not None: for attr in attrs.keys(): self._curport['state_%s' % attr] = attrs[attr] for field in ['state_reason_ttl']: if field in self._curport: self._curport[field] = int(self._curport[field]) for field in ['state_reason_ip']: if field in self._curport: try: self._curport[field] = utils.ip2int( self._curport[field]) except utils.socket.error: pass elif name == 'service' and self._curport is not None: if attrs.get("method") == "table": # discard information from nmap-services return if self.scanner == "masscan": # create fake scripts from masscan "service" tags self._curport.setdefault('scripts', []).append({ "id": MASSCAN_SERVICES_NMAP_SCRIPTS.get(attrs['name'], attrs['name']), "output": MASSCAN_ENCODING.sub( _masscan_decode, attrs["banner"], ) }) # get service name service = MASSCAN_SERVICES_NMAP_SERVICES.get(attrs['name']) if service is not None: self._curport['service_name'] = service return for attr in attrs.keys(): self._curport['service_%s' % attr] = attrs[attr] for field in [ 'service_conf', 'service_rpcnum', 'service_lowver', 'service_highver' ]: if field in self._curport: self._curport[field] = int(self._curport[field]) elif name == 'script': if self._curscript is not None: sys.stderr.write("WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript) self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys()) elif name in ['table', 'elem']: if self._curscript.get('id') in IGNORE_TABLE_ELEMS: return if name == 'elem': # start recording characters if self._curdata is not None: sys.stderr.write("WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata) self._curdata = '' if 'key' in attrs: key = attrs['key'].replace('.', '_') obj = {key: {}} else: key = None obj = [] if not self._curtablepath: if not self._curtable: self._curtable = obj elif key is not None: self._curtable.update(obj) if key is None: key = len(self._curtable) self._curtablepath.append(key) return lastlevel = self._curtable for k in self._curtablepath[:-1]: lastlevel = lastlevel[k] k = self._curtablepath[-1] if type(k) is int: if k < len(lastlevel): if key is not None: lastlevel[k].update(obj) else: lastlevel.append(obj) if key is None: key = len(lastlevel[k]) else: if key is None: if lastlevel[k]: key = len(lastlevel[k]) else: key = 0 lastlevel[k] = obj else: lastlevel[k].update(obj) self._curtablepath.append(key) elif name == 'os': self._curhost['os'] = {} elif name == 'portused' and 'os' in self._curhost: self._curhost['os']['portused'] = { 'port': '%s_%s' % (attrs['proto'], attrs['portid']), 'state': attrs['state'], } elif name in ['osclass', 'osmatch'] and 'os' in self._curhost: self._curhost['os'].setdefault(name, []).append(dict(attrs)) elif name == 'osfingerprint' and 'os' in self._curhost: self._curhost['os']['fingerprint'] = attrs['fingerprint'] elif name == 'trace': if self._curtrace is not None: sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace) if 'proto' not in attrs: self._curtrace = {'protocol': None} elif attrs['proto'] in ['tcp', 'udp']: self._curtrace = { 'protocol': attrs['proto'], 'port': int(attrs['port']) } else: self._curtrace = {'protocol': attrs['proto']} self._curtrace['hops'] = [] elif name == 'hop' and self._curtrace is not None: attrsdict = dict(attrs) try: attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr']) except utils.socket.error: pass try: attrsdict['rtt'] = float(attrs['rtt']) except ValueError: pass try: attrsdict['ttl'] = int(attrs['ttl']) except ValueError: pass if 'host' in attrsdict: attrsdict['domains'] = list( utils.get_domains(attrsdict['host'])) self._curtrace['hops'].append(attrsdict) elif name == 'cpe': # start recording self._curdata = ''
def startElement(self, name, attrs): if name == 'nmaprun': if self._curscan is not None: sys.stderr.write("WARNING, self._curscan should be None at " "this point(got %r)\n" % self._curscan) self._curscan = dict(attrs) self._curscan['_id'] = self._filehash elif name == 'scaninfo' and self._curscan is not None: self._addscaninfo(dict(attrs)) elif name == 'host': if self._curhost is not None: sys.stderr.write("WARNING, self._curhost should be None at " "this point (got %r)\n" % self._curhost) self._curhost = {"schema_version": SCHEMA_VERSION} if self._curscan: self._curhost['scanid'] = self._curscan['_id'] for attr in attrs.keys(): self._curhost[attr] = attrs[attr] for field in ['starttime', 'endtime']: if field in self._curhost: self._curhost[field] = datetime.datetime.utcfromtimestamp( int(self._curhost[field])) elif name == 'address' and self._curhost is not None: if attrs['addrtype'] != 'ipv4': if 'addresses' not in self._curhost: self._curhost['addresses'] = { attrs['addrtype']: [attrs['addr']] } elif attrs['addrtype'] not in self._curhost: self._curhost['addresses'].update( {attrs['addrtype']: [attrs['addr']]}) else: addresses = self._curhost['addresses'][attrs['addrtype']] addresses.append(attrs['addr']) self._curhost['addresses'][attrs['addrtype']] = addresses else: try: self._curhost['addr'] = utils.ip2int(attrs['addr']) except utils.socket.error: self._curhost['addr'] = attrs['addr'] elif name == 'hostnames': if self._curhostnames is not None: sys.stderr.write("WARNING, self._curhostnames should be None " "at this point " "(got %r)\n" % self._curhostnames) self._curhostnames = [] elif name == 'hostname': if self._curhostnames is None: sys.stderr.write("WARNING, self._curhostnames should NOT be " "None at this point\n") self._curhostnames = [] hostname = dict(attrs) if 'name' in attrs: hostname['domains'] = list(utils.get_domains(attrs['name'])) self._curhostnames.append(hostname) elif name == 'status' and self._curhost is not None: self._curhost['state'] = attrs['state'] if 'reason' in attrs: self._curhost['state_reason'] = attrs['reason'] if 'reason_ttl' in attrs: self._curhost['state_reason_ttl'] = int(attrs['reason_ttl']) elif name == 'extraports': if self._curextraports is not None: sys.stderr.write("WARNING, self._curextraports should be None" " at this point " "(got %r)\n" % self._curextraports) self._curextraports = {attrs['state']: [int(attrs['count']), {}]} elif name == 'extrareasons' and self._curextraports is not None: self._curextraports[self._curextraports.keys()[0]][1][ attrs['reason']] = int(attrs['count']) elif name == 'port': if self._curport is not None: sys.stderr.write("WARNING, self._curport should be None at " "this point (got %r)\n" % self._curport) self._curport = { 'protocol': attrs['protocol'], 'port': int(attrs['portid']) } elif name == 'state' and self._curport is not None: for attr in attrs.keys(): self._curport['state_%s' % attr] = attrs[attr] for field in ['state_reason_ttl']: if field in self._curport: self._curport[field] = int(self._curport[field]) for field in ['state_reason_ip']: if field in self._curport: try: self._curport[field] = utils.ip2int( self._curport[field]) except utils.socket.error: pass elif name == 'service' and self._curport is not None: if attrs.get("method") == "table": # discard information from nmap-services return for attr in attrs.keys(): self._curport['service_%s' % attr] = attrs[attr] for field in [ 'service_conf', 'service_rpcnum', 'service_lowver', 'service_highver' ]: if field in self._curport: self._curport[field] = int(self._curport[field]) elif name == 'script': if self._curscript is not None: sys.stderr.write("WARNING, self._curscript should be None " "at this point (got %r)\n" % self._curscript) self._curscript = dict([attr, attrs[attr]] for attr in attrs.keys()) elif name in ['table', 'elem']: if self._curscript.get('id') in IGNORE_TABLE_ELEMS: return if name == 'elem': # start recording characters if self._curdata is not None: sys.stderr.write("WARNING, self._curdata should be None" " at this point " "(got %r)\n" % self._curdata) self._curdata = '' if 'key' in attrs: key = attrs['key'].replace('.', '_') obj = {key: {}} else: key = None obj = [] if not self._curtablepath: if not self._curtable: self._curtable = obj elif key is not None: self._curtable.update(obj) if key is None: key = len(self._curtable) self._curtablepath.append(key) return lastlevel = self._curtable for k in self._curtablepath[:-1]: lastlevel = lastlevel[k] k = self._curtablepath[-1] if type(k) is int: if k < len(lastlevel): if key is not None: lastlevel[k].update(obj) else: lastlevel.append(obj) if key is None: key = len(lastlevel[k]) else: if key is None: if lastlevel[k]: key = len(lastlevel[k]) else: key = 0 lastlevel[k] = obj else: lastlevel[k].update(obj) self._curtablepath.append(key) elif name == 'os': self._curhost['os'] = {} elif name == 'portused' and 'os' in self._curhost: self._curhost['os']['portused'] = { 'port': '%s_%s' % (attrs['proto'], attrs['portid']), 'state': attrs['state'], } elif name in ['osclass', 'osmatch'] and 'os' in self._curhost: if name not in self._curhost['os']: self._curhost['os'][name] = [dict(attrs)] else: self._curhost['os'][name].append(dict(attrs)) elif name == 'osfingerprint' and 'os' in self._curhost: self._curhost['os']['fingerprint'] = attrs['fingerprint'] elif name == 'trace': if self._curtrace is not None: sys.stderr.write("WARNING, self._curtrace should be None " "at this point (got %r)\n" % self._curtrace) if 'proto' not in attrs: self._curtrace = {'protocol': None} elif attrs['proto'] in ['tcp', 'udp']: self._curtrace = { 'protocol': attrs['proto'], 'port': int(attrs['port']) } else: self._curtrace = {'protocol': attrs['proto']} self._curtrace['hops'] = [] elif name == 'hop' and self._curtrace is not None: attrsdict = dict(attrs) try: attrsdict['ipaddr'] = utils.ip2int(attrs['ipaddr']) except utils.socket.error: pass try: attrsdict['rtt'] = float(attrs['rtt']) except ValueError: pass try: attrsdict['ttl'] = int(attrs['ttl']) except ValueError: pass if 'host' in attrsdict: attrsdict['domains'] = list( utils.get_domains(attrsdict['host'])) self._curtrace['hops'].append(attrsdict) elif name == 'cpe': # start recording self._curdata = ''