def parse_header_line(self, line): if not line: return if line[:1] != b"#": LOGGER.warning("Not a header line") return keyval = line[1:].split(self.sep, 1) if len(keyval) < 2: if line.startswith(b'#separator '): keyval = [b'separator', line[11:]] else: LOGGER.warning("Invalid header line") return directive = keyval[0] arg = keyval[1] if directive == b"separator": self.sep = decode_hex(arg[2:]) if arg.startswith(b'\\x') else arg elif directive == b"set_separator": self.set_sep = arg elif directive == b"empty_field": self.empty_field = arg elif directive == b"unset_field": self.unset_field = arg elif directive == b"path": self.path = arg.decode() elif directive == b"open": pass elif directive == b"fields": self.fields = arg.split(self.sep) elif directive == b"types": self.types = arg.split(self.sep)
def parse_header_line(self, line): if not line: return if line[0] != "#": log.warning("Not a header line") return keyval = line[1:].split(self.sep, 1) if len(keyval) < 2: log.warn("Invalid header line") directive = keyval[0] arg = keyval[1] if directive == "separator": self.sep = decode_hex(arg[2:]) if arg.startswith('\\x') else arg elif directive == "set_separator": self.set_sep = arg elif directive == "empty_field": self.empty_field = arg elif directive == "unset_field": self.unset_field = arg elif directive == "path": self.path = arg elif directive == "open": pass elif directive == "fields": self.fields = arg.split(self.sep) elif directive == "types": self.types = arg.split(self.sep) return None
def getkeys(self, record): yield Key( record['addr'], record["port"], "ssh", record['infos']['algo'][4:], record['infos']['bits'], _rsa_construct(int(record['infos']['exponent']), int(record['infos']['modulus'])), utils.decode_hex(record['infos']['md5']))
def getkeys(self, host): for script in self.getscripts(host): yield Key(host['addr'], script["port"], "ssl", script["script"][self.scriptid]['pubkey']['type'], script["script"][self.scriptid]['pubkey']['bits'], self.pem2key(script["script"][self.scriptid]['pem']), utils.decode_hex(script["script"][self.scriptid]['md5']))
def getkeys(self, record): yield Key( utils.force_int2ip(record['addr']), record["port"], "ssh", record['infos']['algo'][4:], record['infos']['bits'], RSA.construct((long(record['infos']['modulus']), long(record['infos']['exponent']))), utils.decode_hex(record['infos']['md5hash']))
def getkeys(self, host): for script in self.getscripts(host): key = script["script"][self.scriptid]['pubkey'] yield Key( host['addr'], script["port"], "ssl", key['type'], key['bits'], _rsa_construct(long(key['exponent']), long(key['modulus'])), utils.decode_hex(script["script"][self.scriptid]['md5']))
def parse_header_line(self, line): if not line: return if line[:1] != b"#": LOGGER.warning("Not a header line") return keyval = line[1:].split(self.sep, 1) if len(keyval) < 2: if line.startswith(b'#separator '): keyval = [b'separator', line[11:]] else: LOGGER.warn("Invalid header line") return directive = keyval[0] arg = keyval[1] if directive == b"separator": self.sep = decode_hex(arg[2:]) if arg.startswith(b'\\x') else arg elif directive == b"set_separator": self.set_sep = arg elif directive == b"empty_field": self.empty_field = arg elif directive == b"unset_field": self.unset_field = arg elif directive == b"path": self.path = arg.decode() elif directive == b"open": pass elif directive == b"fields": self.fields = arg.split(self.sep) elif directive == b"types": self.types = arg.split(self.sep)
def store_host(self, host): scanid = self._store_host(host) insrt = postgresql.insert(self.tables.association_scan_scanfile) self.db.execute( insrt.values(scan=scanid, scan_file=utils.decode_hex( host['scanid'])).on_conflict_do_nothing())
def getkeys(self, host): for script in self.getscripts(host): yield Key(utils.force_int2ip(host['addr']), script["port"], "ssl", script["script"][self.scriptid]['pubkey']['type'], script["script"][self.scriptid]['pubkey']['bits'], self.pem2key(script["script"][self.scriptid]['pem']), utils.decode_hex(script["script"][self.scriptid]['md5']))
def parse_header_line(self, line): if not line: return if line[0] != "#": log.warning("Not a header line") return keyval = line[1:].split(self.sep, 1) if len(keyval) < 2: log.warn("Invalid header line") directive = keyval[0] arg = keyval[1] if directive == "separator": self.sep = decode_hex(arg[2:]) if arg.startswith('\\x') else arg elif directive == "set_separator": self.set_sep = arg elif directive == "empty_field": self.empty_field = arg elif directive == "unset_field": self.unset_field = arg elif directive == "path": self.path = arg elif directive == "open": pass elif directive == "fields": self.fields = arg.split(self.sep) elif directive == "types": self.types = arg.split(self.sep) return None
def getkeys(self, host): for script in self.getscripts(host): key = script["script"][self.scriptid]['pubkey'] yield Key(host['addr'], script["port"], "ssl", key['type'], key['bits'], RSA.construct((long(key['modulus']), long(key['exponent']),)), utils.decode_hex(script["script"][self.scriptid]['md5']))
def getkeys(self, host): for script in self.getscripts(host): for cert in script["script"].get(self.scriptid, []): key = cert['pubkey'] yield Key( host['addr'], script["port"], "ssl", key['type'], key['bits'], _rsa_construct(int(key['exponent']), int(key['modulus'])), utils.decode_hex(cert['md5']))
def getkeys(self, host): for script in self.getscripts(host): yield Key( host["addr"], script["port"], "ssl", script["script"][self.scriptid]["pubkey"]["type"], script["script"][self.scriptid]["pubkey"]["bits"], self.pem2key(script["script"][self.scriptid]["pem"]), utils.decode_hex(script["script"][self.scriptid]["md5"]), )
def getkeys(record: Record) -> Generator[Key, None, None]: yield Key( record["addr"], record["port"], "ssh", record["infos"]["algo"][4:], record["infos"]["bits"], _rsa_construct(int(record["infos"]["exponent"]), int(record["infos"]["modulus"])), utils.decode_hex(record["infos"]["md5"]), )
def getkeys(self, record): certtext = self._pem2key(record['fullvalue'] if 'fullvalue' in record else record['value']) if certtext is None: return yield Key( utils.int2ip(record['addr']), record["port"], "ssl", certtext['type'], int(certtext['len']), RSA.construct( (long(self.modulus_badchars.sub("", certtext['modulus']), 16), long(certtext['exponent']))), utils.decode_hex(record['infos']['md5hash']))
def getkeys(self, record: Record) -> Generator[Key, None, None]: for script in self.getscripts(record): assert isinstance(script["script"], dict) # NmapRecord yield Key( record["addr"], script["port"], "ssl", script["script"][self.scriptid]["pubkey"]["type"], script["script"][self.scriptid]["pubkey"]["bits"], self.pem2key(script["script"][self.scriptid]["pem"]), utils.decode_hex(script["script"][self.scriptid]["md5"]), )
def getkeys(self, record): certtext = self._der2key(record['value']) if certtext is None: return yield Key( record['addr'], record["port"], "ssl", certtext['type'], int(certtext['len']), _rsa_construct( int(certtext['exponent']), int(self.modulus_badchars.sub(b"", certtext['modulus']), 16)), utils.decode_hex(record['infos']['md5']))
def getkeys(self, record): certtext = self._der2key(record['value']) if certtext is None: return yield Key(record['addr'], record["port"], "ssl", certtext['type'], int(certtext['len']), RSA.construct(( long(self.modulus_badchars.sub( b"", certtext['modulus']), 16), long(certtext['exponent']))), utils.decode_hex(record['infos']['md5']))
def getkeys(self, record): certtext = self._pem2key(record['fullvalue'] if 'fullvalue' in record else record['value']) if certtext is None: return yield Key(utils.int2ip(record['addr']), record["port"], "ssl", certtext['type'], int(certtext['len']), RSA.construct(( long(self.modulus_badchars.sub( "", certtext['modulus']), 16), long(certtext['exponent']))), utils.decode_hex(record['infos']['md5hash']))
def getkeys(self, host): for script in self.getscripts(host): for cert in script["script"].get(self.scriptid, []): key = cert["pubkey"] yield Key( host["addr"], script["port"], "ssl", key["type"], key["bits"], _rsa_construct(int(key["exponent"]), int(key["modulus"])), utils.decode_hex(cert["md5"]), )
def getkeys(self, record: Record) -> Generator[Key, None, None]: for script in self.getscripts(record): assert isinstance(script["script"], dict) for cert in script["script"].get(self.scriptid, []): key = cert["pubkey"] yield Key( record["addr"], script["port"], "ssl", key["type"], key["bits"], _rsa_construct(int(key["exponent"]), int(key["modulus"])), utils.decode_hex(cert["md5"]), )
def getkeys(self, host): for script in self.getscripts(host): for key in script['script'][self.scriptid]: if key['type'][4:] == self.keytype: data = utils.decode_b64(key['key']) # Handle bug (in Nmap?) where data gets encoded # twice. if data[0] != b'\x00': data = utils.decode_b64(data) yield Key( utils.int2ip(host['addr']), script["port"], "ssh", key['type'][4:], int(key['bits']), self.data2key(data), utils.decode_hex(key['fingerprint']), )
def _extract_passive_SSH_SERVER_HOSTKEY(rec): """Handle SSH host keys.""" # TODO: should (probably) be merged, sorted by date/time, keep one # entry per key type. # # (MAYBE) we should add a "lastseen" tag to every intel in view. value = utils.encode_b64(utils.nmap_decode_data(rec["value"])).decode() fingerprint = rec["infos"]["md5"] key = { "type": rec["infos"]["algo"], "key": value, "fingerprint": fingerprint } if "bits" in rec["infos"]: # FIXME key["bits"] = rec["infos"]["bits"] fingerprint = utils.decode_hex(fingerprint) script = { "id": "ssh-hostkey", "ssh-hostkey": [key], "output": "\n %s %s (%s)\n%s %s" % ( key.get("bits", "-"), # FIXME ":".join("%02x" % (ord(i) if isinstance(i, (bytes, str)) else i) for i in fingerprint), _KEYS.get( key["type"], (key["type"][4:] if key["type"][:4] == "ssh-" else key["type"]).upper(), ), key["type"], value, ), "key": key, } return { "ports": [{ "state_state": "open", "state_reason": "passive", "port": rec["port"], "protocol": rec.get("protocol", "tcp"), "service_name": "ssh", "scripts": [script], }] }
def getkeys(self, record: Filter) -> Generator[Key, None, None]: certtext = self._der2key(record["value"]) if certtext is None: return yield Key( record["addr"], record["port"], "ssl", certtext["type"], int(certtext["len"]), _rsa_construct( int(certtext["exponent"]), int(MODULUS_BADCHARS.sub(b"", certtext["modulus"]), 16), ), utils.decode_hex(record["infos"]["md5"]), )
def getkeys(self, record): certtext = self._der2key(record["value"]) if certtext is None: return yield Key( record["addr"], record["port"], "ssl", certtext["type"], int(certtext["len"]), _rsa_construct( int(certtext["exponent"]), int(self.modulus_badchars.sub(b"", certtext["modulus"]), 16), ), utils.decode_hex(record["infos"]["md5"]), )
def getkeys(self, host): for script in self.getscripts(host): for key in script['script'][self.scriptid]: if key['type'][4:] == self.keytype: data = utils.decode_b64(key['key'].encode()) # Handle bug (in Nmap?) where data gets encoded # twice. if data[:1] != b'\x00': data = utils.decode_b64(data) yield Key( host['addr'], script["port"], "ssh", key['type'][4:], int(float(key['bits'])), # for some reason, # Nmap sometimes # outputs 1024.0 self.data2key(data), utils.decode_hex(key['fingerprint']), )
def getkeys(self, host): for script in self.getscripts(host): for key in script['script'][self.scriptid]: if key['type'][4:] == self.keytype: data = utils.decode_b64(key['key'].encode()) # Handle bug (in Nmap?) where data gets encoded # twice. if data[:1] != b'\x00': data = utils.decode_b64(data) yield Key( host['addr'], script["port"], "ssh", key['type'][4:], int(float(key['bits'])), # for some reason, # Nmap sometimes # outputs 1024.0 self.data2key(data), utils.decode_hex(key['fingerprint']), )
def getkeys(self, host): for script in self.getscripts(host): for key in script['script'][self.scriptid]: if key['type'][4:] == self.keytype: data = utils.decode_b64(key['key']) # Handle bug (in Nmap?) where data gets encoded # twice. if data[0] != b'\x00': data = utils.decode_b64(data) yield Key( utils.int2ip(host['addr']), script["port"], "ssh", key['type'][4:], int(key['bits']), self.data2key(data), utils.decode_hex(key['fingerprint']), )
def _extract_passive_SSH_SERVER_HOSTKEY(rec): """Handle SSH host keys.""" # TODO: should (probably) be merged, sorted by date/time, keep one # entry per key type. # # (MAYBE) we should add a "lastseen" tag to every intel in view. value = utils.encode_b64( utils.nmap_decode_data(rec.get('fullvalue', rec['value']))).decode() fingerprint = rec['infos']['md5'] key = { 'type': rec['infos']['algo'], 'key': value, 'fingerprint': fingerprint } if 'bits' in rec['infos']: # FIXME key['bits'] = rec['infos']['bits'] fingerprint = utils.decode_hex(fingerprint) script = { 'id': 'ssh-hostkey', 'ssh-hostkey': [key], 'output': '\n %s %s (%s)\n%s %s' % ( key.get('bits', '-'), # FIXME ':'.join('%02x' % (ord(i) if isinstance(i, (bytes, str)) else i) for i in fingerprint), _KEYS.get(key['type'], (key['type'][4:] if key['type'][:4] == 'ssh-' else key['type']).upper()), key['type'], value), 'key': key } return { 'ports': [{ 'state_state': 'open', 'state_reason': "passive", 'port': rec['port'], 'protocol': rec.get('protocol', 'tcp'), 'service_name': 'ssh', 'scripts': [script], }] }
def store_scan_doc(self, scan): scan = scan.copy() if 'start' in scan: scan['start'] = datetime.datetime.utcfromtimestamp( int(scan['start'])) if 'scaninfos' in scan: scan["scaninfo"] = scan.pop('scaninfos') scan["sha256"] = utils.decode_hex(scan.pop('_id')) insrt = insert(self.tables.scanfile).values(**dict( (key, scan[key]) for key in [ 'sha256', 'args', 'scaninfo', 'scanner', 'start', 'version', 'xmloutputversion' ] if key in scan)) if config.DEBUG: scanfileid = self.db.execute( insrt.returning(self.tables.scanfile.sha256)).fetchone()[0] utils.LOGGER.debug("SCAN STORED: %r", utils.encode_hex(scanfileid)) else: self.db.execute(insrt)
def _extract_passive_SSH_SERVER_HOSTKEY(rec): """Handle SSH host keys.""" # TODO: should (probably) be merged, sorted by date/time, keep one # entry per key type. # # (MAYBE) we should add a "lastseen" tag to every intel in view. value = utils.encode_b64( utils.nmap_decode_data(rec['value']) ).decode() fingerprint = rec['infos']['md5'] key = {'type': rec['infos']['algo'], 'key': value, 'fingerprint': fingerprint} if 'bits' in rec['infos']: # FIXME key['bits'] = rec['infos']['bits'] fingerprint = utils.decode_hex(fingerprint) script = { 'id': 'ssh-hostkey', 'ssh-hostkey': [key], 'output': '\n %s %s (%s)\n%s %s' % ( key.get('bits', '-'), # FIXME ':'.join('%02x' % ( ord(i) if isinstance(i, (bytes, str)) else i ) for i in fingerprint), _KEYS.get( key['type'], (key['type'][4:] if key['type'][:4] == 'ssh-' else key['type']).upper() ), key['type'], value ), 'key': key } return {'ports': [{ 'state_state': 'open', 'state_reason': "passive", 'port': rec['port'], 'protocol': rec.get('protocol', 'tcp'), 'service_name': 'ssh', 'scripts': [script], }]}
def getkeys(self, record: Record) -> Generator[Key, None, None]: for script in self.getscripts(record): assert isinstance(script["script"], dict) for key in script["script"][self.scriptid]: if key["type"][4:] == self.keytype: data = utils.decode_b64(key["key"].encode()) # Handle bug (in Nmap?) where data gets encoded # twice. if data[:1] != b"\x00": data = utils.decode_b64(data) yield Key( record["addr"], script["port"], "ssh", key["type"][4:], int(float(key["bits"])), # for some reason, # Nmap sometimes # outputs 1024.0 self.data2key(data), utils.decode_hex(key["fingerprint"]), )
def process(value): return None if not value else utils.bin2ip( utils.decode_hex(value) )
def getkeys(self, record): yield Key(record['addr'], record["port"], "ssh", record['infos']['algo'][4:], record['infos']['bits'], RSA.construct((long(record['infos']['modulus']), long(record['infos']['exponent']))), utils.decode_hex(record['infos']['md5']))
def store_host(self, host, merge=False): addr = self.convert_ip(host['addr']) info = host.get('infos') if 'coordinates' in (info or {}).get('loc', {}): info['coordinates'] = info.pop('loc')['coordinates'][::-1] source = host.get('source', '') if merge: insrt = postgresql.insert(Scan) scanid, scan_tstop, merge = self.db.execute( insrt.values( addr=addr, source=source, info=info, time_start=host['starttime'], time_stop=host['endtime'], archive=0, merge=False, **dict( (key, host.get(key)) for key in ['state', 'state_reason', 'state_reason_ttl'] if key in host ) ) .on_conflict_do_update( index_elements=['addr', 'source', 'archive'], set_={ 'time_start': func.least( Scan.time_start, insrt.excluded.time_start, ), 'time_stop': func.greatest( Scan.time_stop, insrt.excluded.time_stop, ), 'merge': True, }, ) .returning(Scan.id, Scan.time_stop, Scan.merge)).fetchone() if merge: # Test should be ==, using <= in case of rounding # issues. newest = scan_tstop <= host['endtime'] else: newest = None else: curarchive = self.db.execute(select([func.max(Scan.archive)]) .where(and_(Scan.addr == addr, Scan.source == source)))\ .fetchone()[0] if curarchive is not None: self.db.execute(update(Scan).where(and_( Scan.addr == addr, Scan.source == source, Scan.archive == 0, )).values(archive=curarchive + 1)) scanid = self.db.execute( insert(Scan).values( addr=addr, source=source, info=info, time_start=host['starttime'], time_stop=host['endtime'], state=host['state'], state_reason=host['state_reason'], state_reason_ttl=host.get('state_reason_ttl'), archive=0, merge=False, ).returning(Scan.id) ).fetchone()[0] insrt = postgresql.insert(Association_Scan_ScanFile) self.db.execute(insrt .values(scan=scanid, scan_file=utils.decode_hex(host['scanid'])) .on_conflict_do_nothing()) for category in host.get("categories", []): insrt = postgresql.insert(Category) catid = self.db.execute( insrt.values(name=category) .on_conflict_do_update( index_elements=['name'], set_={'name': insrt.excluded.name} ) .returning(Category.id) ).fetchone()[0] self.db.execute(postgresql.insert(Association_Scan_Category) .values(scan=scanid, category=catid) .on_conflict_do_nothing()) for port in host.get('ports', []): scripts = port.pop('scripts', []) # FIXME: handle screenshots for fld in ['screendata', 'screenshot', 'screenwords', 'service_method']: try: del port[fld] except KeyError: pass if 'service_servicefp' in port: port['service_fp'] = port.pop('service_servicefp') if 'state_state' in port: port['state'] = port.pop('state_state') if 'state_reason_ip' in port: port['state_reason_ip'] = self.convert_ip( port['state_reason_ip'] ) if merge: insrt = postgresql.insert(Port) portid = self.db.execute( insrt.values(scan=scanid, **port) .on_conflict_do_update( index_elements=['scan', 'port', 'protocol'], set_=dict( scan=scanid, **(port if newest else {}) ) ) .returning(Port.id) ).fetchone()[0] else: portid = self.db.execute(insert(Port).values(scan=scanid, **port) .returning(Port.id)).fetchone()[0] for script in scripts: name, output = script.pop('id'), script.pop('output') if merge: if newest: insrt = postgresql.insert(Script) self.bulk.append(insrt .values( port=portid, name=name, output=output, data=script ) .on_conflict_do_update( index_elements=['port', 'name'], set_={ "output": insrt.excluded.output, "data": insrt.excluded.data, }, )) else: insrt = postgresql.insert(Script) self.bulk.append(insrt .values( port=portid, name=name, output=output, data=script ) .on_conflict_do_nothing()) else: self.bulk.append(insert(Script).values( port=portid, name=name, output=output, data=script )) if not merge: # FIXME: handle traceroutes on merge for trace in host.get('traces', []): traceid = self.db.execute(insert(Trace).values( scan=scanid, port=trace.get('port'), protocol=trace['protocol'] ).returning(Trace.id)).fetchone()[0] for hop in trace.get('hops'): hop['ipaddr'] = self.convert_ip(hop['ipaddr']) self.bulk.append(insert(Hop).values( trace=traceid, ipaddr=self.convert_ip(hop['ipaddr']), ttl=hop["ttl"], rtt=None if hop["rtt"] == '--' else hop["rtt"], host=hop.get("host"), domains=hop.get("domains"), )) # FIXME: handle hostnames on merge for hostname in host.get('hostnames', []): self.bulk.append(insert(Hostname).values( scan=scanid, domains=hostname.get('domains'), name=hostname.get('name'), type=hostname.get('type'), )) utils.LOGGER.debug("HOST STORED: %r", scanid)