def parse_header_line(self, line):
if not line:
return
if line[:1] != b"#":
LOGGER.warning("Not a header line")
return
keyval = line[1:].split(self.sep, 1)
if len(keyval) < 2:
if line.startswith(b'#separator '):
keyval = [b'separator', line[11:]]
else:
LOGGER.warning("Invalid header line")
return
directive = keyval[0]
arg = keyval[1]
if directive == b"separator":
self.sep = decode_hex(arg[2:]) if arg.startswith(b'\\x') else arg
elif directive == b"set_separator":
self.set_sep = arg
elif directive == b"empty_field":
self.empty_field = arg
elif directive == b"unset_field":
self.unset_field = arg
elif directive == b"path":
self.path = arg.decode()
elif directive == b"open":
pass
elif directive == b"fields":
self.fields = arg.split(self.sep)
elif directive == b"types":
self.types = arg.split(self.sep)
def parse_header_line(self, line):
if not line:
return
if line[0] != "#":
log.warning("Not a header line")
return
keyval = line[1:].split(self.sep, 1)
if len(keyval) < 2:
log.warn("Invalid header line")
directive = keyval[0]
arg = keyval[1]
if directive == "separator":
self.sep = decode_hex(arg[2:]) if arg.startswith('\\x') else arg
elif directive == "set_separator":
self.set_sep = arg
elif directive == "empty_field":
self.empty_field = arg
elif directive == "unset_field":
self.unset_field = arg
elif directive == "path":
self.path = arg
elif directive == "open":
pass
elif directive == "fields":
self.fields = arg.split(self.sep)
elif directive == "types":
self.types = arg.split(self.sep)
return None
def getkeys(self, record):
yield Key(
record['addr'], record["port"], "ssh", record['infos']['algo'][4:],
record['infos']['bits'],
_rsa_construct(int(record['infos']['exponent']),
int(record['infos']['modulus'])),
utils.decode_hex(record['infos']['md5']))
def getkeys(self, host):
for script in self.getscripts(host):
yield Key(host['addr'], script["port"], "ssl",
script["script"][self.scriptid]['pubkey']['type'],
script["script"][self.scriptid]['pubkey']['bits'],
self.pem2key(script["script"][self.scriptid]['pem']),
utils.decode_hex(script["script"][self.scriptid]['md5']))
def getkeys(self, record):
yield Key(
utils.force_int2ip(record['addr']), record["port"], "ssh",
record['infos']['algo'][4:], record['infos']['bits'],
RSA.construct((long(record['infos']['modulus']),
long(record['infos']['exponent']))),
utils.decode_hex(record['infos']['md5hash']))
def getkeys(self, host):
for script in self.getscripts(host):
key = script["script"][self.scriptid]['pubkey']
yield Key(
host['addr'], script["port"], "ssl", key['type'], key['bits'],
_rsa_construct(long(key['exponent']), long(key['modulus'])),
utils.decode_hex(script["script"][self.scriptid]['md5']))
def parse_header_line(self, line):
if not line:
return
if line[:1] != b"#":
LOGGER.warning("Not a header line")
return
keyval = line[1:].split(self.sep, 1)
if len(keyval) < 2:
if line.startswith(b'#separator '):
keyval = [b'separator', line[11:]]
else:
LOGGER.warn("Invalid header line")
return
directive = keyval[0]
arg = keyval[1]
if directive == b"separator":
self.sep = decode_hex(arg[2:]) if arg.startswith(b'\\x') else arg
elif directive == b"set_separator":
self.set_sep = arg
elif directive == b"empty_field":
self.empty_field = arg
elif directive == b"unset_field":
self.unset_field = arg
elif directive == b"path":
self.path = arg.decode()
elif directive == b"open":
pass
elif directive == b"fields":
self.fields = arg.split(self.sep)
elif directive == b"types":
self.types = arg.split(self.sep)
def store_host(self, host):
scanid = self._store_host(host)
insrt = postgresql.insert(self.tables.association_scan_scanfile)
self.db.execute(
insrt.values(scan=scanid,
scan_file=utils.decode_hex(
host['scanid'])).on_conflict_do_nothing())
def getkeys(self, host):
for script in self.getscripts(host):
yield Key(utils.force_int2ip(host['addr']), script["port"], "ssl",
script["script"][self.scriptid]['pubkey']['type'],
script["script"][self.scriptid]['pubkey']['bits'],
self.pem2key(script["script"][self.scriptid]['pem']),
utils.decode_hex(script["script"][self.scriptid]['md5']))
def parse_header_line(self, line):
if not line:
return
if line[0] != "#":
log.warning("Not a header line")
return
keyval = line[1:].split(self.sep, 1)
if len(keyval) < 2:
log.warn("Invalid header line")
directive = keyval[0]
arg = keyval[1]
if directive == "separator":
self.sep = decode_hex(arg[2:]) if arg.startswith('\\x') else arg
elif directive == "set_separator":
self.set_sep = arg
elif directive == "empty_field":
self.empty_field = arg
elif directive == "unset_field":
self.unset_field = arg
elif directive == "path":
self.path = arg
elif directive == "open":
pass
elif directive == "fields":
self.fields = arg.split(self.sep)
elif directive == "types":
self.types = arg.split(self.sep)
return None
def getkeys(self, host):
for script in self.getscripts(host):
key = script["script"][self.scriptid]['pubkey']
yield Key(host['addr'], script["port"], "ssl", key['type'],
key['bits'],
RSA.construct((long(key['modulus']),
long(key['exponent']),)),
utils.decode_hex(script["script"][self.scriptid]['md5']))
def getkeys(self, host):
for script in self.getscripts(host):
for cert in script["script"].get(self.scriptid, []):
key = cert['pubkey']
yield Key(
host['addr'], script["port"], "ssl", key['type'],
key['bits'],
_rsa_construct(int(key['exponent']), int(key['modulus'])),
utils.decode_hex(cert['md5']))
def getkeys(self, host):
for script in self.getscripts(host):
yield Key(
host["addr"],
script["port"],
"ssl",
script["script"][self.scriptid]["pubkey"]["type"],
script["script"][self.scriptid]["pubkey"]["bits"],
self.pem2key(script["script"][self.scriptid]["pem"]),
utils.decode_hex(script["script"][self.scriptid]["md5"]),
)
def getkeys(record: Record) -> Generator[Key, None, None]:
yield Key(
record["addr"],
record["port"],
"ssh",
record["infos"]["algo"][4:],
record["infos"]["bits"],
_rsa_construct(int(record["infos"]["exponent"]),
int(record["infos"]["modulus"])),
utils.decode_hex(record["infos"]["md5"]),
)
def getkeys(self, record):
certtext = self._pem2key(record['fullvalue'] if 'fullvalue' in
record else record['value'])
if certtext is None:
return
yield Key(
utils.int2ip(record['addr']), record["port"], "ssl",
certtext['type'], int(certtext['len']),
RSA.construct(
(long(self.modulus_badchars.sub("", certtext['modulus']),
16), long(certtext['exponent']))),
utils.decode_hex(record['infos']['md5hash']))
def getkeys(self, record: Record) -> Generator[Key, None, None]:
for script in self.getscripts(record):
assert isinstance(script["script"], dict) # NmapRecord
yield Key(
record["addr"],
script["port"],
"ssl",
script["script"][self.scriptid]["pubkey"]["type"],
script["script"][self.scriptid]["pubkey"]["bits"],
self.pem2key(script["script"][self.scriptid]["pem"]),
utils.decode_hex(script["script"][self.scriptid]["md5"]),
)
def getkeys(self, record):
certtext = self._der2key(record['value'])
if certtext is None:
return
yield Key(
record['addr'], record["port"], "ssl", certtext['type'],
int(certtext['len']),
_rsa_construct(
int(certtext['exponent']),
int(self.modulus_badchars.sub(b"", certtext['modulus']), 16)),
utils.decode_hex(record['infos']['md5']))
def getkeys(self, record):
certtext = self._der2key(record['value'])
if certtext is None:
return
yield Key(record['addr'], record["port"], "ssl", certtext['type'],
int(certtext['len']),
RSA.construct((
long(self.modulus_badchars.sub(
b"", certtext['modulus']), 16),
long(certtext['exponent']))),
utils.decode_hex(record['infos']['md5']))
def getkeys(self, record):
certtext = self._pem2key(record['fullvalue']
if 'fullvalue' in record
else record['value'])
if certtext is None:
return
yield Key(utils.int2ip(record['addr']), record["port"], "ssl",
certtext['type'], int(certtext['len']),
RSA.construct((
long(self.modulus_badchars.sub(
"", certtext['modulus']), 16),
long(certtext['exponent']))),
utils.decode_hex(record['infos']['md5hash']))
def getkeys(self, host):
for script in self.getscripts(host):
for cert in script["script"].get(self.scriptid, []):
key = cert["pubkey"]
yield Key(
host["addr"],
script["port"],
"ssl",
key["type"],
key["bits"],
_rsa_construct(int(key["exponent"]), int(key["modulus"])),
utils.decode_hex(cert["md5"]),
)
def getkeys(self, record: Record) -> Generator[Key, None, None]:
for script in self.getscripts(record):
assert isinstance(script["script"], dict)
for cert in script["script"].get(self.scriptid, []):
key = cert["pubkey"]
yield Key(
record["addr"],
script["port"],
"ssl",
key["type"],
key["bits"],
_rsa_construct(int(key["exponent"]), int(key["modulus"])),
utils.decode_hex(cert["md5"]),
)
def getkeys(self, host):
for script in self.getscripts(host):
for key in script['script'][self.scriptid]:
if key['type'][4:] == self.keytype:
data = utils.decode_b64(key['key'])
# Handle bug (in Nmap?) where data gets encoded
# twice.
if data[0] != b'\x00':
data = utils.decode_b64(data)
yield Key(
utils.int2ip(host['addr']), script["port"], "ssh",
key['type'][4:],
int(key['bits']),
self.data2key(data),
utils.decode_hex(key['fingerprint']),
)
def _extract_passive_SSH_SERVER_HOSTKEY(rec):
"""Handle SSH host keys."""
# TODO: should (probably) be merged, sorted by date/time, keep one
# entry per key type.
#
# (MAYBE) we should add a "lastseen" tag to every intel in view.
value = utils.encode_b64(utils.nmap_decode_data(rec["value"])).decode()
fingerprint = rec["infos"]["md5"]
key = {
"type": rec["infos"]["algo"],
"key": value,
"fingerprint": fingerprint
}
if "bits" in rec["infos"]: # FIXME
key["bits"] = rec["infos"]["bits"]
fingerprint = utils.decode_hex(fingerprint)
script = {
"id":
"ssh-hostkey",
"ssh-hostkey": [key],
"output":
"\n %s %s (%s)\n%s %s" % (
key.get("bits", "-"), # FIXME
":".join("%02x" % (ord(i) if isinstance(i, (bytes, str)) else i)
for i in fingerprint),
_KEYS.get(
key["type"],
(key["type"][4:]
if key["type"][:4] == "ssh-" else key["type"]).upper(),
),
key["type"],
value,
),
"key":
key,
}
return {
"ports": [{
"state_state": "open",
"state_reason": "passive",
"port": rec["port"],
"protocol": rec.get("protocol", "tcp"),
"service_name": "ssh",
"scripts": [script],
}]
}
def getkeys(self, record: Filter) -> Generator[Key, None, None]:
certtext = self._der2key(record["value"])
if certtext is None:
return
yield Key(
record["addr"],
record["port"],
"ssl",
certtext["type"],
int(certtext["len"]),
_rsa_construct(
int(certtext["exponent"]),
int(MODULUS_BADCHARS.sub(b"", certtext["modulus"]), 16),
),
utils.decode_hex(record["infos"]["md5"]),
)
def getkeys(self, record):
certtext = self._der2key(record["value"])
if certtext is None:
return
yield Key(
record["addr"],
record["port"],
"ssl",
certtext["type"],
int(certtext["len"]),
_rsa_construct(
int(certtext["exponent"]),
int(self.modulus_badchars.sub(b"", certtext["modulus"]), 16),
),
utils.decode_hex(record["infos"]["md5"]),
)
def getkeys(self, host):
for script in self.getscripts(host):
for key in script['script'][self.scriptid]:
if key['type'][4:] == self.keytype:
data = utils.decode_b64(key['key'].encode())
# Handle bug (in Nmap?) where data gets encoded
# twice.
if data[:1] != b'\x00':
data = utils.decode_b64(data)
yield Key(
host['addr'], script["port"], "ssh", key['type'][4:],
int(float(key['bits'])), # for some reason,
# Nmap sometimes
# outputs 1024.0
self.data2key(data),
utils.decode_hex(key['fingerprint']),
)
def getkeys(self, host):
for script in self.getscripts(host):
for key in script['script'][self.scriptid]:
if key['type'][4:] == self.keytype:
data = utils.decode_b64(key['key'].encode())
# Handle bug (in Nmap?) where data gets encoded
# twice.
if data[:1] != b'\x00':
data = utils.decode_b64(data)
yield Key(
host['addr'], script["port"], "ssh", key['type'][4:],
int(float(key['bits'])), # for some reason,
# Nmap sometimes
# outputs 1024.0
self.data2key(data),
utils.decode_hex(key['fingerprint']),
)
def getkeys(self, host):
for script in self.getscripts(host):
for key in script['script'][self.scriptid]:
if key['type'][4:] == self.keytype:
data = utils.decode_b64(key['key'])
# Handle bug (in Nmap?) where data gets encoded
# twice.
if data[0] != b'\x00':
data = utils.decode_b64(data)
yield Key(
utils.int2ip(host['addr']),
script["port"],
"ssh",
key['type'][4:],
int(key['bits']),
self.data2key(data),
utils.decode_hex(key['fingerprint']),
)
def _extract_passive_SSH_SERVER_HOSTKEY(rec):
"""Handle SSH host keys."""
# TODO: should (probably) be merged, sorted by date/time, keep one
# entry per key type.
#
# (MAYBE) we should add a "lastseen" tag to every intel in view.
value = utils.encode_b64(
utils.nmap_decode_data(rec.get('fullvalue', rec['value']))).decode()
fingerprint = rec['infos']['md5']
key = {
'type': rec['infos']['algo'],
'key': value,
'fingerprint': fingerprint
}
if 'bits' in rec['infos']: # FIXME
key['bits'] = rec['infos']['bits']
fingerprint = utils.decode_hex(fingerprint)
script = {
'id':
'ssh-hostkey',
'ssh-hostkey': [key],
'output':
'\n %s %s (%s)\n%s %s' % (
key.get('bits', '-'), # FIXME
':'.join('%02x' % (ord(i) if isinstance(i, (bytes, str)) else i)
for i in fingerprint),
_KEYS.get(key['type'], (key['type'][4:] if key['type'][:4]
== 'ssh-' else key['type']).upper()),
key['type'],
value),
'key':
key
}
return {
'ports': [{
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'ssh',
'scripts': [script],
}]
}
def store_scan_doc(self, scan):
scan = scan.copy()
if 'start' in scan:
scan['start'] = datetime.datetime.utcfromtimestamp(
int(scan['start']))
if 'scaninfos' in scan:
scan["scaninfo"] = scan.pop('scaninfos')
scan["sha256"] = utils.decode_hex(scan.pop('_id'))
insrt = insert(self.tables.scanfile).values(**dict(
(key, scan[key]) for key in [
'sha256', 'args', 'scaninfo', 'scanner', 'start', 'version',
'xmloutputversion'
] if key in scan))
if config.DEBUG:
scanfileid = self.db.execute(
insrt.returning(self.tables.scanfile.sha256)).fetchone()[0]
utils.LOGGER.debug("SCAN STORED: %r", utils.encode_hex(scanfileid))
else:
self.db.execute(insrt)
def _extract_passive_SSH_SERVER_HOSTKEY(rec):
"""Handle SSH host keys."""
# TODO: should (probably) be merged, sorted by date/time, keep one
# entry per key type.
#
# (MAYBE) we should add a "lastseen" tag to every intel in view.
value = utils.encode_b64(
utils.nmap_decode_data(rec['value'])
).decode()
fingerprint = rec['infos']['md5']
key = {'type': rec['infos']['algo'],
'key': value,
'fingerprint': fingerprint}
if 'bits' in rec['infos']: # FIXME
key['bits'] = rec['infos']['bits']
fingerprint = utils.decode_hex(fingerprint)
script = {
'id': 'ssh-hostkey', 'ssh-hostkey': [key],
'output': '\n %s %s (%s)\n%s %s' % (
key.get('bits', '-'), # FIXME
':'.join('%02x' % (
ord(i) if isinstance(i, (bytes, str)) else i
) for i in fingerprint),
_KEYS.get(
key['type'],
(key['type'][4:] if key['type'][:4] == 'ssh-'
else key['type']).upper()
),
key['type'],
value
),
'key': key
}
return {'ports': [{
'state_state': 'open',
'state_reason': "passive",
'port': rec['port'],
'protocol': rec.get('protocol', 'tcp'),
'service_name': 'ssh',
'scripts': [script],
}]}
def getkeys(self, record: Record) -> Generator[Key, None, None]:
for script in self.getscripts(record):
assert isinstance(script["script"], dict)
for key in script["script"][self.scriptid]:
if key["type"][4:] == self.keytype:
data = utils.decode_b64(key["key"].encode())
# Handle bug (in Nmap?) where data gets encoded
# twice.
if data[:1] != b"\x00":
data = utils.decode_b64(data)
yield Key(
record["addr"],
script["port"],
"ssh",
key["type"][4:],
int(float(key["bits"])), # for some reason,
# Nmap sometimes
# outputs 1024.0
self.data2key(data),
utils.decode_hex(key["fingerprint"]),
)
def process(value):
return None if not value else utils.bin2ip(
utils.decode_hex(value)
)
def getkeys(self, record):
yield Key(record['addr'], record["port"], "ssh",
record['infos']['algo'][4:], record['infos']['bits'],
RSA.construct((long(record['infos']['modulus']),
long(record['infos']['exponent']))),
utils.decode_hex(record['infos']['md5']))
def store_host(self, host, merge=False):
addr = self.convert_ip(host['addr'])
info = host.get('infos')
if 'coordinates' in (info or {}).get('loc', {}):
info['coordinates'] = info.pop('loc')['coordinates'][::-1]
source = host.get('source', '')
if merge:
insrt = postgresql.insert(Scan)
scanid, scan_tstop, merge = self.db.execute(
insrt.values(
addr=addr,
source=source,
info=info,
time_start=host['starttime'],
time_stop=host['endtime'],
archive=0,
merge=False,
**dict(
(key, host.get(key)) for key in
['state', 'state_reason', 'state_reason_ttl']
if key in host
)
)
.on_conflict_do_update(
index_elements=['addr', 'source', 'archive'],
set_={
'time_start': func.least(
Scan.time_start,
insrt.excluded.time_start,
),
'time_stop': func.greatest(
Scan.time_stop,
insrt.excluded.time_stop,
),
'merge': True,
},
)
.returning(Scan.id, Scan.time_stop,
Scan.merge)).fetchone()
if merge:
# Test should be ==, using <= in case of rounding
# issues.
newest = scan_tstop <= host['endtime']
else:
newest = None
else:
curarchive = self.db.execute(select([func.max(Scan.archive)])
.where(and_(Scan.addr == addr,
Scan.source == source)))\
.fetchone()[0]
if curarchive is not None:
self.db.execute(update(Scan).where(and_(
Scan.addr == addr,
Scan.source == source,
Scan.archive == 0,
)).values(archive=curarchive + 1))
scanid = self.db.execute(
insert(Scan).values(
addr=addr,
source=source,
info=info,
time_start=host['starttime'],
time_stop=host['endtime'],
state=host['state'],
state_reason=host['state_reason'],
state_reason_ttl=host.get('state_reason_ttl'),
archive=0,
merge=False,
).returning(Scan.id)
).fetchone()[0]
insrt = postgresql.insert(Association_Scan_ScanFile)
self.db.execute(insrt
.values(scan=scanid,
scan_file=utils.decode_hex(host['scanid']))
.on_conflict_do_nothing())
for category in host.get("categories", []):
insrt = postgresql.insert(Category)
catid = self.db.execute(
insrt.values(name=category)
.on_conflict_do_update(
index_elements=['name'],
set_={'name': insrt.excluded.name}
)
.returning(Category.id)
).fetchone()[0]
self.db.execute(postgresql.insert(Association_Scan_Category)
.values(scan=scanid, category=catid)
.on_conflict_do_nothing())
for port in host.get('ports', []):
scripts = port.pop('scripts', [])
# FIXME: handle screenshots
for fld in ['screendata', 'screenshot', 'screenwords',
'service_method']:
try:
del port[fld]
except KeyError:
pass
if 'service_servicefp' in port:
port['service_fp'] = port.pop('service_servicefp')
if 'state_state' in port:
port['state'] = port.pop('state_state')
if 'state_reason_ip' in port:
port['state_reason_ip'] = self.convert_ip(
port['state_reason_ip']
)
if merge:
insrt = postgresql.insert(Port)
portid = self.db.execute(
insrt.values(scan=scanid, **port)
.on_conflict_do_update(
index_elements=['scan', 'port',
'protocol'],
set_=dict(
scan=scanid,
**(port if newest else {})
)
)
.returning(Port.id)
).fetchone()[0]
else:
portid = self.db.execute(insert(Port).values(scan=scanid,
**port)
.returning(Port.id)).fetchone()[0]
for script in scripts:
name, output = script.pop('id'), script.pop('output')
if merge:
if newest:
insrt = postgresql.insert(Script)
self.bulk.append(insrt
.values(
port=portid,
name=name,
output=output,
data=script
)
.on_conflict_do_update(
index_elements=['port', 'name'],
set_={
"output":
insrt.excluded.output,
"data": insrt.excluded.data,
},
))
else:
insrt = postgresql.insert(Script)
self.bulk.append(insrt
.values(
port=portid,
name=name,
output=output,
data=script
)
.on_conflict_do_nothing())
else:
self.bulk.append(insert(Script).values(
port=portid,
name=name,
output=output,
data=script
))
if not merge:
# FIXME: handle traceroutes on merge
for trace in host.get('traces', []):
traceid = self.db.execute(insert(Trace).values(
scan=scanid,
port=trace.get('port'),
protocol=trace['protocol']
).returning(Trace.id)).fetchone()[0]
for hop in trace.get('hops'):
hop['ipaddr'] = self.convert_ip(hop['ipaddr'])
self.bulk.append(insert(Hop).values(
trace=traceid,
ipaddr=self.convert_ip(hop['ipaddr']),
ttl=hop["ttl"],
rtt=None if hop["rtt"] == '--' else hop["rtt"],
host=hop.get("host"),
domains=hop.get("domains"),
))
# FIXME: handle hostnames on merge
for hostname in host.get('hostnames', []):
self.bulk.append(insert(Hostname).values(
scan=scanid,
domains=hostname.get('domains'),
name=hostname.get('name'),
type=hostname.get('type'),
))
utils.LOGGER.debug("HOST STORED: %r", scanid)
