def list_ips_by_data(datafile, parseline, data,
listall=True, listcidrs=False,
skip=0, maxnbr=None, multiple=False):
if ((not listall) or listcidrs) and ((skip != 0) or (maxnbr is not None)):
sys.stderr.write('WARNING: skip and maxnbr parameters have no effect '
'when listall == False or listcidrs == True.\n')
if listcidrs:
listall = False
with open(datafile) as fdesc:
for line in fdesc:
start, stop, curdata = parseline(line)
if (multiple and curdata in data) or curdata == data:
if listall:
curaddrs = map(utils.int2ip, xrange(start, stop + 1))
if skip > 0:
skip -= len(curaddrs)
if skip <= 0:
curaddrs = curaddrs[skip:]
else:
curaddrs = []
if maxnbr is not None:
maxnbr -= len(curaddrs)
if maxnbr < 0:
curaddrs = curaddrs[:maxnbr]
for addr in curaddrs:
print addr
if maxnbr is not None and maxnbr <= 0:
return
elif listcidrs:
for net in utils.range2nets((start, stop)):
print net
else:
print "%s - %s" % (utils.int2ip(start),
utils.int2ip(stop))
def bgp_raw_to_csv(fname, out):
out = open(os.path.join(config.GEOIP_PATH, out), 'wb')
cur = []
with open(os.path.join(config.GEOIP_PATH, fname), 'rb') as fdesc:
for line in fdesc:
start, stop = (utils.ip2int(elt) for elt in
utils.net2range(line[:-1].split()[0]))
if cur:
if start >= cur[0] and stop <= cur[1]:
continue
if start >= cur[0] and start <= cur[1]:
cur = [cur[0], stop]
continue
if stop >= cur[0] and stop <= cur[1]:
cur = [start, cur[1]]
continue
if start <= cur[0] and stop >= cur[1]:
cur = [start, stop]
continue
if start == cur[1] + 1:
cur = [cur[0], stop]
continue
if stop == cur[0] + 1:
cur = [start, cur[1]]
continue
out.write(('"%s","%s","%d","%d"\n' % (
utils.int2ip(cur[0]), utils.int2ip(cur[1]), cur[0], cur[1],
)).encode())
cur = [start, stop]
if cur:
out.write(('"%s","%s","%d","%d"\n' % (
utils.int2ip(cur[0]), utils.int2ip(cur[1]), cur[0], cur[1],
)).encode())
def list_ips_by_data(datafile, condition,
listall=True, listcidrs=False,
skip=0, maxnbr=None, multiple=False):
if ((not listall) or listcidrs) and ((skip != 0) or (maxnbr is not None)):
utils.LOGGER.warning('Skip and maxnbr parameters have no effect '
'when listall == False or listcidrs == True.')
if listcidrs:
listall = False
for start, stop in _get_by_data(datafile, condition):
if listall:
curaddrs = [utils.int2ip(addr) for addr in
range(start, stop + 1)]
if skip > 0:
skip -= len(curaddrs)
if skip <= 0:
curaddrs = curaddrs[skip:]
else:
curaddrs = []
if maxnbr is not None:
maxnbr -= len(curaddrs)
if maxnbr < 0:
curaddrs = curaddrs[:maxnbr]
for addr in curaddrs:
print(addr)
if maxnbr is not None and maxnbr <= 0:
return
elif listcidrs:
for net in utils.range2nets((start, stop)):
print(net)
else:
print("%s - %s" % (utils.int2ip(start),
utils.int2ip(stop)))
def list_ips_by_data(datafile, parseline, data,
listall=True, listcidrs=False,
skip=0, maxnbr=None, multiple=False):
if ((not listall) or listcidrs) and ((skip != 0) or (maxnbr is not None)):
utils.LOGGER.warning('Skip and maxnbr parameters have no effect '
'when listall == False or listcidrs == True.')
if listcidrs:
listall = False
with open(datafile, 'rb') as fdesc:
for line in fdesc:
start, stop, curdata = parseline(line)
if (multiple and curdata in data) or curdata == data:
if listall:
curaddrs = [utils.int2ip(addr) for addr in range(start, stop + 1)]
if skip > 0:
skip -= len(curaddrs)
if skip <= 0:
curaddrs = curaddrs[skip:]
else:
curaddrs = []
if maxnbr is not None:
maxnbr -= len(curaddrs)
if maxnbr < 0:
curaddrs = curaddrs[:maxnbr]
for addr in curaddrs:
print(addr)
if maxnbr is not None and maxnbr <= 0:
return
elif listcidrs:
for net in utils.range2nets((start, stop)):
print(net)
else:
print("%s - %s" % (utils.int2ip(start), utils.int2ip(stop)))
def getkeys(self, host):
for script in self.getscripts(host):
yield Key(utils.int2ip(host['addr']), script["port"], "ssl",
script["script"][self.scriptid]['pubkey']['type'],
script["script"][self.scriptid]['pubkey']['bits'],
self.pem2key(script["script"][self.scriptid]['pem']),
utils.decode_hex(script["script"][self.scriptid]['md5']))
def feed(self, agent, maxnbr=None):
for _ in xrange(max(agent.may_receive(), maxnbr)):
addr = utils.int2ip(self.targiter.next())
with open(os.path.join(agent.get_local_path('input'),
'%s.%s' % (self.visiblecategory, addr)),
'w') as fdesc:
fdesc.write('%s\n' % addr)
def displayfunction_graphroute(cur, arg, gr_include, gr_dont_reset):
graph, entry_nodes = graphroute.buildgraph(
cur,
include_last_hop=gr_include == "last-hop",
include_target=gr_include == "target",
)
if arg == "dot":
if arg == "AS":
def cluster(ipaddr):
res = db.db.data.as_byip(ipaddr)
if res is None:
return
return (res['as_num'],
"%(as_num)d\n[%(as_name)s]" % res)
elif arg == "Country":
def cluster(ipaddr):
res = db.db.data.country_byip(ipaddr)
if res is None:
return
return (res['country_code'],
"%(country_code)s - %(country_name)s" % res)
else:
cluster = None
graphroute.writedotgraph(graph, sys.stdout,
cluster=cluster)
elif arg == "rtgraph3d":
g = graphroute.display3dgraph(
graph,
reset_world=not gr_dont_reset
)
for n in entry_nodes:
g.glow(utils.int2ip(n))
def writedotgraph(graph, out, cluster=None):
"""From a graph produced by buildgraph(), produces an output in
the (Graphiz) Dot format.
`cluster`, if provided, should be a function receiving an IP
address (as an integer) and returning either a cluster the address
belongs to or None if the address do not belong to a cluster. The
`cluster` returned may be either a string or a tuple of two
strings (label, name).
"""
out.write('digraph traceroute {\n')
nodes = set()
edges = set()
if cluster is None:
def _add_node(node):
if node not in nodes:
nodes.add(node)
out.write('\t%d [label="%s"];\n' % (node, utils.int2ip(node)))
else:
clusters = {}
def _add_node(node):
if node not in nodes:
nodes.add(node)
clusters.setdefault(cluster(node), set()).update([node])
for node, node_edges in viewitems(graph):
_add_node(node)
for destnode in node_edges:
_add_node(destnode)
if (node, destnode) not in edges:
out.write("\t%d -> %d;\n" % (node, destnode))
edges.add((node, destnode))
if cluster is not None:
if None in clusters:
for node in clusters.pop(None):
out.write('\t%d [label="%s"];\n' % (node, utils.int2ip(node)))
for clu, nodes in viewitems(clusters):
if isinstance(clu, basestring):
clu = (clu, clu)
out.write('\tsubgraph cluster_%s {\n' % clu[0])
out.write('\t\tlabel = "%s";\n' % clu[1])
for node in nodes:
out.write('\t\t%d [label="%s"];\n' % (node,
utils.int2ip(node)))
out.write('\t}\n')
out.write('}\n')
def _display_honeyd_conf(host, honeyd_routes, honeyd_entries, out=sys.stdout):
addr = utils.int2ip(host['addr'])
hname = "host_%s" % addr.replace('.', '_')
out.write("create %s\n" % hname)
defaction = HONEYD_DEFAULT_ACTION
if 'extraports' in host:
extra = host['extraports']
defaction = max(
max(viewvalues(extra),
key=lambda state: viewitems(state['total'])['reasons']),
key=lambda reason: reason[1],
)[0]
defaction = HONEYD_ACTION_FROM_NMAP_STATE.get(defaction)
out.write('set %s default tcp action %s\n' % (hname, defaction))
for p in host.get('ports', []):
try:
out.write('add %s %s port %d %s\n' % (
hname, p['protocol'], p['port'],
_nmap_port2honeyd_action(p),
))
except KeyError:
# let's skip pseudo-port records that are only containers for host
# scripts.
pass
if 'traces' in host and len(host['traces']) > 0:
trace = max(host['traces'], key=lambda x: len(x['hops']))['hops']
if trace:
trace.sort(key=lambda x: x['ttl'])
curhop = trace[0]
honeyd_entries.add(curhop['ipaddr'])
for t in trace[1:]:
key = (curhop['ipaddr'], t['ipaddr'])
latency = max(t['rtt'] - curhop['rtt'], 0)
route = honeyd_routes.get(key)
if route is None:
honeyd_routes[key] = {
'count': 1,
'high': latency,
'low': latency,
'mean': latency,
'targets': set([host['addr']])
}
else:
route['targets'].add(host['addr'])
honeyd_routes[key] = {
'count': route['count'] + 1,
'high': max(route['high'], latency),
'low': min(route['low'], latency),
'mean': (route['mean'] * route['count'] +
latency) / (route['count'] + 1),
'targets': route['targets'],
}
curhop = t
out.write('bind %s %s\n\n' % (addr, hname))
return honeyd_routes, honeyd_entries
def feed(self, agent, maxnbr=None):
"""Send targets to scan to `agent`, depending on how many it
can receive.
"""
for _ in xrange(max(agent.may_receive(), maxnbr)):
addr = utils.int2ip(self.targiter.next())
with open(os.path.join(agent.get_local_path('input'),
'%s.%s' % (self.visiblecategory, addr)),
'w') as fdesc:
fdesc.write('%s\n' % addr)
def writedotgraph(graph, out):
"""From a graph produced by buildgraph(), produces an output in
the (Graphiz) Dot format.
"""
out.write('digraph traceroute {\n')
nodes = set()
edges = set()
for node, node_edges in graph.iteritems():
if node not in nodes:
out.write('\t%d [label="%s"];\n' % (node, utils.int2ip(node)))
nodes.add(node)
for destnode in node_edges:
if destnode not in nodes:
out.write('\t%d [label="%s"];\n' % (destnode,
utils.int2ip(destnode)))
nodes.add(destnode)
if (node, destnode) not in edges:
out.write("\t%d -> %d;\n" % (node, destnode))
edges.add((node, destnode))
out.write('}\n')
def display3dgraph(graph, reset_world=True):
"""Send the graph (produced by buildgraph()) to a running
rtgraph3d instance.
"""
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
bus = dbus.SessionBus()
control = bus.get_object("org.secdev.rtgraph3d", "/control")
graph3d = dbus.Interface(control, "org.secdev.rtgraph3d.command")
if reset_world:
graph3d.reset_world()
for node, node_edges in viewitems(graph):
for destnode in node_edges:
if destnode == node:
continue
try:
graph3d.new_edge(utils.int2ip(node), {},
utils.int2ip(destnode), {})
except Exception:
utils.LOGGER.warning('Exception', exc_info=True)
return graph3d
def add_target(self, agentid, scanid, addr):
agent = self.get_agent(agentid)
try:
addr = int(addr)
addr = utils.int2ip(addr)
except (ValueError, struct.error):
pass
with tempfile.NamedTemporaryFile(
prefix=str(scanid) + "-", dir=self.get_local_path(agent, "input"), delete=False
) as fdesc:
fdesc.write("%s\n" % addr)
return True
return False
def getkeys(self, record):
certtext = self._pem2key(record['fullvalue']
if 'fullvalue' in record
else record['value'])
if certtext is None:
return
yield Key(utils.int2ip(record['addr']), record["port"], "ssl",
certtext['type'], int(certtext['len']),
RSA.construct((
long(self.modulus_badchars.sub(
"", certtext['modulus']), 16),
long(certtext['exponent']))),
utils.decode_hex(record['infos']['md5hash']))
def _display_honeyd_epilogue(honeyd_routes, honeyd_entries, out=sys.stdout):
for r in honeyd_entries:
out.write('route entry %s\n' % utils.int2ip(r))
out.write('route %s link %s/32\n' % (utils.int2ip(r),
utils.int2ip(r)))
out.write('\n')
for r in honeyd_routes:
out.write('route %s link %s/32\n' % (utils.int2ip(r[0]),
utils.int2ip(r[1])))
for t in honeyd_routes[r]['targets']:
out.write('route %s add net %s/32 %s latency %dms\n' % (
utils.int2ip(r[0]), utils.int2ip(t),
utils.int2ip(r[1]),
int(round(honeyd_routes[r]['mean'])),
))
def display_honeyd_conf(host, honeyd_routes, honeyd_entries, out=sys.stdout):
addr = utils.int2ip(host['addr'])
hname = "host_%s" % addr.replace('.', '_')
out.write("create %s\n" % hname)
defaction = HONEYD_DEFAULT_ACTION
if 'extraports' in host:
extra = host['extraports']
defstate = extra[max(extra, key=lambda x: extra[x][0])][1]
defaction = HONEYD_ACTION_FROM_NMAP_STATE.get(
max(defstate, key=lambda x: defstate[x]),
HONEYD_DEFAULT_ACTION
)
out.write('set %s default tcp action %s\n' % (hname, defaction))
for p in host.get('ports', []):
out.write('add %s %s port %d %s\n' % (
hname, p['protocol'], p['port'],
nmap_port2honeyd_action(p))
)
if 'traces' in host and len(host['traces']) > 0:
trace = max(host['traces'], key=lambda x: len(x['hops']))['hops']
if trace:
trace.sort(key=lambda x: x['ttl'])
curhop = trace[0]
honeyd_entries.add(curhop['ipaddr'])
for t in trace[1:]:
key = (curhop['ipaddr'], t['ipaddr'])
latency = max(t['rtt'] - curhop['rtt'], 0)
route = honeyd_routes.get(key)
if route is None:
honeyd_routes[key] = {
'count': 1,
'high': latency,
'low': latency,
'mean': latency,
'targets': set([host['addr']])
}
else:
route['targets'].add(host['addr'])
honeyd_routes[key] = {
'count': route['count'] + 1,
'high': max(route['high'], latency),
'low': min(route['low'], latency),
'mean': (route['mean'] * route['count']
+ latency) / (route['count'] + 1),
'targets': route['targets'],
}
curhop = t
out.write('bind %s %s\n\n' % (addr, hname))
return honeyd_routes, honeyd_entries
def getkeys(self, host):
for script in self.getscripts(host):
for key in script['script'][self.scriptid]:
if key['type'][4:] == self.keytype:
data = key['key'].decode('base64')
# Handle bug (in Nmap?) where data gets encoded
# twice.
if data[0] != '\x00':
data = data.decode('base64')
yield Key(
int2ip(host['addr']), script["port"], "ssh",
key['type'][4:],
int(key['bits']),
self.data2key(data),
key['fingerprint'].decode('hex'))
def getkeys(self, host):
for script in self.getscripts(host):
for key in script['script'][self.scriptid]:
if key['type'][4:] == self.keytype:
data = utils.decode_b64(key['key'])
# Handle bug (in Nmap?) where data gets encoded
# twice.
if data[0] != b'\x00':
data = utils.decode_b64(data)
yield Key(
utils.int2ip(host['addr']), script["port"], "ssh",
key['type'][4:],
int(key['bits']),
self.data2key(data),
utils.decode_hex(key['fingerprint']),
)
def get_ips_by_data(datafile, condition, skip=0, maxnbr=None):
res = []
for start, stop in _get_by_data(datafile, condition):
curaddrs = [utils.int2ip(addr) for addr in range(start, stop + 1)]
if skip > 0:
skip -= len(curaddrs)
if skip <= 0:
curaddrs = curaddrs[skip:]
else:
curaddrs = []
if maxnbr is not None:
maxnbr -= len(curaddrs)
if maxnbr < 0:
return res + curaddrs[:maxnbr]
elif maxnbr == 0:
return res + curaddrs
res += curaddrs
return res
def get_ips_by_data(datafile, parseline, data, skip=0, maxnbr=None,
multiple=False):
res = []
with open(datafile, 'rb') as fdesc:
for line in fdesc:
start, stop, curdata = parseline(line)
if (multiple and curdata in data) or curdata == data:
curaddrs = [utils.int2ip(addr) for addr in range(start, stop + 1)]
if skip > 0:
skip -= len(curaddrs)
if skip <= 0:
curaddrs = curaddrs[skip:]
else:
curaddrs = []
if maxnbr is not None:
maxnbr -= len(curaddrs)
if maxnbr < 0:
return res + curaddrs[:maxnbr]
elif maxnbr == 0:
return res + curaddrs
res += curaddrs
return res
def displayhost(record, showscripts=True, showtraceroute=True, showos=True,
out=sys.stdout):
"""Displays (on `out`, by default `sys.stdout`) the Nmap scan
result contained in `record`.
"""
try:
line = "Host %s" % utils.int2ip(record['addr'])
except:
line = "Host %s" % record['addr']
if record.get('hostnames'):
line += " (%s)" % '/'.join(x['name'] for x in record['hostnames'])
if 'source' in record:
line += ' from %s' % record['source']
if record.get('categories'):
line += ' (%s)' % ', '.join(record['categories'])
if 'state' in record:
line += ' (%s' % record['state']
if 'state_reason' in record:
line += ': %s' % record['state_reason']
line += ')\n'
out.write(line)
if 'infos' in record:
infos = record['infos']
if 'country_code' in infos or 'country_name' in infos:
out.write("\t%s - %s" % (infos.get('country_code', '?'),
infos.get('country_name', '?')))
if 'city' in infos:
out.write(' - %s' % infos['city'])
out.write('\n')
if 'as_num' in infos or 'as_name' in infos:
out.write("\tAS%s - %s\n" % (infos.get('as_num', '?'),
infos.get('as_name', '?')))
if 'starttime' in record and 'endtime' in record:
out.write("\tscan %s - %s\n" %
(record['starttime'], record['endtime']))
if 'extraports' in record:
d = record['extraports']
for k in d:
out.write("\t%d ports %s (%s)\n" %
(d[k][0], k, ', '.join(['%d %s' % (d[k][1][kk], kk)
for kk in d[k][1].keys()])))
ports = record.get('ports', [])
ports.sort(key=lambda x: (x.get('protocol'), x['port']))
for port in ports:
if port.get('port') == 'host':
record['scripts'] = port['scripts']
continue
if 'state_reason' in port:
reason = " (%s)" % ', '.join(
[port['state_reason']] +
["%s=%s" % (field[13:], value)
for field, value in port.iteritems()
if field.startswith('state_reason_')]
)
else:
reason = ""
if 'service_name' in port:
srv = port['service_name']
if 'service_method' in port:
srv += ' (%s)' % port['service_method']
for field in ['service_product', 'service_version',
'service_extrainfo', 'service_ostype',
'service_hostname']:
if field in port:
srv += ' %s' % port[field]
else:
srv = ""
out.write("\t%-10s%-8s%-22s%s\n" %
('%s/%d' % (port.get('protocol'), port['port']),
port['state_state'], reason, srv))
if showscripts:
out.writelines(_scriptoutput(port))
if showscripts:
scripts = _scriptoutput(record)
if scripts:
out.write('\tHost scripts:\n')
out.writelines(scripts)
if showtraceroute and record.get('traces'):
for trace in record['traces']:
proto = trace['protocol']
if proto in ['tcp', 'udp']:
proto += '/%d' % trace['port']
out.write('\tTraceroute (using %s)\n' % proto)
hops = trace['hops']
hops.sort(key=lambda hop: hop['ttl'])
for hop in hops:
try:
out.write('\t\t%3s %15s %7s\n' %
(hop['ttl'], utils.int2ip(hop['ipaddr']),
hop['rtt']))
except:
out.write('\t\t%3s %15s %7s\n' %
(hop['ttl'], hop['ipaddr'], hop['rtt']))
if showos and record.get('os', {}).get('osclass'):
osclasses = record['os']['osclass']
maxacc = str(max(int(x['accuracy']) for x in osclasses))
osclasses = [osclass for osclass in osclasses
if osclass['accuracy'] == maxacc]
out.write('\tOS fingerprint\n')
for osclass in osclasses:
out.write(
'\t\t%(osfamily)s / %(type)s / %(vendor)s / '
'accuracy = %(accuracy)s\n' % osclass)
def main():
# write headers
sys.stdout.write(webutils.JS_HEADERS)
params = webutils.parse_query_string()
query = webutils.query_from_params(params)
flt, archive, sortby, unused, skip, limit = webutils.flt_from_query(query)
if limit is None:
limit = config.WEB_LIMIT
if config.WEB_MAXRESULTS is not None:
limit = min(limit, config.WEB_MAXRESULTS)
callback = params.get("callback")
# type of result
action = params.get("action", "")
# top values
if action.startswith('topvalues:'):
field = action[10:]
if field[0] in '-!':
field = field[1:]
least = True
else:
least = False
topnbr = 15
if ':' in field:
field, topnbr = field.rsplit(':', 1)
try:
topnbr = int(topnbr)
except ValueError:
field = '%s:%s' % (field, topnbr)
topnbr = 15
series = [{"label": t['_id'], "value": t['count']} for t in
db.nmap.topvalues(field, flt=flt,
least=least, topnbr=topnbr,
archive=archive)]
if callback is None:
sys.stdout.write("%s;\n" % json.dumps(series))
else:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(series)))
exit(0)
# extract info
if action in ["onlyips", "ipsports", "timeline", "coordinates",
"countopenports"]:
preamble = "[\n"
postamble = "]\n"
if action == "timeline":
result = db.nmap.get(
flt, archive=archive,
fields=['addr', 'starttime', 'openports.count']
)
count = result.count()
if params.get("modulo") is None:
r2time = lambda r: int(r['starttime'].strftime('%s'))
else:
r2time = lambda r: (int(r['starttime'].strftime('%s'))
% int(params.get("modulo")))
if params.get("ipsasnumbers"):
r2res = lambda r: [r2time(r), r['addr'],
r['openports']['count']]
else:
r2res = lambda r: [r2time(r),
utils.int2ip(r['addr']),
r['openports']['count']]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": ['
postamble = ']}'
result = list(db.nmap.getlocations(flt, archive=archive))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {"count": r['count']},
}
elif action == "countopenports":
result = db.nmap.get(flt, archive=archive,
fields=['addr', 'openports.count'])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: [r['addr'], r['openports']['count']]
else:
r2res = lambda r: [utils.int2ip(r['addr']),
r['openports']['count']]
elif action == "ipsports":
result = db.nmap.get(
flt, archive=archive,
fields=['addr', 'ports.port', 'ports.state_state']
)
count = sum(len(host.get('ports', [])) for host in result)
result.rewind()
if params.get("ipsasnumbers"):
r2res = lambda r: [
r['addr'],
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
else:
r2res = lambda r: [
utils.int2ip(r['addr']),
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
elif action == "onlyips":
result = db.nmap.get(flt, archive=archive, fields=['addr'])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: r['addr']
else:
r2res = lambda r: utils.int2ip(r['addr'])
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write(
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count
)
if callback is not None:
sys.stdout.write("%s(\n" % callback)
sys.stdout.write(preamble)
for rec in result:
sys.stdout.write(json.dumps(r2res(rec)) + ",\n")
sys.stdout.write(postamble)
if callback is not None:
sys.stdout.write(")")
sys.stdout.write(";\n")
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write('}\n')
exit(0)
# generic request
result = db.nmap.get(flt, archive=archive,
limit=limit, skip=skip, sort=sortby)
if action == "count":
if callback is not None:
sys.stdout.write("%s(%d);\n" % (callback, result.count()))
else:
sys.stdout.write("%d;\n" % result.count())
exit(0)
if unused:
msg = 'Option%s not understood: %s' % (
's' if len(unused) > 1 else '',
', '.join(unused),
)
sys.stdout.write(webutils.js_alert("param-unused", "warning", msg))
sys.stderr.write('IVRE: WARNING: %r\n' % msg)
else:
sys.stdout.write(webutils.js_del_alert("param-unused"))
if config.DEBUG:
msg = "filter: %r" % flt
sys.stdout.write(webutils.js_alert("filter", "info", msg))
sys.stderr.write('IVRE: INFO: %r\n' % msg)
msg = "user: %r" % webutils.get_user()
sys.stdout.write(webutils.js_alert("user", "info", msg))
sys.stderr.write('IVRE: INFO: %r\n' % msg)
version_mismatch = {}
for rec in result:
del rec['_id']
try:
rec['addr'] = utils.int2ip(rec['addr'])
except:
pass
for field in ['starttime', 'endtime']:
if field in rec:
rec[field] = int(rec[field].strftime('%s'))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = port['screendata'].encode('base64')
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
try:
hop['ipaddr'] = utils.int2ip(hop['ipaddr'])
except:
pass
if callback is not None:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(rec)))
else:
sys.stdout.write("%s;\n" % json.dumps(rec))
check = db.nmap.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
messages = {
1: lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following commands: 'scancli --update-schema; "
"scancli --update-schema --archives'"
"" % (count, 's' if count > 1 else '',
'are' if count > 1 else 'is')),
-1: lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '',
've' if count > 1 else 's')),
}
for mismatch, count in version_mismatch.iteritems():
message = messages[mismatch](count)
sys.stdout.write(
webutils.js_alert("version-mismatch-%d" % ((mismatch + 1) / 2),
"warning", message)
)
sys.stderr.write('IVRE: WARNING: %r\n' % message)
def iter_nets(self):
for start, length in sorted(viewvalues(self.ranges)):
for net in utils.range2nets(
(utils.int2ip(start), utils.int2ip(start + length - 1))):
yield net
def main():
# write headers
sys.stdout.write(webutils.JS_HEADERS)
params = webutils.parse_query_string()
query = webutils.query_from_params(params)
flt, archive, sortby, unused, skip, limit = webutils.flt_from_query(query)
if limit is None:
limit = config.WEB_LIMIT
if config.WEB_MAXRESULTS is not None:
limit = min(limit, config.WEB_MAXRESULTS)
callback = params.get("callback")
# type of result
action = params.get("action", "")
ipsasnumbers = params.get("ipsasnumbers")
datesasstrings = params.get("datesasstrings")
if callback is None:
sys.stdout.write('Content-Disposition: attachment; '
'filename="IVRE-results.json"\r\n')
sys.stdout.write("\r\n")
# top values
if action.startswith('topvalues:'):
field = action[10:]
if field[0] in '-!':
field = field[1:]
least = True
else:
least = False
topnbr = 15
if ':' in field:
field, topnbr = field.rsplit(':', 1)
try:
topnbr = int(topnbr)
except ValueError:
field = '%s:%s' % (field, topnbr)
topnbr = 15
series = [{"label": t['_id'], "value": t['count']} for t in
db.nmap.topvalues(field, flt=flt,
least=least, topnbr=topnbr,
archive=archive)]
if callback is None:
sys.stdout.write("%s\n" % json.dumps(series))
else:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(series)))
exit(0)
# extract info
if action in ["onlyips", "ipsports", "timeline", "coordinates",
"countopenports", "diffcats"]:
preamble = "[\n"
postamble = "]\n"
r2res = lambda x: x
if action == "timeline":
if hasattr(db.nmap, "get_open_port_count"):
result = list(db.nmap.get_open_port_count(flt, archive=archive))
count = len(result)
else:
result = db.nmap.get(
flt, archive=archive,
fields=['addr', 'starttime', 'openports.count']
)
count = result.count()
if params.get("modulo") is None:
r2time = lambda r: int(r['starttime'].strftime('%s'))
else:
r2time = lambda r: (int(r['starttime'].strftime('%s'))
% int(params.get("modulo")))
if ipsasnumbers:
r2res = lambda r: [r2time(r), force_ip_int(r['addr']),
r['openports']['count']]
else:
r2res = lambda r: [r2time(r), force_ip_str(r['addr']),
r['openports']['count']]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": ['
postamble = ']}'
result = list(db.nmap.getlocations(flt, archive=archive))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {"count": r['count']},
}
elif action == "countopenports":
if hasattr(db.nmap, "get_open_port_count"):
result = db.nmap.get_open_port_count(flt, archive=archive)
else:
result = db.nmap.get(flt, archive=archive,
fields=['addr', 'openports.count'])
if hasattr(result, "count"):
count = result.count()
else:
count = db.nmap.count(flt, archive=archive,
fields=['addr', 'openports.count'])
if ipsasnumbers:
r2res = lambda r: [force_ip_int(r['addr']),
r['openports']['count']]
else:
r2res = lambda r: [force_ip_str(r['addr']),
r['openports']['count']]
elif action == "ipsports":
if hasattr(db.nmap, "get_ips_ports"):
result = list(db.nmap.get_ips_ports(flt, archive=archive))
count = sum(len(host.get('ports', [])) for host in result)
else:
result = db.nmap.get(
flt, archive=archive,
fields=['addr', 'ports.port', 'ports.state_state']
)
count = sum(len(host.get('ports', [])) for host in result)
result.rewind()
if ipsasnumbers:
r2res = lambda r: [
force_ip_int(r['addr']),
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
else:
r2res = lambda r: [
force_ip_str(r['addr']),
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
elif action == "onlyips":
result = db.nmap.get(flt, archive=archive, fields=['addr'])
if hasattr(result, "count"):
count = result.count()
else:
count = db.nmap.count(flt, archive=archive, fields=['addr'])
if ipsasnumbers:
r2res = lambda r: r['addr']
else:
r2res = lambda r: utils.int2ip(r['addr'])
elif action == "diffcats":
if params.get("onlydiff"):
output = db.nmap.diff_categories(params.get("cat1"),
params.get("cat2"),
flt=flt,
include_both_open=False)
else:
output = db.nmap.diff_categories(params.get("cat1"),
params.get("cat2"),
flt=flt)
count = 0
result = {}
if ipsasnumbers:
for res in output:
result.setdefault(res["addr"], []).append([res['port'],
res['value']])
count += 1
else:
for res in output:
result.setdefault(utils.int2ip(res["addr"]),
[]).append([res['port'], res['value']])
count += 1
result = viewitems(result)
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write(
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count
)
if callback is not None:
sys.stdout.write("%s(\n" % callback)
sys.stdout.write(preamble)
for rec in result:
sys.stdout.write(json.dumps(r2res(rec)) + ",\n")
sys.stdout.write(postamble)
if callback is not None:
sys.stdout.write(");")
sys.stdout.write("\n")
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write('}\n')
exit(0)
# generic request
if action == "count":
if callback is None:
sys.stdout.write("%d\n" % db.nmap.count(flt, archive=archive))
else:
sys.stdout.write("%s(%d);\n" % (callback,
db.nmap.count(flt,
archive=archive)))
exit(0)
## PostgreSQL: the query plan if affected by the limit and gives
## really poor results. This is a temporary workaround (look for
## XXX-WORKAROUND-PGSQL)
# result = db.nmap.get(flt, archive=archive,
# limit=limit, skip=skip, sort=sortby)
result = db.nmap.get(flt, archive=archive,
skip=skip, sort=sortby)
if unused:
msg = 'Option%s not understood: %s' % (
's' if len(unused) > 1 else '',
', '.join(unused),
)
sys.stdout.write(webutils.js_alert("param-unused", "warning", msg))
utils.LOGGER.warning(msg)
elif callback is not None:
sys.stdout.write(webutils.js_del_alert("param-unused"))
if config.DEBUG:
msg = "filter: %r" % flt
sys.stdout.write(webutils.js_alert("filter", "info", msg))
utils.LOGGER.debug(msg)
msg = "user: %r" % webutils.get_user()
sys.stdout.write(webutils.js_alert("user", "info", msg))
utils.LOGGER.debug(msg)
version_mismatch = {}
if callback is None:
tab, sep = "", "\n"
else:
tab, sep = "\t", ",\n"
sys.stdout.write("%s([\n" % callback)
## XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not ipsasnumbers:
try:
rec['addr'] = utils.int2ip(rec['addr'])
except:
pass
for field in ['starttime', 'endtime']:
if field in rec:
if not datesasstrings:
rec[field] = int(rec[field].strftime('%s'))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = utils.encode_b64(port['screendata'])
for script in port.get('scripts', []):
if "masscan" in script:
try: del script['masscan']['raw']
except KeyError: pass
if not ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
try:
hop['ipaddr'] = utils.int2ip(hop['ipaddr'])
except:
pass
sys.stdout.write("%s%s%s" % (
tab, json.dumps(rec, default=utils.serialize), sep
))
check = db.nmap.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if i + 1>= limit:
break
if callback is not None:
sys.stdout.write("]);\n")
messages = {
1: lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following commands: 'ivre scancli "
"--update-schema; ivre scancli --update-schema "
"--archives'" % (count, 's' if count > 1 else '',
'are' if count > 1 else 'is')),
-1: lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '',
've' if count > 1 else 's')),
}
for mismatch, count in viewitems(version_mismatch):
message = messages[mismatch](count)
sys.stdout.write(
webutils.js_alert("version-mismatch-%d" % ((mismatch + 1) / 2),
"warning", message)
)
utils.LOGGER.warning(message)
def iter_ranges(self):
for start, length in sorted(viewvalues(self.ranges)):
yield utils.int2ip(start), utils.int2ip(start + length - 1)
def main():
# write headers
sys.stdout.write(webutils.JS_HEADERS)
params = webutils.parse_query_string()
query = webutils.query_from_params(params)
flt, archive, sortby, unused, skip, limit = webutils.flt_from_query(query)
if limit is None:
limit = config.WEB_LIMIT
if config.WEB_MAXRESULTS is not None:
limit = min(limit, config.WEB_MAXRESULTS)
callback = params.get("callback")
# type of result
action = params.get("action", "")
ipsasnumbers = params.get("ipsasnumbers")
datesasstrings = params.get("datesasstrings")
if callback is None:
sys.stdout.write('Content-Disposition: attachment; '
'filename="IVRE-results.json"\r\n')
sys.stdout.write("\r\n")
# top values
if action.startswith('topvalues:'):
field = action[10:]
if field[0] in '-!':
field = field[1:]
least = True
else:
least = False
topnbr = 15
if ':' in field:
field, topnbr = field.rsplit(':', 1)
try:
topnbr = int(topnbr)
except ValueError:
field = '%s:%s' % (field, topnbr)
topnbr = 15
series = [{
"label": t['_id'],
"value": t['count']
} for t in db.nmap.topvalues(
field, flt=flt, least=least, topnbr=topnbr, archive=archive)]
if callback is None:
sys.stdout.write("%s\n" % json.dumps(series))
else:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(series)))
exit(0)
# extract info
if action in [
"onlyips", "ipsports", "timeline", "coordinates", "countopenports",
"diffcats"
]:
preamble = "[\n"
postamble = "]\n"
r2res = lambda x: x
if action == "timeline":
result = db.nmap.get(
flt,
archive=archive,
fields=['addr', 'starttime', 'openports.count'])
count = result.count()
if params.get("modulo") is None:
r2time = lambda r: int(r['starttime'].strftime('%s'))
else:
r2time = lambda r: (int(r['starttime'].strftime('%s')) % int(
params.get("modulo")))
if ipsasnumbers:
r2res = lambda r: [
r2time(r), r['addr'], r['openports']['count']
]
else:
r2res = lambda r: [
r2time(r),
utils.int2ip(r['addr']), r['openports']['count']
]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": ['
postamble = ']}'
result = list(db.nmap.getlocations(flt, archive=archive))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {
"count": r['count']
},
}
elif action == "countopenports":
result = db.nmap.get(flt,
archive=archive,
fields=['addr', 'openports.count'])
count = result.count()
if ipsasnumbers:
r2res = lambda r: [r['addr'], r['openports']['count']]
else:
r2res = lambda r: [
utils.int2ip(r['addr']), r['openports']['count']
]
elif action == "ipsports":
result = db.nmap.get(
flt,
archive=archive,
fields=['addr', 'ports.port', 'ports.state_state'])
count = sum(len(host.get('ports', [])) for host in result)
result.rewind()
if ipsasnumbers:
r2res = lambda r: [
r['addr'],
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
else:
r2res = lambda r: [
utils.int2ip(r['addr']),
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
elif action == "onlyips":
result = db.nmap.get(flt, archive=archive, fields=['addr'])
count = result.count()
if ipsasnumbers:
r2res = lambda r: r['addr']
else:
r2res = lambda r: utils.int2ip(r['addr'])
elif action == "diffcats":
if params.get("onlydiff"):
output = db.nmap.diff_categories(params.get("cat1"),
params.get("cat2"),
flt=flt,
include_both_open=False)
else:
output = db.nmap.diff_categories(params.get("cat1"),
params.get("cat2"),
flt=flt)
count = 0
result = {}
if ipsasnumbers:
for res in output:
result.setdefault(res["addr"],
[]).append([res['port'], res['value']])
count += 1
else:
for res in output:
result.setdefault(utils.int2ip(res["addr"]),
[]).append([res['port'], res['value']])
count += 1
result = result.iteritems()
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write(
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count)
if callback is not None:
sys.stdout.write("%s(\n" % callback)
sys.stdout.write(preamble)
for rec in result:
sys.stdout.write(json.dumps(r2res(rec)) + ",\n")
sys.stdout.write(postamble)
if callback is not None:
sys.stdout.write(");")
sys.stdout.write("\n")
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write('}\n')
exit(0)
# generic request
result = db.nmap.get(flt,
archive=archive,
limit=limit,
skip=skip,
sort=sortby)
if action == "count":
if callback is None:
sys.stdout.write("%d\n" % result.count())
else:
sys.stdout.write("%s(%d);\n" % (callback, result.count()))
exit(0)
if unused:
msg = 'Option%s not understood: %s' % (
's' if len(unused) > 1 else '',
', '.join(unused),
)
sys.stdout.write(webutils.js_alert("param-unused", "warning", msg))
sys.stderr.write('IVRE: WARNING: %r\n' % msg)
elif callback is not None:
sys.stdout.write(webutils.js_del_alert("param-unused"))
if config.DEBUG:
msg = "filter: %r" % flt
sys.stdout.write(webutils.js_alert("filter", "info", msg))
sys.stderr.write('IVRE: INFO: %r\n' % msg)
msg = "user: %r" % webutils.get_user()
sys.stdout.write(webutils.js_alert("user", "info", msg))
sys.stderr.write('IVRE: INFO: %r\n' % msg)
version_mismatch = {}
if callback is None:
tab, sep = "", "\n"
else:
tab, sep = "\t", ",\n"
sys.stdout.write("%s([\n" % callback)
for rec in result:
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not ipsasnumbers:
try:
rec['addr'] = utils.int2ip(rec['addr'])
except:
pass
for field in ['starttime', 'endtime']:
if field in rec:
if not datesasstrings:
rec[field] = int(rec[field].strftime('%s'))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = port['screendata'].encode('base64')
for script in port.get('scripts', []):
if "masscan" in script:
try:
del script['masscan']['raw']
except KeyError:
pass
if not ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
try:
hop['ipaddr'] = utils.int2ip(hop['ipaddr'])
except:
pass
sys.stdout.write("%s%s%s" %
(tab, json.dumps(rec, default=utils.serialize), sep))
check = db.nmap.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
if callback is not None:
sys.stdout.write("]);\n")
messages = {
1:
lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following commands: 'ivre scancli "
"--update-schema; ivre scancli --update-schema "
"--archives'" % (count, 's' if count > 1 else '', 'are'
if count > 1 else 'is')),
-1:
lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '', 've'
if count > 1 else 's')),
}
for mismatch, count in version_mismatch.iteritems():
message = messages[mismatch](count)
sys.stdout.write(
webutils.js_alert("version-mismatch-%d" % ((mismatch + 1) / 2),
"warning", message))
sys.stderr.write('IVRE: WARNING: %r\n' % message)
def get_nmap_action(action):
flt_params = get_nmap_base()
preamble = "[\n"
postamble = "]\n"
r2res = lambda x: x
if action == "timeline":
if hasattr(db.view, "get_open_port_count"):
result = list(db.view.get_open_port_count(flt_params.flt))
count = len(result)
else:
result = db.view.get(
flt_params.flt,
fields=['addr', 'starttime', 'openports.count'])
count = result.count()
if request.params.get("modulo") is None:
r2time = lambda r: int(utils.datetime2timestamp(r['starttime']))
else:
r2time = lambda r: (int(utils.datetime2timestamp(r['starttime'])) %
int(request.params.get("modulo")))
if flt_params.ipsasnumbers:
r2res = lambda r: [
r2time(r),
utils.force_ip2int(r['addr']), r['openports']['count']
]
else:
r2res = lambda r: [
r2time(r),
utils.force_int2ip(r['addr']), r['openports']['count']
]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": [\n'
postamble = ']}\n'
result = list(db.view.getlocations(flt_params.flt))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {
"count": r['count']
},
}
elif action == "countopenports":
if hasattr(db.view, "get_open_port_count"):
result = db.view.get_open_port_count(flt_params.flt)
else:
result = db.view.get(flt_params.flt,
fields=['addr', 'openports.count'])
if hasattr(result, "count"):
count = result.count()
else:
count = db.view.count(flt_params.flt,
fields=['addr', 'openports.count'])
if flt_params.ipsasnumbers:
r2res = lambda r: [
utils.force_ip2int(r['addr']), r['openports']['count']
]
else:
r2res = lambda r: [
utils.force_int2ip(r['addr']), r['openports']['count']
]
elif action == "ipsports":
if hasattr(db.view, "get_ips_ports"):
result = list(db.view.get_ips_ports(flt_params.flt))
count = sum(len(host.get('ports', [])) for host in result)
else:
result = db.view.get(
flt_params.flt,
fields=['addr', 'ports.port', 'ports.state_state'])
count = sum(len(host.get('ports', [])) for host in result)
result.rewind()
if flt_params.ipsasnumbers:
r2res = lambda r: [
utils.force_ip2int(r['addr']),
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
else:
r2res = lambda r: [
utils.force_int2ip(r['addr']),
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
elif action == "onlyips":
result = db.view.get(flt_params.flt, fields=['addr'])
if hasattr(result, "count"):
count = result.count()
else:
count = db.view.count(flt_params.flt, fields=['addr'])
if flt_params.ipsasnumbers:
r2res = lambda r: utils.force_ip2int(r['addr'])
else:
r2res = lambda r: utils.force_int2ip(r['addr'])
elif action == "diffcats":
if request.params.get("onlydiff"):
output = db.view.diff_categories(request.params.get("cat1"),
request.params.get("cat2"),
flt=flt_params.flt,
include_both_open=False)
else:
output = db.view.diff_categories(request.params.get("cat1"),
request.params.get("cat2"),
flt=flt_params.flt)
count = 0
result = {}
if flt_params.ipsasnumbers:
for res in output:
result.setdefault(res["addr"],
[]).append([res['port'], res['value']])
count += 1
else:
for res in output:
result.setdefault(utils.int2ip(res["addr"]),
[]).append([res['port'], res['value']])
count += 1
result = viewitems(result)
if flt_params.callback is not None:
if count >= config.WEB_WARN_DOTS_COUNT:
yield (
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count)
yield '%s(\n' % flt_params.callback
yield preamble
# hack to avoid a trailing comma
result = iter(result)
try:
rec = next(result)
except StopIteration:
pass
else:
yield json.dumps(r2res(rec))
for rec in result:
yield ",\n" + json.dumps(r2res(rec))
yield "\n"
yield postamble
if flt_params.callback is not None:
yield ");"
if count >= config.WEB_WARN_DOTS_COUNT:
yield '}\n'
else:
yield "\n"
def _display_xml_host(h, out=sys.stdout):
out.write('<host')
for k in ["timedout", "timeoutcounter"]:
if k in h:
out.write(' %s=%s' % (k, saxutils.quoteattr(h[k])))
for k in ["starttime", "endtime"]:
if k in h:
out.write(' %s=%s' % (k, saxutils.quoteattr(h[k].strftime('%s'))))
out.write('>')
if 'state' in h:
out.write('<status state="%s"' % h['state'])
for k in ["reason", "reason_ttl"]:
kk = "state_%s" % k
if kk in h:
out.write(' %s="%s"' % (k, h[kk]))
out.write('/>')
out.write('\n')
if 'addr' in h:
out.write('<address addr="%s" addrtype="ipv4"/>\n' % h['addr'])
for t in h.get('addresses', []):
for a in h['addresses'][t]:
out.write('<address addr="%s" addrtype="%s"/>\n' % (a, t))
if 'hostnames' in h:
out.write('<hostnames>\n')
for hostname in h['hostnames']:
out.write('<hostname')
for k in ['name', 'type']:
if k in hostname:
out.write(' %s="%s"' % (k, hostname[k]))
out.write('/>\n')
out.write('</hostnames>\n')
out.write('<ports>')
for state, counts in viewitems(h.get('extraports', {})):
out.write('<extraports state="%s" count="%d">\n' %
(state, counts['total']))
for reason, count in viewitems(counts['reasons']):
out.write('<extrareasons reason="%s" count="%d"/>\n' %
(reason, count))
out.write('</extraports>\n')
for p in h.get('ports', []):
if p.get('port') == -1:
h['scripts'] = p['scripts']
continue
out.write('<port')
if 'protocol' in p:
out.write(' protocol="%s"' % p['protocol'])
if 'port' in p:
out.write(' portid="%s"' % p['port'])
out.write('><state')
for k in ['state', 'reason', 'reason_ttl']:
kk = 'state_%s' % k
if kk in p:
out.write(' %s=%s' % (k, saxutils.quoteattr(str(p[kk]))))
out.write('/>')
if 'service_name' in p:
out.write('<service name="%s"' % p['service_name'])
for k in [
'servicefp', 'product', 'version', 'extrainfo', 'ostype',
'method', 'conf'
]:
kk = "service_%s" % k
if kk in p:
if isinstance(p[kk], basestring):
out.write(' %s=%s' % (k, saxutils.quoteattr(p[kk])))
else:
out.write(' %s="%s"' % (k, p[kk]))
# TODO: CPE
out.write('></service>')
for s in p.get('scripts', []):
_display_xml_script(s, out=out)
out.write('</port>\n')
out.write('</ports>\n')
if 'scripts' in h:
out.write('<hostscript>')
for s in h['scripts']:
_display_xml_script(s, out=out)
out.write('</hostscript>')
for trace in h.get('traces', []):
out.write('<trace')
if 'port' in trace:
out.write(' port=%s' % (saxutils.quoteattr(str(trace['port']))))
if 'protocol' in trace:
out.write(' proto=%s' % (saxutils.quoteattr(trace['protocol'])))
out.write('>\n')
for hop in sorted(trace.get('hops', []), key=lambda hop: hop['ttl']):
out.write('<hop')
if 'ttl' in hop:
out.write(' ttl=%s' % (saxutils.quoteattr(str(hop['ttl']))))
if 'ipaddr' in hop:
out.write(' ipaddr=%s' %
(saxutils.quoteattr(utils.int2ip(hop['ipaddr']))))
if 'rtt' in hop:
out.write(
' rtt=%s' %
(saxutils.quoteattr('%.2f' % hop['rtt'] if isinstance(
hop['rtt'], float) else hop['rtt'])))
if 'host' in hop:
out.write(' host=%s' % (saxutils.quoteattr(hop['host'])))
out.write('/>\n')
out.write('</trace>\n')
out.write('</host>\n')
def main():
# write headers
sys.stdout.write(webutils.JS_HEADERS)
params = webutils.parse_query_string()
query = webutils.query_from_params(params)
flt, archive, sortby, unused, skip, limit = webutils.flt_from_query(query)
if limit is None:
limit = config.WEB_LIMIT
if config.WEB_MAXRESULTS is not None:
limit = min(limit, config.WEB_MAXRESULTS)
callback = params.get("callback")
# type of result
action = params.get("action", "")
# top values
if action.startswith("topvalues:"):
field = action[10:]
if field[0] in "-!":
field = field[1:]
least = True
else:
least = False
topnbr = 15
if ":" in field:
field, topnbr = field.rsplit(":", 1)
try:
topnbr = int(topnbr)
except ValueError:
field = "%s:%s" % (field, topnbr)
topnbr = 15
series = [
{"label": t["_id"], "value": t["count"]}
for t in db.nmap.topvalues(field, flt=flt, least=least, topnbr=topnbr, archive=archive)
]
if callback is None:
sys.stdout.write("%s\n" % json.dumps(series))
else:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(series)))
exit(0)
# extract info
if action in ["onlyips", "ipsports", "timeline", "coordinates", "countopenports", "diffcats"]:
preamble = "[\n"
postamble = "]\n"
r2res = lambda x: x
if action == "timeline":
result = db.nmap.get(flt, archive=archive, fields=["addr", "starttime", "openports.count"])
count = result.count()
if params.get("modulo") is None:
r2time = lambda r: int(r["starttime"].strftime("%s"))
else:
r2time = lambda r: (int(r["starttime"].strftime("%s")) % int(params.get("modulo")))
if params.get("ipsasnumbers"):
r2res = lambda r: [r2time(r), r["addr"], r["openports"]["count"]]
else:
r2res = lambda r: [r2time(r), utils.int2ip(r["addr"]), r["openports"]["count"]]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": ['
postamble = "]}"
result = list(db.nmap.getlocations(flt, archive=archive))
count = len(result)
r2res = lambda r: {"type": "Point", "coordinates": r["_id"], "properties": {"count": r["count"]}}
elif action == "countopenports":
result = db.nmap.get(flt, archive=archive, fields=["addr", "openports.count"])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: [r["addr"], r["openports"]["count"]]
else:
r2res = lambda r: [utils.int2ip(r["addr"]), r["openports"]["count"]]
elif action == "ipsports":
result = db.nmap.get(flt, archive=archive, fields=["addr", "ports.port", "ports.state_state"])
count = sum(len(host.get("ports", [])) for host in result)
result.rewind()
if params.get("ipsasnumbers"):
r2res = lambda r: [
r["addr"],
[[p["port"], p["state_state"]] for p in r.get("ports", []) if "state_state" in p],
]
else:
r2res = lambda r: [
utils.int2ip(r["addr"]),
[[p["port"], p["state_state"]] for p in r.get("ports", []) if "state_state" in p],
]
elif action == "onlyips":
result = db.nmap.get(flt, archive=archive, fields=["addr"])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: r["addr"]
else:
r2res = lambda r: utils.int2ip(r["addr"])
elif action == "diffcats":
if params.get("onlydiff"):
output = db.nmap.diff_categories(
params.get("cat1"), params.get("cat2"), flt=flt, include_both_open=False
)
else:
output = db.nmap.diff_categories(params.get("cat1"), params.get("cat2"), flt=flt)
count = 0
result = {}
if params.get("ipsasnumbers"):
for res in output:
result.setdefault(res["addr"], []).append([res["port"], res["value"]])
count += 1
else:
for res in output:
result.setdefault(utils.int2ip(res["addr"]), []).append([res["port"], res["value"]])
count += 1
result = result.iteritems()
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write(
'if(confirm("You are about to ask your browser to display %d '
"dots, which is a lot and might slow down, freeze or crash "
'your browser. Do you want to continue?")) {\n' % count
)
if callback is not None:
sys.stdout.write("%s(\n" % callback)
sys.stdout.write(preamble)
for rec in result:
sys.stdout.write(json.dumps(r2res(rec)) + ",\n")
sys.stdout.write(postamble)
if callback is not None:
sys.stdout.write(");")
sys.stdout.write("\n")
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write("}\n")
exit(0)
# generic request
result = db.nmap.get(flt, archive=archive, limit=limit, skip=skip, sort=sortby)
if action == "count":
if callback is None:
sys.stdout.write("%d\n" % result.count())
else:
sys.stdout.write("%s(%d);\n" % (callback, result.count()))
exit(0)
if unused:
msg = "Option%s not understood: %s" % ("s" if len(unused) > 1 else "", ", ".join(unused))
sys.stdout.write(webutils.js_alert("param-unused", "warning", msg))
sys.stderr.write("IVRE: WARNING: %r\n" % msg)
else:
sys.stdout.write(webutils.js_del_alert("param-unused"))
if config.DEBUG:
msg = "filter: %r" % flt
sys.stdout.write(webutils.js_alert("filter", "info", msg))
sys.stderr.write("IVRE: INFO: %r\n" % msg)
msg = "user: %r" % webutils.get_user()
sys.stdout.write(webutils.js_alert("user", "info", msg))
sys.stderr.write("IVRE: INFO: %r\n" % msg)
version_mismatch = {}
if callback is None:
sys.stdout.write("[\n")
else:
sys.stdout.write("%s([\n" % callback)
for rec in result:
del rec["_id"]
try:
rec["addr"] = utils.int2ip(rec["addr"])
except:
pass
for field in ["starttime", "endtime"]:
if field in rec:
rec[field] = int(rec[field].strftime("%s"))
for port in rec.get("ports", []):
if "screendata" in port:
port["screendata"] = port["screendata"].encode("base64")
if "traces" in rec:
for trace in rec["traces"]:
trace["hops"].sort(key=lambda x: x["ttl"])
for hop in trace["hops"]:
try:
hop["ipaddr"] = utils.int2ip(hop["ipaddr"])
except:
pass
sys.stdout.write("\t%s,\n" % json.dumps(rec))
check = db.nmap.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
if callback is None:
sys.stdout.write("]\n")
else:
sys.stdout.write("]);\n")
messages = {
1: lambda count: (
"%d document%s displayed %s out-of-date. Please run "
"the following commands: 'ivre scancli "
"--update-schema; ivre scancli --update-schema "
"--archives'" % (count, "s" if count > 1 else "", "are" if count > 1 else "is")
),
-1: lambda count: (
"%d document%s displayed ha%s been inserted by "
"a more recent version of IVRE. Please update "
"IVRE!" % (count, "s" if count > 1 else "", "ve" if count > 1 else "s")
),
}
for mismatch, count in version_mismatch.iteritems():
message = messages[mismatch](count)
sys.stdout.write(webutils.js_alert("version-mismatch-%d" % ((mismatch + 1) / 2), "warning", message))
sys.stderr.write("IVRE: WARNING: %r\n" % message)
def main():
try:
import argparse
parser = argparse.ArgumentParser(
description='Access and query the active scans database.',
parents=[db.db.nmap.argparser])
USING_ARGPARSE = True
except ImportError:
import optparse
parser = optparse.OptionParser(
description='Access and query the active scans database.')
for args, kargs in db.db.nmap.argparser.args:
parser.add_option(*args, **kargs)
parser.parse_args_orig = parser.parse_args
parser.parse_args = lambda: parser.parse_args_orig()[0]
parser.add_argument = parser.add_option
USING_ARGPARSE = False
parser.add_argument('--init', '--purgedb', action='store_true',
help='Purge or create and initialize the database.')
parser.add_argument('--ensure-indexes', action='store_true',
help='Create missing indexes (will lock the database).')
parser.add_argument('--short', action='store_true',
help='Output only IP addresses, one per line.')
parser.add_argument('--json', action='store_true',
help='Output results as JSON documents.')
parser.add_argument('--no-screenshots', action='store_true',
help='When used with --json, do not output '
'screenshots data.')
parser.add_argument('--honeyd', action='store_true',
help='Output results as a honeyd config file.')
parser.add_argument('--nmap-xml', action='store_true',
help='Output results as a nmap XML output file.')
parser.add_argument(
'--graphroute',
choices=["dot", "rtgraph3d"] if graphroute.HAVE_DBUS else ["dot"],
help='Create a graph from traceroute results. '
'dot: output result as Graphviz "dot" format to stdout.'
'%s' % (" rtgraph3d: send results to rtgraph3d."
if graphroute.HAVE_DBUS else "")
)
parser.add_argument('--graphroute-cluster', choices=['AS', 'Country'],
help='Cluster IP according to the specified criteria'
'(only for --graphroute dot)')
if graphroute.HAVE_DBUS:
parser.add_argument('--graphroute-dont-reset', action='store_true',
help='Do NOT reset graph (only for '
'--graphroute rtgraph3d)')
parser.add_argument('--graphroute-include', choices=['last-hop', 'target'],
help='How far should graphroute go? Default if to '
'exclude the last hop and the target for each result.')
parser.add_argument('--count', action='store_true',
help='Count matched results.')
parser.add_argument('--explain', action='store_true',
help='MongoDB specific: .explain() the query.')
parser.add_argument('--distinct', metavar='FIELD',
help='Output only unique FIELD part of the '
'results, one per line.')
parser.add_argument('--top', metavar='FIELD / ~FIELD',
help='Output most common (least common: ~) values for '
'FIELD, by default 10, use --limit to change that, '
'--limit 0 means unlimited.')
parser.add_argument('--delete', action='store_true',
help='DELETE the matched results instead of '
'displaying them.')
parser.add_argument('--move-to-archives', action='store_true',
help='ARCHIVE the matched results instead of '
'displaying them (i.e., move the results to '
'the archive collections).')
parser.add_argument('--move-from-archives', action='store_true',
help='UNARCHIVE the matched results instead of '
'displaying them (i.e., move the results from '
'the archive collections to the "fresh" results '
'collections).')
parser.add_argument('--update-schema', action='store_true',
help='update (host) schema. Use with --version to '
'specify your current version and run twice, once '
'with --archive.')
parser.add_argument('--csv', metavar='TYPE',
help='Output result as a CSV file',
choices=['ports', 'hops'])
parser.add_argument('--csv-separator', metavar='SEPARATOR',
default=",",
help='Select separator for --csv output')
parser.add_argument('--csv-add-infos', action='store_true',
help="Include country_code and as_number"
"fields to CSV file")
parser.add_argument('--csv-na-str', default="NA",
help='String to use for "Not Applicable" value '
'(defaults to "NA")')
if USING_ARGPARSE:
parser.add_argument('--sort', metavar='FIELD / ~FIELD', nargs='+',
help='Sort results according to FIELD; use ~FIELD '
'to reverse sort order.')
else:
parser.add_argument('--sort', metavar='FIELD / ~FIELD',
help='Sort results according to FIELD; use ~FIELD '
'to reverse sort order.')
parser.add_argument('--limit', type=int,
help='Ouput at most LIMIT results.')
parser.add_argument('--skip', type=int,
help='Skip first SKIP results.')
args = parser.parse_args()
out = sys.stdout
hostfilter = db.db.nmap.parse_args(args)
sortkeys = []
if args.init:
if os.isatty(sys.stdin.fileno()):
sys.stdout.write(
'This will remove any scan result in your database. '
'Process ? [y/N] ')
ans = raw_input()
if ans.lower() != 'y':
sys.exit(-1)
db.db.nmap.init()
sys.exit(0)
if args.ensure_indexes:
if os.isatty(sys.stdin.fileno()):
sys.stdout.write(
'This will lock your database. '
'Process ? [y/N] ')
ans = raw_input()
if ans.lower() != 'y':
sys.exit(-1)
db.db.nmap.ensure_indexes()
sys.exit(0)
if args.top is not None:
field, least = ((args.top[1:], True)
if args.top[:1] in '!-~' else
(args.top, False))
topnbr = {0: None, None: 10}.get(args.limit, args.limit)
for entry in db.db.nmap.topvalues(field, flt=hostfilter,
topnbr=topnbr,
archive=args.archives):
if isinstance(entry['_id'], list):
if entry['_id']:
entry['_id'] = ' / '.join(str(elt) for elt in entry['_id'])
else:
entry['_id'] = "None"
print "%(_id)s: %(count)d" % entry
sys.exit(0)
if args.sort is not None:
sortkeys = [(field[1:], -1) if field.startswith('~') else (field, 1)
for field in args.sort]
if args.short:
for val in db.db.nmap.distinct("addr", flt=hostfilter, sortby=sortkeys,
limit=args.limit, skip=args.skip,
archive=args.archives):
try:
out.write(utils.int2ip(val) + '\n')
except:
out.write(str(val) + '\n')
sys.exit(0)
elif args.distinct is not None:
for val in db.db.nmap.distinct(args.distinct, flt=hostfilter,
sortby=sortkeys, limit=args.limit,
skip=args.skip, archive=args.archives):
out.write(str(val) + '\n')
sys.exit(0)
if args.json:
import json
def displayfunction(x):
if os.isatty(sys.stdout.fileno()):
indent = 4
else:
indent = None
for h in x:
for fld in ['_id', 'scanid']:
try:
del h[fld]
except KeyError:
pass
for port in h.get('ports', []):
if args.no_screenshots:
for fname in ['screenshot', 'screendata']:
if fname in port:
del port[fname]
elif 'screendata' in port:
port['screendata'] = port['screendata'].encode(
'base64')
for script in port.get('scripts', []):
if 'masscan' in script and 'raw' in script['masscan']:
script['masscan']['raw'] = script['masscan'][
'raw'].encode('base64')
print json.dumps(h, indent=indent,
default=db.db.nmap.serialize)
elif args.honeyd:
def displayfunction(x):
display_honeyd_preamble(out)
honeyd_routes = {}
honeyd_entries = set()
for h in x:
honeyd_routes, honeyd_entries = display_honeyd_conf(
h,
honeyd_routes,
honeyd_entries,
out
)
display_honeyd_epilogue(honeyd_routes, honeyd_entries, out)
elif args.nmap_xml:
def displayfunction(x):
display_xml_preamble(out=out)
if x.count() == 1 and not isinstance(x[0]['scanid'], list):
scan = db.db.nmap.getscan(x[0]['scanid'], archive=args.archives)
if 'scaninfos' in scan and scan['scaninfos']:
for k in scan['scaninfos'][0]:
scan['scaninfo.%s' % k] = scan['scaninfos'][0][k]
del scan['scaninfos']
else:
scan = {}
display_xml_scan(scan, out=out)
for h in x:
display_xml_host(h, out=out)
display_xml_epilogue(out=out)
elif args.graphroute is not None:
def displayfunction(cursor):
graph, entry_nodes = graphroute.buildgraph(
cursor,
include_last_hop=args.graphroute_include == "last-hop",
include_target=args.graphroute_include == "target",
)
if args.graphroute == "dot":
if args.graphroute_cluster == "AS":
def cluster(ipaddr):
res = db.db.data.as_byip(ipaddr)
if res is None:
return
return (res['as_num'],
"%(as_num)d\n[%(as_name)s]" % res)
elif args.graphroute_cluster == "Country":
def cluster(ipaddr):
res = db.db.data.country_byip(ipaddr)
if res is None:
return
return (res['country_code'],
"%(country_code)s - %(country_name)s" % res)
else:
cluster = None
graphroute.writedotgraph(graph, sys.stdout,
cluster=cluster)
elif args.graphroute == "rtgraph3d":
g = graphroute.display3dgraph(
graph,
reset_world=not args.graphroute_dont_reset
)
for n in entry_nodes:
g.glow(utils.int2ip(n))
elif args.count:
def displayfunction(x):
out.write(str(x.count()) + '\n')
elif args.explain:
def displayfunction(x):
out.write(db.db.nmap.explain(x, indent=4) + '\n')
elif args.delete:
def displayfunction(x):
for h in x:
db.db.nmap.remove(h, archive=args.archives)
elif args.move_to_archives:
args.archives = False
def displayfunction(x):
for h in x:
db.db.nmap.archive(h)
elif args.move_from_archives:
args.archives = True
def displayfunction(x):
for h in x:
db.db.nmap.archive(h, unarchive=True)
elif args.csv is not None:
fields = {
"ports": OrderedDict([
["addr", utils.int2ip],
["ports", OrderedDict([
["port", str],
["state_state", True]])]]),
"hops": OrderedDict([
["addr", utils.int2ip],
["traces", OrderedDict([
["hops", OrderedDict([
["ipaddr", utils.int2ip],
["ttl", str],
["rtt", lambda x: (args.csv_na_str if x == '--'
else str(x))],
])]
])]
]),
"rtt": OrderedDict([
["addr", utils.int2ip],
["traces", OrderedDict([
["hops", OrderedDict([
["rtt", lambda x: (args.csv_na_str if x == '--'
else str(x))],
])]
])]
]),
}.get(args.csv)
if fields is None:
parser.error("Invalid choice for --csv.")
if args.csv_add_infos:
fields['infos'] = OrderedDict([
["country_code", True],
["city", True],
["as_num", str],
])
def displayfunction(x):
out.write(args.csv_separator.join(utils.fields2csv_head(fields)))
out.write('\n')
for h in x:
displayhost_csv(fields, args.csv_separator, args.csv_na_str,
h, out=out)
else:
def displayfunction(cursor):
nmapout.displayhosts(cursor, out=out)
if args.update_schema:
db.db.nmap.migrate_schema(
db.db.nmap.colname_oldhosts if args.archives
else db.db.nmap.colname_hosts, args.version
)
else:
cursor = db.db.nmap.get(hostfilter, archive=args.archives)
if sortkeys:
cursor = cursor.sort(sortkeys)
if args.skip is not None:
cursor = cursor.skip(args.skip)
if args.limit is not None:
cursor = cursor.limit(args.limit)
displayfunction(cursor)
sys.exit(0)
def _display_honeyd_conf(host, honeyd_routes, honeyd_entries, out=sys.stdout):
addr = utils.int2ip(host['addr'])
hname = "host_%s" % addr.replace('.', '_')
out.write("create %s\n" % hname)
defaction = HONEYD_DEFAULT_ACTION
if 'extraports' in host:
extra = host['extraports']
defaction = max(
max(viewvalues(extra),
key=lambda state: viewitems(state['total'])['reasons']),
key=lambda reason: reason[1],
)[0]
defaction = HONEYD_ACTION_FROM_NMAP_STATE.get(defaction)
out.write('set %s default tcp action %s\n' % (hname, defaction))
for p in host.get('ports', []):
try:
out.write('add %s %s port %d %s\n' % (
hname,
p['protocol'],
p['port'],
_nmap_port2honeyd_action(p),
))
except KeyError:
# let's skip pseudo-port records that are only containers for host
# scripts.
pass
if 'traces' in host and len(host['traces']) > 0:
trace = max(host['traces'], key=lambda x: len(x['hops']))['hops']
if trace:
trace.sort(key=lambda x: x['ttl'])
curhop = trace[0]
honeyd_entries.add(curhop['ipaddr'])
for t in trace[1:]:
key = (curhop['ipaddr'], t['ipaddr'])
latency = max(t['rtt'] - curhop['rtt'], 0)
route = honeyd_routes.get(key)
if route is None:
honeyd_routes[key] = {
'count': 1,
'high': latency,
'low': latency,
'mean': latency,
'targets': set([host['addr']])
}
else:
route['targets'].add(host['addr'])
honeyd_routes[key] = {
'count':
route['count'] + 1,
'high':
max(route['high'], latency),
'low':
min(route['low'], latency),
'mean': (route['mean'] * route['count'] + latency) /
(route['count'] + 1),
'targets':
route['targets'],
}
curhop = t
out.write('bind %s %s\n\n' % (addr, hname))
return honeyd_routes, honeyd_entries
def _add_node(node):
if node not in nodes:
nodes.add(node)
out.write('\t%d [label="%s"];\n' % (node, utils.int2ip(node)))
def iter_ranges(self):
for start, length in sorted(viewvalues(self.ranges)):
yield utils.int2ip(start), utils.int2ip(start + length - 1)
def displayhost(dic,
showscripts=True,
showtraceroute=True,
showos=True,
out=sys.stdout):
try:
h = "Host %s" % utils.int2ip(dic['addr'])
except:
h = "Host %s" % dic['addr']
if 'hostnames' in dic and dic['hostnames']:
h += " (%s)" % '/'.join(x['name'] for x in dic['hostnames'])
if 'source' in dic:
h += ' from %s' % dic['source']
if 'categories' in dic and dic['categories']:
h += ' (%s)' % ', '.join(dic['categories'])
if 'state' in dic:
h += ' (%s' % dic['state']
if 'state_reason' in dic:
h += ': %s' % dic['state_reason']
h += ')\n'
out.write(h)
if 'infos' in dic:
infos = dic['infos']
if 'country_code' in infos or 'country_name' in infos:
out.write("\t%s - %s" % (infos.get(
'country_code', '?'), infos.get('country_name', '?')))
if 'city' in infos:
out.write(' - %s' % infos['city'])
out.write('\n')
if 'as_num' in infos or 'as_name' in infos:
out.write("\tAS%s - %s\n" %
(infos.get('as_num', '?'), infos.get('as_name', '?')))
if 'starttime' in dic and 'endtime' in dic:
out.write("\tscan %s - %s\n" % (dic['starttime'], dic['endtime']))
if 'extraports' in dic:
d = dic['extraports']
for k in d:
out.write("\t%d ports %s (%s)\n" % (d[k][0], k, ', '.join(
['%d %s' % (d[k][1][kk], kk) for kk in d[k][1].keys()])))
if 'ports' in dic:
d = dic['ports']
d.sort(key=lambda x: (x.get('protocol'), x['port']))
for k in d:
if k.get('port') == 'host':
dic['scripts'] = k['scripts']
continue
reason = ""
if 'state_reason' in k:
reason = " (%s" % k['state_reason']
for kk in filter(lambda x: x.startswith('state_reason_'),
k.keys()):
reason += ", %s=%s" % (kk[13:], k[kk])
reason += ')'
srv = ""
if 'service_name' in k:
srv = "" + k['service_name']
if 'service_method' in k:
srv += ' (%s)' % k['service_method']
for kk in [
'service_product', 'service_version',
'service_extrainfo', 'service_ostype',
'service_hostname'
]:
if kk in k:
srv += ' %s' % k[kk]
out.write("\t%-10s%-8s%-22s%s\n" %
('%s/%d' % (k.get('protocol'), k['port']),
k['state_state'], reason, srv))
if showscripts and 'scripts' in k:
for s in k['scripts']:
if 'output' not in s:
out.write('\t\t' + s['id'] + ':\n')
else:
o = filter(
lambda x: x,
map(lambda x: x.strip(), s['output'].split('\n')))
if len(o) == 0:
out.write('\t\t' + s['id'] + ':\n')
elif len(o) == 1:
out.write('\t\t' + s['id'] + ': ' + o[0] + '\n')
elif len(o) > 1:
out.write('\t\t' + s['id'] + ': \n')
for oo in o:
out.write('\t\t\t' + oo + '\n')
if showscripts and 'scripts' in dic:
out.write('\tHost scripts:\n')
for s in dic['scripts']:
if 'output' not in s:
out.write('\t\t' + s['id'] + ':\n')
else:
o = [x.strip() for x in s['output'].split('\n') if x]
if len(o) == 0:
out.write('\t\t' + s['id'] + ':\n')
elif len(o) == 1:
out.write('\t\t' + s['id'] + ': ' + o[0] + '\n')
elif len(o) > 1:
out.write('\t\t' + s['id'] + ': \n')
for oo in o:
out.write('\t\t\t' + oo + '\n')
if showtraceroute and 'traces' in dic and dic['traces']:
for k in dic['traces']:
proto = k['protocol']
if proto in ['tcp', 'udp']:
proto += '/%d' % k['port']
out.write('\tTraceroute (using %s)\n' % proto)
hops = k['hops']
hops.sort(key=lambda x: x['ttl'])
for i in hops:
try:
out.write('\t\t%3s %15s %7s\n' %
(i['ttl'], utils.int2ip(i['ipaddr']), i['rtt']))
except:
out.write('\t\t%3s %15s %7s\n' %
(i['ttl'], i['ipaddr'], i['rtt']))
if showos and 'os' in dic and 'osclass' in dic['os'] and \
dic['os']['osclass']:
o = dic['os']['osclass']
maxacc = str(max([int(x['accuracy']) for x in o]))
o = filter(lambda x: x['accuracy'] == maxacc, o)
out.write('\tOS fingerprint\n')
for oo in o:
out.write('\t\t%(osfamily)s / %(type)s / %(vendor)s / '
'accuracy = %(accuracy)s\n' % oo)
def displayfunction(x):
for h in x.distinct('addr'):
try:
out.write(utils.int2ip(h) + '\n')
except:
out.write(str(h) + '\n')
def iter_addrs(self):
for start, length in sorted(viewvalues(self.ranges)):
for val in range(start, start + length):
yield utils.int2ip(val)
def displayfunction(x):
for h in x.distinct('addr'):
try:
out.write(utils.int2ip(h) + '\n')
except:
out.write(str(h) + '\n')
def displayhost(record,
showscripts=True,
showtraceroute=True,
showos=True,
out=sys.stdout):
"""Displays (on `out`, by default `sys.stdout`) the Nmap scan
result contained in `record`.
"""
try:
line = "Host %s" % utils.int2ip(record['addr'])
except:
line = "Host %s" % record['addr']
if record.get('hostnames'):
line += " (%s)" % '/'.join(x['name'] for x in record['hostnames'])
if 'source' in record:
line += ' from %s' % record['source']
if record.get('categories'):
line += ' (%s)' % ', '.join(record['categories'])
if 'state' in record:
line += ' (%s' % record['state']
if 'state_reason' in record:
line += ': %s' % record['state_reason']
line += ')\n'
out.write(line)
if 'infos' in record:
infos = record['infos']
if 'country_code' in infos or 'country_name' in infos:
out.write("\t%s - %s" % (infos.get(
'country_code', '?'), infos.get('country_name', '?')))
if 'city' in infos:
out.write(' - %s' % infos['city'])
out.write('\n')
if 'as_num' in infos or 'as_name' in infos:
out.write("\tAS%s - %s\n" %
(infos.get('as_num', '?'), infos.get('as_name', '?')))
if 'starttime' in record and 'endtime' in record:
out.write("\tscan %s - %s\n" %
(record['starttime'], record['endtime']))
for state, counts in record.get('extraports', {}).iteritems():
out.write("\t%d ports %s (%s)\n" % (counts["total"], state, ", ".join(
"%d %s" % (count, reason)
for reason, count in counts["reasons"].iteritems()
if reason != "total")))
ports = record.get('ports', [])
ports.sort(key=lambda x: (x.get('protocol'), x['port']))
for port in ports:
if port.get('port') == -1:
record['scripts'] = port['scripts']
continue
if 'state_reason' in port:
reason = " (%s)" % ', '.join([port['state_reason']] + [
"%s=%s" % (field[13:], value)
for field, value in port.iteritems()
if field.startswith('state_reason_')
])
else:
reason = ""
if 'service_name' in port:
srv = port['service_name']
if 'service_method' in port:
srv += ' (%s)' % port['service_method']
for field in [
'service_product', 'service_version', 'service_extrainfo',
'service_ostype', 'service_hostname'
]:
if field in port:
srv += ' %s' % port[field]
else:
srv = ""
out.write("\t%-10s%-8s%-22s%s\n" %
('%s/%d' % (port.get('protocol'), port['port']),
port['state_state'], reason, srv))
if showscripts:
out.writelines(_scriptoutput(port))
if showscripts:
scripts = _scriptoutput(record)
if scripts:
out.write('\tHost scripts:\n')
out.writelines(scripts)
if showtraceroute and record.get('traces'):
for trace in record['traces']:
proto = trace['protocol']
if proto in ['tcp', 'udp']:
proto += '/%d' % trace['port']
out.write('\tTraceroute (using %s)\n' % proto)
hops = trace['hops']
hops.sort(key=lambda hop: hop['ttl'])
for hop in hops:
try:
out.write(
'\t\t%3s %15s %7s\n' %
(hop['ttl'], utils.int2ip(hop['ipaddr']), hop['rtt']))
except:
out.write('\t\t%3s %15s %7s\n' %
(hop['ttl'], hop['ipaddr'], hop['rtt']))
if showos and record.get('os', {}).get('osclass'):
osclasses = record['os']['osclass']
maxacc = str(max(int(x['accuracy']) for x in osclasses))
osclasses = [
osclass for osclass in osclasses if osclass['accuracy'] == maxacc
]
out.write('\tOS fingerprint\n')
for osclass in osclasses:
out.write('\t\t%(osfamily)s / %(type)s / %(vendor)s / '
'accuracy = %(accuracy)s\n' % osclass)
def _display_xml_host(h, out=sys.stdout):
out.write('<host')
for k in ["timedout", "timeoutcounter"]:
if k in h:
out.write(' %s=%s' % (k, saxutils.quoteattr(h[k])))
for k in ["starttime", "endtime"]:
if k in h:
out.write(' %s=%s' % (k, saxutils.quoteattr(h[k].strftime('%s'))))
out.write('>')
if 'state' in h:
out.write('<status state="%s"' % h['state'])
for k in ["reason", "reason_ttl"]:
kk = "state_%s" % k
if kk in h:
out.write(' %s="%s"' % (k, h[kk]))
out.write('/>')
out.write('\n')
if 'addr' in h:
out.write('<address addr="%s" addrtype="ipv4"/>\n' % h['addr'])
for t in h.get('addresses', []):
for a in h['addresses'][t]:
out.write('<address addr="%s" addrtype="%s"/>\n' % (a, t))
if 'hostnames' in h:
out.write('<hostnames>\n')
for hostname in h['hostnames']:
out.write('<hostname')
for k in ['name', 'type']:
if k in hostname:
out.write(' %s="%s"' % (k, hostname[k]))
out.write('/>\n')
out.write('</hostnames>\n')
out.write('<ports>')
for state, counts in viewitems(h.get('extraports', {})):
out.write('<extraports state="%s" count="%d">\n' % (
state, counts['total']
))
for reason, count in viewitems(counts['reasons']):
out.write('<extrareasons reason="%s" count="%d"/>\n' % (
reason, count
))
out.write('</extraports>\n')
for p in h.get('ports', []):
if p.get('port') == -1:
h['scripts'] = p['scripts']
continue
out.write('<port')
if 'protocol' in p:
out.write(' protocol="%s"' % p['protocol'])
if 'port' in p:
out.write(' portid="%s"' % p['port'])
out.write('><state')
for k in ['state', 'reason', 'reason_ttl']:
kk = 'state_%s' % k
if kk in p:
out.write(' %s=%s' % (k, saxutils.quoteattr(str(p[kk]))))
out.write('/>')
if 'service_name' in p:
out.write('<service name="%s"' % p['service_name'])
for k in ['servicefp', 'product', 'version', 'extrainfo',
'ostype', 'method', 'conf']:
kk = "service_%s" % k
if kk in p:
if isinstance(p[kk], basestring):
out.write(' %s=%s' % (
k, saxutils.quoteattr(p[kk])
))
else:
out.write(' %s="%s"' % (k, p[kk]))
# TODO: CPE
out.write('></service>')
for s in p.get('scripts', []):
_display_xml_script(s, out=out)
out.write('</port>\n')
out.write('</ports>\n')
if 'scripts' in h:
out.write('<hostscript>')
for s in h['scripts']:
_display_xml_script(s, out=out)
out.write('</hostscript>')
for trace in h.get('traces', []):
out.write('<trace')
if 'port' in trace:
out.write(' port=%s' % (saxutils.quoteattr(str(trace['port']))))
if 'protocol' in trace:
out.write(' proto=%s' % (saxutils.quoteattr(trace['protocol'])))
out.write('>\n')
for hop in sorted(trace.get('hops', []),
key=lambda hop: hop['ttl']):
out.write('<hop')
if 'ttl' in hop:
out.write(' ttl=%s' % (
saxutils.quoteattr(str(hop['ttl']))
))
if 'ipaddr' in hop:
out.write(' ipaddr=%s' % (
saxutils.quoteattr(utils.int2ip(hop['ipaddr']))
))
if 'rtt' in hop:
out.write(' rtt=%s' % (
saxutils.quoteattr('%.2f' % hop['rtt']
if isinstance(hop['rtt'], float) else
hop['rtt'])
))
if 'host' in hop:
out.write(' host=%s' % (
saxutils.quoteattr(hop['host'])
))
out.write('/>\n')
out.write('</trace>\n')
out.write('</host>\n')
def _add_node(node):
if node not in nodes:
nodes.add(node)
out.write('\t%d [label="%s"];\n' % (node, utils.int2ip(node)))
def main():
try:
import argparse
parser = argparse.ArgumentParser(
description='Access and query the active scans database.',
parents=[db.db.nmap.argparser])
USING_ARGPARSE = True
except ImportError:
import optparse
parser = optparse.OptionParser(
description='Access and query the active scans database.')
for args, kargs in db.db.nmap.argparser.args:
parser.add_option(*args, **kargs)
parser.parse_args_orig = parser.parse_args
parser.parse_args = lambda: parser.parse_args_orig()[0]
parser.add_argument = parser.add_option
USING_ARGPARSE = False
parser.add_argument('--init',
'--purgedb',
action='store_true',
help='Purge or create and initialize the database.')
parser.add_argument(
'--ensure-indexes',
action='store_true',
help='Create missing indexes (will lock the database).')
parser.add_argument('--short',
action='store_true',
help='Output only IP addresses, one per line.')
parser.add_argument('--json',
action='store_true',
help='Output results as JSON documents.')
parser.add_argument('--no-screenshots',
action='store_true',
help='When used with --json, do not output '
'screenshots data.')
parser.add_argument('--honeyd',
action='store_true',
help='Output results as a honeyd config file.')
parser.add_argument('--nmap-xml',
action='store_true',
help='Output results as a nmap XML output file.')
parser.add_argument(
'--graphroute',
choices=["dot", "rtgraph3d"] if graphroute.HAVE_DBUS else ["dot"],
help='Create a graph from traceroute results. '
'dot: output result as Graphviz "dot" format to stdout.'
'%s' % (" rtgraph3d: send results to rtgraph3d."
if graphroute.HAVE_DBUS else ""))
parser.add_argument('--graphroute-cluster',
choices=['AS', 'Country'],
help='Cluster IP according to the specified criteria'
'(only for --graphroute dot)')
if graphroute.HAVE_DBUS:
parser.add_argument('--graphroute-dont-reset',
action='store_true',
help='Do NOT reset graph (only for '
'--graphroute rtgraph3d)')
parser.add_argument('--graphroute-include',
choices=['last-hop', 'target'],
help='How far should graphroute go? Default if to '
'exclude the last hop and the target for each result.')
parser.add_argument('--count',
action='store_true',
help='Count matched results.')
parser.add_argument('--explain',
action='store_true',
help='MongoDB specific: .explain() the query.')
parser.add_argument('--distinct',
metavar='FIELD',
help='Output only unique FIELD part of the '
'results, one per line.')
parser.add_argument('--top',
metavar='FIELD / ~FIELD',
help='Output most common (least common: ~) values for '
'FIELD, by default 10, use --limit to change that, '
'--limit 0 means unlimited.')
parser.add_argument('--delete',
action='store_true',
help='DELETE the matched results instead of '
'displaying them.')
parser.add_argument('--move-to-archives',
action='store_true',
help='ARCHIVE the matched results instead of '
'displaying them (i.e., move the results to '
'the archive collections).')
parser.add_argument('--move-from-archives',
action='store_true',
help='UNARCHIVE the matched results instead of '
'displaying them (i.e., move the results from '
'the archive collections to the "fresh" results '
'collections).')
parser.add_argument('--update-schema',
action='store_true',
help='update (host) schema. Use with --version to '
'specify your current version and run twice, once '
'with --archive.')
parser.add_argument('--csv',
metavar='TYPE',
help='Output result as a CSV file',
choices=['ports', 'hops'])
parser.add_argument('--csv-separator',
metavar='SEPARATOR',
default=",",
help='Select separator for --csv output')
parser.add_argument('--csv-add-infos',
action='store_true',
help="Include country_code and as_number"
"fields to CSV file")
parser.add_argument('--csv-na-str',
default="NA",
help='String to use for "Not Applicable" value '
'(defaults to "NA")')
if USING_ARGPARSE:
parser.add_argument('--sort',
metavar='FIELD / ~FIELD',
nargs='+',
help='Sort results according to FIELD; use ~FIELD '
'to reverse sort order.')
else:
parser.add_argument('--sort',
metavar='FIELD / ~FIELD',
help='Sort results according to FIELD; use ~FIELD '
'to reverse sort order.')
parser.add_argument('--limit',
type=int,
help='Ouput at most LIMIT results.')
parser.add_argument('--skip', type=int, help='Skip first SKIP results.')
args = parser.parse_args()
out = sys.stdout
hostfilter = db.db.nmap.parse_args(args)
sortkeys = []
if args.init:
if os.isatty(sys.stdin.fileno()):
sys.stdout.write(
'This will remove any scan result in your database. '
'Process ? [y/N] ')
ans = raw_input()
if ans.lower() != 'y':
sys.exit(-1)
db.db.nmap.init()
sys.exit(0)
if args.ensure_indexes:
if os.isatty(sys.stdin.fileno()):
sys.stdout.write('This will lock your database. '
'Process ? [y/N] ')
ans = raw_input()
if ans.lower() != 'y':
sys.exit(-1)
db.db.nmap.ensure_indexes()
sys.exit(0)
if args.top is not None:
field, least = ((args.top[1:], True) if args.top[:1] in '!-~' else
(args.top, False))
topnbr = {0: None, None: 10}.get(args.limit, args.limit)
for entry in db.db.nmap.topvalues(field,
flt=hostfilter,
topnbr=topnbr,
archive=args.archives):
if isinstance(entry['_id'], list):
if entry['_id']:
entry['_id'] = ' / '.join(str(elt) for elt in entry['_id'])
else:
entry['_id'] = "None"
print "%(_id)s: %(count)d" % entry
sys.exit(0)
if args.sort is not None:
sortkeys = [(field[1:], -1) if field.startswith('~') else (field, 1)
for field in args.sort]
if args.short:
for val in db.db.nmap.distinct("addr",
flt=hostfilter,
sortby=sortkeys,
limit=args.limit,
skip=args.skip,
archive=args.archives):
try:
out.write(utils.int2ip(val) + '\n')
except:
out.write(str(val) + '\n')
sys.exit(0)
elif args.distinct is not None:
for val in db.db.nmap.distinct(args.distinct,
flt=hostfilter,
sortby=sortkeys,
limit=args.limit,
skip=args.skip,
archive=args.archives):
out.write(str(val) + '\n')
sys.exit(0)
if args.json:
import json
def displayfunction(x):
if os.isatty(sys.stdout.fileno()):
indent = 4
else:
indent = None
for h in x:
for fld in ['_id', 'scanid']:
try:
del h[fld]
except KeyError:
pass
for port in h.get('ports', []):
if args.no_screenshots:
for fname in ['screenshot', 'screendata']:
if fname in port:
del port[fname]
elif 'screendata' in port:
port['screendata'] = port['screendata'].encode(
'base64')
print json.dumps(h,
indent=indent,
default=db.db.nmap.serialize)
elif args.honeyd:
def displayfunction(x):
display_honeyd_preamble(out)
honeyd_routes = {}
honeyd_entries = set()
for h in x:
honeyd_routes, honeyd_entries = display_honeyd_conf(
h, honeyd_routes, honeyd_entries, out)
display_honeyd_epilogue(honeyd_routes, honeyd_entries, out)
elif args.nmap_xml:
def displayfunction(x):
display_xml_preamble(out=out)
if x.count() == 1 and not isinstance(x[0]['scanid'], list):
scan = db.db.nmap.getscan(x[0]['scanid'],
archive=args.archives)
if 'scaninfos' in scan and scan['scaninfos']:
for k in scan['scaninfos'][0]:
scan['scaninfo.%s' % k] = scan['scaninfos'][0][k]
del scan['scaninfos']
else:
scan = {}
display_xml_scan(scan, out=out)
for h in x:
display_xml_host(h, out=out)
display_xml_epilogue(out=out)
elif args.graphroute is not None:
def displayfunction(cursor):
graph, entry_nodes = graphroute.buildgraph(
cursor,
include_last_hop=args.graphroute_include == "last-hop",
include_target=args.graphroute_include == "target",
)
if args.graphroute == "dot":
if args.graphroute_cluster == "AS":
def cluster(ipaddr):
res = db.db.data.as_byip(ipaddr)
if res is None:
return
return (res['as_num'],
"%(as_num)d\n[%(as_name)s]" % res)
elif args.graphroute_cluster == "Country":
def cluster(ipaddr):
res = db.db.data.country_byip(ipaddr)
if res is None:
return
return (res['country_code'],
"%(country_code)s - %(country_name)s" % res)
else:
cluster = None
graphroute.writedotgraph(graph, sys.stdout, cluster=cluster)
elif args.graphroute == "rtgraph3d":
g = graphroute.display3dgraph(
graph, reset_world=not args.graphroute_dont_reset)
for n in entry_nodes:
g.glow(utils.int2ip(n))
elif args.count:
def displayfunction(x):
out.write(str(x.count()) + '\n')
elif args.explain:
def displayfunction(x):
out.write(db.db.nmap.explain(x, indent=4) + '\n')
elif args.delete:
def displayfunction(x):
for h in x:
db.db.nmap.remove(h, archive=args.archives)
elif args.move_to_archives:
args.archives = False
def displayfunction(x):
for h in x:
db.db.nmap.archive(h)
elif args.move_from_archives:
args.archives = True
def displayfunction(x):
for h in x:
db.db.nmap.archive(h, unarchive=True)
elif args.csv is not None:
fields = {
"ports":
OrderedDict([["addr", utils.int2ip],
[
"ports",
OrderedDict([["port", str], ["state_state",
True]])
]]),
"hops":
OrderedDict([["addr", utils.int2ip],
[
"traces",
OrderedDict([[
"hops",
OrderedDict([
["ipaddr", utils.int2ip],
["ttl", str],
[
"rtt", lambda x:
(args.csv_na_str
if x == '--' else str(x))
],
])
]])
]]),
"rtt":
OrderedDict([["addr", utils.int2ip],
[
"traces",
OrderedDict([[
"hops",
OrderedDict([
[
"rtt", lambda x:
(args.csv_na_str
if x == '--' else str(x))
],
])
]])
]]),
}.get(args.csv)
if fields is None:
parser.error("Invalid choice for --csv.")
if args.csv_add_infos:
fields['infos'] = OrderedDict([
["country_code", True],
["city", True],
["as_num", str],
])
def displayfunction(x):
out.write(args.csv_separator.join(utils.fields2csv_head(fields)))
out.write('\n')
for h in x:
displayhost_csv(fields,
args.csv_separator,
args.csv_na_str,
h,
out=out)
else:
def displayfunction(cursor):
nmapout.displayhosts(cursor, out=out)
if args.update_schema:
db.db.nmap.migrate_schema(
db.db.nmap.colname_oldhosts
if args.archives else db.db.nmap.colname_hosts, args.version)
else:
cursor = db.db.nmap.get(hostfilter, archive=args.archives)
if sortkeys:
cursor = cursor.sort(sortkeys)
if args.skip is not None:
cursor = cursor.skip(args.skip)
if args.limit is not None:
cursor = cursor.limit(args.limit)
displayfunction(cursor)
sys.exit(0)
def force_ip_str(addr):
try:
return utils.int2ip(addr)
except (TypeError, socket.error):
return addr