def _build_filter(self, cmd):
"""Build a db.nmap.flt from the string `cmd`"""
query = webutils.query_from_params({'q': cmd})
(self._dfilter['filter'], _, _, _,
self._dfilter['skip'],
self._dfilter['limit']) = webutils.flt_from_query(
query, base_flt=db.nmap.flt_empty)
def main():
# write headers
sys.stdout.write(webutils.JS_HEADERS)
params = webutils.parse_query_string()
query = webutils.query_from_params(params)
flt, archive, sortby, unused, skip, limit = webutils.flt_from_query(query)
callback = params.get("callback")
# type of result
action = params.get("action", "")
# top values
if action.startswith('topvalues:'):
field = action[10:]
if field[0] in '-!':
field = field[1:]
least = True
else:
least = False
topnbr = 15
if ':' in field:
field, topnbr = field.rsplit(':', 1)
try:
topnbr = int(topnbr)
except ValueError:
field = '%s:%s' % (field, topnbr)
topnbr = 15
series = [{
"label": t['_id'],
"value": t['count']
} for t in db.nmap.topvalues(
field, flt=flt, least=least, topnbr=topnbr, archive=archive)]
if callback is None:
sys.stdout.write("%s;\n" % json.dumps(series))
else:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(series)))
exit(0)
# extract info
if action in [
"onlyips", "ipsports", "timeline", "coordinates", "countopenports"
]:
preamble = "[\n"
postamble = "]\n"
if action == "timeline":
result = db.nmap.get(
flt,
archive=archive,
fields=['addr', 'starttime', 'openports.count'])
count = result.count()
if params.get("modulo") is None:
r2time = lambda r: int(r['starttime'].strftime('%s'))
else:
r2time = lambda r: (int(r['starttime'].strftime('%s')) % int(
params.get("modulo")))
if params.get("ipsasnumbers"):
r2res = lambda r: [
r2time(r), r['addr'], r['openports']['count']
]
else:
r2res = lambda r: [
r2time(r),
utils.int2ip(r['addr']), r['openports']['count']
]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": ['
postamble = ']}'
result = list(db.nmap.getlocations(flt, archive=archive))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {
"count": r['count']
},
}
elif action == "countopenports":
result = db.nmap.get(flt,
archive=archive,
fields=['addr', 'openports.count'])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: [r['addr'], r['openports']['count']]
else:
r2res = lambda r: [
utils.int2ip(r['addr']), r['openports']['count']
]
elif action == "ipsports":
result = db.nmap.get(
flt,
archive=archive,
fields=['addr', 'ports.port', 'ports.state_state'])
count = sum(len(host.get('ports', [])) for host in result)
result.rewind()
if params.get("ipsasnumbers"):
r2res = lambda r: [
r['addr'],
[[p['port'], p['state_state']] for p in r.get('ports', [])]
]
else:
r2res = lambda r: [
utils.int2ip(r['addr']),
[[p['port'], p['state_state']] for p in r.get('ports', [])]
]
elif action == "onlyips":
result = db.nmap.get(flt, archive=archive, fields=['addr'])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: r['addr']
else:
r2res = lambda r: utils.int2ip(r['addr'])
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write(
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count)
if callback is not None:
sys.stdout.write("%s(\n" % callback)
sys.stdout.write(preamble)
for rec in result:
sys.stdout.write(json.dumps(r2res(rec)) + ",\n")
sys.stdout.write(postamble)
if callback is not None:
sys.stdout.write(")")
sys.stdout.write(";\n")
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write('}\n')
exit(0)
# generic request
result = db.nmap.get(flt,
archive=archive,
limit=limit,
skip=skip,
sort=sortby)
if action == "count":
if callback is not None:
sys.stdout.write("%s(%d);\n" % (callback, result.count()))
else:
sys.stdout.write("%d;\n" % result.count())
exit(0)
if unused:
msg = 'Option%s not understood: %s' % (
's' if len(unused) > 1 else '',
', '.join(unused),
)
sys.stdout.write(webutils.js_alert("param-unused", "warning", msg))
sys.stderr.write('IVRE: WARNING: %r\n' % msg)
if config.DEBUG:
msg = "filter: %r" % flt
sys.stdout.write(webutils.js_alert("filter", "info", msg))
sys.stderr.write('IVRE: INFO: %r\n' % msg)
version_mismatch = {}
for rec in result:
del rec['_id']
try:
rec['addr'] = utils.int2ip(rec['addr'])
except:
pass
for field in ['starttime', 'endtime']:
if field in rec:
rec[field] = int(rec[field].strftime('%s'))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = port['screendata'].encode('base64')
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
try:
hop['ipaddr'] = utils.int2ip(hop['ipaddr'])
except:
pass
if callback is not None:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(rec)))
else:
sys.stdout.write("%s;\n" % json.dumps(rec))
check = db.nmap.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
messages = {
1:
lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following commands: 'scancli --update-schema; "
"scancli --update-schema --archives'"
"" % (count, 's' if count > 1 else '', 'are'
if count > 1 else 'is')),
-1:
lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '', 've'
if count > 1 else 's')),
}
for mismatch, count in version_mismatch.iteritems():
message = messages[mismatch](count)
sys.stdout.write(
webutils.js_alert("version-mismatch-%d" % ((mismatch + 1) / 2),
"warning", message))
sys.stderr.write('IVRE: WARNING: %r\n' % message)
def main():
# write headers
sys.stdout.write(webutils.JS_HEADERS)
params = webutils.parse_query_string()
query = webutils.query_from_params(params)
flt, archive, sortby, unused, skip, limit = webutils.flt_from_query(query)
if limit is None:
limit = config.WEB_LIMIT
if config.WEB_MAXRESULTS is not None:
limit = min(limit, config.WEB_MAXRESULTS)
callback = params.get("callback")
# type of result
action = params.get("action", "")
# top values
if action.startswith('topvalues:'):
field = action[10:]
if field[0] in '-!':
field = field[1:]
least = True
else:
least = False
topnbr = 15
if ':' in field:
field, topnbr = field.rsplit(':', 1)
try:
topnbr = int(topnbr)
except ValueError:
field = '%s:%s' % (field, topnbr)
topnbr = 15
series = [{"label": t['_id'], "value": t['count']} for t in
db.nmap.topvalues(field, flt=flt,
least=least, topnbr=topnbr,
archive=archive)]
if callback is None:
sys.stdout.write("%s;\n" % json.dumps(series))
else:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(series)))
exit(0)
# extract info
if action in ["onlyips", "ipsports", "timeline", "coordinates",
"countopenports"]:
preamble = "[\n"
postamble = "]\n"
if action == "timeline":
result = db.nmap.get(
flt, archive=archive,
fields=['addr', 'starttime', 'openports.count']
)
count = result.count()
if params.get("modulo") is None:
r2time = lambda r: int(r['starttime'].strftime('%s'))
else:
r2time = lambda r: (int(r['starttime'].strftime('%s'))
% int(params.get("modulo")))
if params.get("ipsasnumbers"):
r2res = lambda r: [r2time(r), r['addr'],
r['openports']['count']]
else:
r2res = lambda r: [r2time(r),
utils.int2ip(r['addr']),
r['openports']['count']]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": ['
postamble = ']}'
result = list(db.nmap.getlocations(flt, archive=archive))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {"count": r['count']},
}
elif action == "countopenports":
result = db.nmap.get(flt, archive=archive,
fields=['addr', 'openports.count'])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: [r['addr'], r['openports']['count']]
else:
r2res = lambda r: [utils.int2ip(r['addr']),
r['openports']['count']]
elif action == "ipsports":
result = db.nmap.get(
flt, archive=archive,
fields=['addr', 'ports.port', 'ports.state_state']
)
count = sum(len(host.get('ports', [])) for host in result)
result.rewind()
if params.get("ipsasnumbers"):
r2res = lambda r: [
r['addr'],
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
else:
r2res = lambda r: [
utils.int2ip(r['addr']),
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
elif action == "onlyips":
result = db.nmap.get(flt, archive=archive, fields=['addr'])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: r['addr']
else:
r2res = lambda r: utils.int2ip(r['addr'])
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write(
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count
)
if callback is not None:
sys.stdout.write("%s(\n" % callback)
sys.stdout.write(preamble)
for rec in result:
sys.stdout.write(json.dumps(r2res(rec)) + ",\n")
sys.stdout.write(postamble)
if callback is not None:
sys.stdout.write(")")
sys.stdout.write(";\n")
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write('}\n')
exit(0)
# generic request
result = db.nmap.get(flt, archive=archive,
limit=limit, skip=skip, sort=sortby)
if action == "count":
if callback is not None:
sys.stdout.write("%s(%d);\n" % (callback, result.count()))
else:
sys.stdout.write("%d;\n" % result.count())
exit(0)
if unused:
msg = 'Option%s not understood: %s' % (
's' if len(unused) > 1 else '',
', '.join(unused),
)
sys.stdout.write(webutils.js_alert("param-unused", "warning", msg))
sys.stderr.write('IVRE: WARNING: %r\n' % msg)
else:
sys.stdout.write(webutils.js_del_alert("param-unused"))
if config.DEBUG:
msg = "filter: %r" % flt
sys.stdout.write(webutils.js_alert("filter", "info", msg))
sys.stderr.write('IVRE: INFO: %r\n' % msg)
msg = "user: %r" % webutils.get_user()
sys.stdout.write(webutils.js_alert("user", "info", msg))
sys.stderr.write('IVRE: INFO: %r\n' % msg)
version_mismatch = {}
for rec in result:
del rec['_id']
try:
rec['addr'] = utils.int2ip(rec['addr'])
except:
pass
for field in ['starttime', 'endtime']:
if field in rec:
rec[field] = int(rec[field].strftime('%s'))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = port['screendata'].encode('base64')
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
try:
hop['ipaddr'] = utils.int2ip(hop['ipaddr'])
except:
pass
if callback is not None:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(rec)))
else:
sys.stdout.write("%s;\n" % json.dumps(rec))
check = db.nmap.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
messages = {
1: lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following commands: 'scancli --update-schema; "
"scancli --update-schema --archives'"
"" % (count, 's' if count > 1 else '',
'are' if count > 1 else 'is')),
-1: lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '',
've' if count > 1 else 's')),
}
for mismatch, count in version_mismatch.iteritems():
message = messages[mismatch](count)
sys.stdout.write(
webutils.js_alert("version-mismatch-%d" % ((mismatch + 1) / 2),
"warning", message)
)
sys.stderr.write('IVRE: WARNING: %r\n' % message)
def _build_filter(self, cmd):
"""Build a db.nmap.flt from the string `cmd`"""
query = webutils.query_from_params({'q': cmd})
(self._dfilter['filter'], _, _, _, self._dfilter['skip'],
self._dfilter['limit']) = webutils.flt_from_query(
query, base_flt=db.nmap.flt_empty)
def main():
# write headers
sys.stdout.write(webutils.JS_HEADERS)
params = webutils.parse_query_string()
query = webutils.query_from_params(params)
flt, archive, sortby, unused, skip, limit = webutils.flt_from_query(query)
if limit is None:
limit = config.WEB_LIMIT
if config.WEB_MAXRESULTS is not None:
limit = min(limit, config.WEB_MAXRESULTS)
callback = params.get("callback")
# type of result
action = params.get("action", "")
ipsasnumbers = params.get("ipsasnumbers")
datesasstrings = params.get("datesasstrings")
if callback is None:
sys.stdout.write('Content-Disposition: attachment; '
'filename="IVRE-results.json"\r\n')
sys.stdout.write("\r\n")
# top values
if action.startswith('topvalues:'):
field = action[10:]
if field[0] in '-!':
field = field[1:]
least = True
else:
least = False
topnbr = 15
if ':' in field:
field, topnbr = field.rsplit(':', 1)
try:
topnbr = int(topnbr)
except ValueError:
field = '%s:%s' % (field, topnbr)
topnbr = 15
series = [{
"label": t['_id'],
"value": t['count']
} for t in db.nmap.topvalues(
field, flt=flt, least=least, topnbr=topnbr, archive=archive)]
if callback is None:
sys.stdout.write("%s\n" % json.dumps(series))
else:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(series)))
exit(0)
# extract info
if action in [
"onlyips", "ipsports", "timeline", "coordinates", "countopenports",
"diffcats"
]:
preamble = "[\n"
postamble = "]\n"
r2res = lambda x: x
if action == "timeline":
if hasattr(db.nmap, "get_open_port_count"):
result = list(db.nmap.get_open_port_count(flt,
archive=archive))
count = len(result)
else:
result = db.nmap.get(
flt,
archive=archive,
fields=['addr', 'starttime', 'openports.count'])
count = result.count()
if params.get("modulo") is None:
r2time = lambda r: int(r['starttime'].strftime('%s'))
else:
r2time = lambda r: (int(r['starttime'].strftime('%s')) % int(
params.get("modulo")))
if ipsasnumbers:
r2res = lambda r: [
r2time(r),
force_ip_int(r['addr']), r['openports']['count']
]
else:
r2res = lambda r: [
r2time(r),
force_ip_str(r['addr']), r['openports']['count']
]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": ['
postamble = ']}'
result = list(db.nmap.getlocations(flt, archive=archive))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {
"count": r['count']
},
}
elif action == "countopenports":
if hasattr(db.nmap, "get_open_port_count"):
result = db.nmap.get_open_port_count(flt, archive=archive)
else:
result = db.nmap.get(flt,
archive=archive,
fields=['addr', 'openports.count'])
if hasattr(result, "count"):
count = result.count()
else:
count = db.nmap.count(flt,
archive=archive,
fields=['addr', 'openports.count'])
if ipsasnumbers:
r2res = lambda r: [
force_ip_int(r['addr']), r['openports']['count']
]
else:
r2res = lambda r: [
force_ip_str(r['addr']), r['openports']['count']
]
elif action == "ipsports":
if hasattr(db.nmap, "get_ips_ports"):
result = list(db.nmap.get_ips_ports(flt, archive=archive))
count = sum(len(host.get('ports', [])) for host in result)
else:
result = db.nmap.get(
flt,
archive=archive,
fields=['addr', 'ports.port', 'ports.state_state'])
count = sum(len(host.get('ports', [])) for host in result)
result.rewind()
if ipsasnumbers:
r2res = lambda r: [
force_ip_int(r['addr']),
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
else:
r2res = lambda r: [
force_ip_str(r['addr']),
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
elif action == "onlyips":
result = db.nmap.get(flt, archive=archive, fields=['addr'])
if hasattr(result, "count"):
count = result.count()
else:
count = db.nmap.count(flt, archive=archive, fields=['addr'])
if ipsasnumbers:
r2res = lambda r: r['addr']
else:
r2res = lambda r: utils.int2ip(r['addr'])
elif action == "diffcats":
if params.get("onlydiff"):
output = db.nmap.diff_categories(params.get("cat1"),
params.get("cat2"),
flt=flt,
include_both_open=False)
else:
output = db.nmap.diff_categories(params.get("cat1"),
params.get("cat2"),
flt=flt)
count = 0
result = {}
if ipsasnumbers:
for res in output:
result.setdefault(res["addr"],
[]).append([res['port'], res['value']])
count += 1
else:
for res in output:
result.setdefault(utils.int2ip(res["addr"]),
[]).append([res['port'], res['value']])
count += 1
result = viewitems(result)
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write(
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count)
if callback is not None:
sys.stdout.write("%s(\n" % callback)
sys.stdout.write(preamble)
for rec in result:
sys.stdout.write(json.dumps(r2res(rec)) + ",\n")
sys.stdout.write(postamble)
if callback is not None:
sys.stdout.write(");")
sys.stdout.write("\n")
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write('}\n')
exit(0)
# generic request
if action == "count":
if callback is None:
sys.stdout.write("%d\n" % db.nmap.count(flt, archive=archive))
else:
sys.stdout.write("%s(%d);\n" %
(callback, db.nmap.count(flt, archive=archive)))
exit(0)
## PostgreSQL: the query plan if affected by the limit and gives
## really poor results. This is a temporary workaround (look for
## XXX-WORKAROUND-PGSQL)
# result = db.nmap.get(flt, archive=archive,
# limit=limit, skip=skip, sort=sortby)
result = db.nmap.get(flt, archive=archive, skip=skip, sort=sortby)
if unused:
msg = 'Option%s not understood: %s' % (
's' if len(unused) > 1 else '',
', '.join(unused),
)
sys.stdout.write(webutils.js_alert("param-unused", "warning", msg))
utils.LOGGER.warning(msg)
elif callback is not None:
sys.stdout.write(webutils.js_del_alert("param-unused"))
if config.DEBUG:
msg = "filter: %r" % flt
sys.stdout.write(webutils.js_alert("filter", "info", msg))
utils.LOGGER.debug(msg)
msg = "user: %r" % webutils.get_user()
sys.stdout.write(webutils.js_alert("user", "info", msg))
utils.LOGGER.debug(msg)
version_mismatch = {}
if callback is None:
tab, sep = "", "\n"
else:
tab, sep = "\t", ",\n"
sys.stdout.write("%s([\n" % callback)
## XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not ipsasnumbers:
try:
rec['addr'] = utils.int2ip(rec['addr'])
except:
pass
for field in ['starttime', 'endtime']:
if field in rec:
if not datesasstrings:
rec[field] = int(rec[field].strftime('%s'))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = utils.encode_b64(port['screendata'])
for script in port.get('scripts', []):
if "masscan" in script:
try:
del script['masscan']['raw']
except KeyError:
pass
if not ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
try:
hop['ipaddr'] = utils.int2ip(hop['ipaddr'])
except:
pass
sys.stdout.write("%s%s%s" %
(tab, json.dumps(rec, default=utils.serialize), sep))
check = db.nmap.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if i + 1 >= limit:
break
if callback is not None:
sys.stdout.write("]);\n")
messages = {
1:
lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following commands: 'ivre scancli "
"--update-schema; ivre scancli --update-schema "
"--archives'" % (count, 's' if count > 1 else '', 'are'
if count > 1 else 'is')),
-1:
lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '', 've'
if count > 1 else 's')),
}
for mismatch, count in viewitems(version_mismatch):
message = messages[mismatch](count)
sys.stdout.write(
webutils.js_alert("version-mismatch-%d" % ((mismatch + 1) / 2),
"warning", message))
utils.LOGGER.warning(message)
def main():
# write headers
sys.stdout.write(webutils.JS_HEADERS)
params = webutils.parse_query_string()
query = webutils.query_from_params(params)
flt, archive, sortby, unused, skip, limit = webutils.flt_from_query(query)
if limit is None:
limit = config.WEB_LIMIT
if config.WEB_MAXRESULTS is not None:
limit = min(limit, config.WEB_MAXRESULTS)
callback = params.get("callback")
# type of result
action = params.get("action", "")
ipsasnumbers = params.get("ipsasnumbers")
datesasstrings = params.get("datesasstrings")
if callback is None:
sys.stdout.write('Content-Disposition: attachment; '
'filename="IVRE-results.json"\r\n')
sys.stdout.write("\r\n")
# top values
if action.startswith('topvalues:'):
field = action[10:]
if field[0] in '-!':
field = field[1:]
least = True
else:
least = False
topnbr = 15
if ':' in field:
field, topnbr = field.rsplit(':', 1)
try:
topnbr = int(topnbr)
except ValueError:
field = '%s:%s' % (field, topnbr)
topnbr = 15
series = [{"label": t['_id'], "value": t['count']} for t in
db.nmap.topvalues(field, flt=flt,
least=least, topnbr=topnbr,
archive=archive)]
if callback is None:
sys.stdout.write("%s\n" % json.dumps(series))
else:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(series)))
exit(0)
# extract info
if action in ["onlyips", "ipsports", "timeline", "coordinates",
"countopenports", "diffcats"]:
preamble = "[\n"
postamble = "]\n"
r2res = lambda x: x
if action == "timeline":
if hasattr(db.nmap, "get_open_port_count"):
result = list(db.nmap.get_open_port_count(flt, archive=archive))
count = len(result)
else:
result = db.nmap.get(
flt, archive=archive,
fields=['addr', 'starttime', 'openports.count']
)
count = result.count()
if params.get("modulo") is None:
r2time = lambda r: int(r['starttime'].strftime('%s'))
else:
r2time = lambda r: (int(r['starttime'].strftime('%s'))
% int(params.get("modulo")))
if ipsasnumbers:
r2res = lambda r: [r2time(r), force_ip_int(r['addr']),
r['openports']['count']]
else:
r2res = lambda r: [r2time(r), force_ip_str(r['addr']),
r['openports']['count']]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": ['
postamble = ']}'
result = list(db.nmap.getlocations(flt, archive=archive))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {"count": r['count']},
}
elif action == "countopenports":
if hasattr(db.nmap, "get_open_port_count"):
result = db.nmap.get_open_port_count(flt, archive=archive)
else:
result = db.nmap.get(flt, archive=archive,
fields=['addr', 'openports.count'])
if hasattr(result, "count"):
count = result.count()
else:
count = db.nmap.count(flt, archive=archive,
fields=['addr', 'openports.count'])
if ipsasnumbers:
r2res = lambda r: [force_ip_int(r['addr']),
r['openports']['count']]
else:
r2res = lambda r: [force_ip_str(r['addr']),
r['openports']['count']]
elif action == "ipsports":
if hasattr(db.nmap, "get_ips_ports"):
result = list(db.nmap.get_ips_ports(flt, archive=archive))
count = sum(len(host.get('ports', [])) for host in result)
else:
result = db.nmap.get(
flt, archive=archive,
fields=['addr', 'ports.port', 'ports.state_state']
)
count = sum(len(host.get('ports', [])) for host in result)
result.rewind()
if ipsasnumbers:
r2res = lambda r: [
force_ip_int(r['addr']),
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
else:
r2res = lambda r: [
force_ip_str(r['addr']),
[[p['port'], p['state_state']]
for p in r.get('ports', [])
if 'state_state' in p]
]
elif action == "onlyips":
result = db.nmap.get(flt, archive=archive, fields=['addr'])
if hasattr(result, "count"):
count = result.count()
else:
count = db.nmap.count(flt, archive=archive, fields=['addr'])
if ipsasnumbers:
r2res = lambda r: r['addr']
else:
r2res = lambda r: utils.int2ip(r['addr'])
elif action == "diffcats":
if params.get("onlydiff"):
output = db.nmap.diff_categories(params.get("cat1"),
params.get("cat2"),
flt=flt,
include_both_open=False)
else:
output = db.nmap.diff_categories(params.get("cat1"),
params.get("cat2"),
flt=flt)
count = 0
result = {}
if ipsasnumbers:
for res in output:
result.setdefault(res["addr"], []).append([res['port'],
res['value']])
count += 1
else:
for res in output:
result.setdefault(utils.int2ip(res["addr"]),
[]).append([res['port'], res['value']])
count += 1
result = viewitems(result)
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write(
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count
)
if callback is not None:
sys.stdout.write("%s(\n" % callback)
sys.stdout.write(preamble)
for rec in result:
sys.stdout.write(json.dumps(r2res(rec)) + ",\n")
sys.stdout.write(postamble)
if callback is not None:
sys.stdout.write(");")
sys.stdout.write("\n")
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write('}\n')
exit(0)
# generic request
if action == "count":
if callback is None:
sys.stdout.write("%d\n" % db.nmap.count(flt, archive=archive))
else:
sys.stdout.write("%s(%d);\n" % (callback,
db.nmap.count(flt,
archive=archive)))
exit(0)
## PostgreSQL: the query plan if affected by the limit and gives
## really poor results. This is a temporary workaround (look for
## XXX-WORKAROUND-PGSQL)
# result = db.nmap.get(flt, archive=archive,
# limit=limit, skip=skip, sort=sortby)
result = db.nmap.get(flt, archive=archive,
skip=skip, sort=sortby)
if unused:
msg = 'Option%s not understood: %s' % (
's' if len(unused) > 1 else '',
', '.join(unused),
)
sys.stdout.write(webutils.js_alert("param-unused", "warning", msg))
utils.LOGGER.warning(msg)
elif callback is not None:
sys.stdout.write(webutils.js_del_alert("param-unused"))
if config.DEBUG:
msg = "filter: %r" % flt
sys.stdout.write(webutils.js_alert("filter", "info", msg))
utils.LOGGER.debug(msg)
msg = "user: %r" % webutils.get_user()
sys.stdout.write(webutils.js_alert("user", "info", msg))
utils.LOGGER.debug(msg)
version_mismatch = {}
if callback is None:
tab, sep = "", "\n"
else:
tab, sep = "\t", ",\n"
sys.stdout.write("%s([\n" % callback)
## XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not ipsasnumbers:
try:
rec['addr'] = utils.int2ip(rec['addr'])
except:
pass
for field in ['starttime', 'endtime']:
if field in rec:
if not datesasstrings:
rec[field] = int(rec[field].strftime('%s'))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = utils.encode_b64(port['screendata'])
for script in port.get('scripts', []):
if "masscan" in script:
try: del script['masscan']['raw']
except KeyError: pass
if not ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
try:
hop['ipaddr'] = utils.int2ip(hop['ipaddr'])
except:
pass
sys.stdout.write("%s%s%s" % (
tab, json.dumps(rec, default=utils.serialize), sep
))
check = db.nmap.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if i + 1>= limit:
break
if callback is not None:
sys.stdout.write("]);\n")
messages = {
1: lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following commands: 'ivre scancli "
"--update-schema; ivre scancli --update-schema "
"--archives'" % (count, 's' if count > 1 else '',
'are' if count > 1 else 'is')),
-1: lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '',
've' if count > 1 else 's')),
}
for mismatch, count in viewitems(version_mismatch):
message = messages[mismatch](count)
sys.stdout.write(
webutils.js_alert("version-mismatch-%d" % ((mismatch + 1) / 2),
"warning", message)
)
utils.LOGGER.warning(message)
def main():
# write headers
sys.stdout.write(webutils.JS_HEADERS)
params = webutils.parse_query_string()
query = webutils.query_from_params(params)
flt, archive, sortby, unused, skip, limit = webutils.flt_from_query(query)
if limit is None:
limit = config.WEB_LIMIT
if config.WEB_MAXRESULTS is not None:
limit = min(limit, config.WEB_MAXRESULTS)
callback = params.get("callback")
# type of result
action = params.get("action", "")
# top values
if action.startswith("topvalues:"):
field = action[10:]
if field[0] in "-!":
field = field[1:]
least = True
else:
least = False
topnbr = 15
if ":" in field:
field, topnbr = field.rsplit(":", 1)
try:
topnbr = int(topnbr)
except ValueError:
field = "%s:%s" % (field, topnbr)
topnbr = 15
series = [
{"label": t["_id"], "value": t["count"]}
for t in db.nmap.topvalues(field, flt=flt, least=least, topnbr=topnbr, archive=archive)
]
if callback is None:
sys.stdout.write("%s\n" % json.dumps(series))
else:
sys.stdout.write("%s(%s);\n" % (callback, json.dumps(series)))
exit(0)
# extract info
if action in ["onlyips", "ipsports", "timeline", "coordinates", "countopenports", "diffcats"]:
preamble = "[\n"
postamble = "]\n"
r2res = lambda x: x
if action == "timeline":
result = db.nmap.get(flt, archive=archive, fields=["addr", "starttime", "openports.count"])
count = result.count()
if params.get("modulo") is None:
r2time = lambda r: int(r["starttime"].strftime("%s"))
else:
r2time = lambda r: (int(r["starttime"].strftime("%s")) % int(params.get("modulo")))
if params.get("ipsasnumbers"):
r2res = lambda r: [r2time(r), r["addr"], r["openports"]["count"]]
else:
r2res = lambda r: [r2time(r), utils.int2ip(r["addr"]), r["openports"]["count"]]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": ['
postamble = "]}"
result = list(db.nmap.getlocations(flt, archive=archive))
count = len(result)
r2res = lambda r: {"type": "Point", "coordinates": r["_id"], "properties": {"count": r["count"]}}
elif action == "countopenports":
result = db.nmap.get(flt, archive=archive, fields=["addr", "openports.count"])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: [r["addr"], r["openports"]["count"]]
else:
r2res = lambda r: [utils.int2ip(r["addr"]), r["openports"]["count"]]
elif action == "ipsports":
result = db.nmap.get(flt, archive=archive, fields=["addr", "ports.port", "ports.state_state"])
count = sum(len(host.get("ports", [])) for host in result)
result.rewind()
if params.get("ipsasnumbers"):
r2res = lambda r: [
r["addr"],
[[p["port"], p["state_state"]] for p in r.get("ports", []) if "state_state" in p],
]
else:
r2res = lambda r: [
utils.int2ip(r["addr"]),
[[p["port"], p["state_state"]] for p in r.get("ports", []) if "state_state" in p],
]
elif action == "onlyips":
result = db.nmap.get(flt, archive=archive, fields=["addr"])
count = result.count()
if params.get("ipsasnumbers"):
r2res = lambda r: r["addr"]
else:
r2res = lambda r: utils.int2ip(r["addr"])
elif action == "diffcats":
if params.get("onlydiff"):
output = db.nmap.diff_categories(
params.get("cat1"), params.get("cat2"), flt=flt, include_both_open=False
)
else:
output = db.nmap.diff_categories(params.get("cat1"), params.get("cat2"), flt=flt)
count = 0
result = {}
if params.get("ipsasnumbers"):
for res in output:
result.setdefault(res["addr"], []).append([res["port"], res["value"]])
count += 1
else:
for res in output:
result.setdefault(utils.int2ip(res["addr"]), []).append([res["port"], res["value"]])
count += 1
result = result.iteritems()
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write(
'if(confirm("You are about to ask your browser to display %d '
"dots, which is a lot and might slow down, freeze or crash "
'your browser. Do you want to continue?")) {\n' % count
)
if callback is not None:
sys.stdout.write("%s(\n" % callback)
sys.stdout.write(preamble)
for rec in result:
sys.stdout.write(json.dumps(r2res(rec)) + ",\n")
sys.stdout.write(postamble)
if callback is not None:
sys.stdout.write(");")
sys.stdout.write("\n")
if count >= config.WEB_WARN_DOTS_COUNT:
sys.stdout.write("}\n")
exit(0)
# generic request
result = db.nmap.get(flt, archive=archive, limit=limit, skip=skip, sort=sortby)
if action == "count":
if callback is None:
sys.stdout.write("%d\n" % result.count())
else:
sys.stdout.write("%s(%d);\n" % (callback, result.count()))
exit(0)
if unused:
msg = "Option%s not understood: %s" % ("s" if len(unused) > 1 else "", ", ".join(unused))
sys.stdout.write(webutils.js_alert("param-unused", "warning", msg))
sys.stderr.write("IVRE: WARNING: %r\n" % msg)
else:
sys.stdout.write(webutils.js_del_alert("param-unused"))
if config.DEBUG:
msg = "filter: %r" % flt
sys.stdout.write(webutils.js_alert("filter", "info", msg))
sys.stderr.write("IVRE: INFO: %r\n" % msg)
msg = "user: %r" % webutils.get_user()
sys.stdout.write(webutils.js_alert("user", "info", msg))
sys.stderr.write("IVRE: INFO: %r\n" % msg)
version_mismatch = {}
if callback is None:
sys.stdout.write("[\n")
else:
sys.stdout.write("%s([\n" % callback)
for rec in result:
del rec["_id"]
try:
rec["addr"] = utils.int2ip(rec["addr"])
except:
pass
for field in ["starttime", "endtime"]:
if field in rec:
rec[field] = int(rec[field].strftime("%s"))
for port in rec.get("ports", []):
if "screendata" in port:
port["screendata"] = port["screendata"].encode("base64")
if "traces" in rec:
for trace in rec["traces"]:
trace["hops"].sort(key=lambda x: x["ttl"])
for hop in trace["hops"]:
try:
hop["ipaddr"] = utils.int2ip(hop["ipaddr"])
except:
pass
sys.stdout.write("\t%s,\n" % json.dumps(rec))
check = db.nmap.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
if callback is None:
sys.stdout.write("]\n")
else:
sys.stdout.write("]);\n")
messages = {
1: lambda count: (
"%d document%s displayed %s out-of-date. Please run "
"the following commands: 'ivre scancli "
"--update-schema; ivre scancli --update-schema "
"--archives'" % (count, "s" if count > 1 else "", "are" if count > 1 else "is")
),
-1: lambda count: (
"%d document%s displayed ha%s been inserted by "
"a more recent version of IVRE. Please update "
"IVRE!" % (count, "s" if count > 1 else "", "ve" if count > 1 else "s")
),
}
for mismatch, count in version_mismatch.iteritems():
message = messages[mismatch](count)
sys.stdout.write(webutils.js_alert("version-mismatch-%d" % ((mismatch + 1) / 2), "warning", message))
sys.stderr.write("IVRE: WARNING: %r\n" % message)