def _die(referer):
utils.LOGGER.critical("Invalid Referer header [%r]", referer)
response.set_header("Content-Type", "application/javascript")
response.status = "400 Bad Request"
return webutils.js_alert(
"referer", "error",
"Invalid Referer header. Check your configuration.")
def _die(referer):
utils.LOGGER.critical("Invalid Referer header [%r]", referer)
response.set_header('Content-Type', 'application/javascript')
response.status = '400 Bad Request'
return webutils.js_alert(
"referer", "error",
"Invalid Referer header. Check your configuration."
)
def get_nmap(subdb):
"""Get records from Nmap & View databases
:param str subdb: database to query (must be "scans" or "view")
:query str q: query (including limit/skip and sort)
:query str f: filter
:query str callback: callback to use for JSONP results
:query bool ipsasnumbers: to get IP addresses as numbers rather than as
strings
:query bool datesasstrings: to get dates as strings rather than as
timestamps
:query str format: "json" (the default) or "ndjson"
:status 200: no error
:status 400: invalid referer
:>jsonarr object: results
"""
subdb_tool = "view" if subdb == "view" else "scancli"
subdb = db.view if subdb == "view" else db.nmap
flt_params = get_base(subdb)
# PostgreSQL: the query plan if affected by the limit and gives
# really poor results. This is a temporary workaround (look for
# XXX-WORKAROUND-PGSQL).
# result = subdb.get(flt_params.flt, limit=flt_params.limit,
# skip=flt_params.skip, sort=flt_params.sortby)
result = subdb.get(
flt_params.flt,
skip=flt_params.skip,
sort=flt_params.sortby,
fields=flt_params.fields,
)
if flt_params.unused:
msg = "Option%s not understood: %s" % (
"s" if len(flt_params.unused) > 1 else "",
", ".join(flt_params.unused),
)
if flt_params.callback is not None:
yield webutils.js_alert("param-unused", "warning", msg)
utils.LOGGER.warning(msg)
elif flt_params.callback is not None:
yield webutils.js_del_alert("param-unused")
if config.DEBUG:
msg1 = "filter: %r" % subdb.flt2str(flt_params.flt)
msg2 = "user: %r" % webutils.get_user()
utils.LOGGER.debug(msg1)
utils.LOGGER.debug(msg2)
if flt_params.callback is not None:
yield webutils.js_alert("filter", "info", msg1)
yield webutils.js_alert("user", "info", msg2)
version_mismatch = {}
if flt_params.callback is None:
if flt_params.fmt == "json":
yield "[\n"
else:
yield "%s([\n" % flt_params.callback
# XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ["_id", "scanid"]:
try:
del rec[fld]
except KeyError:
pass
if flt_params.ipsasnumbers:
rec["addr"] = utils.force_ip2int(rec["addr"])
if not flt_params.datesasstrings:
for field in subdb.datetime_fields:
_set_datetime_field(subdb, rec, field)
for port in rec.get("ports", []):
if "screendata" in port:
port["screendata"] = utils.encode_b64(port["screendata"])
for script in port.get("scripts", []):
if "masscan" in script:
try:
del script["masscan"]["raw"]
except KeyError:
pass
if not flt_params.ipsasnumbers:
if "traces" in rec:
for trace in rec["traces"]:
trace["hops"].sort(key=lambda x: x["ttl"])
for hop in trace["hops"]:
hop["ipaddr"] = utils.force_int2ip(hop["ipaddr"])
addresses = rec.get("addresses", {}).get("mac")
if addresses:
newaddresses = []
for addr in addresses:
manuf = utils.mac2manuf(addr)
if manuf and manuf[0]:
newaddresses.append({"addr": addr, "manuf": manuf[0]})
else:
newaddresses.append({"addr": addr})
rec["addresses"]["mac"] = newaddresses
if flt_params.fmt == "ndjson":
yield "%s\n" % json.dumps(rec, default=utils.serialize)
else:
yield "%s\t%s" % (
"" if i == 0 else ",\n",
json.dumps(rec, default=utils.serialize),
)
check = subdb.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if flt_params.limit and i + 1 >= flt_params.limit:
break
if flt_params.callback is None:
if flt_params.fmt == "json":
yield "\n]\n"
else:
yield "\n]);\n"
messages = {
1:
lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following command: 'ivre %s "
"--update-schema;" % (
count,
"s" if count > 1 else "",
"are" if count > 1 else "is",
subdb_tool,
)),
-1:
lambda count: ("%d document%s displayed ha%s been inserted by "
"a more recent version of IVRE. Please update "
"IVRE!" % (count, "s" if count > 1 else "", "ve"
if count > 1 else "s")),
}
for mismatch, count in version_mismatch.items():
message = messages[mismatch](count)
if flt_params.callback is not None:
yield webutils.js_alert(
"version-mismatch-%d" % ((mismatch + 1) // 2), "warning",
message)
utils.LOGGER.warning(message)
def get_nmap(subdb):
"""Get records from Nmap & View databases
:param str subdb: database to query (must be "scans" or "view")
:query str q: query (including limit/skip and sort)
:query str callback: callback to use for JSONP results
:query bool ipsasnumbers: to get IP addresses as numbers rather than as
strings
:query bool datesasstrings: to get dates as strings rather than as
timestamps
:status 200: no error
:status 400: invalid referer
:>jsonarr object: results
"""
subdb_tool = "view" if subdb == 'view' else "scancli"
subdb = db.view if subdb == 'view' else db.nmap
flt_params = get_nmap_base(subdb)
# PostgreSQL: the query plan if affected by the limit and gives
# really poor results. This is a temporary workaround (look for
# XXX-WORKAROUND-PGSQL).
# result = subdb.get(flt_params.flt, limit=flt_params.limit,
# skip=flt_params.skip, sort=flt_params.sortby)
result = subdb.get(flt_params.flt,
skip=flt_params.skip,
sort=flt_params.sortby)
if flt_params.unused:
msg = 'Option%s not understood: %s' % (
's' if len(flt_params.unused) > 1 else '',
', '.join(flt_params.unused),
)
if flt_params.callback is not None:
yield webutils.js_alert("param-unused", "warning", msg)
utils.LOGGER.warning(msg)
elif flt_params.callback is not None:
yield webutils.js_del_alert("param-unused")
if config.DEBUG:
msg1 = "filter: %s" % subdb.flt2str(flt_params.flt)
msg2 = "user: %r" % webutils.get_user()
utils.LOGGER.debug(msg1)
utils.LOGGER.debug(msg2)
if flt_params.callback is not None:
yield webutils.js_alert("filter", "info", msg1)
yield webutils.js_alert("user", "info", msg2)
version_mismatch = {}
if flt_params.callback is None:
yield "[\n"
else:
yield "%s([\n" % flt_params.callback
# XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not flt_params.ipsasnumbers:
rec['addr'] = utils.force_int2ip(rec['addr'])
for field in ['starttime', 'endtime']:
if field in rec:
if not flt_params.datesasstrings:
rec[field] = int(utils.datetime2timestamp(rec[field]))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = utils.encode_b64(port['screendata'])
for script in port.get('scripts', []):
if "masscan" in script:
try:
del script['masscan']['raw']
except KeyError:
pass
if not flt_params.ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
hop['ipaddr'] = utils.force_int2ip(hop['ipaddr'])
addresses = rec.get('addresses', {}).get('mac')
if addresses:
newaddresses = []
for addr in addresses:
manuf = utils.mac2manuf(addr)
if manuf and manuf[0]:
newaddresses.append({'addr': addr, 'manuf': manuf[0]})
else:
newaddresses.append({'addr': addr})
rec['addresses']['mac'] = newaddresses
yield "%s\t%s" % ('' if i == 0 else ',\n',
json.dumps(rec, default=utils.serialize))
check = subdb.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if i + 1 >= flt_params.limit:
break
if flt_params.callback is None:
yield "\n]\n"
else:
yield "\n]);\n"
messages = {
1:
lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following command: 'ivre %s "
"--update-schema;" %
(count, 's' if count > 1 else '', 'are'
if count > 1 else 'is', subdb_tool)),
-1:
lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '', 've'
if count > 1 else 's')),
}
for mismatch, count in viewitems(version_mismatch):
message = messages[mismatch](count)
if flt_params.callback is not None:
yield webutils.js_alert(
"version-mismatch-%d" % ((mismatch + 1) // 2), "warning",
message)
utils.LOGGER.warning(message)
def get_nmap():
flt_params = get_nmap_base()
## PostgreSQL: the query plan if affected by the limit and gives
## really poor results. This is a temporary workaround (look for
## XXX-WORKAROUND-PGSQL)
# result = db.view.get(flt_params.flt, limit=flt_params.limit,
# skip=flt_params.skip, sort=flt_params.sortby)
result = db.view.get(flt_params.flt,
skip=flt_params.skip,
sort=flt_params.sortby)
if flt_params.unused:
msg = 'Option%s not understood: %s' % (
's' if len(flt_params.unused) > 1 else '',
', '.join(flt_params.unused),
)
if flt_params.callback is not None:
yield webutils.js_alert("param-unused", "warning", msg)
utils.LOGGER.warning(msg)
elif flt_params.callback is not None:
yield webutils.js_del_alert("param-unused")
if config.DEBUG:
msg1 = "filter: %s" % db.view.flt2str(flt_params.flt)
msg2 = "user: %r" % webutils.get_user()
utils.LOGGER.debug(msg1)
utils.LOGGER.debug(msg2)
if flt_params.callback is not None:
yield webutils.js_alert("filter", "info", msg1)
yield webutils.js_alert("user", "info", msg2)
version_mismatch = {}
if flt_params.callback is None:
yield "[\n"
else:
yield "%s([\n" % flt_params.callback
## XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not flt_params.ipsasnumbers:
rec['addr'] = utils.force_int2ip(rec['addr'])
for field in ['starttime', 'endtime']:
if field in rec:
if not flt_params.datesasstrings:
rec[field] = int(utils.datetime2timestamp(rec[field]))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = utils.encode_b64(port['screendata'])
for script in port.get('scripts', []):
if "masscan" in script:
try:
del script['masscan']['raw']
except KeyError:
pass
if not flt_params.ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
hop['ipaddr'] = utils.force_int2ip(hop['ipaddr'])
yield "%s\t%s" % ('' if i == 0 else ',\n',
json.dumps(rec, default=utils.serialize))
check = db.view.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if i + 1 >= flt_params.limit:
break
if flt_params.callback is None:
yield "\n]\n"
else:
yield "\n]);\n"
messages = {
1:
lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following command: 'ivre scancli "
"--update-schema;" % (count, 's'
if count > 1 else '', 'are'
if count > 1 else 'is')),
-1:
lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '', 've'
if count > 1 else 's')),
}
for mismatch, count in viewitems(version_mismatch):
message = messages[mismatch](count)
if flt_params.callback is not None:
yield webutils.js_alert(
"version-mismatch-%d" % ((mismatch + 1) // 2), "warning",
message)
utils.LOGGER.warning(message)
def get_nmap():
flt_params = get_nmap_base()
# PostgreSQL: the query plan if affected by the limit and gives
# really poor results. This is a temporary workaround (look for
# XXX-WORKAROUND-PGSQL).
# result = db.view.get(flt_params.flt, limit=flt_params.limit,
# skip=flt_params.skip, sort=flt_params.sortby)
result = db.view.get(flt_params.flt, skip=flt_params.skip,
sort=flt_params.sortby)
if flt_params.unused:
msg = 'Option%s not understood: %s' % (
's' if len(flt_params.unused) > 1 else '',
', '.join(flt_params.unused),
)
if flt_params.callback is not None:
yield webutils.js_alert("param-unused", "warning", msg)
utils.LOGGER.warning(msg)
elif flt_params.callback is not None:
yield webutils.js_del_alert("param-unused")
if config.DEBUG:
msg1 = "filter: %s" % db.view.flt2str(flt_params.flt)
msg2 = "user: %r" % webutils.get_user()
utils.LOGGER.debug(msg1)
utils.LOGGER.debug(msg2)
if flt_params.callback is not None:
yield webutils.js_alert("filter", "info", msg1)
yield webutils.js_alert("user", "info", msg2)
version_mismatch = {}
if flt_params.callback is None:
yield "[\n"
else:
yield "%s([\n" % flt_params.callback
# XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not flt_params.ipsasnumbers:
rec['addr'] = utils.force_int2ip(rec['addr'])
for field in ['starttime', 'endtime']:
if field in rec:
if not flt_params.datesasstrings:
rec[field] = int(utils.datetime2timestamp(rec[field]))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = utils.encode_b64(port['screendata'])
for script in port.get('scripts', []):
if "masscan" in script:
try:
del script['masscan']['raw']
except KeyError:
pass
if not flt_params.ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
hop['ipaddr'] = utils.force_int2ip(hop['ipaddr'])
yield "%s\t%s" % ('' if i == 0 else ',\n',
json.dumps(rec, default=utils.serialize))
check = db.view.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if i + 1 >= flt_params.limit:
break
if flt_params.callback is None:
yield "\n]\n"
else:
yield "\n]);\n"
messages = {
1: lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following command: 'ivre scancli "
"--update-schema;" % (count,
's' if count > 1 else '',
'are' if count > 1 else 'is')),
-1: lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '',
've' if count > 1 else 's')),
}
for mismatch, count in viewitems(version_mismatch):
message = messages[mismatch](count)
if flt_params.callback is not None:
yield webutils.js_alert(
"version-mismatch-%d" % ((mismatch + 1) // 2),
"warning", message
)
utils.LOGGER.warning(message)
