def handle_rec(sensor, ignorenets, neverignore,
# these argmuments are provided by **bro_line
timestamp=None, uid=None, host=None, srvport=None,
recon_type=None, source=None, value=None, targetval=None):
if host is None:
spec = {
'targetval': targetval,
'recontype': recon_type,
'value': value
}
else:
spec = {
'addr': utils.force_ip2int(host),
'recontype': recon_type,
'value': value
}
if sensor is not None:
spec.update({'sensor': sensor})
if srvport is not None:
spec.update({'port': srvport})
if source is not None:
spec.update({'source': source})
spec = _prepare_rec(spec, ignorenets, neverignore)
float_ts = utils.datetime2timestamp(timestamp)
return float_ts, spec
def to_dbprop(cls, prop, val):
if prop in cls.DATE_FIELDS and isinstance(val, basestring):
val = datetime.strptime(val, cls.TIMEFMT)
# Intentional double if: str -> datetime -> float
if isinstance(val, datetime):
val = utils.datetime2timestamp(val)
return val
def to_dbprop(cls, prop, val):
if prop in cls.DATE_FIELDS and isinstance(val, basestring):
val = datetime.datetime.strptime(val, cls.TIMEFMT)
# Intentional double if: str -> datetime -> float
if isinstance(val, datetime.datetime):
val = utils.datetime2timestamp(val)
return val
def _date_round(cls, date):
if isinstance(date, datetime):
ts = utils.datetime2timestamp(date)
else:
ts = date
ts = ts - (ts % config.FLOW_TIME_PRECISION)
if isinstance(date, datetime):
return datetime.fromtimestamp(ts)
else:
return ts
def _date_round(cls, date):
if isinstance(date, datetime.datetime):
ts = utils.datetime2timestamp(date)
else:
ts = date
ts = ts - (ts % config.FLOW_TIME_PRECISION)
if isinstance(date, datetime.datetime):
return datetime.datetime.fromtimestamp(ts)
else:
return ts
def get_nmap(subdb):
"""Get records from Nmap & View databases
:param str subdb: database to query (must be "scans" or "view")
:query str q: query (including limit/skip and sort)
:query str callback: callback to use for JSONP results
:query bool ipsasnumbers: to get IP addresses as numbers rather than as
strings
:query bool datesasstrings: to get dates as strings rather than as
timestamps
:status 200: no error
:status 400: invalid referer
:>jsonarr object: results
"""
subdb_tool = "view" if subdb == 'view' else "scancli"
subdb = db.view if subdb == 'view' else db.nmap
flt_params = get_nmap_base(subdb)
# PostgreSQL: the query plan if affected by the limit and gives
# really poor results. This is a temporary workaround (look for
# XXX-WORKAROUND-PGSQL).
# result = subdb.get(flt_params.flt, limit=flt_params.limit,
# skip=flt_params.skip, sort=flt_params.sortby)
result = subdb.get(flt_params.flt,
skip=flt_params.skip,
sort=flt_params.sortby)
if flt_params.unused:
msg = 'Option%s not understood: %s' % (
's' if len(flt_params.unused) > 1 else '',
', '.join(flt_params.unused),
)
if flt_params.callback is not None:
yield webutils.js_alert("param-unused", "warning", msg)
utils.LOGGER.warning(msg)
elif flt_params.callback is not None:
yield webutils.js_del_alert("param-unused")
if config.DEBUG:
msg1 = "filter: %s" % subdb.flt2str(flt_params.flt)
msg2 = "user: %r" % webutils.get_user()
utils.LOGGER.debug(msg1)
utils.LOGGER.debug(msg2)
if flt_params.callback is not None:
yield webutils.js_alert("filter", "info", msg1)
yield webutils.js_alert("user", "info", msg2)
version_mismatch = {}
if flt_params.callback is None:
yield "[\n"
else:
yield "%s([\n" % flt_params.callback
# XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not flt_params.ipsasnumbers:
rec['addr'] = utils.force_int2ip(rec['addr'])
for field in ['starttime', 'endtime']:
if field in rec:
if not flt_params.datesasstrings:
rec[field] = int(utils.datetime2timestamp(rec[field]))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = utils.encode_b64(port['screendata'])
for script in port.get('scripts', []):
if "masscan" in script:
try:
del script['masscan']['raw']
except KeyError:
pass
if not flt_params.ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
hop['ipaddr'] = utils.force_int2ip(hop['ipaddr'])
addresses = rec.get('addresses', {}).get('mac')
if addresses:
newaddresses = []
for addr in addresses:
manuf = utils.mac2manuf(addr)
if manuf and manuf[0]:
newaddresses.append({'addr': addr, 'manuf': manuf[0]})
else:
newaddresses.append({'addr': addr})
rec['addresses']['mac'] = newaddresses
yield "%s\t%s" % ('' if i == 0 else ',\n',
json.dumps(rec, default=utils.serialize))
check = subdb.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if i + 1 >= flt_params.limit:
break
if flt_params.callback is None:
yield "\n]\n"
else:
yield "\n]);\n"
messages = {
1:
lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following command: 'ivre %s "
"--update-schema;" %
(count, 's' if count > 1 else '', 'are'
if count > 1 else 'is', subdb_tool)),
-1:
lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '', 've'
if count > 1 else 's')),
}
for mismatch, count in viewitems(version_mismatch):
message = messages[mismatch](count)
if flt_params.callback is not None:
yield webutils.js_alert(
"version-mismatch-%d" % ((mismatch + 1) // 2), "warning",
message)
utils.LOGGER.warning(message)
def r2time(r):
return (int(utils.datetime2timestamp(r['starttime'])) %
int(request.params.get("modulo")))
def r2time(r):
return int(utils.datetime2timestamp(r['starttime']))
def get_nmap():
flt_params = get_nmap_base()
## PostgreSQL: the query plan if affected by the limit and gives
## really poor results. This is a temporary workaround (look for
## XXX-WORKAROUND-PGSQL)
# result = db.view.get(flt_params.flt, limit=flt_params.limit,
# skip=flt_params.skip, sort=flt_params.sortby)
result = db.view.get(flt_params.flt,
skip=flt_params.skip,
sort=flt_params.sortby)
if flt_params.unused:
msg = 'Option%s not understood: %s' % (
's' if len(flt_params.unused) > 1 else '',
', '.join(flt_params.unused),
)
if flt_params.callback is not None:
yield webutils.js_alert("param-unused", "warning", msg)
utils.LOGGER.warning(msg)
elif flt_params.callback is not None:
yield webutils.js_del_alert("param-unused")
if config.DEBUG:
msg1 = "filter: %s" % db.view.flt2str(flt_params.flt)
msg2 = "user: %r" % webutils.get_user()
utils.LOGGER.debug(msg1)
utils.LOGGER.debug(msg2)
if flt_params.callback is not None:
yield webutils.js_alert("filter", "info", msg1)
yield webutils.js_alert("user", "info", msg2)
version_mismatch = {}
if flt_params.callback is None:
yield "[\n"
else:
yield "%s([\n" % flt_params.callback
## XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not flt_params.ipsasnumbers:
rec['addr'] = utils.force_int2ip(rec['addr'])
for field in ['starttime', 'endtime']:
if field in rec:
if not flt_params.datesasstrings:
rec[field] = int(utils.datetime2timestamp(rec[field]))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = utils.encode_b64(port['screendata'])
for script in port.get('scripts', []):
if "masscan" in script:
try:
del script['masscan']['raw']
except KeyError:
pass
if not flt_params.ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
hop['ipaddr'] = utils.force_int2ip(hop['ipaddr'])
yield "%s\t%s" % ('' if i == 0 else ',\n',
json.dumps(rec, default=utils.serialize))
check = db.view.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if i + 1 >= flt_params.limit:
break
if flt_params.callback is None:
yield "\n]\n"
else:
yield "\n]);\n"
messages = {
1:
lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following command: 'ivre scancli "
"--update-schema;" % (count, 's'
if count > 1 else '', 'are'
if count > 1 else 'is')),
-1:
lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '', 've'
if count > 1 else 's')),
}
for mismatch, count in viewitems(version_mismatch):
message = messages[mismatch](count)
if flt_params.callback is not None:
yield webutils.js_alert(
"version-mismatch-%d" % ((mismatch + 1) // 2), "warning",
message)
utils.LOGGER.warning(message)
def get_nmap_action(action):
flt_params = get_nmap_base()
preamble = "[\n"
postamble = "]\n"
r2res = lambda x: x
if action == "timeline":
result, count = db.view.get_open_port_count(flt_params.flt)
if request.params.get("modulo") is None:
r2time = lambda r: int(utils.datetime2timestamp(r['starttime']))
else:
r2time = lambda r: (int(utils.datetime2timestamp(r['starttime'])) %
int(request.params.get("modulo")))
if flt_params.ipsasnumbers:
r2res = lambda r: [
r2time(r),
utils.ip2int(r['addr']), r['openports']['count']
]
else:
r2res = lambda r: [r2time(r), r['addr'], r['openports']['count']]
elif action == "coordinates":
preamble = '{"type": "GeometryCollection", "geometries": [\n'
postamble = ']}\n'
result = list(db.view.getlocations(flt_params.flt))
count = len(result)
r2res = lambda r: {
"type": "Point",
"coordinates": r['_id'],
"properties": {
"count": r['count']
},
}
elif action == "countopenports":
result, count = db.view.get_open_port_count(flt_params.flt)
if flt_params.ipsasnumbers:
r2res = lambda r: [
utils.ip2int(r['addr']), r['openports']['count']
]
else:
r2res = lambda r: [r['addr'], r['openports']['count']]
elif action == "ipsports":
result, count = db.view.get_ips_ports(flt_params.flt)
if flt_params.ipsasnumbers:
r2res = lambda r: [
utils.ip2int(r['addr']),
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
else:
r2res = lambda r: [
r['addr'],
[[p['port'], p['state_state']] for p in r.get('ports', [])
if 'state_state' in p]
]
elif action == "onlyips":
result, count = db.view.get_ips(flt_params.flt)
if flt_params.ipsasnumbers:
r2res = lambda r: utils.ip2int(r['addr'])
else:
r2res = lambda r: r['addr']
elif action == "diffcats":
if request.params.get("onlydiff"):
output = db.view.diff_categories(request.params.get("cat1"),
request.params.get("cat2"),
flt=flt_params.flt,
include_both_open=False)
else:
output = db.view.diff_categories(request.params.get("cat1"),
request.params.get("cat2"),
flt=flt_params.flt)
count = 0
result = {}
if flt_params.ipsasnumbers:
for res in output:
result.setdefault(res["addr"],
[]).append([res['port'], res['value']])
count += 1
else:
for res in output:
result.setdefault(utils.int2ip(res["addr"]),
[]).append([res['port'], res['value']])
count += 1
result = viewitems(result)
if flt_params.callback is not None:
if count >= config.WEB_WARN_DOTS_COUNT:
yield (
'if(confirm("You are about to ask your browser to display %d '
'dots, which is a lot and might slow down, freeze or crash '
'your browser. Do you want to continue?")) {\n' % count)
yield '%s(\n' % flt_params.callback
yield preamble
# hack to avoid a trailing comma
result = iter(result)
try:
rec = next(result)
except StopIteration:
pass
else:
yield json.dumps(r2res(rec))
for rec in result:
yield ",\n" + json.dumps(r2res(rec))
yield "\n"
yield postamble
if flt_params.callback is not None:
yield ");"
if count >= config.WEB_WARN_DOTS_COUNT:
yield '}\n'
else:
yield "\n"
def get_nmap():
flt_params = get_nmap_base()
# PostgreSQL: the query plan if affected by the limit and gives
# really poor results. This is a temporary workaround (look for
# XXX-WORKAROUND-PGSQL).
# result = db.view.get(flt_params.flt, limit=flt_params.limit,
# skip=flt_params.skip, sort=flt_params.sortby)
result = db.view.get(flt_params.flt, skip=flt_params.skip,
sort=flt_params.sortby)
if flt_params.unused:
msg = 'Option%s not understood: %s' % (
's' if len(flt_params.unused) > 1 else '',
', '.join(flt_params.unused),
)
if flt_params.callback is not None:
yield webutils.js_alert("param-unused", "warning", msg)
utils.LOGGER.warning(msg)
elif flt_params.callback is not None:
yield webutils.js_del_alert("param-unused")
if config.DEBUG:
msg1 = "filter: %s" % db.view.flt2str(flt_params.flt)
msg2 = "user: %r" % webutils.get_user()
utils.LOGGER.debug(msg1)
utils.LOGGER.debug(msg2)
if flt_params.callback is not None:
yield webutils.js_alert("filter", "info", msg1)
yield webutils.js_alert("user", "info", msg2)
version_mismatch = {}
if flt_params.callback is None:
yield "[\n"
else:
yield "%s([\n" % flt_params.callback
# XXX-WORKAROUND-PGSQL
# for rec in result:
for i, rec in enumerate(result):
for fld in ['_id', 'scanid']:
try:
del rec[fld]
except KeyError:
pass
if not flt_params.ipsasnumbers:
rec['addr'] = utils.force_int2ip(rec['addr'])
for field in ['starttime', 'endtime']:
if field in rec:
if not flt_params.datesasstrings:
rec[field] = int(utils.datetime2timestamp(rec[field]))
for port in rec.get('ports', []):
if 'screendata' in port:
port['screendata'] = utils.encode_b64(port['screendata'])
for script in port.get('scripts', []):
if "masscan" in script:
try:
del script['masscan']['raw']
except KeyError:
pass
if not flt_params.ipsasnumbers:
if 'traces' in rec:
for trace in rec['traces']:
trace['hops'].sort(key=lambda x: x['ttl'])
for hop in trace['hops']:
hop['ipaddr'] = utils.force_int2ip(hop['ipaddr'])
yield "%s\t%s" % ('' if i == 0 else ',\n',
json.dumps(rec, default=utils.serialize))
check = db.view.cmp_schema_version_host(rec)
if check:
version_mismatch[check] = version_mismatch.get(check, 0) + 1
# XXX-WORKAROUND-PGSQL
if i + 1 >= flt_params.limit:
break
if flt_params.callback is None:
yield "\n]\n"
else:
yield "\n]);\n"
messages = {
1: lambda count: ("%d document%s displayed %s out-of-date. Please run "
"the following command: 'ivre scancli "
"--update-schema;" % (count,
's' if count > 1 else '',
'are' if count > 1 else 'is')),
-1: lambda count: ('%d document%s displayed ha%s been inserted by '
'a more recent version of IVRE. Please update '
'IVRE!' % (count, 's' if count > 1 else '',
've' if count > 1 else 's')),
}
for mismatch, count in viewitems(version_mismatch):
message = messages[mismatch](count)
if flt_params.callback is not None:
yield webutils.js_alert(
"version-mismatch-%d" % ((mismatch + 1) // 2),
"warning", message
)
utils.LOGGER.warning(message)
def r2time(r):
return (int(utils.datetime2timestamp(r['starttime']))
% int(request.params.get("modulo")))
def r2time(r):
return int(utils.datetime2timestamp(r['starttime']))