def handle_rec(sensor, ignorenets, neverignore, # these argmuments are provided by **bro_line timestamp=None, uid=None, host=None, srvport=None, recon_type=None, source=None, value=None, targetval=None): if host is None: spec = { 'targetval': targetval, 'recontype': recon_type, 'value': value } else: spec = { 'addr': utils.force_ip2int(host), 'recontype': recon_type, 'value': value } if sensor is not None: spec.update({'sensor': sensor}) if srvport is not None: spec.update({'port': srvport}) if source is not None: spec.update({'source': source}) spec = _prepare_rec(spec, ignorenets, neverignore) float_ts = utils.datetime2timestamp(timestamp) return float_ts, spec
def to_dbprop(cls, prop, val): if prop in cls.DATE_FIELDS and isinstance(val, basestring): val = datetime.strptime(val, cls.TIMEFMT) # Intentional double if: str -> datetime -> float if isinstance(val, datetime): val = utils.datetime2timestamp(val) return val
def to_dbprop(cls, prop, val): if prop in cls.DATE_FIELDS and isinstance(val, basestring): val = datetime.datetime.strptime(val, cls.TIMEFMT) # Intentional double if: str -> datetime -> float if isinstance(val, datetime.datetime): val = utils.datetime2timestamp(val) return val
def _date_round(cls, date): if isinstance(date, datetime): ts = utils.datetime2timestamp(date) else: ts = date ts = ts - (ts % config.FLOW_TIME_PRECISION) if isinstance(date, datetime): return datetime.fromtimestamp(ts) else: return ts
def _date_round(cls, date): if isinstance(date, datetime.datetime): ts = utils.datetime2timestamp(date) else: ts = date ts = ts - (ts % config.FLOW_TIME_PRECISION) if isinstance(date, datetime.datetime): return datetime.datetime.fromtimestamp(ts) else: return ts
def get_nmap(subdb): """Get records from Nmap & View databases :param str subdb: database to query (must be "scans" or "view") :query str q: query (including limit/skip and sort) :query str callback: callback to use for JSONP results :query bool ipsasnumbers: to get IP addresses as numbers rather than as strings :query bool datesasstrings: to get dates as strings rather than as timestamps :status 200: no error :status 400: invalid referer :>jsonarr object: results """ subdb_tool = "view" if subdb == 'view' else "scancli" subdb = db.view if subdb == 'view' else db.nmap flt_params = get_nmap_base(subdb) # PostgreSQL: the query plan if affected by the limit and gives # really poor results. This is a temporary workaround (look for # XXX-WORKAROUND-PGSQL). # result = subdb.get(flt_params.flt, limit=flt_params.limit, # skip=flt_params.skip, sort=flt_params.sortby) result = subdb.get(flt_params.flt, skip=flt_params.skip, sort=flt_params.sortby) if flt_params.unused: msg = 'Option%s not understood: %s' % ( 's' if len(flt_params.unused) > 1 else '', ', '.join(flt_params.unused), ) if flt_params.callback is not None: yield webutils.js_alert("param-unused", "warning", msg) utils.LOGGER.warning(msg) elif flt_params.callback is not None: yield webutils.js_del_alert("param-unused") if config.DEBUG: msg1 = "filter: %s" % subdb.flt2str(flt_params.flt) msg2 = "user: %r" % webutils.get_user() utils.LOGGER.debug(msg1) utils.LOGGER.debug(msg2) if flt_params.callback is not None: yield webutils.js_alert("filter", "info", msg1) yield webutils.js_alert("user", "info", msg2) version_mismatch = {} if flt_params.callback is None: yield "[\n" else: yield "%s([\n" % flt_params.callback # XXX-WORKAROUND-PGSQL # for rec in result: for i, rec in enumerate(result): for fld in ['_id', 'scanid']: try: del rec[fld] except KeyError: pass if not flt_params.ipsasnumbers: rec['addr'] = utils.force_int2ip(rec['addr']) for field in ['starttime', 'endtime']: if field in rec: if not flt_params.datesasstrings: rec[field] = int(utils.datetime2timestamp(rec[field])) for port in rec.get('ports', []): if 'screendata' in port: port['screendata'] = utils.encode_b64(port['screendata']) for script in port.get('scripts', []): if "masscan" in script: try: del script['masscan']['raw'] except KeyError: pass if not flt_params.ipsasnumbers: if 'traces' in rec: for trace in rec['traces']: trace['hops'].sort(key=lambda x: x['ttl']) for hop in trace['hops']: hop['ipaddr'] = utils.force_int2ip(hop['ipaddr']) addresses = rec.get('addresses', {}).get('mac') if addresses: newaddresses = [] for addr in addresses: manuf = utils.mac2manuf(addr) if manuf and manuf[0]: newaddresses.append({'addr': addr, 'manuf': manuf[0]}) else: newaddresses.append({'addr': addr}) rec['addresses']['mac'] = newaddresses yield "%s\t%s" % ('' if i == 0 else ',\n', json.dumps(rec, default=utils.serialize)) check = subdb.cmp_schema_version_host(rec) if check: version_mismatch[check] = version_mismatch.get(check, 0) + 1 # XXX-WORKAROUND-PGSQL if i + 1 >= flt_params.limit: break if flt_params.callback is None: yield "\n]\n" else: yield "\n]);\n" messages = { 1: lambda count: ("%d document%s displayed %s out-of-date. Please run " "the following command: 'ivre %s " "--update-schema;" % (count, 's' if count > 1 else '', 'are' if count > 1 else 'is', subdb_tool)), -1: lambda count: ('%d document%s displayed ha%s been inserted by ' 'a more recent version of IVRE. Please update ' 'IVRE!' % (count, 's' if count > 1 else '', 've' if count > 1 else 's')), } for mismatch, count in viewitems(version_mismatch): message = messages[mismatch](count) if flt_params.callback is not None: yield webutils.js_alert( "version-mismatch-%d" % ((mismatch + 1) // 2), "warning", message) utils.LOGGER.warning(message)
def r2time(r): return (int(utils.datetime2timestamp(r['starttime'])) % int(request.params.get("modulo")))
def r2time(r): return int(utils.datetime2timestamp(r['starttime']))
def get_nmap(): flt_params = get_nmap_base() ## PostgreSQL: the query plan if affected by the limit and gives ## really poor results. This is a temporary workaround (look for ## XXX-WORKAROUND-PGSQL) # result = db.view.get(flt_params.flt, limit=flt_params.limit, # skip=flt_params.skip, sort=flt_params.sortby) result = db.view.get(flt_params.flt, skip=flt_params.skip, sort=flt_params.sortby) if flt_params.unused: msg = 'Option%s not understood: %s' % ( 's' if len(flt_params.unused) > 1 else '', ', '.join(flt_params.unused), ) if flt_params.callback is not None: yield webutils.js_alert("param-unused", "warning", msg) utils.LOGGER.warning(msg) elif flt_params.callback is not None: yield webutils.js_del_alert("param-unused") if config.DEBUG: msg1 = "filter: %s" % db.view.flt2str(flt_params.flt) msg2 = "user: %r" % webutils.get_user() utils.LOGGER.debug(msg1) utils.LOGGER.debug(msg2) if flt_params.callback is not None: yield webutils.js_alert("filter", "info", msg1) yield webutils.js_alert("user", "info", msg2) version_mismatch = {} if flt_params.callback is None: yield "[\n" else: yield "%s([\n" % flt_params.callback ## XXX-WORKAROUND-PGSQL # for rec in result: for i, rec in enumerate(result): for fld in ['_id', 'scanid']: try: del rec[fld] except KeyError: pass if not flt_params.ipsasnumbers: rec['addr'] = utils.force_int2ip(rec['addr']) for field in ['starttime', 'endtime']: if field in rec: if not flt_params.datesasstrings: rec[field] = int(utils.datetime2timestamp(rec[field])) for port in rec.get('ports', []): if 'screendata' in port: port['screendata'] = utils.encode_b64(port['screendata']) for script in port.get('scripts', []): if "masscan" in script: try: del script['masscan']['raw'] except KeyError: pass if not flt_params.ipsasnumbers: if 'traces' in rec: for trace in rec['traces']: trace['hops'].sort(key=lambda x: x['ttl']) for hop in trace['hops']: hop['ipaddr'] = utils.force_int2ip(hop['ipaddr']) yield "%s\t%s" % ('' if i == 0 else ',\n', json.dumps(rec, default=utils.serialize)) check = db.view.cmp_schema_version_host(rec) if check: version_mismatch[check] = version_mismatch.get(check, 0) + 1 # XXX-WORKAROUND-PGSQL if i + 1 >= flt_params.limit: break if flt_params.callback is None: yield "\n]\n" else: yield "\n]);\n" messages = { 1: lambda count: ("%d document%s displayed %s out-of-date. Please run " "the following command: 'ivre scancli " "--update-schema;" % (count, 's' if count > 1 else '', 'are' if count > 1 else 'is')), -1: lambda count: ('%d document%s displayed ha%s been inserted by ' 'a more recent version of IVRE. Please update ' 'IVRE!' % (count, 's' if count > 1 else '', 've' if count > 1 else 's')), } for mismatch, count in viewitems(version_mismatch): message = messages[mismatch](count) if flt_params.callback is not None: yield webutils.js_alert( "version-mismatch-%d" % ((mismatch + 1) // 2), "warning", message) utils.LOGGER.warning(message)
def get_nmap_action(action): flt_params = get_nmap_base() preamble = "[\n" postamble = "]\n" r2res = lambda x: x if action == "timeline": result, count = db.view.get_open_port_count(flt_params.flt) if request.params.get("modulo") is None: r2time = lambda r: int(utils.datetime2timestamp(r['starttime'])) else: r2time = lambda r: (int(utils.datetime2timestamp(r['starttime'])) % int(request.params.get("modulo"))) if flt_params.ipsasnumbers: r2res = lambda r: [ r2time(r), utils.ip2int(r['addr']), r['openports']['count'] ] else: r2res = lambda r: [r2time(r), r['addr'], r['openports']['count']] elif action == "coordinates": preamble = '{"type": "GeometryCollection", "geometries": [\n' postamble = ']}\n' result = list(db.view.getlocations(flt_params.flt)) count = len(result) r2res = lambda r: { "type": "Point", "coordinates": r['_id'], "properties": { "count": r['count'] }, } elif action == "countopenports": result, count = db.view.get_open_port_count(flt_params.flt) if flt_params.ipsasnumbers: r2res = lambda r: [ utils.ip2int(r['addr']), r['openports']['count'] ] else: r2res = lambda r: [r['addr'], r['openports']['count']] elif action == "ipsports": result, count = db.view.get_ips_ports(flt_params.flt) if flt_params.ipsasnumbers: r2res = lambda r: [ utils.ip2int(r['addr']), [[p['port'], p['state_state']] for p in r.get('ports', []) if 'state_state' in p] ] else: r2res = lambda r: [ r['addr'], [[p['port'], p['state_state']] for p in r.get('ports', []) if 'state_state' in p] ] elif action == "onlyips": result, count = db.view.get_ips(flt_params.flt) if flt_params.ipsasnumbers: r2res = lambda r: utils.ip2int(r['addr']) else: r2res = lambda r: r['addr'] elif action == "diffcats": if request.params.get("onlydiff"): output = db.view.diff_categories(request.params.get("cat1"), request.params.get("cat2"), flt=flt_params.flt, include_both_open=False) else: output = db.view.diff_categories(request.params.get("cat1"), request.params.get("cat2"), flt=flt_params.flt) count = 0 result = {} if flt_params.ipsasnumbers: for res in output: result.setdefault(res["addr"], []).append([res['port'], res['value']]) count += 1 else: for res in output: result.setdefault(utils.int2ip(res["addr"]), []).append([res['port'], res['value']]) count += 1 result = viewitems(result) if flt_params.callback is not None: if count >= config.WEB_WARN_DOTS_COUNT: yield ( 'if(confirm("You are about to ask your browser to display %d ' 'dots, which is a lot and might slow down, freeze or crash ' 'your browser. Do you want to continue?")) {\n' % count) yield '%s(\n' % flt_params.callback yield preamble # hack to avoid a trailing comma result = iter(result) try: rec = next(result) except StopIteration: pass else: yield json.dumps(r2res(rec)) for rec in result: yield ",\n" + json.dumps(r2res(rec)) yield "\n" yield postamble if flt_params.callback is not None: yield ");" if count >= config.WEB_WARN_DOTS_COUNT: yield '}\n' else: yield "\n"
def get_nmap(): flt_params = get_nmap_base() # PostgreSQL: the query plan if affected by the limit and gives # really poor results. This is a temporary workaround (look for # XXX-WORKAROUND-PGSQL). # result = db.view.get(flt_params.flt, limit=flt_params.limit, # skip=flt_params.skip, sort=flt_params.sortby) result = db.view.get(flt_params.flt, skip=flt_params.skip, sort=flt_params.sortby) if flt_params.unused: msg = 'Option%s not understood: %s' % ( 's' if len(flt_params.unused) > 1 else '', ', '.join(flt_params.unused), ) if flt_params.callback is not None: yield webutils.js_alert("param-unused", "warning", msg) utils.LOGGER.warning(msg) elif flt_params.callback is not None: yield webutils.js_del_alert("param-unused") if config.DEBUG: msg1 = "filter: %s" % db.view.flt2str(flt_params.flt) msg2 = "user: %r" % webutils.get_user() utils.LOGGER.debug(msg1) utils.LOGGER.debug(msg2) if flt_params.callback is not None: yield webutils.js_alert("filter", "info", msg1) yield webutils.js_alert("user", "info", msg2) version_mismatch = {} if flt_params.callback is None: yield "[\n" else: yield "%s([\n" % flt_params.callback # XXX-WORKAROUND-PGSQL # for rec in result: for i, rec in enumerate(result): for fld in ['_id', 'scanid']: try: del rec[fld] except KeyError: pass if not flt_params.ipsasnumbers: rec['addr'] = utils.force_int2ip(rec['addr']) for field in ['starttime', 'endtime']: if field in rec: if not flt_params.datesasstrings: rec[field] = int(utils.datetime2timestamp(rec[field])) for port in rec.get('ports', []): if 'screendata' in port: port['screendata'] = utils.encode_b64(port['screendata']) for script in port.get('scripts', []): if "masscan" in script: try: del script['masscan']['raw'] except KeyError: pass if not flt_params.ipsasnumbers: if 'traces' in rec: for trace in rec['traces']: trace['hops'].sort(key=lambda x: x['ttl']) for hop in trace['hops']: hop['ipaddr'] = utils.force_int2ip(hop['ipaddr']) yield "%s\t%s" % ('' if i == 0 else ',\n', json.dumps(rec, default=utils.serialize)) check = db.view.cmp_schema_version_host(rec) if check: version_mismatch[check] = version_mismatch.get(check, 0) + 1 # XXX-WORKAROUND-PGSQL if i + 1 >= flt_params.limit: break if flt_params.callback is None: yield "\n]\n" else: yield "\n]);\n" messages = { 1: lambda count: ("%d document%s displayed %s out-of-date. Please run " "the following command: 'ivre scancli " "--update-schema;" % (count, 's' if count > 1 else '', 'are' if count > 1 else 'is')), -1: lambda count: ('%d document%s displayed ha%s been inserted by ' 'a more recent version of IVRE. Please update ' 'IVRE!' % (count, 's' if count > 1 else '', 've' if count > 1 else 's')), } for mismatch, count in viewitems(version_mismatch): message = messages[mismatch](count) if flt_params.callback is not None: yield webutils.js_alert( "version-mismatch-%d" % ((mismatch + 1) // 2), "warning", message ) utils.LOGGER.warning(message)