def parse_form(): categories = request.forms.get("categories") categories = set(categories.split(",")) if categories else set() source = request.forms.get("source") if not source: utils.LOGGER.critical("source is mandatory") abort(400, "ERROR: source is mandatory\n") files = request.files.getall("result") if config.WEB_PUBLIC_SRV: if webutils.get_user() is None: utils.LOGGER.critical("username is mandatory on public instances") abort(400, "ERROR: username is mandatory on public instances") if request.forms.get("public") == "on": categories.add("Shared") user = webutils.get_anonymized_user() categories.add(user) source = "%s-%s" % (user, source) return (request.forms.get("referer"), source, categories, files)
def parse_form(): categories = request.forms.get("categories") categories = (set(categories.split(',')) if categories else set()) source = request.forms.get("source") if not source: utils.LOGGER.critical("source is mandatory") abort(400, "ERROR: source is mandatory\n") files = request.files.getall("result") if config.WEB_PUBLIC_SRV: if webutils.get_user() is None: utils.LOGGER.critical("username is mandatory on public instances") abort(400, "ERROR: username is mandatory on public instances") if request.forms.get('public') == 'on': categories.add('Shared') user = webutils.get_anonymized_user() categories.add(user) source = "%s-%s" % (user, source) return (request.forms.get('referer'), source, categories, files)
def get_nmap(subdb): """Get records from Nmap & View databases :param str subdb: database to query (must be "scans" or "view") :query str q: query (including limit/skip and sort) :query str f: filter :query str callback: callback to use for JSONP results :query bool ipsasnumbers: to get IP addresses as numbers rather than as strings :query bool datesasstrings: to get dates as strings rather than as timestamps :query str format: "json" (the default) or "ndjson" :status 200: no error :status 400: invalid referer :>jsonarr object: results """ subdb_tool = "view" if subdb == "view" else "scancli" subdb = db.view if subdb == "view" else db.nmap flt_params = get_base(subdb) # PostgreSQL: the query plan if affected by the limit and gives # really poor results. This is a temporary workaround (look for # XXX-WORKAROUND-PGSQL). # result = subdb.get(flt_params.flt, limit=flt_params.limit, # skip=flt_params.skip, sort=flt_params.sortby) result = subdb.get( flt_params.flt, skip=flt_params.skip, sort=flt_params.sortby, fields=flt_params.fields, ) if flt_params.unused: msg = "Option%s not understood: %s" % ( "s" if len(flt_params.unused) > 1 else "", ", ".join(flt_params.unused), ) if flt_params.callback is not None: yield webutils.js_alert("param-unused", "warning", msg) utils.LOGGER.warning(msg) elif flt_params.callback is not None: yield webutils.js_del_alert("param-unused") if config.DEBUG: msg1 = "filter: %r" % subdb.flt2str(flt_params.flt) msg2 = "user: %r" % webutils.get_user() utils.LOGGER.debug(msg1) utils.LOGGER.debug(msg2) if flt_params.callback is not None: yield webutils.js_alert("filter", "info", msg1) yield webutils.js_alert("user", "info", msg2) version_mismatch = {} if flt_params.callback is None: if flt_params.fmt == "json": yield "[\n" else: yield "%s([\n" % flt_params.callback # XXX-WORKAROUND-PGSQL # for rec in result: for i, rec in enumerate(result): for fld in ["_id", "scanid"]: try: del rec[fld] except KeyError: pass if flt_params.ipsasnumbers: rec["addr"] = utils.force_ip2int(rec["addr"]) if not flt_params.datesasstrings: for field in subdb.datetime_fields: _set_datetime_field(subdb, rec, field) for port in rec.get("ports", []): if "screendata" in port: port["screendata"] = utils.encode_b64(port["screendata"]) for script in port.get("scripts", []): if "masscan" in script: try: del script["masscan"]["raw"] except KeyError: pass if not flt_params.ipsasnumbers: if "traces" in rec: for trace in rec["traces"]: trace["hops"].sort(key=lambda x: x["ttl"]) for hop in trace["hops"]: hop["ipaddr"] = utils.force_int2ip(hop["ipaddr"]) addresses = rec.get("addresses", {}).get("mac") if addresses: newaddresses = [] for addr in addresses: manuf = utils.mac2manuf(addr) if manuf and manuf[0]: newaddresses.append({"addr": addr, "manuf": manuf[0]}) else: newaddresses.append({"addr": addr}) rec["addresses"]["mac"] = newaddresses if flt_params.fmt == "ndjson": yield "%s\n" % json.dumps(rec, default=utils.serialize) else: yield "%s\t%s" % ( "" if i == 0 else ",\n", json.dumps(rec, default=utils.serialize), ) check = subdb.cmp_schema_version_host(rec) if check: version_mismatch[check] = version_mismatch.get(check, 0) + 1 # XXX-WORKAROUND-PGSQL if flt_params.limit and i + 1 >= flt_params.limit: break if flt_params.callback is None: if flt_params.fmt == "json": yield "\n]\n" else: yield "\n]);\n" messages = { 1: lambda count: ("%d document%s displayed %s out-of-date. Please run " "the following command: 'ivre %s " "--update-schema;" % ( count, "s" if count > 1 else "", "are" if count > 1 else "is", subdb_tool, )), -1: lambda count: ("%d document%s displayed ha%s been inserted by " "a more recent version of IVRE. Please update " "IVRE!" % (count, "s" if count > 1 else "", "ve" if count > 1 else "s")), } for mismatch, count in version_mismatch.items(): message = messages[mismatch](count) if flt_params.callback is not None: yield webutils.js_alert( "version-mismatch-%d" % ((mismatch + 1) // 2), "warning", message) utils.LOGGER.warning(message)
def get_nmap(subdb): """Get records from Nmap & View databases :param str subdb: database to query (must be "scans" or "view") :query str q: query (including limit/skip and sort) :query str callback: callback to use for JSONP results :query bool ipsasnumbers: to get IP addresses as numbers rather than as strings :query bool datesasstrings: to get dates as strings rather than as timestamps :status 200: no error :status 400: invalid referer :>jsonarr object: results """ subdb_tool = "view" if subdb == 'view' else "scancli" subdb = db.view if subdb == 'view' else db.nmap flt_params = get_nmap_base(subdb) # PostgreSQL: the query plan if affected by the limit and gives # really poor results. This is a temporary workaround (look for # XXX-WORKAROUND-PGSQL). # result = subdb.get(flt_params.flt, limit=flt_params.limit, # skip=flt_params.skip, sort=flt_params.sortby) result = subdb.get(flt_params.flt, skip=flt_params.skip, sort=flt_params.sortby) if flt_params.unused: msg = 'Option%s not understood: %s' % ( 's' if len(flt_params.unused) > 1 else '', ', '.join(flt_params.unused), ) if flt_params.callback is not None: yield webutils.js_alert("param-unused", "warning", msg) utils.LOGGER.warning(msg) elif flt_params.callback is not None: yield webutils.js_del_alert("param-unused") if config.DEBUG: msg1 = "filter: %s" % subdb.flt2str(flt_params.flt) msg2 = "user: %r" % webutils.get_user() utils.LOGGER.debug(msg1) utils.LOGGER.debug(msg2) if flt_params.callback is not None: yield webutils.js_alert("filter", "info", msg1) yield webutils.js_alert("user", "info", msg2) version_mismatch = {} if flt_params.callback is None: yield "[\n" else: yield "%s([\n" % flt_params.callback # XXX-WORKAROUND-PGSQL # for rec in result: for i, rec in enumerate(result): for fld in ['_id', 'scanid']: try: del rec[fld] except KeyError: pass if not flt_params.ipsasnumbers: rec['addr'] = utils.force_int2ip(rec['addr']) for field in ['starttime', 'endtime']: if field in rec: if not flt_params.datesasstrings: rec[field] = int(utils.datetime2timestamp(rec[field])) for port in rec.get('ports', []): if 'screendata' in port: port['screendata'] = utils.encode_b64(port['screendata']) for script in port.get('scripts', []): if "masscan" in script: try: del script['masscan']['raw'] except KeyError: pass if not flt_params.ipsasnumbers: if 'traces' in rec: for trace in rec['traces']: trace['hops'].sort(key=lambda x: x['ttl']) for hop in trace['hops']: hop['ipaddr'] = utils.force_int2ip(hop['ipaddr']) addresses = rec.get('addresses', {}).get('mac') if addresses: newaddresses = [] for addr in addresses: manuf = utils.mac2manuf(addr) if manuf and manuf[0]: newaddresses.append({'addr': addr, 'manuf': manuf[0]}) else: newaddresses.append({'addr': addr}) rec['addresses']['mac'] = newaddresses yield "%s\t%s" % ('' if i == 0 else ',\n', json.dumps(rec, default=utils.serialize)) check = subdb.cmp_schema_version_host(rec) if check: version_mismatch[check] = version_mismatch.get(check, 0) + 1 # XXX-WORKAROUND-PGSQL if i + 1 >= flt_params.limit: break if flt_params.callback is None: yield "\n]\n" else: yield "\n]);\n" messages = { 1: lambda count: ("%d document%s displayed %s out-of-date. Please run " "the following command: 'ivre %s " "--update-schema;" % (count, 's' if count > 1 else '', 'are' if count > 1 else 'is', subdb_tool)), -1: lambda count: ('%d document%s displayed ha%s been inserted by ' 'a more recent version of IVRE. Please update ' 'IVRE!' % (count, 's' if count > 1 else '', 've' if count > 1 else 's')), } for mismatch, count in viewitems(version_mismatch): message = messages[mismatch](count) if flt_params.callback is not None: yield webutils.js_alert( "version-mismatch-%d" % ((mismatch + 1) // 2), "warning", message) utils.LOGGER.warning(message)
def get_nmap(): flt_params = get_nmap_base() ## PostgreSQL: the query plan if affected by the limit and gives ## really poor results. This is a temporary workaround (look for ## XXX-WORKAROUND-PGSQL) # result = db.view.get(flt_params.flt, limit=flt_params.limit, # skip=flt_params.skip, sort=flt_params.sortby) result = db.view.get(flt_params.flt, skip=flt_params.skip, sort=flt_params.sortby) if flt_params.unused: msg = 'Option%s not understood: %s' % ( 's' if len(flt_params.unused) > 1 else '', ', '.join(flt_params.unused), ) if flt_params.callback is not None: yield webutils.js_alert("param-unused", "warning", msg) utils.LOGGER.warning(msg) elif flt_params.callback is not None: yield webutils.js_del_alert("param-unused") if config.DEBUG: msg1 = "filter: %s" % db.view.flt2str(flt_params.flt) msg2 = "user: %r" % webutils.get_user() utils.LOGGER.debug(msg1) utils.LOGGER.debug(msg2) if flt_params.callback is not None: yield webutils.js_alert("filter", "info", msg1) yield webutils.js_alert("user", "info", msg2) version_mismatch = {} if flt_params.callback is None: yield "[\n" else: yield "%s([\n" % flt_params.callback ## XXX-WORKAROUND-PGSQL # for rec in result: for i, rec in enumerate(result): for fld in ['_id', 'scanid']: try: del rec[fld] except KeyError: pass if not flt_params.ipsasnumbers: rec['addr'] = utils.force_int2ip(rec['addr']) for field in ['starttime', 'endtime']: if field in rec: if not flt_params.datesasstrings: rec[field] = int(utils.datetime2timestamp(rec[field])) for port in rec.get('ports', []): if 'screendata' in port: port['screendata'] = utils.encode_b64(port['screendata']) for script in port.get('scripts', []): if "masscan" in script: try: del script['masscan']['raw'] except KeyError: pass if not flt_params.ipsasnumbers: if 'traces' in rec: for trace in rec['traces']: trace['hops'].sort(key=lambda x: x['ttl']) for hop in trace['hops']: hop['ipaddr'] = utils.force_int2ip(hop['ipaddr']) yield "%s\t%s" % ('' if i == 0 else ',\n', json.dumps(rec, default=utils.serialize)) check = db.view.cmp_schema_version_host(rec) if check: version_mismatch[check] = version_mismatch.get(check, 0) + 1 # XXX-WORKAROUND-PGSQL if i + 1 >= flt_params.limit: break if flt_params.callback is None: yield "\n]\n" else: yield "\n]);\n" messages = { 1: lambda count: ("%d document%s displayed %s out-of-date. Please run " "the following command: 'ivre scancli " "--update-schema;" % (count, 's' if count > 1 else '', 'are' if count > 1 else 'is')), -1: lambda count: ('%d document%s displayed ha%s been inserted by ' 'a more recent version of IVRE. Please update ' 'IVRE!' % (count, 's' if count > 1 else '', 've' if count > 1 else 's')), } for mismatch, count in viewitems(version_mismatch): message = messages[mismatch](count) if flt_params.callback is not None: yield webutils.js_alert( "version-mismatch-%d" % ((mismatch + 1) // 2), "warning", message) utils.LOGGER.warning(message)
def get_nmap(): flt_params = get_nmap_base() # PostgreSQL: the query plan if affected by the limit and gives # really poor results. This is a temporary workaround (look for # XXX-WORKAROUND-PGSQL). # result = db.view.get(flt_params.flt, limit=flt_params.limit, # skip=flt_params.skip, sort=flt_params.sortby) result = db.view.get(flt_params.flt, skip=flt_params.skip, sort=flt_params.sortby) if flt_params.unused: msg = 'Option%s not understood: %s' % ( 's' if len(flt_params.unused) > 1 else '', ', '.join(flt_params.unused), ) if flt_params.callback is not None: yield webutils.js_alert("param-unused", "warning", msg) utils.LOGGER.warning(msg) elif flt_params.callback is not None: yield webutils.js_del_alert("param-unused") if config.DEBUG: msg1 = "filter: %s" % db.view.flt2str(flt_params.flt) msg2 = "user: %r" % webutils.get_user() utils.LOGGER.debug(msg1) utils.LOGGER.debug(msg2) if flt_params.callback is not None: yield webutils.js_alert("filter", "info", msg1) yield webutils.js_alert("user", "info", msg2) version_mismatch = {} if flt_params.callback is None: yield "[\n" else: yield "%s([\n" % flt_params.callback # XXX-WORKAROUND-PGSQL # for rec in result: for i, rec in enumerate(result): for fld in ['_id', 'scanid']: try: del rec[fld] except KeyError: pass if not flt_params.ipsasnumbers: rec['addr'] = utils.force_int2ip(rec['addr']) for field in ['starttime', 'endtime']: if field in rec: if not flt_params.datesasstrings: rec[field] = int(utils.datetime2timestamp(rec[field])) for port in rec.get('ports', []): if 'screendata' in port: port['screendata'] = utils.encode_b64(port['screendata']) for script in port.get('scripts', []): if "masscan" in script: try: del script['masscan']['raw'] except KeyError: pass if not flt_params.ipsasnumbers: if 'traces' in rec: for trace in rec['traces']: trace['hops'].sort(key=lambda x: x['ttl']) for hop in trace['hops']: hop['ipaddr'] = utils.force_int2ip(hop['ipaddr']) yield "%s\t%s" % ('' if i == 0 else ',\n', json.dumps(rec, default=utils.serialize)) check = db.view.cmp_schema_version_host(rec) if check: version_mismatch[check] = version_mismatch.get(check, 0) + 1 # XXX-WORKAROUND-PGSQL if i + 1 >= flt_params.limit: break if flt_params.callback is None: yield "\n]\n" else: yield "\n]);\n" messages = { 1: lambda count: ("%d document%s displayed %s out-of-date. Please run " "the following command: 'ivre scancli " "--update-schema;" % (count, 's' if count > 1 else '', 'are' if count > 1 else 'is')), -1: lambda count: ('%d document%s displayed ha%s been inserted by ' 'a more recent version of IVRE. Please update ' 'IVRE!' % (count, 's' if count > 1 else '', 've' if count > 1 else 's')), } for mismatch, count in viewitems(version_mismatch): message = messages[mismatch](count) if flt_params.callback is not None: yield webutils.js_alert( "version-mismatch-%d" % ((mismatch + 1) // 2), "warning", message ) utils.LOGGER.warning(message)