def main() -> None: baseflt = db.passive.searchrecontype("DNS_ANSWER") subdomains = False try: opts, args = getopt.getopt( sys.argv[1:], "s:h", [ # filters "sensor=", # subdomains "sub", "help", ], ) except getopt.GetoptError as err: sys.stderr.write(str(err) + "\n") sys.exit(-1) for o, a in opts: if o in ["-s", "--sensor"]: baseflt = db.passive.flt_and( baseflt, db.passive.searchsensor(utils.str2list(a))) elif o == "--sub": subdomains = True elif o in ["-h", "--help"]: sys.stdout.write("usage: %s [-h] [-s SENSOR] [--sub]" "\n\n" % (sys.argv[0])) sys.stdout.write(__doc__) sys.stdout.write("\n\n") sys.exit(0) else: sys.stderr.write( "%r %r not understood (this is probably a bug).\n" % (o, a)) sys.exit(-1) first = True flts = [] for a in args: if first: first = False else: print() if utils.IPADDR.search(a) or a.isdigit(): flts.append(db.passive.flt_and(baseflt, db.passive.searchhost(a))) else: flts += [ db.passive.flt_and( baseflt, db.passive.searchdns(utils.str2regexp(a), subdomains=subdomains), ), db.passive.flt_and( baseflt, db.passive.searchdns(utils.str2regexp(a), reverse=True, subdomains=subdomains), ), ] for flt in flts: for r in db.passive.get(flt, sort=[("source", 1)]): disp_rec(r)
def main(): baseflt = db.passive.searchrecontype('DNS_ANSWER') subdomains = False try: opts, args = getopt.getopt( sys.argv[1:], "s:h", [ # filters "sensor=", # subdomains "sub", "help", ], ) except getopt.GetoptError as err: sys.stderr.write(str(err) + '\n') sys.exit(-1) for o, a in opts: if o in ['-s', '--sensor']: baseflt = db.passive.flt_and(baseflt, db.passive.searchsensor(a)) elif o == '--sub': subdomains = True elif o in ['-h', '--help']: sys.stdout.write('usage: %s [-h] [-s SENSOR] [--sub]' '\n\n' % (sys.argv[0])) sys.stdout.write(__doc__) sys.stdout.write("\n\n") sys.exit(0) else: sys.stderr.write( '%r %r not undestood (this is probably a bug).\n' % (o, a)) sys.exit(-1) first = True flts = [] for a in args: if first: first = False else: print() if IPADDR.match(a) or a.isdigit(): flts.append(db.passive.flt_and(baseflt, db.passive.searchhost(a))) else: flts += [ db.passive.flt_and( baseflt, db.passive.searchdns(utils.str2regexp(a), subdomains=subdomains)), db.passive.flt_and( baseflt, db.passive.searchdns(utils.str2regexp(a), reverse=True, subdomains=subdomains)) ] for flt in flts: for r in db.passive.get(flt, sort=[('source', 1)]): disp_rec(r)
def main(): baseflt = db.passive.searchrecontype('DNS_ANSWER') subdomains = False try: opts, args = getopt.getopt( sys.argv[1:], "s:h", [ # filters "sensor=", # subdomains "sub", "help", ], ) except getopt.GetoptError as err: sys.stderr.write(str(err) + '\n') sys.exit(-1) for o, a in opts: if o in ['-s', '--sensor']: baseflt = db.passive.flt_and(baseflt, db.passive.searchsensor(a)) elif o == '--sub': subdomains = True elif o in ['-h', '--help']: sys.stdout.write('usage: %s [-h] [-s SENSOR] [--sub]' '\n\n' % (sys.argv[0])) sys.stdout.write(__doc__) sys.stdout.write("\n\n") sys.exit(0) else: sys.stderr.write( '%r %r not understood (this is probably a bug).\n' % (o, a)) sys.exit(-1) first = True flts = [] for a in args: if first: first = False else: print() if IPADDR.match(a) or a.isdigit(): flts.append(db.passive.flt_and(baseflt, db.passive.searchhost(a))) else: flts += [ db.passive.flt_and( baseflt, db.passive.searchdns( utils.str2regexp(a), subdomains=subdomains)), db.passive.flt_and( baseflt, db.passive.searchdns( utils.str2regexp(a), reverse=True, subdomains=subdomains)) ] for flt in flts: for r in db.passive.get(flt, sort=[('source', 1)]): disp_rec(r)
def str2regexpnone(value): """Just like str2regexp, but handle special '-' value, which means False. """ if value == "-": return False return utils.str2regexp(value)
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "ips_or_macs", nargs="*", help=("Display results for specified IP (or networks) or MAC addresses" " (or MAC address regexps)."), ) parser.add_argument("-s", "--sensor") parser.add_argument("-c", "--count", action="store_true") parser.add_argument("-r", "--resolve", action="store_true", help="Resolve MAC manufacturer") args = parser.parse_args() flts = ([], []) # MAC & IP filters for arg in args.ips_or_macs: if arg[:1] in "-!~": neg = True arg = arg[1:] else: neg = False match = MAC_ADDR.search(arg) if match: flts[0].append(db.passive.searchmac(mac=arg, neg=neg)) elif arg.startswith("/") and "/" in arg[1:]: flts[0].append( db.passive.searchmac(mac=utils.str2regexp(arg), neg=neg)) elif "/" in arg: flts[1].append(db.passive.searchnet(arg, neg=neg)) else: flts[1].append(db.passive.searchhost(arg, neg=neg)) if not flts[0]: flts[0].append(db.passive.searchmac()) flt = db.passive.flt_or(*flts[0]) if flts[1]: flt = db.passive.flt_and(flt, db.passive.flt_or(*flts[1])) if args.sensor is not None: flt = db.passive.flt_and(flt, db.passive.searchsensor(args.sensor)) if args.count: print(db.passive.count(flt)) return for rec in db.passive.get(flt, sort=[("addr", 1), ("value", 1), ("source", 1)]): rec["times"] = "s" if rec["count"] > 1 else "" if not rec.get("sensor"): rec["sensor"] = "-" if args.resolve: try: manuf = utils.mac2manuf(rec["value"])[0] except (TypeError, ValueError): pass else: rec["value"] = "%s (%s)" % (rec["value"], manuf) print("%(addr)s at %(value)s on %(sensor)s (%(source)s %(count)s " "time%(times)s, %(firstseen)s - %(lastseen)s)" % rec)
def flt_from_query(query, base_flt=None): """Return a tuple (`flt`, `archive`, `sortby`, `unused`, `skip`, `limit`): - a filter based on the query - a boolean (`True` iff the filter should be applied to the archive collection) - a list of [`key`, `order`] to sort results - a list of the unused elements of the query (errors) - an integer for the number of results to skip - an integer for the maximum number of results to return """ unused = [] sortby = [] archive = False skip = 0 limit = None flt = get_init_flt() if base_flt is None else base_flt def add_unused(neg, param, value): """Add to the `unused` list a string representing (neg, param, value). """ unused.append("%s%s" % ('-' if neg else '', "%s=%s" % (param, value) if value is not None else param)) for (neg, param, value) in query: if not neg and param == 'skip': skip = int(value) elif not neg and param == 'limit': limit = int(value) elif param == "archives": archive = not neg elif param == "id": flt = db.nmap.flt_and( flt, db.nmap.searchobjectid(value.replace('-', ',').split(','), neg=neg)) elif param == "host": flt = db.nmap.flt_and(flt, db.nmap.searchhost(value, neg=neg)) elif param == "net": flt = db.nmap.flt_and(flt, db.nmap.searchnet(value, neg=neg)) elif param == "range": flt = db.nmap.flt_and( flt, db.nmap.searchrange(*value.replace('-', ',').split(',', 1), neg=neg)) elif param == "label": group, lab = ((None, None) if value is None else map( utils.str2regexp, value.split(':', 1)) if ':' in value else (utils.str2regexp(value), None)) flt = db.nmap.flt_and( flt, db.nmap.searchlabel(group=group, label=lab, neg=neg)) elif param == "countports": vals = [int(val) for val in value.replace('-', ',').split(',', 1)] if len(vals) == 1: flt = db.nmap.flt_and( flt, db.nmap.searchcountopenports(minn=vals[0], maxn=vals[0], neg=neg)) else: flt = db.nmap.flt_and( flt, db.nmap.searchcountopenports(minn=vals[0], maxn=vals[1], neg=neg)) elif param == "hostname": flt = db.nmap.flt_and( flt, db.nmap.searchhostname(utils.str2regexp(value), neg=neg)) elif param == "domain": flt = db.nmap.flt_and( flt, db.nmap.searchdomain(utils.str2regexp(value), neg=neg)) elif param == "category": flt = db.nmap.flt_and( flt, db.nmap.searchcategory(utils.str2regexp(value), neg=neg)) elif param == "country": flt = db.nmap.flt_and( flt, db.nmap.searchcountry(utils.str2list(value.upper()), neg=neg)) elif param == "city": flt = db.nmap.flt_and( flt, db.nmap.searchcity(utils.str2regexp(value), neg=neg)) elif param == "asnum": flt = db.nmap.flt_and( flt, db.nmap.searchasnum(utils.str2list(value), neg=neg)) elif param == "asname": flt = db.nmap.flt_and( flt, db.nmap.searchasname(utils.str2regexp(value), neg=neg)) elif param == "source": flt = db.nmap.flt_and(flt, db.nmap.searchsource(value, neg=neg)) elif param == "timerange": flt = db.nmap.flt_and( flt, db.nmap.searchtimerange(*map( float, value.replace('-', ',').split(',')), neg=neg)) elif param == 'timeago': if value and value[-1].isalpha(): unit = { 's': 1, 'm': 60, 'h': 3600, 'd': 86400, 'y': 31557600, }[value[-1]] timeago = int(value[:-1]) * unit else: timeago = int(value) flt = db.nmap.flt_and( flt, db.nmap.searchtimeago(datetime.timedelta(0, timeago), neg=neg)) elif not neg and param == "service": if ':' in value: req, port = value.split(':', 1) port = int(port) flt = db.nmap.flt_and( flt, db.nmap.searchservice(utils.str2regexp(req), port=port)) else: flt = db.nmap.flt_and( flt, db.nmap.searchservice(utils.str2regexp(value))) elif not neg and param == "product" and ":" in value: product = value.split(':', 2) if len(product) == 2: flt = db.nmap.flt_and( flt, db.nmap.searchproduct(utils.str2regexp(product[1]), service=utils.str2regexp( product[0]))) else: flt = db.nmap.flt_and( flt, db.nmap.searchproduct(utils.str2regexp(product[1]), service=utils.str2regexp(product[0]), port=int(product[2]))) elif not neg and param == "version" and value.count(":") >= 2: product = value.split(':', 3) if len(product) == 3: flt = db.nmap.flt_and( flt, db.nmap.searchproduct( utils.str2regexp(product[1]), version=utils.str2regexp(product[2]), service=utils.str2regexp(product[0]), )) else: flt = db.nmap.flt_and( flt, db.nmap.searchproduct(utils.str2regexp(product[1]), version=utils.str2regexp(product[2]), service=utils.str2regexp(product[0]), port=int(product[3]))) elif not neg and param == "script": value = value.split(':', 1) if len(value) == 1: flt = db.nmap.flt_and( flt, db.nmap.searchscript(name=utils.str2regexp(value[0])), ) else: flt = db.nmap.flt_and( flt, db.nmap.searchscript( name=utils.str2regexp(value[0]), output=utils.str2regexp(value[1]), ), ) # results of scripts or version scans elif not neg and param == "anonftp": flt = db.nmap.flt_and(flt, db.nmap.searchftpanon()) elif not neg and param == 'anonldap': flt = db.nmap.flt_and(flt, db.nmap.searchldapanon()) elif not neg and param == 'authbypassvnc': flt = db.nmap.flt_and(flt, db.nmap.searchvncauthbypass()) elif not neg and param == "authhttp": flt = db.nmap.flt_and(flt, db.nmap.searchhttpauth()) elif not neg and param == 'banner': flt = db.nmap.flt_and( flt, db.nmap.searchbanner(utils.str2regexp(value))) elif param == 'cookie': flt = db.nmap.flt_and(flt, db.nmap.searchcookie(value)) elif param == 'file': if value is None: flt = db.nmap.flt_and(flt, db.nmap.searchfile()) else: value = value.split(':', 1) if len(value) == 1: flt = db.nmap.flt_and( flt, db.nmap.searchfile(fname=utils.str2regexp(value[0]))) else: flt = db.nmap.flt_and( flt, db.nmap.searchfile(fname=utils.str2regexp(value[1]), scripts=value[0].split(','))) elif not neg and param == 'geovision': flt = db.nmap.flt_and(flt, db.nmap.searchgeovision()) elif param == 'httptitle': flt = db.nmap.flt_and( flt, db.nmap.searchhttptitle(utils.str2regexp(value))) elif not neg and param == "nfs": flt = db.nmap.flt_and(flt, db.nmap.searchnfs()) elif not neg and param in ["nis", "yp"]: flt = db.nmap.flt_and(flt, db.nmap.searchypserv()) elif not neg and param == 'mssqlemptypwd': flt = db.nmap.flt_and(flt, db.nmap.searchmssqlemptypwd()) elif not neg and param == 'mysqlemptypwd': flt = db.nmap.flt_and(flt, db.nmap.searchmysqlemptypwd()) elif not neg and param == 'sshkey': if value: flt = db.nmap.flt_and( flt, db.nmap.searchsshkey(output=utils.str2regexp(value))) else: flt = db.nmap.flt_and(flt, db.nmap.searchsshkey()) elif not neg and param.startswith('sshkey.'): subfield = param.split('.', 1)[1] if subfield in ['fingerprint', 'key', 'type', 'bits']: if subfield == 'type': subfield = 'keytype' flt = db.nmap.flt_and( flt, db.nmap.searchsshkey( **{subfield: utils.str2regexp(value)})) else: add_unused(neg, param, value) elif not neg and param == 'owa': flt = db.nmap.flt_and(flt, db.nmap.searchowa()) elif param == 'phpmyadmin': flt = db.nmap.flt_and(flt, db.nmap.searchphpmyadmin()) elif not neg and param.startswith('smb.'): flt = db.nmap.flt_and( flt, db.nmap.searchsmb(**{param[4:]: utils.str2regexp(value)})) elif not neg and param == 'smbshare': flt = db.nmap.flt_and( flt, db.nmap.searchsmbshares(access="" if value is None else value), ) elif param == 'torcert': flt = db.nmap.flt_and(flt, db.nmap.searchtorcert()) elif not neg and param == 'webfiles': flt = db.nmap.flt_and(flt, db.nmap.searchwebfiles()) elif not neg and param == "webmin": flt = db.nmap.flt_and(flt, db.nmap.searchwebmin()) elif not neg and param == 'x11srv': flt = db.nmap.flt_and(flt, db.nmap.searchx11()) elif not neg and param == 'x11open': flt = db.nmap.flt_and(flt, db.nmap.searchx11access()) elif not neg and param == 'xp445': flt = db.nmap.flt_and(flt, db.nmap.searchxp445()) # OS fingerprint elif not neg and param == "os": flt = db.nmap.flt_and(flt, db.nmap.searchos(utils.str2regexp(value))) # device types elif param in ['devicetype', 'devtype']: flt = db.nmap.flt_and( flt, db.nmap.searchdevicetype(utils.str2regexp(value))) elif param in ['netdev', 'networkdevice']: flt = db.nmap.flt_and(flt, db.nmap.searchnetdev()) elif param == 'phonedev': flt = db.nmap.flt_and(flt, db.nmap.searchphonedev()) # traceroute elif param == 'hop': if ':' in value: hop, ttl = value.split(':', 1) flt = db.nmap.flt_and( flt, db.nmap.searchhop(hop, ttl=int(ttl), neg=neg)) else: flt = db.nmap.flt_and(flt, db.nmap.searchhop(value, neg=neg)) elif param == 'hopname': flt = db.nmap.flt_and(flt, db.nmap.searchhopname(value, neg=neg)) elif param == 'hopdomain': flt = db.nmap.flt_and(flt, db.nmap.searchhopdomain(value, neg=neg)) # sort elif param == 'sortby': if neg: sortby.append((value, -1)) else: sortby.append((value, 1)) elif param in ['open', 'filtered', 'closed']: if '_' in value: value = value.replace('_', '/') if '/' in value: proto, port = value.split('/') else: proto, port = "tcp", value port = port.split(',') if len(port) > 1: flt = db.nmap.flt_and( flt, db.nmap.searchports([int(p) for p in port], protocol=proto, state=param)) else: flt = db.nmap.flt_and( flt, db.nmap.searchport(int(port[0]), protocol=proto, state=param)) elif param == 'otheropenport': flt = db.nmap.flt_and( flt, db.nmap.searchportsother(map(int, value.split(',')))) elif param == "screenshot": if value is None: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(neg=neg)) elif value.isdigit(): flt = db.nmap.flt_and( flt, db.nmap.searchscreenshot(port=int(value), neg=neg)) elif value.startswith('tcp/') or value.startswith('udp/'): value = value.split('/', 1) flt = db.nmap.flt_and( flt, db.nmap.searchscreenshot(port=int(value[1]), protocol=value[0], neg=neg)) else: flt = db.nmap.flt_and( flt, db.nmap.searchscreenshot(service=value, neg=neg)) elif param == "screenwords": if value is None: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(words=not neg)) else: params = value.split(':', 1) words = (map(utils.str2regexp, params[0].split(',')) if ',' in params[0] else utils.str2regexp(params[0])) if len(params) == 1: flt = db.nmap.flt_and( flt, db.nmap.searchscreenshot(words=words, neg=neg)) elif params[1].isdigit(): flt = db.nmap.flt_and( flt, db.nmap.searchscreenshot(port=int(value), neg=neg, words=words)) elif (params[1].startswith('tcp/') or params[1].startswith('udp/')): params[1] = params[1].split('/', 1) flt = db.nmap.flt_and( flt, db.nmap.searchscreenshot(port=int(params[1][1]), protocol=params[1][0], neg=neg, words=words)) else: flt = db.nmap.flt_and( flt, db.nmap.searchscreenshot(service=value, neg=neg, words=words)) elif param == "cpe": if value: cpe_kwargs = {} cpe_fields = ["cpe_type", "vendor", "product", "version"] for field, cpe_arg in zip(cpe_fields, value.split(':', 3)): cpe_kwargs[field] = utils.str2regexp(cpe_arg) flt = db.nmap.flt_and(flt, db.nmap.searchcpe(**cpe_kwargs)) else: flt = db.nmap.flt_and(flt, db.nmap.searchcpe()) elif param == 'display': # ignore this parameter pass elif value is None: if param.startswith('tcp_') or param.startswith('tcp/') or \ param.startswith('udp_') or param.startswith('udp/'): proto, port = param.replace('_', '/').split('/', 1) port = int(port) flt = db.nmap.flt_and( flt, db.nmap.searchport(port, protocol=proto, neg=neg)) elif param == "openport": flt = db.nmap.flt_and(flt, db.nmap.searchopenport(neg=neg)) elif param.isdigit(): flt = db.nmap.flt_and(flt, db.nmap.searchport(int(param), neg=neg)) elif all(x.isdigit() for x in param.split(',')): flt = db.nmap.flt_and( flt, db.nmap.searchports(map(int, param.split(',')), neg=neg)) elif IPADDR.match(param): flt = db.nmap.flt_and(flt, db.nmap.searchhost(param, neg=neg)) elif NETADDR.match(param): flt = db.nmap.flt_and(flt, db.nmap.searchnet(param, neg=neg)) elif get_notepad_pages is not None and param == 'notes': flt = db.nmap.flt_and( flt, db.nmap.searchhosts(get_notepad_pages(), neg=neg)) elif '<' in param: param = param.split('<', 1) if param[1] and param[1][0] == '=': flt = db.nmap.flt_and( flt, db.nmap.searchcmp(param[0], int(param[1][1:]), '>' if neg else '<=')) else: flt = db.nmap.flt_and( flt, db.nmap.searchcmp(param[0], int(param[1]), '>=' if neg else '<')) elif '>' in param: param = param.split('>', 1) if param[1] and param[1][0] == '=': flt = db.nmap.flt_and( flt, db.nmap.searchcmp(param[0], int(param[1][1:]), '<' if neg else '>=')) else: flt = db.nmap.flt_and( flt, db.nmap.searchcmp(param[0], int(param[1]), '<=' if neg else '>')) else: add_unused(neg, param, value) else: add_unused(neg, param, value) return flt, archive, sortby, unused, skip, limit
def flt_from_query(query, base_flt=None): """Return a tuple (`flt`, `sortby`, `unused`, `skip`, `limit`): - a filter based on the query - a list of [`key`, `order`] to sort results - a list of the unused elements of the query (errors) - an integer for the number of results to skip - an integer for the maximum number of results to return """ unused = [] sortby = [] skip = 0 limit = None flt = get_init_flt() if base_flt is None else base_flt def add_unused(neg, param, value): """Add to the `unused` list a string representing (neg, param, value). """ unused.append("%s%s" % ('-' if neg else '', "%s=%s" % (param, value) if value is not None else param)) for (neg, param, value) in query: if not neg and param == 'skip': skip = int(value) elif not neg and param == 'limit': limit = int(value) elif param == "id": flt = db.view.flt_and( flt, db.view.searchobjectid(value.replace('-', ',').split(','), neg=neg)) elif param == "host": flt = db.view.flt_and(flt, db.view.searchhost(value, neg=neg)) elif param == "net": flt = db.view.flt_and(flt, db.view.searchnet(value, neg=neg)) elif param == "range": flt = db.view.flt_and( flt, db.view.searchrange(*value.replace('-', ',').split(',', 1), neg=neg)) elif param == "countports": vals = [int(val) for val in value.replace('-', ',').split(',', 1)] if len(vals) == 1: flt = db.view.flt_and( flt, db.view.searchcountopenports(minn=vals[0], maxn=vals[0], neg=neg)) else: flt = db.view.flt_and( flt, db.view.searchcountopenports(minn=vals[0], maxn=vals[1], neg=neg)) elif param == "hostname": flt = db.view.flt_and( flt, db.view.searchhostname(utils.str2regexp(value), neg=neg)) elif param == "domain": flt = db.view.flt_and( flt, db.view.searchdomain(utils.str2regexp(value), neg=neg)) elif param == "category": flt = db.view.flt_and( flt, db.view.searchcategory(utils.str2regexp(value), neg=neg)) elif param == "country": flt = db.view.flt_and( flt, db.view.searchcountry(utils.str2list(value.upper()), neg=neg)) elif param == "city": flt = db.view.flt_and( flt, db.view.searchcity(utils.str2regexp(value), neg=neg)) elif param == "asnum": flt = db.view.flt_and( flt, db.view.searchasnum(utils.str2list(value), neg=neg)) elif param == "asname": flt = db.view.flt_and( flt, db.view.searchasname(utils.str2regexp(value), neg=neg)) elif param == "source": flt = db.view.flt_and(flt, db.view.searchsource(value, neg=neg)) elif param == "timerange": flt = db.view.flt_and( flt, db.view.searchtimerange( *(float(val) for val in value.replace('-', ',').split(',')), neg=neg)) elif param == 'timeago': if value and value[-1].isalpha(): unit = { 's': 1, 'm': 60, 'h': 3600, 'd': 86400, 'y': 31557600, }[value[-1]] timeago = int(value[:-1]) * unit else: timeago = int(value) flt = db.view.flt_and( flt, db.view.searchtimeago(datetime.timedelta(0, timeago), neg=neg)) elif not neg and param == "service": if ':' in value: req, port = value.split(':', 1) port = int(port) flt = db.view.flt_and( flt, db.view.searchservice(utils.str2regexp(req), port=port)) else: flt = db.view.flt_and( flt, db.view.searchservice(utils.str2regexp(value))) elif not neg and param == "product" and ":" in value: product = value.split(':', 2) if len(product) == 2: flt = db.view.flt_and( flt, db.view.searchproduct(utils.str2regexp(product[1]), service=utils.str2regexp( product[0]))) else: flt = db.view.flt_and( flt, db.view.searchproduct(utils.str2regexp(product[1]), service=utils.str2regexp(product[0]), port=int(product[2]))) elif not neg and param == "version" and value.count(":") >= 2: product = value.split(':', 3) if len(product) == 3: flt = db.view.flt_and( flt, db.view.searchproduct( utils.str2regexp(product[1]), version=utils.str2regexp(product[2]), service=utils.str2regexp(product[0]), )) else: flt = db.view.flt_and( flt, db.view.searchproduct(utils.str2regexp(product[1]), version=utils.str2regexp(product[2]), service=utils.str2regexp(product[0]), port=int(product[3]))) elif param == "script": value = value.split(':', 1) if len(value) == 1: flt = db.view.flt_and( flt, db.view.searchscript(name=utils.str2regexp(value[0]), neg=neg), ) else: flt = db.view.flt_and( flt, db.view.searchscript(name=utils.str2regexp(value[0]), output=utils.str2regexp(value[1]), neg=neg), ) # results of scripts or version scans elif not neg and param == "anonftp": flt = db.view.flt_and(flt, db.view.searchftpanon()) elif not neg and param == 'anonldap': flt = db.view.flt_and(flt, db.view.searchldapanon()) elif not neg and param == 'authbypassvnc': flt = db.view.flt_and(flt, db.view.searchvncauthbypass()) elif not neg and param == "authhttp": flt = db.view.flt_and(flt, db.view.searchhttpauth()) elif not neg and param == 'banner': flt = db.view.flt_and( flt, db.view.searchbanner(utils.str2regexp(value))) elif param == 'cookie': flt = db.view.flt_and(flt, db.view.searchcookie(value)) elif param == 'file': if value is None: flt = db.view.flt_and(flt, db.view.searchfile()) else: value = value.split(':', 1) if len(value) == 1: flt = db.view.flt_and( flt, db.view.searchfile(fname=utils.str2regexp(value[0]))) else: flt = db.view.flt_and( flt, db.view.searchfile(fname=utils.str2regexp(value[1]), scripts=value[0].split(','))) elif param == 'vuln': try: vulnid, status = value.split(':', 1) except ValueError: vulnid = value status = None except AttributeError: vulnid = None status = None flt = db.view.flt_and( flt, db.view.searchvuln(vulnid=vulnid, status=status)) elif not neg and param == 'geovision': flt = db.view.flt_and(flt, db.view.searchgeovision()) elif param == 'httptitle': flt = db.view.flt_and( flt, db.view.searchhttptitle(utils.str2regexp(value))) elif not neg and param == "nfs": flt = db.view.flt_and(flt, db.view.searchnfs()) elif not neg and param in ["nis", "yp"]: flt = db.view.flt_and(flt, db.view.searchypserv()) elif not neg and param == 'mssqlemptypwd': flt = db.view.flt_and(flt, db.view.searchmssqlemptypwd()) elif not neg and param == 'mysqlemptypwd': flt = db.view.flt_and(flt, db.view.searchmysqlemptypwd()) elif not neg and param == 'sshkey': if value: flt = db.view.flt_and( flt, db.view.searchsshkey(output=utils.str2regexp(value))) else: flt = db.view.flt_and(flt, db.view.searchsshkey()) elif not neg and param.startswith('sshkey.'): subfield = param.split('.', 1)[1] if subfield in ['fingerprint', 'key', 'type', 'bits']: if subfield == 'type': subfield = 'keytype' elif subfield == 'bits': try: value = int(value) except (ValueError, TypeError): pass else: value = utils.str2regexp(value) flt = db.view.flt_and( flt, db.view.searchsshkey(**{subfield: value})) else: add_unused(neg, param, value) elif not neg and param == 'httphdr': if value is None: flt = db.view.flt_and(flt, db.view.searchhttphdr()) elif ':' in value: name, value = (utils.str2regexp(string) for string in value.split(':', 1)) flt = db.view.flt_and( flt, db.view.searchhttphdr(name=name, value=value)) else: flt = db.view.flt_and( flt, db.view.searchhttphdr(name=utils.str2regexp(value))) elif not neg and param == 'owa': flt = db.view.flt_and(flt, db.view.searchowa()) elif param == 'phpmyadmin': flt = db.view.flt_and(flt, db.view.searchphpmyadmin()) elif not neg and param.startswith('smb.'): flt = db.view.flt_and( flt, db.view.searchsmb(**{param[4:]: utils.str2regexp(value)})) elif not neg and param == 'smbshare': flt = db.view.flt_and( flt, db.view.searchsmbshares(access="" if value is None else value), ) elif param == 'torcert': flt = db.view.flt_and(flt, db.view.searchtorcert()) elif not neg and param == 'webfiles': flt = db.view.flt_and(flt, db.view.searchwebfiles()) elif not neg and param == "webmin": flt = db.view.flt_and(flt, db.view.searchwebmin()) elif not neg and param == 'x11srv': flt = db.view.flt_and(flt, db.view.searchx11()) elif not neg and param == 'x11open': flt = db.view.flt_and(flt, db.view.searchx11access()) elif not neg and param == 'xp445': flt = db.view.flt_and(flt, db.view.searchxp445()) elif param == "ssl-ja3-client": flt = db.view.flt_and( flt, db.view.searchja3client( value_or_hash=(None if value is None else utils.str2regexp(value)), neg=neg)) elif param == "ssl-ja3-server": if value is None: # There are no additional arguments flt = db.view.flt_and(flt, db.view.searchja3server(neg=neg)) else: split = [ utils.str2regexp(v) if v else None for v in value.split(':', 1) ] if len(split) == 1: # Only a JA3 server is given flt = db.view.flt_and( flt, db.view.searchja3server( value_or_hash=(split[0]), neg=neg, )) else: # Both client and server JA3 are specified flt = db.view.flt_and( flt, db.view.searchja3server( value_or_hash=split[0], client_value_or_hash=split[1], neg=neg, )) elif param == "useragent": if value: flt = db.view.flt_and( flt, db.view.searchuseragent(useragent=utils.str2regexp(value))) else: flt = db.view.flt_and(flt, db.view.searchuseragent()) # OS fingerprint elif not neg and param == "os": flt = db.view.flt_and(flt, db.view.searchos(utils.str2regexp(value))) # device types elif param in ['devicetype', 'devtype']: flt = db.view.flt_and( flt, db.view.searchdevicetype(utils.str2regexp(value))) elif param in ['netdev', 'networkdevice']: flt = db.view.flt_and(flt, db.view.searchnetdev()) elif param == 'phonedev': flt = db.view.flt_and(flt, db.view.searchphonedev()) # traceroute elif param == 'hop': if ':' in value: hop, ttl = value.split(':', 1) flt = db.view.flt_and( flt, db.view.searchhop(hop, ttl=int(ttl), neg=neg)) else: flt = db.view.flt_and(flt, db.view.searchhop(value, neg=neg)) elif param == 'hopname': flt = db.view.flt_and(flt, db.view.searchhopname(value, neg=neg)) elif param == 'hopdomain': flt = db.view.flt_and(flt, db.view.searchhopdomain(value, neg=neg)) elif not neg and param in [ "ike.vendor_id.name", "ike.vendor_id.value" ]: flt = db.view.flt_and( flt, db.view.searchscript( name="ike-info", values={ 'vendor_ids.%s' % param[14:]: utils.str2regexp(value) }, ), ) elif not neg and param == "ike.notification": flt = db.view.flt_and( flt, db.view.searchscript( name="ike-info", values={'notification_type': utils.str2regexp(value)}, ), ) # sort elif param == 'sortby': if neg: sortby.append((value, -1)) else: sortby.append((value, 1)) elif param in ['open', 'filtered', 'closed']: value = value.replace('_', '/').split(',') protos = {} for port in value: if '/' in port: proto, port = port.split('/') else: proto, port = "tcp", port protos.setdefault(proto, []).append(int(port)) for proto, ports in viewitems(protos): flt = db.view.flt_and( flt, db.view.searchport(ports[0], protocol=proto, state=param) if len(ports) == 1 else db.view.searchports( ports, protocol=proto, state=param)) elif param == 'otheropenport': flt = db.view.flt_and( flt, db.view.searchportsother( [int(val) for val in value.split(',')])) elif param == "screenshot": if value is None: flt = db.view.flt_and(flt, db.view.searchscreenshot(neg=neg)) elif value.isdigit(): flt = db.view.flt_and( flt, db.view.searchscreenshot(port=int(value), neg=neg)) elif value.startswith('tcp/') or value.startswith('udp/'): value = value.split('/', 1) flt = db.view.flt_and( flt, db.view.searchscreenshot(port=int(value[1]), protocol=value[0], neg=neg)) else: flt = db.view.flt_and( flt, db.view.searchscreenshot(service=value, neg=neg)) elif param == "screenwords": if value is None: flt = db.view.flt_and(flt, db.view.searchscreenshot(words=not neg)) else: params = value.split(':', 1) words = ([ utils.str2regexp(elt) for elt in params[0].split(',') ] if ',' in params[0] else utils.str2regexp(params[0])) if len(params) == 1: flt = db.view.flt_and( flt, db.view.searchscreenshot(words=words, neg=neg)) elif params[1].isdigit(): flt = db.view.flt_and( flt, db.view.searchscreenshot(port=int(value), neg=neg, words=words)) elif (params[1].startswith('tcp/') or params[1].startswith('udp/')): params[1] = params[1].split('/', 1) flt = db.view.flt_and( flt, db.view.searchscreenshot(port=int(params[1][1]), protocol=params[1][0], neg=neg, words=words)) else: flt = db.view.flt_and( flt, db.view.searchscreenshot(service=value, neg=neg, words=words)) elif param == "cpe": if value: cpe_kwargs = {} cpe_fields = ["cpe_type", "vendor", "product", "version"] for field, cpe_arg in zip(cpe_fields, value.split(':', 3)): cpe_kwargs[field] = utils.str2regexp(cpe_arg) flt = db.view.flt_and(flt, db.view.searchcpe(**cpe_kwargs)) else: flt = db.view.flt_and(flt, db.view.searchcpe()) elif param == 'display': # ignore this parameter pass elif value is None: if param.startswith('tcp_') or param.startswith('tcp/') or \ param.startswith('udp_') or param.startswith('udp/'): proto, port = param.replace('_', '/').split('/', 1) port = int(port) flt = db.view.flt_and( flt, db.view.searchport(port, protocol=proto, neg=neg)) elif param == "openport": flt = db.view.flt_and(flt, db.view.searchopenport(neg=neg)) elif param.isdigit(): flt = db.view.flt_and(flt, db.view.searchport(int(param), neg=neg)) elif all(x.isdigit() for x in param.split(',')): flt = db.view.flt_and( flt, db.view.searchports([int(val) for val in param.split(',')], neg=neg)) elif IPADDR.match(param): flt = db.view.flt_and(flt, db.view.searchhost(param, neg=neg)) elif NETADDR.match(param): flt = db.view.flt_and(flt, db.view.searchnet(param, neg=neg)) elif get_notepad_pages is not None and param == 'notes': flt = db.view.flt_and( flt, db.view.searchhosts(get_notepad_pages(), neg=neg)) elif '<' in param: param = param.split('<', 1) if param[1] and param[1][0] == '=': flt = db.view.flt_and( flt, db.view.searchcmp(param[0], int(param[1][1:]), '>' if neg else '<=')) else: flt = db.view.flt_and( flt, db.view.searchcmp(param[0], int(param[1]), '>=' if neg else '<')) elif '>' in param: param = param.split('>', 1) if param[1] and param[1][0] == '=': flt = db.view.flt_and( flt, db.view.searchcmp(param[0], int(param[1][1:]), '<' if neg else '>=')) else: flt = db.view.flt_and( flt, db.view.searchcmp(param[0], int(param[1]), '<=' if neg else '>')) else: add_unused(neg, param, value) else: add_unused(neg, param, value) return flt, sortby, unused, skip, limit
def flt_from_query(query, base_flt=None): """Return a tuple (`flt`, `archive`, `sortby`, `unused`, `skip`, `limit`): - a filter based on the query - a boolean (`True` iff the filter should be applied to the archive collection) - a list of [`key`, `order`] to sort results - a list of the unused elements of the query (errors) - an integer for the number of results to skip - an integer for the maximum number of results to return """ unused = [] sortby = [] archive = False skip = 0 limit = None flt = get_init_flt() if base_flt is None else base_flt def add_unused(neg, param, value): """Add to the `unused` list a string representing (neg, param, value). """ unused.append("%s%s" % ( '-' if neg else '', "%s=%s" % (param, value) if value is not None else param )) for (neg, param, value) in query: if not neg and param == 'skip': skip = int(value) elif not neg and param == 'limit': limit = int(value) elif param == "archives": archive = not neg elif param == "id": flt = db.nmap.flt_and(flt, db.nmap.searchobjectid( value.replace('-', ',').split(','), neg=neg)) elif param == "host": flt = db.nmap.flt_and(flt, db.nmap.searchhost(value, neg=neg)) elif param == "net": flt = db.nmap.flt_and(flt, db.nmap.searchnet(value, neg=neg)) elif param == "range": flt = db.nmap.flt_and(flt, db.nmap.searchrange( *value.replace('-', ',').split(',', 1), neg=neg)) elif param == "label": group, lab = ((None, None) if value is None else map(utils.str2regexp, value.split(':', 1)) if ':' in value else (utils.str2regexp(value), None)) flt = db.nmap.flt_and(flt, db.nmap.searchlabel(group=group, label=lab, neg=neg)) elif param == "countports": vals = [int(val) for val in value.replace('-', ',').split(',', 1)] if len(vals) == 1: flt = db.nmap.flt_and(flt, db.nmap.searchcountopenports( minn=vals[0], maxn=vals[0], neg=neg)) else: flt = db.nmap.flt_and(flt, db.nmap.searchcountopenports( minn=vals[0], maxn=vals[1], neg=neg)) elif param == "hostname": flt = db.nmap.flt_and( flt, db.nmap.searchhostname(utils.str2regexp(value), neg=neg)) elif param == "domain": flt = db.nmap.flt_and( flt, db.nmap.searchdomain(utils.str2regexp(value), neg=neg)) elif param == "category": flt = db.nmap.flt_and(flt, db.nmap.searchcategory( utils.str2regexp(value), neg=neg)) elif param == "country": flt = db.nmap.flt_and(flt, db.nmap.searchcountry( utils.str2list(value.upper()), neg=neg)) elif param == "city": flt = db.nmap.flt_and(flt, db.nmap.searchcity( utils.str2regexp(value), neg=neg)) elif param == "asnum": flt = db.nmap.flt_and(flt, db.nmap.searchasnum( utils.str2list(value), neg=neg)) elif param == "asname": flt = db.nmap.flt_and(flt, db.nmap.searchasname( utils.str2regexp(value), neg=neg)) elif param == "source": flt = db.nmap.flt_and(flt, db.nmap.searchsource(value, neg=neg)) elif param == "timerange": flt = db.nmap.flt_and(flt, db.nmap.searchtimerange( *map(float, value.replace('-', ',').split(',')), neg=neg)) elif param == 'timeago': if value and value[-1].isalpha(): unit = { 's': 1, 'm': 60, 'h': 3600, 'd': 86400, 'y': 31557600, }[value[-1]] timeago = int(value[:-1]) * unit else: timeago = int(value) flt = db.nmap.flt_and(flt, db.nmap.searchtimeago( datetime.timedelta(0, timeago), neg=neg)) elif not neg and param == "service": if ':' in value: req, port = value.split(':', 1) port = int(port) flt = db.nmap.flt_and( flt, db.nmap.searchservice(utils.str2regexp(req), port=port)) else: flt = db.nmap.flt_and( flt, db.nmap.searchservice(utils.str2regexp(value))) elif not neg and param == "product" and ":" in value: product = value.split(':', 2) if len(product) == 2: flt = db.nmap.flt_and( flt, db.nmap.searchproduct( utils.str2regexp(product[1]), service=utils.str2regexp(product[0]) ) ) else: flt = db.nmap.flt_and( flt, db.nmap.searchproduct( utils.str2regexp(product[1]), service=utils.str2regexp(product[0]), port=int(product[2]) ) ) elif not neg and param == "version" and value.count(":") >= 2: product = value.split(':', 3) if len(product) == 3: flt = db.nmap.flt_and( flt, db.nmap.searchproduct( utils.str2regexp(product[1]), version=utils.str2regexp(product[2]), service=utils.str2regexp(product[0]), ) ) else: flt = db.nmap.flt_and( flt, db.nmap.searchproduct( utils.str2regexp(product[1]), version=utils.str2regexp(product[2]), service=utils.str2regexp(product[0]), port=int(product[3]) ) ) elif not neg and param == "script": value = value.split(':', 1) if len(value) == 1: flt = db.nmap.flt_and( flt, db.nmap.searchscript(name=utils.str2regexp(value[0])), ) else: flt = db.nmap.flt_and( flt, db.nmap.searchscript( name=utils.str2regexp(value[0]), output=utils.str2regexp(value[1]), ), ) # results of scripts or version scans elif not neg and param == "anonftp": flt = db.nmap.flt_and(flt, db.nmap.searchftpanon()) elif not neg and param == 'anonldap': flt = db.nmap.flt_and(flt, db.nmap.searchldapanon()) elif not neg and param == 'authbypassvnc': flt = db.nmap.flt_and(flt, db.nmap.searchvncauthbypass()) elif not neg and param == "authhttp": flt = db.nmap.flt_and(flt, db.nmap.searchhttpauth()) elif not neg and param == 'banner': flt = db.nmap.flt_and( flt, db.nmap.searchbanner(utils.str2regexp(value))) elif param == 'cookie': flt = db.nmap.flt_and(flt, db.nmap.searchcookie(value)) elif param == 'file': if value is None: flt = db.nmap.flt_and(flt, db.nmap.searchfile()) else: value = value.split(':', 1) if len(value) == 1: flt = db.nmap.flt_and(flt, db.nmap.searchfile( fname=utils.str2regexp(value[0]))) else: flt = db.nmap.flt_and(flt, db.nmap.searchfile( fname=utils.str2regexp(value[1]), scripts=value[0].split(','))) elif not neg and param == 'geovision': flt = db.nmap.flt_and(flt, db.nmap.searchgeovision()) elif param == 'httptitle': flt = db.nmap.flt_and( flt, db.nmap.searchhttptitle(utils.str2regexp(value))) elif not neg and param == "nfs": flt = db.nmap.flt_and(flt, db.nmap.searchnfs()) elif not neg and param in ["nis", "yp"]: flt = db.nmap.flt_and(flt, db.nmap.searchypserv()) elif not neg and param == 'mssqlemptypwd': flt = db.nmap.flt_and(flt, db.nmap.searchmssqlemptypwd()) elif not neg and param == 'mysqlemptypwd': flt = db.nmap.flt_and(flt, db.nmap.searchmysqlemptypwd()) elif not neg and param == 'sshkey': if value: flt = db.nmap.flt_and(flt, db.nmap.searchsshkey( output=utils.str2regexp(value))) else: flt = db.nmap.flt_and(flt, db.nmap.searchsshkey()) elif not neg and param.startswith('sshkey.'): subfield = param.split('.', 1)[1] if subfield in ['fingerprint', 'key', 'type', 'bits']: if subfield == 'type': subfield = 'keytype' flt = db.nmap.flt_and(flt, db.nmap.searchsshkey( **{subfield: utils.str2regexp(value)})) else: add_unused(neg, param, value) elif not neg and param == 'owa': flt = db.nmap.flt_and(flt, db.nmap.searchowa()) elif param == 'phpmyadmin': flt = db.nmap.flt_and(flt, db.nmap.searchphpmyadmin()) elif not neg and param.startswith('smb.'): flt = db.nmap.flt_and(flt, db.nmap.searchsmb( **{param[4:]: utils.str2regexp(value)})) elif not neg and param == 'smbshare': flt = db.nmap.flt_and( flt, db.nmap.searchsmbshares(access="" if value is None else value), ) elif param == 'torcert': flt = db.nmap.flt_and(flt, db.nmap.searchtorcert()) elif not neg and param == 'webfiles': flt = db.nmap.flt_and(flt, db.nmap.searchwebfiles()) elif not neg and param == "webmin": flt = db.nmap.flt_and(flt, db.nmap.searchwebmin()) elif not neg and param == 'x11srv': flt = db.nmap.flt_and(flt, db.nmap.searchx11()) elif not neg and param == 'x11open': flt = db.nmap.flt_and(flt, db.nmap.searchx11access()) elif not neg and param == 'xp445': flt = db.nmap.flt_and(flt, db.nmap.searchxp445()) # OS fingerprint elif not neg and param == "os": flt = db.nmap.flt_and( flt, db.nmap.searchos(utils.str2regexp(value))) # device types elif param in ['devicetype', 'devtype']: flt = db.nmap.flt_and( flt, db.nmap.searchdevicetype(utils.str2regexp(value))) elif param in ['netdev', 'networkdevice']: flt = db.nmap.flt_and(flt, db.nmap.searchnetdev()) elif param == 'phonedev': flt = db.nmap.flt_and(flt, db.nmap.searchphonedev()) # traceroute elif param == 'hop': if ':' in value: hop, ttl = value.split(':', 1) flt = db.nmap.flt_and(flt, db.nmap.searchhop(hop, ttl=int(ttl), neg=neg)) else: flt = db.nmap.flt_and(flt, db.nmap.searchhop(value, neg=neg)) elif param == 'hopname': flt = db.nmap.flt_and(flt, db.nmap.searchhopname(value, neg=neg)) elif param == 'hopdomain': flt = db.nmap.flt_and(flt, db.nmap.searchhopdomain(value, neg=neg)) # sort elif param == 'sortby': if neg: sortby.append((value, -1)) else: sortby.append((value, 1)) elif param in ['open', 'filtered', 'closed']: value = value.replace('_', '/').split(',') protos = {} for port in value: if '/' in port: proto, port = port.split('/') else: proto, port = "tcp", port protos.setdefault(proto, []).append(int(port)) for proto, ports in protos.iteritems(): flt = db.nmap.flt_and( flt, db.nmap.searchport(ports[0], protocol=proto, state=param) if len(ports) == 1 else db.nmap.searchports(ports, protocol=proto, state=param) ) elif param == 'otheropenport': flt = db.nmap.flt_and( flt, db.nmap.searchportsother(map(int, value.split(',')))) elif param == "screenshot": if value is None: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(neg=neg)) elif value.isdigit(): flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot( port=int(value), neg=neg)) elif value.startswith('tcp/') or value.startswith('udp/'): value = value.split('/', 1) flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot( port=int(value[1]), protocol=value[0], neg=neg)) else: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot( service=value, neg=neg)) elif param == "screenwords": if value is None: flt = db.nmap.flt_and( flt, db.nmap.searchscreenshot(words=not neg) ) else: params = value.split(':', 1) words = (map(utils.str2regexp, params[0].split(',')) if ',' in params[0] else utils.str2regexp(params[0])) if len(params) == 1: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot( words=words, neg=neg)) elif params[1].isdigit(): flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot( port=int(value), neg=neg, words=words)) elif (params[1].startswith('tcp/') or params[1].startswith('udp/')): params[1] = params[1].split('/', 1) flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot( port=int(params[1][1]), protocol=params[1][0], neg=neg, words=words)) else: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot( service=value, neg=neg, words=words)) elif param == "cpe": if value: cpe_kwargs = {} cpe_fields = ["cpe_type", "vendor", "product", "version"] for field, cpe_arg in zip(cpe_fields, value.split(':', 3)): cpe_kwargs[field] = utils.str2regexp(cpe_arg) flt = db.nmap.flt_and(flt, db.nmap.searchcpe(**cpe_kwargs)) else: flt = db.nmap.flt_and(flt, db.nmap.searchcpe()) elif param == 'display': # ignore this parameter pass elif value is None: if param.startswith('tcp_') or param.startswith('tcp/') or \ param.startswith('udp_') or param.startswith('udp/'): proto, port = param.replace('_', '/').split('/', 1) port = int(port) flt = db.nmap.flt_and(flt, db.nmap.searchport(port, protocol=proto, neg=neg)) elif param == "openport": flt = db.nmap.flt_and(flt, db.nmap.searchopenport(neg=neg)) elif param.isdigit(): flt = db.nmap.flt_and(flt, db.nmap.searchport(int(param), neg=neg)) elif all(x.isdigit() for x in param.split(',')): flt = db.nmap.flt_and( flt, db.nmap.searchports(map(int, param.split(',')), neg=neg) ) elif IPADDR.match(param): flt = db.nmap.flt_and(flt, db.nmap.searchhost(param, neg=neg)) elif NETADDR.match(param): flt = db.nmap.flt_and(flt, db.nmap.searchnet(param, neg=neg)) elif get_notepad_pages is not None and param == 'notes': flt = db.nmap.flt_and(flt, db.nmap.searchhosts( get_notepad_pages(), neg=neg)) elif '<' in param: param = param.split('<', 1) if param[1] and param[1][0] == '=': flt = db.nmap.flt_and(flt, db.nmap.searchcmp( param[0], int(param[1][1:]), '>' if neg else '<=')) else: flt = db.nmap.flt_and(flt, db.nmap.searchcmp( param[0], int(param[1]), '>=' if neg else '<')) elif '>' in param: param = param.split('>', 1) if param[1] and param[1][0] == '=': flt = db.nmap.flt_and(flt, db.nmap.searchcmp( param[0], int(param[1][1:]), '<' if neg else '>=')) else: flt = db.nmap.flt_and(flt, db.nmap.searchcmp( param[0], int(param[1]), '<=' if neg else '>')) else: add_unused(neg, param, value) else: add_unused(neg, param, value) return flt, archive, sortby, unused, skip, limit
def flt_from_query(query, base_flt=None): """Return a tuple (`flt`, `archive`, `sortby`, `unused`, `skip`, `limit`): - a filter based on the query - a boolean (`True` iff the filter should be applied to the archive collection) - a list of [`key`, `order`] to sort results - a list of the unused elements of the query (errors) - an integer for the number of results to skip - an integer for the maximum number of results to return """ unused = [] sortby = [] archive = False skip = 0 limit = None flt = get_init_flt() if base_flt is None else base_flt def add_unused(neg, param, value): """Add to the `unused` list a string representing (neg, param, value). """ unused.append("%s%s" % ("-" if neg else "", "%s=%s" % (param, value) if value is not None else param)) for (neg, param, value) in query: if not neg and param == "skip": skip = int(value) elif not neg and param == "limit": limit = int(value) elif param == "archives": archive = not neg elif param == "id": flt = db.nmap.flt_and(flt, db.nmap.searchobjectid(value.replace("-", ",").split(","), neg=neg)) elif param == "host": flt = db.nmap.flt_and(flt, db.nmap.searchhost(value, neg=neg)) elif param == "net": flt = db.nmap.flt_and(flt, db.nmap.searchnet(value, neg=neg)) elif param == "range": flt = db.nmap.flt_and(flt, db.nmap.searchrange(*value.replace("-", ",").split(",", 1), neg=neg)) elif param == "label": group, lab = ( (None, None) if value is None else map(utils.str2regexp, value.split(":", 1)) if ":" in value else (utils.str2regexp(value), None) ) flt = db.nmap.flt_and(flt, db.nmap.searchlabel(group=group, label=lab, neg=neg)) elif param == "countports": vals = [int(val) for val in value.replace("-", ",").split(",", 1)] if len(vals) == 1: flt = db.nmap.flt_and(flt, db.nmap.searchcountopenports(minn=vals[0], maxn=vals[0], neg=neg)) else: flt = db.nmap.flt_and(flt, db.nmap.searchcountopenports(minn=vals[0], maxn=vals[1], neg=neg)) elif param == "hostname": flt = db.nmap.flt_and(flt, db.nmap.searchhostname(utils.str2regexp(value), neg=neg)) elif param == "domain": flt = db.nmap.flt_and(flt, db.nmap.searchdomain(utils.str2regexp(value), neg=neg)) elif param == "category": flt = db.nmap.flt_and(flt, db.nmap.searchcategory(utils.str2regexp(value), neg=neg)) elif param == "country": flt = db.nmap.flt_and(flt, db.nmap.searchcountry(utils.str2list(value.upper()), neg=neg)) elif param == "city": flt = db.nmap.flt_and(flt, db.nmap.searchcity(utils.str2regexp(value), neg=neg)) elif param == "asnum": flt = db.nmap.flt_and(flt, db.nmap.searchasnum(utils.str2list(value), neg=neg)) elif param == "asname": flt = db.nmap.flt_and(flt, db.nmap.searchasname(utils.str2regexp(value), neg=neg)) elif param == "source": flt = db.nmap.flt_and(flt, db.nmap.searchsource(value, neg=neg)) elif param == "timerange": flt = db.nmap.flt_and( flt, db.nmap.searchtimerange(*map(float, value.replace("-", ",").split(",")), neg=neg) ) elif param == "timeago": if value and value[-1].isalpha(): unit = {"s": 1, "m": 60, "h": 3600, "d": 86400, "y": 31557600}[value[-1]] timeago = int(value[:-1]) * unit else: timeago = int(value) flt = db.nmap.flt_and(flt, db.nmap.searchtimeago(datetime.timedelta(0, timeago), neg=neg)) elif not neg and param == "service": if ":" in value: req, port = value.split(":", 1) port = int(port) flt = db.nmap.flt_and(flt, db.nmap.searchservice(utils.str2regexp(req), port=port)) else: flt = db.nmap.flt_and(flt, db.nmap.searchservice(utils.str2regexp(value))) elif not neg and param == "product" and ":" in value: product = value.split(":", 2) if len(product) == 2: flt = db.nmap.flt_and( flt, db.nmap.searchproduct(utils.str2regexp(product[1]), service=utils.str2regexp(product[0])) ) else: flt = db.nmap.flt_and( flt, db.nmap.searchproduct( utils.str2regexp(product[1]), service=utils.str2regexp(product[0]), port=int(product[2]) ), ) elif not neg and param == "version" and value.count(":") >= 2: product = value.split(":", 3) if len(product) == 3: flt = db.nmap.flt_and( flt, db.nmap.searchproduct( utils.str2regexp(product[1]), version=utils.str2regexp(product[2]), service=utils.str2regexp(product[0]), ), ) else: flt = db.nmap.flt_and( flt, db.nmap.searchproduct( utils.str2regexp(product[1]), version=utils.str2regexp(product[2]), service=utils.str2regexp(product[0]), port=int(product[3]), ), ) elif not neg and param == "script": value = value.split(":", 1) if len(value) == 1: flt = db.nmap.flt_and(flt, db.nmap.searchscript(name=utils.str2regexp(value[0]))) else: flt = db.nmap.flt_and( flt, db.nmap.searchscript(name=utils.str2regexp(value[0]), output=utils.str2regexp(value[1])) ) # results of scripts or version scans elif not neg and param == "anonftp": flt = db.nmap.flt_and(flt, db.nmap.searchftpanon()) elif not neg and param == "anonldap": flt = db.nmap.flt_and(flt, db.nmap.searchldapanon()) elif not neg and param == "authbypassvnc": flt = db.nmap.flt_and(flt, db.nmap.searchvncauthbypass()) elif not neg and param == "authhttp": flt = db.nmap.flt_and(flt, db.nmap.searchhttpauth()) elif not neg and param == "banner": flt = db.nmap.flt_and(flt, db.nmap.searchbanner(utils.str2regexp(value))) elif param == "cookie": flt = db.nmap.flt_and(flt, db.nmap.searchcookie(value)) elif param == "file": if value is None: flt = db.nmap.flt_and(flt, db.nmap.searchfile()) else: value = value.split(":", 1) if len(value) == 1: flt = db.nmap.flt_and(flt, db.nmap.searchfile(fname=utils.str2regexp(value[0]))) else: flt = db.nmap.flt_and( flt, db.nmap.searchfile(fname=utils.str2regexp(value[1]), scripts=value[0].split(",")) ) elif not neg and param == "geovision": flt = db.nmap.flt_and(flt, db.nmap.searchgeovision()) elif param == "httptitle": flt = db.nmap.flt_and(flt, db.nmap.searchhttptitle(utils.str2regexp(value))) elif not neg and param == "nfs": flt = db.nmap.flt_and(flt, db.nmap.searchnfs()) elif not neg and param in ["nis", "yp"]: flt = db.nmap.flt_and(flt, db.nmap.searchypserv()) elif not neg and param == "mssqlemptypwd": flt = db.nmap.flt_and(flt, db.nmap.searchmssqlemptypwd()) elif not neg and param == "mysqlemptypwd": flt = db.nmap.flt_and(flt, db.nmap.searchmysqlemptypwd()) elif not neg and param == "sshkey": if value: flt = db.nmap.flt_and(flt, db.nmap.searchsshkey(output=utils.str2regexp(value))) else: flt = db.nmap.flt_and(flt, db.nmap.searchsshkey()) elif not neg and param.startswith("sshkey."): subfield = param.split(".", 1)[1] if subfield in ["fingerprint", "key", "type", "bits"]: if subfield == "type": subfield = "keytype" flt = db.nmap.flt_and(flt, db.nmap.searchsshkey(**{subfield: utils.str2regexp(value)})) else: add_unused(neg, param, value) elif not neg and param == "owa": flt = db.nmap.flt_and(flt, db.nmap.searchowa()) elif param == "phpmyadmin": flt = db.nmap.flt_and(flt, db.nmap.searchphpmyadmin()) elif not neg and param.startswith("smb."): flt = db.nmap.flt_and(flt, db.nmap.searchsmb(**{param[4:]: utils.str2regexp(value)})) elif not neg and param == "smbshare": flt = db.nmap.flt_and(flt, db.nmap.searchsmbshares(access="" if value is None else value)) elif param == "torcert": flt = db.nmap.flt_and(flt, db.nmap.searchtorcert()) elif not neg and param == "webfiles": flt = db.nmap.flt_and(flt, db.nmap.searchwebfiles()) elif not neg and param == "webmin": flt = db.nmap.flt_and(flt, db.nmap.searchwebmin()) elif not neg and param == "x11srv": flt = db.nmap.flt_and(flt, db.nmap.searchx11()) elif not neg and param == "x11open": flt = db.nmap.flt_and(flt, db.nmap.searchx11access()) elif not neg and param == "xp445": flt = db.nmap.flt_and(flt, db.nmap.searchxp445()) # OS fingerprint elif not neg and param == "os": flt = db.nmap.flt_and(flt, db.nmap.searchos(utils.str2regexp(value))) # device types elif param in ["devicetype", "devtype"]: flt = db.nmap.flt_and(flt, db.nmap.searchdevicetype(utils.str2regexp(value))) elif param in ["netdev", "networkdevice"]: flt = db.nmap.flt_and(flt, db.nmap.searchnetdev()) elif param == "phonedev": flt = db.nmap.flt_and(flt, db.nmap.searchphonedev()) # traceroute elif param == "hop": if ":" in value: hop, ttl = value.split(":", 1) flt = db.nmap.flt_and(flt, db.nmap.searchhop(hop, ttl=int(ttl), neg=neg)) else: flt = db.nmap.flt_and(flt, db.nmap.searchhop(value, neg=neg)) elif param == "hopname": flt = db.nmap.flt_and(flt, db.nmap.searchhopname(value, neg=neg)) elif param == "hopdomain": flt = db.nmap.flt_and(flt, db.nmap.searchhopdomain(value, neg=neg)) # sort elif param == "sortby": if neg: sortby.append((value, -1)) else: sortby.append((value, 1)) elif param in ["open", "filtered", "closed"]: value = value.replace("_", "/").split(",") protos = {} for port in value: if "/" in port: proto, port = port.split("/") else: proto, port = "tcp", port protos.setdefault(proto, []).append(int(port)) for proto, ports in protos.iteritems(): flt = db.nmap.flt_and( flt, db.nmap.searchport(ports[0], protocol=proto, state=param) if len(ports) == 1 else db.nmap.searchports(ports, protocol=proto, state=param), ) elif param == "otheropenport": flt = db.nmap.flt_and(flt, db.nmap.searchportsother(map(int, value.split(",")))) elif param == "screenshot": if value is None: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(neg=neg)) elif value.isdigit(): flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(port=int(value), neg=neg)) elif value.startswith("tcp/") or value.startswith("udp/"): value = value.split("/", 1) flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(port=int(value[1]), protocol=value[0], neg=neg)) else: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(service=value, neg=neg)) elif param == "screenwords": if value is None: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(words=not neg)) else: params = value.split(":", 1) words = map(utils.str2regexp, params[0].split(",")) if "," in params[0] else utils.str2regexp(params[0]) if len(params) == 1: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(words=words, neg=neg)) elif params[1].isdigit(): flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(port=int(value), neg=neg, words=words)) elif params[1].startswith("tcp/") or params[1].startswith("udp/"): params[1] = params[1].split("/", 1) flt = db.nmap.flt_and( flt, db.nmap.searchscreenshot(port=int(params[1][1]), protocol=params[1][0], neg=neg, words=words), ) else: flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(service=value, neg=neg, words=words)) elif param == "cpe": if value: cpe_kwargs = {} cpe_fields = ["cpe_type", "vendor", "product", "version"] for field, cpe_arg in zip(cpe_fields, value.split(":", 3)): cpe_kwargs[field] = utils.str2regexp(cpe_arg) flt = db.nmap.flt_and(flt, db.nmap.searchcpe(**cpe_kwargs)) else: flt = db.nmap.flt_and(flt, db.nmap.searchcpe()) elif param == "display": # ignore this parameter pass elif value is None: if ( param.startswith("tcp_") or param.startswith("tcp/") or param.startswith("udp_") or param.startswith("udp/") ): proto, port = param.replace("_", "/").split("/", 1) port = int(port) flt = db.nmap.flt_and(flt, db.nmap.searchport(port, protocol=proto, neg=neg)) elif param == "openport": flt = db.nmap.flt_and(flt, db.nmap.searchopenport(neg=neg)) elif param.isdigit(): flt = db.nmap.flt_and(flt, db.nmap.searchport(int(param), neg=neg)) elif all(x.isdigit() for x in param.split(",")): flt = db.nmap.flt_and(flt, db.nmap.searchports(map(int, param.split(",")), neg=neg)) elif IPADDR.match(param): flt = db.nmap.flt_and(flt, db.nmap.searchhost(param, neg=neg)) elif NETADDR.match(param): flt = db.nmap.flt_and(flt, db.nmap.searchnet(param, neg=neg)) elif get_notepad_pages is not None and param == "notes": flt = db.nmap.flt_and(flt, db.nmap.searchhosts(get_notepad_pages(), neg=neg)) elif "<" in param: param = param.split("<", 1) if param[1] and param[1][0] == "=": flt = db.nmap.flt_and(flt, db.nmap.searchcmp(param[0], int(param[1][1:]), ">" if neg else "<=")) else: flt = db.nmap.flt_and(flt, db.nmap.searchcmp(param[0], int(param[1]), ">=" if neg else "<")) elif ">" in param: param = param.split(">", 1) if param[1] and param[1][0] == "=": flt = db.nmap.flt_and(flt, db.nmap.searchcmp(param[0], int(param[1][1:]), "<" if neg else ">=")) else: flt = db.nmap.flt_and(flt, db.nmap.searchcmp(param[0], int(param[1]), "<=" if neg else ">")) else: add_unused(neg, param, value) else: add_unused(neg, param, value) return flt, archive, sortby, unused, skip, limit
def topvalues(self, field, flt=None, topnbr=10, sort=None, least=False): """ This method uses an aggregation to produce top values for a given field or pseudo-field. Pseudo-fields are: - category / asnum / country / net[:mask] - port - port:open / :closed / :filtered / :<servicename> - portlist:open / :closed / :filtered - countports:open / :closed / :filtered - service / service:<portnbr> - product / product:<portnbr> - cpe / cpe.<part> / cpe:<cpe_spec> / cpe.<part>:<cpe_spec> - devicetype / devicetype:<portnbr> - script:<scriptid> / script:<port>:<scriptid> / script:host:<scriptid> - cert.* / smb.* / sshkey.* / ike.* - httphdr / httphdr.{name,value} / httphdr:<name> - modbus.* / s7.* / enip.* - mongo.dbs.* - vulns.* - screenwords - file.* / file.*:scriptid - hop """ outputproc = None nested = None if flt is None: flt = self.flt_empty if field == "asnum": flt = self.flt_and(flt, Q("exists", field="infos.as_num")) field = {"field": "infos.as_num"} elif field == "as": def outputproc(value): return tuple(val if i else int(val) for i, val in enumerate(value.split(', ', 1))) flt = self.flt_and(flt, Q("exists", field="infos.as_num")) field = {"script": { "lang": "painless", "source": "doc['infos.as_num'].value + ', ' + " "doc['infos.as_name'].value", }} elif field == "port" or field.startswith("port:"): def outputproc(value): return tuple(int(val) if i else val for i, val in enumerate(value.split('/', 1))) if field == "port": flt = self.flt_and(flt, Q('nested', path='ports', query=Q('exists', field="ports.port"))) field = {"script": { "lang": "painless", "source": """List result = new ArrayList(); for(item in params._source.ports) { if(item.port != -1) { result.add(item.protocol + '/' + item.port); } } return result; """, }} else: info = field[5:] if info in ['open', 'filtered', 'closed']: flt = self.flt_and(flt, Q('nested', path='ports', query=Q('match', ports__state_state=info))) matchfield = "state_state" else: flt = self.flt_and(flt, Q('nested', path='ports', query=Q('match', ports__service_name=info))) matchfield = "service_name" field = {"script": { "lang": "painless", "source": """List result = new ArrayList(); for(item in params._source.ports) { if(item[params.field] == params.value) { result.add(item.protocol + '/' + item.port); } } return result; """, "params": { "field": matchfield, "value": info, } }} elif field == 'httphdr': def outputproc(value): return tuple(value.split(':', 1)) flt = self.flt_and(flt, self.searchscript(name="http-headers")) field = {"script": { "lang": "painless", "source": """List result = new ArrayList(); for(item in params._source.ports) { if (item.containsKey('scripts')) { for(script in item.scripts) { if(script.id == 'http-headers' && script.containsKey('http-headers')) { for(hdr in script['http-headers']) { result.add(hdr.name + ':' + hdr.value); } } } } } return result; """, }} elif field.startswith('httphdr.'): flt = self.flt_and(flt, self.searchscript(name="http-headers")) field = {"field": "ports.scripts.http-headers.%s" % field[8:]} nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": {"terms": field}}, }}, } elif field.startswith('httphdr:'): subfield = field[8:].lower() flt = self.flt_and(flt, self.searchscript(name="http-headers", values={"name": subfield})) field = {"script": { "lang": "painless", "source": """List result = new ArrayList(); for(item in params._source.ports) { if (item.containsKey('scripts')) { for(script in item.scripts) { if(script.id == 'http-headers' && script.containsKey('http-headers')) { for(hdr in script['http-headers']) { if (hdr.name == params.name) { result.add(hdr.value); } } } } } } return result; """, "params": {"name": subfield} }} elif field == 'useragent' or field.startswith('useragent:'): terms = {"field": "ports.scripts.http-user-agent"} if field == 'useragent': flt = self.flt_and(flt, self.searchuseragent()) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": {"terms": terms}}, }}, } else: subfield = utils.str2regexp(field[10:]) flt = self.flt_and(flt, self.searchuseragent(useragent=subfield)) if isinstance(subfield, utils.REGEXP_T): subfield = self._get_pattern(subfield) else: subfield = re.escape(subfield) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": { "terms": dict(terms, include=subfield), }}, }}, } else: field = {"field": field} body = {"query": flt.to_dict()} if nested is None: body["aggs"] = {"patterns": {"terms": dict(field, size=topnbr)}} else: body["aggs"] = {"patterns": nested} result = self.db_client.search( body=body, index=self.indexes[0], ignore_unavailable=True, size=0 ) result = result["aggregations"] while 'patterns' in result: result = result['patterns'] result = result['buckets'] if outputproc is None: for res in result: yield {'_id': res['key'], 'count': res['doc_count']} else: for res in result: yield {'_id': outputproc(res['key']), 'count': res['doc_count']}
def flt_from_query(dbase, query, base_flt=None): """Return a tuple (`flt`, `sortby`, `unused`, `skip`, `limit`, `fields`): - a filter based on the query - a list of [`key`, `order`] to sort results - a list of the unused elements of the query (errors) - an integer for the number of results to skip - an integer for the maximum number of results to return """ unused = [] sortby = [] skip = 0 limit = None fields = None flt = get_init_flt(dbase) if base_flt is None else base_flt def add_unused(neg, param, value): """Add to the `unused` list a string representing (neg, param, value). """ unused.append("%s%s" % ( "-" if neg else "", "%s=%s" % (param, value) if value is not None else param, )) for (neg, param, value) in query: if not neg and param == "skip": skip = int(value) elif not neg and param == "limit": limit = int(value) elif not neg and param == "fields": fields = value.split(",") elif param == "id": flt = dbase.flt_and( flt, dbase.searchobjectid(value.replace("-", ",").split(","), neg=neg)) elif param == "host": flt = dbase.flt_and(flt, dbase.searchhost(value, neg=neg)) elif param == "net": flt = dbase.flt_and(flt, dbase.searchnet(value, neg=neg)) elif param == "range": flt = dbase.flt_and( flt, dbase.searchrange(*value.replace("-", ",").split(",", 1), neg=neg)) elif param == "countports": vals = [int(val) for val in value.replace("-", ",").split(",", 1)] if len(vals) == 1: flt = dbase.flt_and( flt, dbase.searchcountopenports(minn=vals[0], maxn=vals[0], neg=neg)) else: flt = dbase.flt_and( flt, dbase.searchcountopenports(minn=vals[0], maxn=vals[1], neg=neg)) elif param == "hostname": flt = dbase.flt_and( flt, dbase.searchhostname(utils.str2regexp(value), neg=neg)) elif param == "domain": flt = dbase.flt_and( flt, dbase.searchdomain(utils.str2regexp(value), neg=neg)) elif param == "category": flt = dbase.flt_and( flt, dbase.searchcategory(utils.str2regexp(value), neg=neg)) elif param == "country": flt = dbase.flt_and( flt, dbase.searchcountry(utils.str2list(value.upper()), neg=neg)) elif param == "city": flt = dbase.flt_and( flt, dbase.searchcity(utils.str2regexp(value), neg=neg)) elif param == "asnum": flt = dbase.flt_and( flt, dbase.searchasnum(utils.str2list(value), neg=neg)) elif param == "asname": flt = dbase.flt_and( flt, dbase.searchasname(utils.str2regexp(value), neg=neg)) elif param == "source": flt = dbase.flt_and( flt, dbase.searchsource(utils.str2regexp(value), neg=neg)) elif param == "timerange": flt = dbase.flt_and( flt, dbase.searchtimerange( *(float(val) for val in value.replace("-", ",").split(",")), neg=neg), ) elif param == "timeago": if value and value[-1].isalpha(): unit = { "s": 1, "m": 60, "h": 3600, "d": 86400, "y": 31557600, }[value[-1]] timeago = int(value[:-1]) * unit else: timeago = int(value) flt = dbase.flt_and( flt, dbase.searchtimeago(datetime.timedelta(0, timeago), neg=neg)) elif not neg and param == "service": if ":" in value: req, port = value.split(":", 1) port = int(port) flt = dbase.flt_and( flt, dbase.searchservice(utils.str2regexpnone(req), port=port)) else: flt = dbase.flt_and( flt, dbase.searchservice(utils.str2regexpnone(value))) elif not neg and param == "product" and ":" in value: product = value.split(":", 2) if len(product) == 2: flt = dbase.flt_and( flt, dbase.searchproduct( product=utils.str2regexpnone(product[1]), service=utils.str2regexpnone(product[0]), ), ) else: flt = dbase.flt_and( flt, dbase.searchproduct( product=utils.str2regexpnone(product[1]), service=utils.str2regexpnone(product[0]), port=int(product[2]), ), ) elif not neg and param == "version" and value.count(":") >= 2: product = value.split(":", 3) if len(product) == 3: flt = dbase.flt_and( flt, dbase.searchproduct( product=utils.str2regexpnone(product[1]), version=utils.str2regexpnone(product[2]), service=utils.str2regexpnone(product[0]), ), ) else: flt = dbase.flt_and( flt, dbase.searchproduct( product=utils.str2regexpnone(product[1]), version=utils.str2regexpnone(product[2]), service=utils.str2regexpnone(product[0]), port=int(product[3]), ), ) elif param == "script": value = value.split(":", 1) if len(value) == 1: flt = dbase.flt_and( flt, dbase.searchscript(name=utils.str2regexp(value[0]), neg=neg), ) else: flt = dbase.flt_and( flt, dbase.searchscript( name=utils.str2regexp(value[0]), output=utils.str2regexp(value[1]), neg=neg, ), ) # results of scripts or version scans elif not neg and param == "anonftp": flt = dbase.flt_and(flt, dbase.searchftpanon()) elif not neg and param == "anonldap": flt = dbase.flt_and(flt, dbase.searchldapanon()) elif not neg and param == "authbypassvnc": flt = dbase.flt_and(flt, dbase.searchvncauthbypass()) elif not neg and param == "authhttp": flt = dbase.flt_and(flt, dbase.searchhttpauth()) elif not neg and param == "banner": flt = dbase.flt_and(flt, dbase.searchbanner(utils.str2regexp(value))) elif param == "cookie": flt = dbase.flt_and(flt, dbase.searchcookie(value)) elif param == "file": if value is None: flt = dbase.flt_and(flt, dbase.searchfile()) else: value = value.split(":", 1) if len(value) == 1: flt = dbase.flt_and( flt, dbase.searchfile(fname=utils.str2regexp(value[0]))) else: flt = dbase.flt_and( flt, dbase.searchfile( fname=utils.str2regexp(value[1]), scripts=value[0].split(","), ), ) elif param == "vuln": try: vulnid, status = value.split(":", 1) except ValueError: vulnid = value status = None except AttributeError: vulnid = None status = None flt = dbase.flt_and(flt, dbase.searchvuln(vulnid=vulnid, status=status)) elif not neg and param == "geovision": flt = dbase.flt_and(flt, dbase.searchgeovision()) elif param == "httptitle": flt = dbase.flt_and(flt, dbase.searchhttptitle(utils.str2regexp(value))) elif not neg and param == "nfs": flt = dbase.flt_and(flt, dbase.searchnfs()) elif not neg and param in ["nis", "yp"]: flt = dbase.flt_and(flt, dbase.searchypserv()) elif not neg and param == "mssqlemptypwd": flt = dbase.flt_and(flt, dbase.searchmssqlemptypwd()) elif not neg and param == "mysqlemptypwd": flt = dbase.flt_and(flt, dbase.searchmysqlemptypwd()) elif not neg and param == "sshkey": if value: flt = dbase.flt_and( flt, dbase.searchsshkey(output=utils.str2regexp(value))) else: flt = dbase.flt_and(flt, dbase.searchsshkey()) elif not neg and param.startswith("sshkey."): subfield = param.split(".", 1)[1] if subfield in ["fingerprint", "key", "type", "bits"]: if subfield == "type": subfield = "keytype" elif subfield == "bits": try: value = int(value) except (ValueError, TypeError): pass else: value = utils.str2regexp(value) flt = dbase.flt_and(flt, dbase.searchsshkey(**{subfield: value})) else: add_unused(neg, param, value) elif not neg and param == "cert": flt = dbase.flt_and(flt, dbase.searchcert()) elif param.startswith("cert."): subfield = param.split(".", 1)[1] if subfield == "self_signed" and value is None: flt = dbase.flt_and(flt, dbase.searchcert(self_signed=not neg)) elif not neg: if subfield in ["md5", "sha1", "sha256", "subject", "issuer"]: flt = dbase.flt_and( flt, dbase.searchcert(**{ subfield: utils.str2regexp(value), }), ) elif subfield in [ "pubkey.md5", "pubkey.sha1", "pubkey.sha256" ]: flt = dbase.flt_and( flt, dbase.searchcert( **{ "pk%s" % subfield[7:]: utils.str2regexp(value), }), ) else: add_unused(neg, param, value) else: add_unused(neg, param, value) elif not neg and param == "httphdr": if value is None: flt = dbase.flt_and(flt, dbase.searchhttphdr()) elif ":" in value: name, value = value.split(":", 1) name = utils.str2regexp(name.lower()) value = utils.str2regexp(value) flt = dbase.flt_and( flt, dbase.searchhttphdr(name=name, value=value)) else: flt = dbase.flt_and( flt, dbase.searchhttphdr(name=utils.str2regexp(value.lower()))) elif not neg and param == "httpapp": if value is None: flt = dbase.flt_and(flt, dbase.searchhttpapp()) elif ":" in value: name, version = (utils.str2regexp(string) for string in value.split(":", 1)) flt = dbase.flt_and( flt, dbase.searchhttpapp(name=name, version=version)) else: flt = dbase.flt_and( flt, dbase.searchhttpapp(name=utils.str2regexp(value))) elif not neg and param == "owa": flt = dbase.flt_and(flt, dbase.searchowa()) elif param == "phpmyadmin": flt = dbase.flt_and(flt, dbase.searchphpmyadmin()) elif not neg and param.startswith("smb."): flt = dbase.flt_and( flt, dbase.searchsmb(**{param[4:]: utils.str2regexp(value)})) elif not neg and param.startswith("ntlm."): key = { "name": "Target_Name", "server": "NetBIOS_Computer_Name", "domain": "NetBIOS_Domain_Name", "workgroup": "Workgroup", "domain_dns": "DNS_Domain_Name", "forest": "DNS_Tree_Name", "fqdn": "DNS_Computer_Name", "os": "Product_Version", "version": "NTLM_Version", } flt = dbase.flt_and( flt, dbase.searchntlm( **{key.get(param[5:], param[5:]): utils.str2regexp(value) }), ) elif not neg and param == "smbshare": flt = dbase.flt_and( flt, dbase.searchsmbshares(access="" if value is None else value), ) elif param == "torcert": flt = dbase.flt_and(flt, dbase.searchtorcert()) elif not neg and param == "webfiles": flt = dbase.flt_and(flt, dbase.searchwebfiles()) elif not neg and param == "webmin": flt = dbase.flt_and(flt, dbase.searchwebmin()) elif not neg and param == "x11srv": flt = dbase.flt_and(flt, dbase.searchx11()) elif not neg and param == "x11open": flt = dbase.flt_and(flt, dbase.searchx11access()) elif not neg and param == "xp445": flt = dbase.flt_and(flt, dbase.searchxp445()) elif param == "ssl-ja3-client": flt = dbase.flt_and( flt, dbase.searchja3client( value_or_hash=(None if value is None else utils.str2regexp(value)), neg=neg, ), ) elif param == "ssl-ja3-server": if value is None: # There are no additional arguments flt = dbase.flt_and(flt, dbase.searchja3server(neg=neg)) else: split = [ utils.str2regexp(v) if v else None for v in value.split(":", 1) ] if len(split) == 1: # Only a JA3 server is given flt = dbase.flt_and( flt, dbase.searchja3server( value_or_hash=(split[0]), neg=neg, ), ) else: # Both client and server JA3 are specified flt = dbase.flt_and( flt, dbase.searchja3server( value_or_hash=split[0], client_value_or_hash=split[1], neg=neg, ), ) elif param == "ssl-jarm": if value is None: flt = dbase.flt_and(flt, dbase.searchjarm(neg=neg)) else: flt = dbase.flt_and( flt, dbase.searchjarm(value=utils.str2regexp(value), neg=neg)) elif param in {"hassh", "hassh-client", "hassh-server"}: server = { "hassh": None, "hassh-client": False, "hassh-server": True }[param] flt = dbase.flt_and( flt, dbase.searchhassh( value_or_hash=(None if value is None else utils.str2regexp(value)), server=server, ), ) elif param == "useragent": if value: flt = dbase.flt_and( flt, dbase.searchuseragent(useragent=utils.str2regexp(value))) else: flt = dbase.flt_and(flt, dbase.searchuseragent()) # OS fingerprint elif not neg and param == "os": flt = dbase.flt_and(flt, dbase.searchos(utils.str2regexp(value))) # device types elif param in ["devicetype", "devtype"]: flt = dbase.flt_and( flt, dbase.searchdevicetype(utils.str2regexp(value))) elif param in ["netdev", "networkdevice"]: flt = dbase.flt_and(flt, dbase.searchnetdev()) elif param == "phonedev": flt = dbase.flt_and(flt, dbase.searchphonedev()) # traceroute elif param == "hop": if ":" in value: hop, ttl = value.split(":", 1) flt = dbase.flt_and( flt, dbase.searchhop(hop, ttl=int(ttl), neg=neg)) else: flt = dbase.flt_and(flt, dbase.searchhop(value, neg=neg)) elif param == "hopname": flt = dbase.flt_and(flt, dbase.searchhopname(value, neg=neg)) elif param == "hopdomain": flt = dbase.flt_and(flt, dbase.searchhopdomain(value, neg=neg)) elif not neg and param in [ "ike.vendor_id.name", "ike.vendor_id.value" ]: flt = dbase.flt_and( flt, dbase.searchscript( name="ike-info", values={ "vendor_ids.%s" % param[14:]: utils.str2regexp(value) }, ), ) elif not neg and param == "ike.notification": flt = dbase.flt_and( flt, dbase.searchscript( name="ike-info", values={"notification_type": utils.str2regexp(value)}, ), ) # sort elif param == "sortby": if neg: sortby.append((value, -1)) else: sortby.append((value, 1)) elif param in ["open", "filtered", "closed"]: value = value.replace("_", "/").split(",") protos = {} for port in value: if "/" in port: proto, port = port.split("/") else: proto = "tcp" protos.setdefault(proto, []).append(int(port)) for proto, ports in protos.items(): flt = dbase.flt_and( flt, dbase.searchport(ports[0], protocol=proto, state=param) if len(ports) == 1 else dbase.searchports( ports, protocol=proto, state=param), ) elif param == "otheropenport": flt = dbase.flt_and( flt, dbase.searchportsother([int(val) for val in value.split(",")])) elif param == "screenshot": if value is None: flt = dbase.flt_and(flt, dbase.searchscreenshot(neg=neg)) elif value.isdigit(): flt = dbase.flt_and( flt, dbase.searchscreenshot(port=int(value), neg=neg)) elif value.startswith("tcp/") or value.startswith("udp/"): value = value.split("/", 1) flt = dbase.flt_and( flt, dbase.searchscreenshot(port=int(value[1]), protocol=value[0], neg=neg), ) else: flt = dbase.flt_and( flt, dbase.searchscreenshot(service=value, neg=neg)) elif param == "screenwords": if value is None: flt = dbase.flt_and(flt, dbase.searchscreenshot(words=not neg)) else: params = value.split(":", 1) words = ([ utils.str2regexp(elt) for elt in params[0].split(",") ] if "," in params[0] else utils.str2regexp(params[0])) if len(params) == 1: flt = dbase.flt_and( flt, dbase.searchscreenshot(words=words, neg=neg)) elif params[1].isdigit(): flt = dbase.flt_and( flt, dbase.searchscreenshot(port=int(value), neg=neg, words=words), ) elif params[1].startswith("tcp/") or params[1].startswith( "udp/"): params[1] = params[1].split("/", 1) flt = dbase.flt_and( flt, dbase.searchscreenshot( port=int(params[1][1]), protocol=params[1][0], neg=neg, words=words, ), ) else: flt = dbase.flt_and( flt, dbase.searchscreenshot(service=value, neg=neg, words=words)) elif param == "cpe": if value: cpe_kwargs = {} cpe_fields = ["cpe_type", "vendor", "product", "version"] for field, cpe_arg in zip(cpe_fields, value.split(":", 3)): cpe_kwargs[field] = utils.str2regexp(cpe_arg) flt = dbase.flt_and(flt, dbase.searchcpe(**cpe_kwargs)) else: flt = dbase.flt_and(flt, dbase.searchcpe()) elif param == "tag": if value: if ":" in value: tag_val, tag_info = value.split(":", 1) tag = {} if tag_val: tag["value"] = utils.str2regexp(tag_val) if tag_info: tag["info"] = utils.str2regexp(tag_info) flt = dbase.flt_and(flt, dbase.searchtag(tag, neg=neg)) else: flt = dbase.flt_and( flt, dbase.searchtag({"value": utils.str2regexp(value)}, neg=neg), ) else: flt = dbase.flt_and(flt, dbase.searchtag(neg=neg)) elif param == "search": try: flt = dbase.flt_and(flt, dbase.searchtext(value)) except AttributeError: # .searchtext() only exists in MongoDB add_unused(neg, param, value) elif param == "display": # ignore this parameter pass elif value is None: if PORT.search(param) is not None: # Python >= 3.8: if (match := PORT.search(param)) is not None proto, port = PORT.search(param).groups() flt = dbase.flt_and( flt, dbase.searchport(int(port), protocol=proto or "tcp", neg=neg)) elif param == "openport": flt = dbase.flt_and(flt, dbase.searchopenport(neg=neg)) elif param == "ipv4": if neg: flt = dbase.flt_and(flt, dbase.searchipv6()) else: flt = dbase.flt_and(flt, dbase.searchipv4()) elif param == "ipv6": if neg: flt = dbase.flt_and(flt, dbase.searchipv4()) else: flt = dbase.flt_and(flt, dbase.searchipv6()) elif all(PORT.search(x) is not None for x in param.split(",")): proto_ports = {} for port in param.split(","): proto, port = PORT.search(port).groups() proto_ports.setdefault(proto or "tcp", set()).add(int(port)) for proto, ports in proto_ports.items(): if len(ports) > 1: flt = dbase.flt_and( flt, dbase.searchports(list(ports), protocol=proto, neg=neg), ) else: flt = dbase.flt_and( flt, dbase.searchport(next(iter(ports)), protocol=proto, neg=neg), ) elif utils.IPADDR.search(param): flt = dbase.flt_and(flt, dbase.searchhost(param, neg=neg)) elif utils.NETADDR.search(param): flt = dbase.flt_and(flt, dbase.searchnet(param, neg=neg)) elif get_notepad_pages is not None and param == "notes": flt = dbase.flt_and( flt, dbase.searchhosts(get_notepad_pages(), neg=neg)) elif "<" in param: param = param.split("<", 1) if param[1] and param[1][0] == "=": flt = dbase.flt_and( flt, dbase.searchcmp(param[0], int(param[1][1:]), ">" if neg else "<="), ) else: flt = dbase.flt_and( flt, dbase.searchcmp(param[0], int(param[1]), ">=" if neg else "<"), ) elif ">" in param: param = param.split(">", 1) if param[1] and param[1][0] == "=": flt = dbase.flt_and( flt, dbase.searchcmp(param[0], int(param[1][1:]), "<" if neg else ">="), ) else: flt = dbase.flt_and( flt, dbase.searchcmp(param[0], int(param[1]), "<=" if neg else ">"), ) else: add_unused(neg, param, value) else: add_unused(neg, param, value) return flt, sortby, unused, skip, limit, fields
def topvalues(self, field, flt=None, topnbr=10, sort=None, least=False): """ This method uses an aggregation to produce top values for a given field or pseudo-field. Pseudo-fields are: - category / asnum / country / net[:mask] - port - port:open / :closed / :filtered / :<servicename> - portlist:open / :closed / :filtered - countports:open / :closed / :filtered - service / service:<portnbr> - product / product:<portnbr> - cpe / cpe.<part> / cpe:<cpe_spec> / cpe.<part>:<cpe_spec> - devicetype / devicetype:<portnbr> - script:<scriptid> / script:<port>:<scriptid> / script:host:<scriptid> - cert.* / smb.* / sshkey.* / ike.* - httphdr / httphdr.{name,value} / httphdr:<name> - modbus.* / s7.* / enip.* - mongo.dbs.* - vulns.* - screenwords - file.* / file.*:scriptid - hop """ baseterms = {"size": topnbr} if least: baseterms["order"] = {"_count": "asc"} outputproc = None nested = None if flt is None: flt = self.flt_empty if field == "category": field = {"field": "categories"} elif field == "asnum": flt = self.flt_and(flt, Q("exists", field="infos.as_num")) field = {"field": "infos.as_num"} elif field == "as": def outputproc(value): return tuple(val if i else int(val) for i, val in enumerate(value.split(',', 1))) flt = self.flt_and(flt, Q("exists", field="infos.as_num")) field = {"script": { "lang": "painless", "source": "doc['infos.as_num'].value + ',' + " "doc['infos.as_name'].value", }} elif field == "port" or field.startswith("port:"): def outputproc(value): return tuple(int(val) if i else val for i, val in enumerate(value.rsplit('/', 1))) if field == "port": flt = self.flt_and(flt, Q('nested', path='ports', query=Q('exists', field="ports.port"))) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "filter": {'bool': {'must_not': [ {'match': {'ports.port': -1}}, ]}}, "aggs": {"patterns": { "terms": dict( baseterms, script={ "lang": "painless", "source": 'doc["ports.protocol"].value + "/" + ' 'doc["ports.port"].value', }, ), }}, }}, } else: info = field[5:] if info in ['open', 'filtered', 'closed']: flt = self.flt_and(flt, Q('nested', path='ports', query=Q('match', ports__state_state=info))) matchfield = "state_state" else: flt = self.flt_and(flt, Q('nested', path='ports', query=Q('match', ports__service_name=info))) matchfield = "service_name" nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "filter": {'bool': { 'must': [{'match': {'ports.%s' % matchfield: info}}], 'must_not': [{'match': {'ports.port': -1}}], }}, "aggs": {"patterns": { "terms": dict( baseterms, script={ "lang": "painless", "source": 'doc["ports.protocol"].value + "/" + ' 'doc["ports.port"].value', }, ), }}, }}, } elif field == 'service': def outputproc(value): return value or None flt = self.flt_and(flt, self.searchopenport()) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "filter": {"match": {"ports.state_state": "open"}}, "aggs": {"patterns": { "terms": dict( baseterms, field="ports.service_name", missing="", ), }}, }}, } elif field.startswith("service:"): port = int(field[8:]) flt = self.flt_and(flt, self.searchport(port)) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "filter": {"bool": {"must": [ {"match": {"ports.state_state": "open"}}, {"match": {"ports.port": port}}, ]}}, "aggs": {"patterns": { "terms": dict( baseterms, field="ports.service_name", missing="", ), }}, }}, } elif field == 'product': def outputproc(value): return tuple(v or None for v in value.split('###', 1)) flt = self.flt_and(flt, self.searchopenport()) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "filter": {"match": {"ports.state_state": "open"}}, "aggs": {"patterns": { "terms": dict( baseterms, script=""" String result = ""; if(doc['ports.service_name'].size() > 0) { result += doc['ports.service_name'].value; } result += "###"; if(doc['ports.service_product'].size() > 0) { result += doc['ports.service_product'].value; } return result; """, missing="", ), }}, }}, } elif field.startswith("product:"): def outputproc(value): return tuple(v or None for v in value.split('###', 1)) info = field[8:] if info.isdigit(): info = int(info) flt = self.flt_and(flt, self.searchport(info)) matchfield = "port" else: flt = self.flt_and(flt, self.searchservice(info)) matchfield = "service_name" nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "filter": {"bool": {"must": [ {"match": {"ports.state_state": "open"}}, {"match": {"ports.%s" % matchfield: info}}, ]}}, "aggs": {"patterns": { "terms": dict( baseterms, script=""" String result = ""; if(doc['ports.service_name'].size() > 0) { result += doc['ports.service_name'].value; } result += "###"; if(doc['ports.service_product'].size() > 0) { result += doc['ports.service_product'].value; } return result; """, ), }}, }}, } elif field == 'version': def outputproc(value): return tuple(v or None for v in value.split('###', 2)) flt = self.flt_and(flt, self.searchopenport()) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "filter": {"match": {"ports.state_state": "open"}}, "aggs": {"patterns": { "terms": dict( baseterms, script=""" String result = ""; if(doc['ports.service_name'].size() > 0) { result += doc['ports.service_name'].value; } result += "###"; if(doc['ports.service_product'].size() > 0) { result += doc['ports.service_product'].value; } result += "###"; if(doc['ports.service_version'].size() > 0) { result += doc['ports.service_version'].value; } return result; """, missing="", ), }}, }}, } elif field.startswith('version:'): def outputproc(value): return tuple(v or None for v in value.split('###', 2)) info = field[8:] if info.isdigit(): port = int(info) flt = self.flt_and(flt, self.searchport(port)) matchflt = Q("match", ports__port=port) elif ":" in info: service, product = info.split(':', 1) flt = self.flt_and(flt, self.searchproduct( product=product, service=service, )) matchflt = ( Q("match", ports__service_name=service) & Q("match", ports__service_product=product) ) else: flt = self.flt_and(flt, self.searchservice(info)) matchflt = Q("match", ports__service_name=info) matchflt &= Q("match", ports__state_state="open") nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "filter": matchflt.to_dict(), "aggs": {"patterns": { "terms": dict( baseterms, script=""" String result = ""; if(doc['ports.service_name'].size() > 0) { result += doc['ports.service_name'].value; } result += "###"; if(doc['ports.service_product'].size() > 0) { result += doc['ports.service_product'].value; } result += "###"; if(doc['ports.service_version'].size() > 0) { result += doc['ports.service_version'].value; } return result; """, ), }}, }}, } elif field == 'httphdr': def outputproc(value): return tuple(value.split(':', 1)) flt = self.flt_and(flt, self.searchscript(name="http-headers")) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts.http-headers"}, "aggs": {"patterns": { "terms": dict( baseterms, script={ "lang": "painless", "source": "doc['ports.scripts.http-headers.name']." "value + ':' + doc['ports.scripts.http-" "headers.value'].value" }, ) }}, }}, }}, } elif field.startswith('httphdr.'): flt = self.flt_and(flt, self.searchscript(name="http-headers")) field = "ports.scripts.http-headers.%s" % field[8:] nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts.http-headers"}, "aggs": {"patterns": { "terms": dict( baseterms, field=field ), }}, }}, }}, } elif field.startswith('httphdr:'): subfield = field[8:].lower() flt = self.flt_and(flt, self.searchscript(name="http-headers", values={"name": subfield})) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts.http-headers"}, "aggs": {"patterns": { "filter": {"match": { "ports.scripts.http-headers.name": subfield, }}, "aggs": {"patterns": { "terms": dict( baseterms, field='ports.scripts.http-headers.value', ), }}, }}, }}, }}, } elif field == 'useragent' or field.startswith('useragent:'): if field == 'useragent': flt = self.flt_and(flt, self.searchuseragent()) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": { "terms": dict( baseterms, field="ports.scripts.http-user-agent", ), }}, }}, } else: subfield = utils.str2regexp(field[10:]) flt = self.flt_and(flt, self.searchuseragent(useragent=subfield)) if isinstance(subfield, utils.REGEXP_T): subfield = self._get_pattern(subfield) else: subfield = re.escape(subfield) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": { "terms": dict( baseterms, field="ports.scripts.http-user-agent", include=subfield, ), }}, }}, } elif field == 'ja3-client' or ( field.startswith('ja3-client') and field[10] in ':.' ): if ':' in field: field, value = field.split(':', 1) subkey, value = self._ja3keyvalue(utils.str2regexp(value)) if isinstance(value, utils.REGEXP_T): include_value = self._get_pattern(value) filter_value = {'regexp': { "ports.scripts.ssl-ja3-client.%s" % subkey: include_value, }} else: include_value = re.escape(value) filter_value = {'match': { "ports.scripts.ssl-ja3-client.%s" % subkey: value, }} else: value = None subkey = None if '.' in field: field, subfield = field.split('.', 1) else: subfield = 'md5' base = { "terms": dict( baseterms, field="ports.scripts.ssl-ja3-client.%s" % subfield, ), } if subkey is not None: if subkey != subfield: base = { "filter": filter_value, "aggs": {"patterns": base}, } else: base["terms"]["include"] = include_value flt = self.flt_and(flt, self.searchja3client(value_or_hash=value)) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts.ssl-ja3-client"}, "aggs": {"patterns": base}, }}, }}, } elif field == 'ja3-server' or ( field.startswith('ja3-server') and field[10] in ':.' ): def outputproc(value): return tuple(value.split('/')) if ':' in field: field, values = field.split(':', 1) if ':' in values: value1, value2 = values.split(':', 1) if value1: subkey1, value1 = self._ja3keyvalue( utils.str2regexp(value1) ) if isinstance(value1, utils.REGEXP_T): filter_value1 = {'regexp': { "ports.scripts.ssl-ja3-server.%s" % subkey1: self._get_pattern(value1), }} else: filter_value1 = {'match': { "ports.scripts.ssl-ja3-server.%s" % subkey1: value1, }} else: subkey1, value1 = None, None if value2: subkey2, value2 = self._ja3keyvalue( utils.str2regexp(value2) ) if isinstance(value2, utils.REGEXP_T): filter_value2 = {'regexp': { "ports.scripts.ssl-ja3-server.client.%s" % subkey2: self._get_pattern(value2), }} else: filter_value2 = {'match': { "ports.scripts.ssl-ja3-server.client.%s" % subkey2: value2, }} else: subkey2, value2 = None, None else: subkey1, value1 = self._ja3keyvalue( utils.str2regexp(values) ) if isinstance(value1, utils.REGEXP_T): filter_value1 = {'regexp': { "ports.scripts.ssl-ja3-server.%s" % subkey1: self._get_pattern(value1), }} else: filter_value1 = {'match': { "ports.scripts.ssl-ja3-server.%s" % subkey1: value1, }} subkey2, value2 = None, None else: subkey1, value1 = None, None subkey2, value2 = None, None if '.' in field: field, subfield = field.split('.', 1) else: subfield = 'md5' flt = self.flt_and(flt, self.searchja3server( value_or_hash=value1, client_value_or_hash=value2, )) base = { "terms": dict( baseterms, script={ "lang": "painless", "source": "doc['ports.scripts.ssl-ja3-server.%s'].value + '/' + " "doc['ports.scripts.ssl-ja3-server.client.%s'].value" % (subfield, subfield), }, ), } if value1 is not None: base = { "filter": filter_value1, "aggs": {"patterns": base}, } if value2 is not None: base = { "filter": filter_value2, "aggs": {"patterns": base}, } flt = self.flt_and(flt, self.searchja3server( value_or_hash=value1, client_value_or_hash=value2, )) nested = { "nested": {"path": "ports"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts"}, "aggs": {"patterns": { "nested": {"path": "ports.scripts.ssl-ja3-server"}, "aggs": {"patterns": base}, }}, }}, } elif field.startswith('s7.'): flt = self.flt_and(flt, self.searchscript(name="s7-info")) subfield = field[3:] field = {'field': 'ports.scripts.s7-info.' + subfield} else: field = {"field": field} body = {"query": flt.to_dict()} if nested is None: body["aggs"] = {"patterns": {"terms": dict(baseterms, **field)}} else: body["aggs"] = {"patterns": nested} utils.LOGGER.debug("DB: Elasticsearch aggregation: %r", body) result = self.db_client.search( body=body, index=self.indexes[0], ignore_unavailable=True, size=0 ) result = result["aggregations"] while 'patterns' in result: result = result['patterns'] result = result['buckets'] if outputproc is None: for res in result: yield {'_id': res['key'], 'count': res['doc_count']} else: for res in result: yield {'_id': outputproc(res['key']), 'count': res['doc_count']}
def main(): if USING_ARGPARSE: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('ips_or_macs', nargs='*', help='Display results for specified IP (or ' 'networks) or MAC addresses (or MAC address ' 'regexps).') else: parser = optparse.OptionParser(description=__doc__) parser.parse_args_orig = parser.parse_args def my_parse_args(): res = parser.parse_args_orig() res[0].ensure_value('ips_or_macs', res[1]) return res[0] parser.parse_args = my_parse_args parser.add_argument = parser.add_option parser.add_argument('-s', '--sensor') parser.add_argument('-c', '--count', action="store_true") parser.add_argument('-r', '--resolve', action="store_true", help="Resolve MAC manufacturer") args = parser.parse_args() flts = ([], []) # MAC & IP filters for arg in args.ips_or_macs: if arg[:1] in "-!~": neg = True arg = arg[1:] else: neg = False match = MAC_ADDR.search(arg) if match: flts[0].append(db.passive.searchmac(mac=arg, neg=neg)) elif arg.startswith('/') and '/' in arg[1:]: flts[0].append( db.passive.searchmac(mac=utils.str2regexp(arg), neg=neg)) elif '/' in arg: flts[1].append(db.passive.searchnet(arg, neg=neg)) else: flts[1].append(db.passive.searchhost(arg, neg=neg)) if not flts[0]: flts[0].append(db.passive.searchmac()) flt = db.passive.flt_or(*flts[0]) if flts[1]: flt = db.passive.flt_and(flt, db.passive.flt_or(*flts[1])) if args.sensor is not None: flt = db.passive.flt_and(flt, args.sensor) if args.count: print(db.passive.count(flt)) return for rec in db.passive.get(flt, sort=[('addr', 1), ('value', 1), ('source', 1)]): rec["times"] = "s" if rec["count"] > 1 else "" if not rec.get("sensor"): rec["sensor"] = "-" if args.resolve: try: manuf = utils.mac2manuf(rec['value'])[0] except (TypeError, ValueError): pass else: rec['value'] = '%s (%s)' % (rec['value'], manuf) print("%(addr)s at %(value)s on %(sensor)s (%(source)s %(count)s " "time%(times)s, %(firstseen)s - %(lastseen)s)" % rec)