def main() -> None:
baseflt = db.passive.searchrecontype("DNS_ANSWER")
subdomains = False
try:
opts, args = getopt.getopt(
sys.argv[1:],
"s:h",
[
# filters
"sensor=",
# subdomains
"sub",
"help",
],
)
except getopt.GetoptError as err:
sys.stderr.write(str(err) + "\n")
sys.exit(-1)
for o, a in opts:
if o in ["-s", "--sensor"]:
baseflt = db.passive.flt_and(
baseflt, db.passive.searchsensor(utils.str2list(a)))
elif o == "--sub":
subdomains = True
elif o in ["-h", "--help"]:
sys.stdout.write("usage: %s [-h] [-s SENSOR] [--sub]"
"\n\n" % (sys.argv[0]))
sys.stdout.write(__doc__)
sys.stdout.write("\n\n")
sys.exit(0)
else:
sys.stderr.write(
"%r %r not understood (this is probably a bug).\n" % (o, a))
sys.exit(-1)
first = True
flts = []
for a in args:
if first:
first = False
else:
print()
if utils.IPADDR.search(a) or a.isdigit():
flts.append(db.passive.flt_and(baseflt, db.passive.searchhost(a)))
else:
flts += [
db.passive.flt_and(
baseflt,
db.passive.searchdns(utils.str2regexp(a),
subdomains=subdomains),
),
db.passive.flt_and(
baseflt,
db.passive.searchdns(utils.str2regexp(a),
reverse=True,
subdomains=subdomains),
),
]
for flt in flts:
for r in db.passive.get(flt, sort=[("source", 1)]):
disp_rec(r)
def main():
baseflt = db.passive.searchrecontype('DNS_ANSWER')
subdomains = False
try:
opts, args = getopt.getopt(
sys.argv[1:],
"s:h",
[
# filters
"sensor=",
# subdomains
"sub",
"help",
],
)
except getopt.GetoptError as err:
sys.stderr.write(str(err) + '\n')
sys.exit(-1)
for o, a in opts:
if o in ['-s', '--sensor']:
baseflt = db.passive.flt_and(baseflt, db.passive.searchsensor(a))
elif o == '--sub':
subdomains = True
elif o in ['-h', '--help']:
sys.stdout.write('usage: %s [-h] [-s SENSOR] [--sub]'
'\n\n' % (sys.argv[0]))
sys.stdout.write(__doc__)
sys.stdout.write("\n\n")
sys.exit(0)
else:
sys.stderr.write(
'%r %r not undestood (this is probably a bug).\n' % (o, a))
sys.exit(-1)
first = True
flts = []
for a in args:
if first:
first = False
else:
print()
if IPADDR.match(a) or a.isdigit():
flts.append(db.passive.flt_and(baseflt, db.passive.searchhost(a)))
else:
flts += [
db.passive.flt_and(
baseflt,
db.passive.searchdns(utils.str2regexp(a),
subdomains=subdomains)),
db.passive.flt_and(
baseflt,
db.passive.searchdns(utils.str2regexp(a),
reverse=True,
subdomains=subdomains))
]
for flt in flts:
for r in db.passive.get(flt, sort=[('source', 1)]):
disp_rec(r)
def main():
baseflt = db.passive.searchrecontype('DNS_ANSWER')
subdomains = False
try:
opts, args = getopt.getopt(
sys.argv[1:],
"s:h",
[
# filters
"sensor=",
# subdomains
"sub",
"help",
],
)
except getopt.GetoptError as err:
sys.stderr.write(str(err) + '\n')
sys.exit(-1)
for o, a in opts:
if o in ['-s', '--sensor']:
baseflt = db.passive.flt_and(baseflt, db.passive.searchsensor(a))
elif o == '--sub':
subdomains = True
elif o in ['-h', '--help']:
sys.stdout.write('usage: %s [-h] [-s SENSOR] [--sub]'
'\n\n' % (sys.argv[0]))
sys.stdout.write(__doc__)
sys.stdout.write("\n\n")
sys.exit(0)
else:
sys.stderr.write(
'%r %r not understood (this is probably a bug).\n' % (o, a))
sys.exit(-1)
first = True
flts = []
for a in args:
if first:
first = False
else:
print()
if IPADDR.match(a) or a.isdigit():
flts.append(db.passive.flt_and(baseflt, db.passive.searchhost(a)))
else:
flts += [
db.passive.flt_and(
baseflt,
db.passive.searchdns(
utils.str2regexp(a), subdomains=subdomains)),
db.passive.flt_and(
baseflt,
db.passive.searchdns(
utils.str2regexp(a),
reverse=True, subdomains=subdomains))
]
for flt in flts:
for r in db.passive.get(flt, sort=[('source', 1)]):
disp_rec(r)
def str2regexpnone(value):
"""Just like str2regexp, but handle special '-' value, which means
False.
"""
if value == "-":
return False
return utils.str2regexp(value)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"ips_or_macs",
nargs="*",
help=("Display results for specified IP (or networks) or MAC addresses"
" (or MAC address regexps)."),
)
parser.add_argument("-s", "--sensor")
parser.add_argument("-c", "--count", action="store_true")
parser.add_argument("-r",
"--resolve",
action="store_true",
help="Resolve MAC manufacturer")
args = parser.parse_args()
flts = ([], []) # MAC & IP filters
for arg in args.ips_or_macs:
if arg[:1] in "-!~":
neg = True
arg = arg[1:]
else:
neg = False
match = MAC_ADDR.search(arg)
if match:
flts[0].append(db.passive.searchmac(mac=arg, neg=neg))
elif arg.startswith("/") and "/" in arg[1:]:
flts[0].append(
db.passive.searchmac(mac=utils.str2regexp(arg), neg=neg))
elif "/" in arg:
flts[1].append(db.passive.searchnet(arg, neg=neg))
else:
flts[1].append(db.passive.searchhost(arg, neg=neg))
if not flts[0]:
flts[0].append(db.passive.searchmac())
flt = db.passive.flt_or(*flts[0])
if flts[1]:
flt = db.passive.flt_and(flt, db.passive.flt_or(*flts[1]))
if args.sensor is not None:
flt = db.passive.flt_and(flt, db.passive.searchsensor(args.sensor))
if args.count:
print(db.passive.count(flt))
return
for rec in db.passive.get(flt,
sort=[("addr", 1), ("value", 1), ("source", 1)]):
rec["times"] = "s" if rec["count"] > 1 else ""
if not rec.get("sensor"):
rec["sensor"] = "-"
if args.resolve:
try:
manuf = utils.mac2manuf(rec["value"])[0]
except (TypeError, ValueError):
pass
else:
rec["value"] = "%s (%s)" % (rec["value"], manuf)
print("%(addr)s at %(value)s on %(sensor)s (%(source)s %(count)s "
"time%(times)s, %(firstseen)s - %(lastseen)s)" % rec)
def flt_from_query(query, base_flt=None):
"""Return a tuple (`flt`, `archive`, `sortby`, `unused`, `skip`,
`limit`):
- a filter based on the query
- a boolean (`True` iff the filter should be applied to the
archive collection)
- a list of [`key`, `order`] to sort results
- a list of the unused elements of the query (errors)
- an integer for the number of results to skip
- an integer for the maximum number of results to return
"""
unused = []
sortby = []
archive = False
skip = 0
limit = None
flt = get_init_flt() if base_flt is None else base_flt
def add_unused(neg, param, value):
"""Add to the `unused` list a string representing (neg, param,
value).
"""
unused.append("%s%s" %
('-' if neg else '', "%s=%s" %
(param, value) if value is not None else param))
for (neg, param, value) in query:
if not neg and param == 'skip':
skip = int(value)
elif not neg and param == 'limit':
limit = int(value)
elif param == "archives":
archive = not neg
elif param == "id":
flt = db.nmap.flt_and(
flt,
db.nmap.searchobjectid(value.replace('-', ',').split(','),
neg=neg))
elif param == "host":
flt = db.nmap.flt_and(flt, db.nmap.searchhost(value, neg=neg))
elif param == "net":
flt = db.nmap.flt_and(flt, db.nmap.searchnet(value, neg=neg))
elif param == "range":
flt = db.nmap.flt_and(
flt,
db.nmap.searchrange(*value.replace('-', ',').split(',', 1),
neg=neg))
elif param == "label":
group, lab = ((None, None) if value is None else map(
utils.str2regexp, value.split(':', 1)) if ':' in value else
(utils.str2regexp(value), None))
flt = db.nmap.flt_and(
flt, db.nmap.searchlabel(group=group, label=lab, neg=neg))
elif param == "countports":
vals = [int(val) for val in value.replace('-', ',').split(',', 1)]
if len(vals) == 1:
flt = db.nmap.flt_and(
flt,
db.nmap.searchcountopenports(minn=vals[0],
maxn=vals[0],
neg=neg))
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchcountopenports(minn=vals[0],
maxn=vals[1],
neg=neg))
elif param == "hostname":
flt = db.nmap.flt_and(
flt, db.nmap.searchhostname(utils.str2regexp(value), neg=neg))
elif param == "domain":
flt = db.nmap.flt_and(
flt, db.nmap.searchdomain(utils.str2regexp(value), neg=neg))
elif param == "category":
flt = db.nmap.flt_and(
flt, db.nmap.searchcategory(utils.str2regexp(value), neg=neg))
elif param == "country":
flt = db.nmap.flt_and(
flt,
db.nmap.searchcountry(utils.str2list(value.upper()), neg=neg))
elif param == "city":
flt = db.nmap.flt_and(
flt, db.nmap.searchcity(utils.str2regexp(value), neg=neg))
elif param == "asnum":
flt = db.nmap.flt_and(
flt, db.nmap.searchasnum(utils.str2list(value), neg=neg))
elif param == "asname":
flt = db.nmap.flt_and(
flt, db.nmap.searchasname(utils.str2regexp(value), neg=neg))
elif param == "source":
flt = db.nmap.flt_and(flt, db.nmap.searchsource(value, neg=neg))
elif param == "timerange":
flt = db.nmap.flt_and(
flt,
db.nmap.searchtimerange(*map(
float,
value.replace('-', ',').split(',')),
neg=neg))
elif param == 'timeago':
if value and value[-1].isalpha():
unit = {
's': 1,
'm': 60,
'h': 3600,
'd': 86400,
'y': 31557600,
}[value[-1]]
timeago = int(value[:-1]) * unit
else:
timeago = int(value)
flt = db.nmap.flt_and(
flt,
db.nmap.searchtimeago(datetime.timedelta(0, timeago), neg=neg))
elif not neg and param == "service":
if ':' in value:
req, port = value.split(':', 1)
port = int(port)
flt = db.nmap.flt_and(
flt, db.nmap.searchservice(utils.str2regexp(req),
port=port))
else:
flt = db.nmap.flt_and(
flt, db.nmap.searchservice(utils.str2regexp(value)))
elif not neg and param == "product" and ":" in value:
product = value.split(':', 2)
if len(product) == 2:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(utils.str2regexp(product[1]),
service=utils.str2regexp(
product[0])))
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(utils.str2regexp(product[1]),
service=utils.str2regexp(product[0]),
port=int(product[2])))
elif not neg and param == "version" and value.count(":") >= 2:
product = value.split(':', 3)
if len(product) == 3:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(
utils.str2regexp(product[1]),
version=utils.str2regexp(product[2]),
service=utils.str2regexp(product[0]),
))
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(utils.str2regexp(product[1]),
version=utils.str2regexp(product[2]),
service=utils.str2regexp(product[0]),
port=int(product[3])))
elif not neg and param == "script":
value = value.split(':', 1)
if len(value) == 1:
flt = db.nmap.flt_and(
flt,
db.nmap.searchscript(name=utils.str2regexp(value[0])),
)
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchscript(
name=utils.str2regexp(value[0]),
output=utils.str2regexp(value[1]),
),
)
# results of scripts or version scans
elif not neg and param == "anonftp":
flt = db.nmap.flt_and(flt, db.nmap.searchftpanon())
elif not neg and param == 'anonldap':
flt = db.nmap.flt_and(flt, db.nmap.searchldapanon())
elif not neg and param == 'authbypassvnc':
flt = db.nmap.flt_and(flt, db.nmap.searchvncauthbypass())
elif not neg and param == "authhttp":
flt = db.nmap.flt_and(flt, db.nmap.searchhttpauth())
elif not neg and param == 'banner':
flt = db.nmap.flt_and(
flt, db.nmap.searchbanner(utils.str2regexp(value)))
elif param == 'cookie':
flt = db.nmap.flt_and(flt, db.nmap.searchcookie(value))
elif param == 'file':
if value is None:
flt = db.nmap.flt_and(flt, db.nmap.searchfile())
else:
value = value.split(':', 1)
if len(value) == 1:
flt = db.nmap.flt_and(
flt,
db.nmap.searchfile(fname=utils.str2regexp(value[0])))
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchfile(fname=utils.str2regexp(value[1]),
scripts=value[0].split(',')))
elif not neg and param == 'geovision':
flt = db.nmap.flt_and(flt, db.nmap.searchgeovision())
elif param == 'httptitle':
flt = db.nmap.flt_and(
flt, db.nmap.searchhttptitle(utils.str2regexp(value)))
elif not neg and param == "nfs":
flt = db.nmap.flt_and(flt, db.nmap.searchnfs())
elif not neg and param in ["nis", "yp"]:
flt = db.nmap.flt_and(flt, db.nmap.searchypserv())
elif not neg and param == 'mssqlemptypwd':
flt = db.nmap.flt_and(flt, db.nmap.searchmssqlemptypwd())
elif not neg and param == 'mysqlemptypwd':
flt = db.nmap.flt_and(flt, db.nmap.searchmysqlemptypwd())
elif not neg and param == 'sshkey':
if value:
flt = db.nmap.flt_and(
flt, db.nmap.searchsshkey(output=utils.str2regexp(value)))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchsshkey())
elif not neg and param.startswith('sshkey.'):
subfield = param.split('.', 1)[1]
if subfield in ['fingerprint', 'key', 'type', 'bits']:
if subfield == 'type':
subfield = 'keytype'
flt = db.nmap.flt_and(
flt,
db.nmap.searchsshkey(
**{subfield: utils.str2regexp(value)}))
else:
add_unused(neg, param, value)
elif not neg and param == 'owa':
flt = db.nmap.flt_and(flt, db.nmap.searchowa())
elif param == 'phpmyadmin':
flt = db.nmap.flt_and(flt, db.nmap.searchphpmyadmin())
elif not neg and param.startswith('smb.'):
flt = db.nmap.flt_and(
flt, db.nmap.searchsmb(**{param[4:]: utils.str2regexp(value)}))
elif not neg and param == 'smbshare':
flt = db.nmap.flt_and(
flt,
db.nmap.searchsmbshares(access="" if value is None else value),
)
elif param == 'torcert':
flt = db.nmap.flt_and(flt, db.nmap.searchtorcert())
elif not neg and param == 'webfiles':
flt = db.nmap.flt_and(flt, db.nmap.searchwebfiles())
elif not neg and param == "webmin":
flt = db.nmap.flt_and(flt, db.nmap.searchwebmin())
elif not neg and param == 'x11srv':
flt = db.nmap.flt_and(flt, db.nmap.searchx11())
elif not neg and param == 'x11open':
flt = db.nmap.flt_and(flt, db.nmap.searchx11access())
elif not neg and param == 'xp445':
flt = db.nmap.flt_and(flt, db.nmap.searchxp445())
# OS fingerprint
elif not neg and param == "os":
flt = db.nmap.flt_and(flt,
db.nmap.searchos(utils.str2regexp(value)))
# device types
elif param in ['devicetype', 'devtype']:
flt = db.nmap.flt_and(
flt, db.nmap.searchdevicetype(utils.str2regexp(value)))
elif param in ['netdev', 'networkdevice']:
flt = db.nmap.flt_and(flt, db.nmap.searchnetdev())
elif param == 'phonedev':
flt = db.nmap.flt_and(flt, db.nmap.searchphonedev())
# traceroute
elif param == 'hop':
if ':' in value:
hop, ttl = value.split(':', 1)
flt = db.nmap.flt_and(
flt, db.nmap.searchhop(hop, ttl=int(ttl), neg=neg))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchhop(value, neg=neg))
elif param == 'hopname':
flt = db.nmap.flt_and(flt, db.nmap.searchhopname(value, neg=neg))
elif param == 'hopdomain':
flt = db.nmap.flt_and(flt, db.nmap.searchhopdomain(value, neg=neg))
# sort
elif param == 'sortby':
if neg:
sortby.append((value, -1))
else:
sortby.append((value, 1))
elif param in ['open', 'filtered', 'closed']:
if '_' in value:
value = value.replace('_', '/')
if '/' in value:
proto, port = value.split('/')
else:
proto, port = "tcp", value
port = port.split(',')
if len(port) > 1:
flt = db.nmap.flt_and(
flt,
db.nmap.searchports([int(p) for p in port],
protocol=proto,
state=param))
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchport(int(port[0]),
protocol=proto,
state=param))
elif param == 'otheropenport':
flt = db.nmap.flt_and(
flt, db.nmap.searchportsother(map(int, value.split(','))))
elif param == "screenshot":
if value is None:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(neg=neg))
elif value.isdigit():
flt = db.nmap.flt_and(
flt, db.nmap.searchscreenshot(port=int(value), neg=neg))
elif value.startswith('tcp/') or value.startswith('udp/'):
value = value.split('/', 1)
flt = db.nmap.flt_and(
flt,
db.nmap.searchscreenshot(port=int(value[1]),
protocol=value[0],
neg=neg))
else:
flt = db.nmap.flt_and(
flt, db.nmap.searchscreenshot(service=value, neg=neg))
elif param == "screenwords":
if value is None:
flt = db.nmap.flt_and(flt,
db.nmap.searchscreenshot(words=not neg))
else:
params = value.split(':', 1)
words = (map(utils.str2regexp, params[0].split(','))
if ',' in params[0] else utils.str2regexp(params[0]))
if len(params) == 1:
flt = db.nmap.flt_and(
flt, db.nmap.searchscreenshot(words=words, neg=neg))
elif params[1].isdigit():
flt = db.nmap.flt_and(
flt,
db.nmap.searchscreenshot(port=int(value),
neg=neg,
words=words))
elif (params[1].startswith('tcp/')
or params[1].startswith('udp/')):
params[1] = params[1].split('/', 1)
flt = db.nmap.flt_and(
flt,
db.nmap.searchscreenshot(port=int(params[1][1]),
protocol=params[1][0],
neg=neg,
words=words))
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchscreenshot(service=value,
neg=neg,
words=words))
elif param == "cpe":
if value:
cpe_kwargs = {}
cpe_fields = ["cpe_type", "vendor", "product", "version"]
for field, cpe_arg in zip(cpe_fields, value.split(':', 3)):
cpe_kwargs[field] = utils.str2regexp(cpe_arg)
flt = db.nmap.flt_and(flt, db.nmap.searchcpe(**cpe_kwargs))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchcpe())
elif param == 'display':
# ignore this parameter
pass
elif value is None:
if param.startswith('tcp_') or param.startswith('tcp/') or \
param.startswith('udp_') or param.startswith('udp/'):
proto, port = param.replace('_', '/').split('/', 1)
port = int(port)
flt = db.nmap.flt_and(
flt, db.nmap.searchport(port, protocol=proto, neg=neg))
elif param == "openport":
flt = db.nmap.flt_and(flt, db.nmap.searchopenport(neg=neg))
elif param.isdigit():
flt = db.nmap.flt_and(flt,
db.nmap.searchport(int(param), neg=neg))
elif all(x.isdigit() for x in param.split(',')):
flt = db.nmap.flt_and(
flt,
db.nmap.searchports(map(int, param.split(',')), neg=neg))
elif IPADDR.match(param):
flt = db.nmap.flt_and(flt, db.nmap.searchhost(param, neg=neg))
elif NETADDR.match(param):
flt = db.nmap.flt_and(flt, db.nmap.searchnet(param, neg=neg))
elif get_notepad_pages is not None and param == 'notes':
flt = db.nmap.flt_and(
flt, db.nmap.searchhosts(get_notepad_pages(), neg=neg))
elif '<' in param:
param = param.split('<', 1)
if param[1] and param[1][0] == '=':
flt = db.nmap.flt_and(
flt,
db.nmap.searchcmp(param[0], int(param[1][1:]),
'>' if neg else '<='))
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchcmp(param[0], int(param[1]),
'>=' if neg else '<'))
elif '>' in param:
param = param.split('>', 1)
if param[1] and param[1][0] == '=':
flt = db.nmap.flt_and(
flt,
db.nmap.searchcmp(param[0], int(param[1][1:]),
'<' if neg else '>='))
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchcmp(param[0], int(param[1]),
'<=' if neg else '>'))
else:
add_unused(neg, param, value)
else:
add_unused(neg, param, value)
return flt, archive, sortby, unused, skip, limit
def flt_from_query(query, base_flt=None):
"""Return a tuple (`flt`, `sortby`, `unused`, `skip`, `limit`):
- a filter based on the query
- a list of [`key`, `order`] to sort results
- a list of the unused elements of the query (errors)
- an integer for the number of results to skip
- an integer for the maximum number of results to return
"""
unused = []
sortby = []
skip = 0
limit = None
flt = get_init_flt() if base_flt is None else base_flt
def add_unused(neg, param, value):
"""Add to the `unused` list a string representing (neg, param,
value).
"""
unused.append("%s%s" %
('-' if neg else '', "%s=%s" %
(param, value) if value is not None else param))
for (neg, param, value) in query:
if not neg and param == 'skip':
skip = int(value)
elif not neg and param == 'limit':
limit = int(value)
elif param == "id":
flt = db.view.flt_and(
flt,
db.view.searchobjectid(value.replace('-', ',').split(','),
neg=neg))
elif param == "host":
flt = db.view.flt_and(flt, db.view.searchhost(value, neg=neg))
elif param == "net":
flt = db.view.flt_and(flt, db.view.searchnet(value, neg=neg))
elif param == "range":
flt = db.view.flt_and(
flt,
db.view.searchrange(*value.replace('-', ',').split(',', 1),
neg=neg))
elif param == "countports":
vals = [int(val) for val in value.replace('-', ',').split(',', 1)]
if len(vals) == 1:
flt = db.view.flt_and(
flt,
db.view.searchcountopenports(minn=vals[0],
maxn=vals[0],
neg=neg))
else:
flt = db.view.flt_and(
flt,
db.view.searchcountopenports(minn=vals[0],
maxn=vals[1],
neg=neg))
elif param == "hostname":
flt = db.view.flt_and(
flt, db.view.searchhostname(utils.str2regexp(value), neg=neg))
elif param == "domain":
flt = db.view.flt_and(
flt, db.view.searchdomain(utils.str2regexp(value), neg=neg))
elif param == "category":
flt = db.view.flt_and(
flt, db.view.searchcategory(utils.str2regexp(value), neg=neg))
elif param == "country":
flt = db.view.flt_and(
flt,
db.view.searchcountry(utils.str2list(value.upper()), neg=neg))
elif param == "city":
flt = db.view.flt_and(
flt, db.view.searchcity(utils.str2regexp(value), neg=neg))
elif param == "asnum":
flt = db.view.flt_and(
flt, db.view.searchasnum(utils.str2list(value), neg=neg))
elif param == "asname":
flt = db.view.flt_and(
flt, db.view.searchasname(utils.str2regexp(value), neg=neg))
elif param == "source":
flt = db.view.flt_and(flt, db.view.searchsource(value, neg=neg))
elif param == "timerange":
flt = db.view.flt_and(
flt,
db.view.searchtimerange(
*(float(val)
for val in value.replace('-', ',').split(',')),
neg=neg))
elif param == 'timeago':
if value and value[-1].isalpha():
unit = {
's': 1,
'm': 60,
'h': 3600,
'd': 86400,
'y': 31557600,
}[value[-1]]
timeago = int(value[:-1]) * unit
else:
timeago = int(value)
flt = db.view.flt_and(
flt,
db.view.searchtimeago(datetime.timedelta(0, timeago), neg=neg))
elif not neg and param == "service":
if ':' in value:
req, port = value.split(':', 1)
port = int(port)
flt = db.view.flt_and(
flt, db.view.searchservice(utils.str2regexp(req),
port=port))
else:
flt = db.view.flt_and(
flt, db.view.searchservice(utils.str2regexp(value)))
elif not neg and param == "product" and ":" in value:
product = value.split(':', 2)
if len(product) == 2:
flt = db.view.flt_and(
flt,
db.view.searchproduct(utils.str2regexp(product[1]),
service=utils.str2regexp(
product[0])))
else:
flt = db.view.flt_and(
flt,
db.view.searchproduct(utils.str2regexp(product[1]),
service=utils.str2regexp(product[0]),
port=int(product[2])))
elif not neg and param == "version" and value.count(":") >= 2:
product = value.split(':', 3)
if len(product) == 3:
flt = db.view.flt_and(
flt,
db.view.searchproduct(
utils.str2regexp(product[1]),
version=utils.str2regexp(product[2]),
service=utils.str2regexp(product[0]),
))
else:
flt = db.view.flt_and(
flt,
db.view.searchproduct(utils.str2regexp(product[1]),
version=utils.str2regexp(product[2]),
service=utils.str2regexp(product[0]),
port=int(product[3])))
elif param == "script":
value = value.split(':', 1)
if len(value) == 1:
flt = db.view.flt_and(
flt,
db.view.searchscript(name=utils.str2regexp(value[0]),
neg=neg),
)
else:
flt = db.view.flt_and(
flt,
db.view.searchscript(name=utils.str2regexp(value[0]),
output=utils.str2regexp(value[1]),
neg=neg),
)
# results of scripts or version scans
elif not neg and param == "anonftp":
flt = db.view.flt_and(flt, db.view.searchftpanon())
elif not neg and param == 'anonldap':
flt = db.view.flt_and(flt, db.view.searchldapanon())
elif not neg and param == 'authbypassvnc':
flt = db.view.flt_and(flt, db.view.searchvncauthbypass())
elif not neg and param == "authhttp":
flt = db.view.flt_and(flt, db.view.searchhttpauth())
elif not neg and param == 'banner':
flt = db.view.flt_and(
flt, db.view.searchbanner(utils.str2regexp(value)))
elif param == 'cookie':
flt = db.view.flt_and(flt, db.view.searchcookie(value))
elif param == 'file':
if value is None:
flt = db.view.flt_and(flt, db.view.searchfile())
else:
value = value.split(':', 1)
if len(value) == 1:
flt = db.view.flt_and(
flt,
db.view.searchfile(fname=utils.str2regexp(value[0])))
else:
flt = db.view.flt_and(
flt,
db.view.searchfile(fname=utils.str2regexp(value[1]),
scripts=value[0].split(',')))
elif param == 'vuln':
try:
vulnid, status = value.split(':', 1)
except ValueError:
vulnid = value
status = None
except AttributeError:
vulnid = None
status = None
flt = db.view.flt_and(
flt, db.view.searchvuln(vulnid=vulnid, status=status))
elif not neg and param == 'geovision':
flt = db.view.flt_and(flt, db.view.searchgeovision())
elif param == 'httptitle':
flt = db.view.flt_and(
flt, db.view.searchhttptitle(utils.str2regexp(value)))
elif not neg and param == "nfs":
flt = db.view.flt_and(flt, db.view.searchnfs())
elif not neg and param in ["nis", "yp"]:
flt = db.view.flt_and(flt, db.view.searchypserv())
elif not neg and param == 'mssqlemptypwd':
flt = db.view.flt_and(flt, db.view.searchmssqlemptypwd())
elif not neg and param == 'mysqlemptypwd':
flt = db.view.flt_and(flt, db.view.searchmysqlemptypwd())
elif not neg and param == 'sshkey':
if value:
flt = db.view.flt_and(
flt, db.view.searchsshkey(output=utils.str2regexp(value)))
else:
flt = db.view.flt_and(flt, db.view.searchsshkey())
elif not neg and param.startswith('sshkey.'):
subfield = param.split('.', 1)[1]
if subfield in ['fingerprint', 'key', 'type', 'bits']:
if subfield == 'type':
subfield = 'keytype'
elif subfield == 'bits':
try:
value = int(value)
except (ValueError, TypeError):
pass
else:
value = utils.str2regexp(value)
flt = db.view.flt_and(
flt, db.view.searchsshkey(**{subfield: value}))
else:
add_unused(neg, param, value)
elif not neg and param == 'httphdr':
if value is None:
flt = db.view.flt_and(flt, db.view.searchhttphdr())
elif ':' in value:
name, value = (utils.str2regexp(string)
for string in value.split(':', 1))
flt = db.view.flt_and(
flt, db.view.searchhttphdr(name=name, value=value))
else:
flt = db.view.flt_and(
flt, db.view.searchhttphdr(name=utils.str2regexp(value)))
elif not neg and param == 'owa':
flt = db.view.flt_and(flt, db.view.searchowa())
elif param == 'phpmyadmin':
flt = db.view.flt_and(flt, db.view.searchphpmyadmin())
elif not neg and param.startswith('smb.'):
flt = db.view.flt_and(
flt, db.view.searchsmb(**{param[4:]: utils.str2regexp(value)}))
elif not neg and param == 'smbshare':
flt = db.view.flt_and(
flt,
db.view.searchsmbshares(access="" if value is None else value),
)
elif param == 'torcert':
flt = db.view.flt_and(flt, db.view.searchtorcert())
elif not neg and param == 'webfiles':
flt = db.view.flt_and(flt, db.view.searchwebfiles())
elif not neg and param == "webmin":
flt = db.view.flt_and(flt, db.view.searchwebmin())
elif not neg and param == 'x11srv':
flt = db.view.flt_and(flt, db.view.searchx11())
elif not neg and param == 'x11open':
flt = db.view.flt_and(flt, db.view.searchx11access())
elif not neg and param == 'xp445':
flt = db.view.flt_and(flt, db.view.searchxp445())
elif param == "ssl-ja3-client":
flt = db.view.flt_and(
flt,
db.view.searchja3client(
value_or_hash=(None if value is None else
utils.str2regexp(value)),
neg=neg))
elif param == "ssl-ja3-server":
if value is None:
# There are no additional arguments
flt = db.view.flt_and(flt, db.view.searchja3server(neg=neg))
else:
split = [
utils.str2regexp(v) if v else None
for v in value.split(':', 1)
]
if len(split) == 1:
# Only a JA3 server is given
flt = db.view.flt_and(
flt,
db.view.searchja3server(
value_or_hash=(split[0]),
neg=neg,
))
else:
# Both client and server JA3 are specified
flt = db.view.flt_and(
flt,
db.view.searchja3server(
value_or_hash=split[0],
client_value_or_hash=split[1],
neg=neg,
))
elif param == "useragent":
if value:
flt = db.view.flt_and(
flt,
db.view.searchuseragent(useragent=utils.str2regexp(value)))
else:
flt = db.view.flt_and(flt, db.view.searchuseragent())
# OS fingerprint
elif not neg and param == "os":
flt = db.view.flt_and(flt,
db.view.searchos(utils.str2regexp(value)))
# device types
elif param in ['devicetype', 'devtype']:
flt = db.view.flt_and(
flt, db.view.searchdevicetype(utils.str2regexp(value)))
elif param in ['netdev', 'networkdevice']:
flt = db.view.flt_and(flt, db.view.searchnetdev())
elif param == 'phonedev':
flt = db.view.flt_and(flt, db.view.searchphonedev())
# traceroute
elif param == 'hop':
if ':' in value:
hop, ttl = value.split(':', 1)
flt = db.view.flt_and(
flt, db.view.searchhop(hop, ttl=int(ttl), neg=neg))
else:
flt = db.view.flt_and(flt, db.view.searchhop(value, neg=neg))
elif param == 'hopname':
flt = db.view.flt_and(flt, db.view.searchhopname(value, neg=neg))
elif param == 'hopdomain':
flt = db.view.flt_and(flt, db.view.searchhopdomain(value, neg=neg))
elif not neg and param in [
"ike.vendor_id.name", "ike.vendor_id.value"
]:
flt = db.view.flt_and(
flt,
db.view.searchscript(
name="ike-info",
values={
'vendor_ids.%s' % param[14:]: utils.str2regexp(value)
},
),
)
elif not neg and param == "ike.notification":
flt = db.view.flt_and(
flt,
db.view.searchscript(
name="ike-info",
values={'notification_type': utils.str2regexp(value)},
),
)
# sort
elif param == 'sortby':
if neg:
sortby.append((value, -1))
else:
sortby.append((value, 1))
elif param in ['open', 'filtered', 'closed']:
value = value.replace('_', '/').split(',')
protos = {}
for port in value:
if '/' in port:
proto, port = port.split('/')
else:
proto, port = "tcp", port
protos.setdefault(proto, []).append(int(port))
for proto, ports in viewitems(protos):
flt = db.view.flt_and(
flt,
db.view.searchport(ports[0], protocol=proto, state=param)
if len(ports) == 1 else db.view.searchports(
ports, protocol=proto, state=param))
elif param == 'otheropenport':
flt = db.view.flt_and(
flt,
db.view.searchportsother(
[int(val) for val in value.split(',')]))
elif param == "screenshot":
if value is None:
flt = db.view.flt_and(flt, db.view.searchscreenshot(neg=neg))
elif value.isdigit():
flt = db.view.flt_and(
flt, db.view.searchscreenshot(port=int(value), neg=neg))
elif value.startswith('tcp/') or value.startswith('udp/'):
value = value.split('/', 1)
flt = db.view.flt_and(
flt,
db.view.searchscreenshot(port=int(value[1]),
protocol=value[0],
neg=neg))
else:
flt = db.view.flt_and(
flt, db.view.searchscreenshot(service=value, neg=neg))
elif param == "screenwords":
if value is None:
flt = db.view.flt_and(flt,
db.view.searchscreenshot(words=not neg))
else:
params = value.split(':', 1)
words = ([
utils.str2regexp(elt) for elt in params[0].split(',')
] if ',' in params[0] else utils.str2regexp(params[0]))
if len(params) == 1:
flt = db.view.flt_and(
flt, db.view.searchscreenshot(words=words, neg=neg))
elif params[1].isdigit():
flt = db.view.flt_and(
flt,
db.view.searchscreenshot(port=int(value),
neg=neg,
words=words))
elif (params[1].startswith('tcp/')
or params[1].startswith('udp/')):
params[1] = params[1].split('/', 1)
flt = db.view.flt_and(
flt,
db.view.searchscreenshot(port=int(params[1][1]),
protocol=params[1][0],
neg=neg,
words=words))
else:
flt = db.view.flt_and(
flt,
db.view.searchscreenshot(service=value,
neg=neg,
words=words))
elif param == "cpe":
if value:
cpe_kwargs = {}
cpe_fields = ["cpe_type", "vendor", "product", "version"]
for field, cpe_arg in zip(cpe_fields, value.split(':', 3)):
cpe_kwargs[field] = utils.str2regexp(cpe_arg)
flt = db.view.flt_and(flt, db.view.searchcpe(**cpe_kwargs))
else:
flt = db.view.flt_and(flt, db.view.searchcpe())
elif param == 'display':
# ignore this parameter
pass
elif value is None:
if param.startswith('tcp_') or param.startswith('tcp/') or \
param.startswith('udp_') or param.startswith('udp/'):
proto, port = param.replace('_', '/').split('/', 1)
port = int(port)
flt = db.view.flt_and(
flt, db.view.searchport(port, protocol=proto, neg=neg))
elif param == "openport":
flt = db.view.flt_and(flt, db.view.searchopenport(neg=neg))
elif param.isdigit():
flt = db.view.flt_and(flt,
db.view.searchport(int(param), neg=neg))
elif all(x.isdigit() for x in param.split(',')):
flt = db.view.flt_and(
flt,
db.view.searchports([int(val) for val in param.split(',')],
neg=neg))
elif IPADDR.match(param):
flt = db.view.flt_and(flt, db.view.searchhost(param, neg=neg))
elif NETADDR.match(param):
flt = db.view.flt_and(flt, db.view.searchnet(param, neg=neg))
elif get_notepad_pages is not None and param == 'notes':
flt = db.view.flt_and(
flt, db.view.searchhosts(get_notepad_pages(), neg=neg))
elif '<' in param:
param = param.split('<', 1)
if param[1] and param[1][0] == '=':
flt = db.view.flt_and(
flt,
db.view.searchcmp(param[0], int(param[1][1:]),
'>' if neg else '<='))
else:
flt = db.view.flt_and(
flt,
db.view.searchcmp(param[0], int(param[1]),
'>=' if neg else '<'))
elif '>' in param:
param = param.split('>', 1)
if param[1] and param[1][0] == '=':
flt = db.view.flt_and(
flt,
db.view.searchcmp(param[0], int(param[1][1:]),
'<' if neg else '>='))
else:
flt = db.view.flt_and(
flt,
db.view.searchcmp(param[0], int(param[1]),
'<=' if neg else '>'))
else:
add_unused(neg, param, value)
else:
add_unused(neg, param, value)
return flt, sortby, unused, skip, limit
def flt_from_query(query, base_flt=None):
"""Return a tuple (`flt`, `archive`, `sortby`, `unused`, `skip`,
`limit`):
- a filter based on the query
- a boolean (`True` iff the filter should be applied to the
archive collection)
- a list of [`key`, `order`] to sort results
- a list of the unused elements of the query (errors)
- an integer for the number of results to skip
- an integer for the maximum number of results to return
"""
unused = []
sortby = []
archive = False
skip = 0
limit = None
flt = get_init_flt() if base_flt is None else base_flt
def add_unused(neg, param, value):
"""Add to the `unused` list a string representing (neg, param,
value).
"""
unused.append("%s%s" % (
'-' if neg else '',
"%s=%s" % (param, value) if value is not None else param
))
for (neg, param, value) in query:
if not neg and param == 'skip':
skip = int(value)
elif not neg and param == 'limit':
limit = int(value)
elif param == "archives":
archive = not neg
elif param == "id":
flt = db.nmap.flt_and(flt, db.nmap.searchobjectid(
value.replace('-', ',').split(','),
neg=neg))
elif param == "host":
flt = db.nmap.flt_and(flt, db.nmap.searchhost(value, neg=neg))
elif param == "net":
flt = db.nmap.flt_and(flt, db.nmap.searchnet(value, neg=neg))
elif param == "range":
flt = db.nmap.flt_and(flt, db.nmap.searchrange(
*value.replace('-', ',').split(',', 1),
neg=neg))
elif param == "label":
group, lab = ((None, None)
if value is None else
map(utils.str2regexp, value.split(':', 1))
if ':' in value else
(utils.str2regexp(value), None))
flt = db.nmap.flt_and(flt, db.nmap.searchlabel(group=group,
label=lab, neg=neg))
elif param == "countports":
vals = [int(val) for val in value.replace('-', ',').split(',', 1)]
if len(vals) == 1:
flt = db.nmap.flt_and(flt, db.nmap.searchcountopenports(
minn=vals[0], maxn=vals[0], neg=neg))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchcountopenports(
minn=vals[0], maxn=vals[1], neg=neg))
elif param == "hostname":
flt = db.nmap.flt_and(
flt, db.nmap.searchhostname(utils.str2regexp(value), neg=neg))
elif param == "domain":
flt = db.nmap.flt_and(
flt, db.nmap.searchdomain(utils.str2regexp(value), neg=neg))
elif param == "category":
flt = db.nmap.flt_and(flt, db.nmap.searchcategory(
utils.str2regexp(value), neg=neg))
elif param == "country":
flt = db.nmap.flt_and(flt, db.nmap.searchcountry(
utils.str2list(value.upper()), neg=neg))
elif param == "city":
flt = db.nmap.flt_and(flt, db.nmap.searchcity(
utils.str2regexp(value), neg=neg))
elif param == "asnum":
flt = db.nmap.flt_and(flt, db.nmap.searchasnum(
utils.str2list(value), neg=neg))
elif param == "asname":
flt = db.nmap.flt_and(flt, db.nmap.searchasname(
utils.str2regexp(value), neg=neg))
elif param == "source":
flt = db.nmap.flt_and(flt, db.nmap.searchsource(value, neg=neg))
elif param == "timerange":
flt = db.nmap.flt_and(flt, db.nmap.searchtimerange(
*map(float, value.replace('-', ',').split(',')),
neg=neg))
elif param == 'timeago':
if value and value[-1].isalpha():
unit = {
's': 1,
'm': 60,
'h': 3600,
'd': 86400,
'y': 31557600,
}[value[-1]]
timeago = int(value[:-1]) * unit
else:
timeago = int(value)
flt = db.nmap.flt_and(flt, db.nmap.searchtimeago(
datetime.timedelta(0, timeago), neg=neg))
elif not neg and param == "service":
if ':' in value:
req, port = value.split(':', 1)
port = int(port)
flt = db.nmap.flt_and(
flt,
db.nmap.searchservice(utils.str2regexp(req), port=port))
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchservice(utils.str2regexp(value)))
elif not neg and param == "product" and ":" in value:
product = value.split(':', 2)
if len(product) == 2:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(
utils.str2regexp(product[1]),
service=utils.str2regexp(product[0])
)
)
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(
utils.str2regexp(product[1]),
service=utils.str2regexp(product[0]),
port=int(product[2])
)
)
elif not neg and param == "version" and value.count(":") >= 2:
product = value.split(':', 3)
if len(product) == 3:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(
utils.str2regexp(product[1]),
version=utils.str2regexp(product[2]),
service=utils.str2regexp(product[0]),
)
)
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(
utils.str2regexp(product[1]),
version=utils.str2regexp(product[2]),
service=utils.str2regexp(product[0]),
port=int(product[3])
)
)
elif not neg and param == "script":
value = value.split(':', 1)
if len(value) == 1:
flt = db.nmap.flt_and(
flt,
db.nmap.searchscript(name=utils.str2regexp(value[0])),
)
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchscript(
name=utils.str2regexp(value[0]),
output=utils.str2regexp(value[1]),
),
)
# results of scripts or version scans
elif not neg and param == "anonftp":
flt = db.nmap.flt_and(flt, db.nmap.searchftpanon())
elif not neg and param == 'anonldap':
flt = db.nmap.flt_and(flt, db.nmap.searchldapanon())
elif not neg and param == 'authbypassvnc':
flt = db.nmap.flt_and(flt, db.nmap.searchvncauthbypass())
elif not neg and param == "authhttp":
flt = db.nmap.flt_and(flt, db.nmap.searchhttpauth())
elif not neg and param == 'banner':
flt = db.nmap.flt_and(
flt,
db.nmap.searchbanner(utils.str2regexp(value)))
elif param == 'cookie':
flt = db.nmap.flt_and(flt, db.nmap.searchcookie(value))
elif param == 'file':
if value is None:
flt = db.nmap.flt_and(flt, db.nmap.searchfile())
else:
value = value.split(':', 1)
if len(value) == 1:
flt = db.nmap.flt_and(flt, db.nmap.searchfile(
fname=utils.str2regexp(value[0])))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchfile(
fname=utils.str2regexp(value[1]),
scripts=value[0].split(',')))
elif not neg and param == 'geovision':
flt = db.nmap.flt_and(flt, db.nmap.searchgeovision())
elif param == 'httptitle':
flt = db.nmap.flt_and(
flt,
db.nmap.searchhttptitle(utils.str2regexp(value)))
elif not neg and param == "nfs":
flt = db.nmap.flt_and(flt, db.nmap.searchnfs())
elif not neg and param in ["nis", "yp"]:
flt = db.nmap.flt_and(flt, db.nmap.searchypserv())
elif not neg and param == 'mssqlemptypwd':
flt = db.nmap.flt_and(flt, db.nmap.searchmssqlemptypwd())
elif not neg and param == 'mysqlemptypwd':
flt = db.nmap.flt_and(flt, db.nmap.searchmysqlemptypwd())
elif not neg and param == 'sshkey':
if value:
flt = db.nmap.flt_and(flt, db.nmap.searchsshkey(
output=utils.str2regexp(value)))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchsshkey())
elif not neg and param.startswith('sshkey.'):
subfield = param.split('.', 1)[1]
if subfield in ['fingerprint', 'key', 'type', 'bits']:
if subfield == 'type':
subfield = 'keytype'
flt = db.nmap.flt_and(flt, db.nmap.searchsshkey(
**{subfield: utils.str2regexp(value)}))
else:
add_unused(neg, param, value)
elif not neg and param == 'owa':
flt = db.nmap.flt_and(flt, db.nmap.searchowa())
elif param == 'phpmyadmin':
flt = db.nmap.flt_and(flt, db.nmap.searchphpmyadmin())
elif not neg and param.startswith('smb.'):
flt = db.nmap.flt_and(flt, db.nmap.searchsmb(
**{param[4:]: utils.str2regexp(value)}))
elif not neg and param == 'smbshare':
flt = db.nmap.flt_and(
flt,
db.nmap.searchsmbshares(access="" if value is None else value),
)
elif param == 'torcert':
flt = db.nmap.flt_and(flt, db.nmap.searchtorcert())
elif not neg and param == 'webfiles':
flt = db.nmap.flt_and(flt, db.nmap.searchwebfiles())
elif not neg and param == "webmin":
flt = db.nmap.flt_and(flt, db.nmap.searchwebmin())
elif not neg and param == 'x11srv':
flt = db.nmap.flt_and(flt, db.nmap.searchx11())
elif not neg and param == 'x11open':
flt = db.nmap.flt_and(flt, db.nmap.searchx11access())
elif not neg and param == 'xp445':
flt = db.nmap.flt_and(flt, db.nmap.searchxp445())
# OS fingerprint
elif not neg and param == "os":
flt = db.nmap.flt_and(
flt,
db.nmap.searchos(utils.str2regexp(value)))
# device types
elif param in ['devicetype', 'devtype']:
flt = db.nmap.flt_and(
flt,
db.nmap.searchdevicetype(utils.str2regexp(value)))
elif param in ['netdev', 'networkdevice']:
flt = db.nmap.flt_and(flt, db.nmap.searchnetdev())
elif param == 'phonedev':
flt = db.nmap.flt_and(flt, db.nmap.searchphonedev())
# traceroute
elif param == 'hop':
if ':' in value:
hop, ttl = value.split(':', 1)
flt = db.nmap.flt_and(flt,
db.nmap.searchhop(hop, ttl=int(ttl),
neg=neg))
else:
flt = db.nmap.flt_and(flt,
db.nmap.searchhop(value, neg=neg))
elif param == 'hopname':
flt = db.nmap.flt_and(flt,
db.nmap.searchhopname(value, neg=neg))
elif param == 'hopdomain':
flt = db.nmap.flt_and(flt,
db.nmap.searchhopdomain(value, neg=neg))
# sort
elif param == 'sortby':
if neg:
sortby.append((value, -1))
else:
sortby.append((value, 1))
elif param in ['open', 'filtered', 'closed']:
value = value.replace('_', '/').split(',')
protos = {}
for port in value:
if '/' in port:
proto, port = port.split('/')
else:
proto, port = "tcp", port
protos.setdefault(proto, []).append(int(port))
for proto, ports in protos.iteritems():
flt = db.nmap.flt_and(
flt,
db.nmap.searchport(ports[0], protocol=proto, state=param)
if len(ports) == 1 else
db.nmap.searchports(ports, protocol=proto, state=param)
)
elif param == 'otheropenport':
flt = db.nmap.flt_and(
flt, db.nmap.searchportsother(map(int, value.split(','))))
elif param == "screenshot":
if value is None:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(neg=neg))
elif value.isdigit():
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(
port=int(value), neg=neg))
elif value.startswith('tcp/') or value.startswith('udp/'):
value = value.split('/', 1)
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(
port=int(value[1]), protocol=value[0], neg=neg))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(
service=value, neg=neg))
elif param == "screenwords":
if value is None:
flt = db.nmap.flt_and(
flt, db.nmap.searchscreenshot(words=not neg)
)
else:
params = value.split(':', 1)
words = (map(utils.str2regexp, params[0].split(','))
if ',' in params[0] else utils.str2regexp(params[0]))
if len(params) == 1:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(
words=words, neg=neg))
elif params[1].isdigit():
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(
port=int(value), neg=neg, words=words))
elif (params[1].startswith('tcp/')
or params[1].startswith('udp/')):
params[1] = params[1].split('/', 1)
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(
port=int(params[1][1]), protocol=params[1][0],
neg=neg, words=words))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(
service=value, neg=neg, words=words))
elif param == "cpe":
if value:
cpe_kwargs = {}
cpe_fields = ["cpe_type", "vendor", "product", "version"]
for field, cpe_arg in zip(cpe_fields, value.split(':', 3)):
cpe_kwargs[field] = utils.str2regexp(cpe_arg)
flt = db.nmap.flt_and(flt, db.nmap.searchcpe(**cpe_kwargs))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchcpe())
elif param == 'display':
# ignore this parameter
pass
elif value is None:
if param.startswith('tcp_') or param.startswith('tcp/') or \
param.startswith('udp_') or param.startswith('udp/'):
proto, port = param.replace('_', '/').split('/', 1)
port = int(port)
flt = db.nmap.flt_and(flt, db.nmap.searchport(port,
protocol=proto,
neg=neg))
elif param == "openport":
flt = db.nmap.flt_and(flt, db.nmap.searchopenport(neg=neg))
elif param.isdigit():
flt = db.nmap.flt_and(flt, db.nmap.searchport(int(param),
neg=neg))
elif all(x.isdigit() for x in param.split(',')):
flt = db.nmap.flt_and(
flt,
db.nmap.searchports(map(int, param.split(',')), neg=neg)
)
elif IPADDR.match(param):
flt = db.nmap.flt_and(flt, db.nmap.searchhost(param, neg=neg))
elif NETADDR.match(param):
flt = db.nmap.flt_and(flt, db.nmap.searchnet(param, neg=neg))
elif get_notepad_pages is not None and param == 'notes':
flt = db.nmap.flt_and(flt, db.nmap.searchhosts(
get_notepad_pages(), neg=neg))
elif '<' in param:
param = param.split('<', 1)
if param[1] and param[1][0] == '=':
flt = db.nmap.flt_and(flt, db.nmap.searchcmp(
param[0],
int(param[1][1:]),
'>' if neg else '<='))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchcmp(
param[0],
int(param[1]),
'>=' if neg else '<'))
elif '>' in param:
param = param.split('>', 1)
if param[1] and param[1][0] == '=':
flt = db.nmap.flt_and(flt, db.nmap.searchcmp(
param[0],
int(param[1][1:]),
'<' if neg else '>='))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchcmp(
param[0],
int(param[1]),
'<=' if neg else '>'))
else:
add_unused(neg, param, value)
else:
add_unused(neg, param, value)
return flt, archive, sortby, unused, skip, limit
def flt_from_query(query, base_flt=None):
"""Return a tuple (`flt`, `archive`, `sortby`, `unused`, `skip`,
`limit`):
- a filter based on the query
- a boolean (`True` iff the filter should be applied to the
archive collection)
- a list of [`key`, `order`] to sort results
- a list of the unused elements of the query (errors)
- an integer for the number of results to skip
- an integer for the maximum number of results to return
"""
unused = []
sortby = []
archive = False
skip = 0
limit = None
flt = get_init_flt() if base_flt is None else base_flt
def add_unused(neg, param, value):
"""Add to the `unused` list a string representing (neg, param,
value).
"""
unused.append("%s%s" % ("-" if neg else "", "%s=%s" % (param, value) if value is not None else param))
for (neg, param, value) in query:
if not neg and param == "skip":
skip = int(value)
elif not neg and param == "limit":
limit = int(value)
elif param == "archives":
archive = not neg
elif param == "id":
flt = db.nmap.flt_and(flt, db.nmap.searchobjectid(value.replace("-", ",").split(","), neg=neg))
elif param == "host":
flt = db.nmap.flt_and(flt, db.nmap.searchhost(value, neg=neg))
elif param == "net":
flt = db.nmap.flt_and(flt, db.nmap.searchnet(value, neg=neg))
elif param == "range":
flt = db.nmap.flt_and(flt, db.nmap.searchrange(*value.replace("-", ",").split(",", 1), neg=neg))
elif param == "label":
group, lab = (
(None, None)
if value is None
else map(utils.str2regexp, value.split(":", 1))
if ":" in value
else (utils.str2regexp(value), None)
)
flt = db.nmap.flt_and(flt, db.nmap.searchlabel(group=group, label=lab, neg=neg))
elif param == "countports":
vals = [int(val) for val in value.replace("-", ",").split(",", 1)]
if len(vals) == 1:
flt = db.nmap.flt_and(flt, db.nmap.searchcountopenports(minn=vals[0], maxn=vals[0], neg=neg))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchcountopenports(minn=vals[0], maxn=vals[1], neg=neg))
elif param == "hostname":
flt = db.nmap.flt_and(flt, db.nmap.searchhostname(utils.str2regexp(value), neg=neg))
elif param == "domain":
flt = db.nmap.flt_and(flt, db.nmap.searchdomain(utils.str2regexp(value), neg=neg))
elif param == "category":
flt = db.nmap.flt_and(flt, db.nmap.searchcategory(utils.str2regexp(value), neg=neg))
elif param == "country":
flt = db.nmap.flt_and(flt, db.nmap.searchcountry(utils.str2list(value.upper()), neg=neg))
elif param == "city":
flt = db.nmap.flt_and(flt, db.nmap.searchcity(utils.str2regexp(value), neg=neg))
elif param == "asnum":
flt = db.nmap.flt_and(flt, db.nmap.searchasnum(utils.str2list(value), neg=neg))
elif param == "asname":
flt = db.nmap.flt_and(flt, db.nmap.searchasname(utils.str2regexp(value), neg=neg))
elif param == "source":
flt = db.nmap.flt_and(flt, db.nmap.searchsource(value, neg=neg))
elif param == "timerange":
flt = db.nmap.flt_and(
flt, db.nmap.searchtimerange(*map(float, value.replace("-", ",").split(",")), neg=neg)
)
elif param == "timeago":
if value and value[-1].isalpha():
unit = {"s": 1, "m": 60, "h": 3600, "d": 86400, "y": 31557600}[value[-1]]
timeago = int(value[:-1]) * unit
else:
timeago = int(value)
flt = db.nmap.flt_and(flt, db.nmap.searchtimeago(datetime.timedelta(0, timeago), neg=neg))
elif not neg and param == "service":
if ":" in value:
req, port = value.split(":", 1)
port = int(port)
flt = db.nmap.flt_and(flt, db.nmap.searchservice(utils.str2regexp(req), port=port))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchservice(utils.str2regexp(value)))
elif not neg and param == "product" and ":" in value:
product = value.split(":", 2)
if len(product) == 2:
flt = db.nmap.flt_and(
flt, db.nmap.searchproduct(utils.str2regexp(product[1]), service=utils.str2regexp(product[0]))
)
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(
utils.str2regexp(product[1]), service=utils.str2regexp(product[0]), port=int(product[2])
),
)
elif not neg and param == "version" and value.count(":") >= 2:
product = value.split(":", 3)
if len(product) == 3:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(
utils.str2regexp(product[1]),
version=utils.str2regexp(product[2]),
service=utils.str2regexp(product[0]),
),
)
else:
flt = db.nmap.flt_and(
flt,
db.nmap.searchproduct(
utils.str2regexp(product[1]),
version=utils.str2regexp(product[2]),
service=utils.str2regexp(product[0]),
port=int(product[3]),
),
)
elif not neg and param == "script":
value = value.split(":", 1)
if len(value) == 1:
flt = db.nmap.flt_and(flt, db.nmap.searchscript(name=utils.str2regexp(value[0])))
else:
flt = db.nmap.flt_and(
flt, db.nmap.searchscript(name=utils.str2regexp(value[0]), output=utils.str2regexp(value[1]))
)
# results of scripts or version scans
elif not neg and param == "anonftp":
flt = db.nmap.flt_and(flt, db.nmap.searchftpanon())
elif not neg and param == "anonldap":
flt = db.nmap.flt_and(flt, db.nmap.searchldapanon())
elif not neg and param == "authbypassvnc":
flt = db.nmap.flt_and(flt, db.nmap.searchvncauthbypass())
elif not neg and param == "authhttp":
flt = db.nmap.flt_and(flt, db.nmap.searchhttpauth())
elif not neg and param == "banner":
flt = db.nmap.flt_and(flt, db.nmap.searchbanner(utils.str2regexp(value)))
elif param == "cookie":
flt = db.nmap.flt_and(flt, db.nmap.searchcookie(value))
elif param == "file":
if value is None:
flt = db.nmap.flt_and(flt, db.nmap.searchfile())
else:
value = value.split(":", 1)
if len(value) == 1:
flt = db.nmap.flt_and(flt, db.nmap.searchfile(fname=utils.str2regexp(value[0])))
else:
flt = db.nmap.flt_and(
flt, db.nmap.searchfile(fname=utils.str2regexp(value[1]), scripts=value[0].split(","))
)
elif not neg and param == "geovision":
flt = db.nmap.flt_and(flt, db.nmap.searchgeovision())
elif param == "httptitle":
flt = db.nmap.flt_and(flt, db.nmap.searchhttptitle(utils.str2regexp(value)))
elif not neg and param == "nfs":
flt = db.nmap.flt_and(flt, db.nmap.searchnfs())
elif not neg and param in ["nis", "yp"]:
flt = db.nmap.flt_and(flt, db.nmap.searchypserv())
elif not neg and param == "mssqlemptypwd":
flt = db.nmap.flt_and(flt, db.nmap.searchmssqlemptypwd())
elif not neg and param == "mysqlemptypwd":
flt = db.nmap.flt_and(flt, db.nmap.searchmysqlemptypwd())
elif not neg and param == "sshkey":
if value:
flt = db.nmap.flt_and(flt, db.nmap.searchsshkey(output=utils.str2regexp(value)))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchsshkey())
elif not neg and param.startswith("sshkey."):
subfield = param.split(".", 1)[1]
if subfield in ["fingerprint", "key", "type", "bits"]:
if subfield == "type":
subfield = "keytype"
flt = db.nmap.flt_and(flt, db.nmap.searchsshkey(**{subfield: utils.str2regexp(value)}))
else:
add_unused(neg, param, value)
elif not neg and param == "owa":
flt = db.nmap.flt_and(flt, db.nmap.searchowa())
elif param == "phpmyadmin":
flt = db.nmap.flt_and(flt, db.nmap.searchphpmyadmin())
elif not neg and param.startswith("smb."):
flt = db.nmap.flt_and(flt, db.nmap.searchsmb(**{param[4:]: utils.str2regexp(value)}))
elif not neg and param == "smbshare":
flt = db.nmap.flt_and(flt, db.nmap.searchsmbshares(access="" if value is None else value))
elif param == "torcert":
flt = db.nmap.flt_and(flt, db.nmap.searchtorcert())
elif not neg and param == "webfiles":
flt = db.nmap.flt_and(flt, db.nmap.searchwebfiles())
elif not neg and param == "webmin":
flt = db.nmap.flt_and(flt, db.nmap.searchwebmin())
elif not neg and param == "x11srv":
flt = db.nmap.flt_and(flt, db.nmap.searchx11())
elif not neg and param == "x11open":
flt = db.nmap.flt_and(flt, db.nmap.searchx11access())
elif not neg and param == "xp445":
flt = db.nmap.flt_and(flt, db.nmap.searchxp445())
# OS fingerprint
elif not neg and param == "os":
flt = db.nmap.flt_and(flt, db.nmap.searchos(utils.str2regexp(value)))
# device types
elif param in ["devicetype", "devtype"]:
flt = db.nmap.flt_and(flt, db.nmap.searchdevicetype(utils.str2regexp(value)))
elif param in ["netdev", "networkdevice"]:
flt = db.nmap.flt_and(flt, db.nmap.searchnetdev())
elif param == "phonedev":
flt = db.nmap.flt_and(flt, db.nmap.searchphonedev())
# traceroute
elif param == "hop":
if ":" in value:
hop, ttl = value.split(":", 1)
flt = db.nmap.flt_and(flt, db.nmap.searchhop(hop, ttl=int(ttl), neg=neg))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchhop(value, neg=neg))
elif param == "hopname":
flt = db.nmap.flt_and(flt, db.nmap.searchhopname(value, neg=neg))
elif param == "hopdomain":
flt = db.nmap.flt_and(flt, db.nmap.searchhopdomain(value, neg=neg))
# sort
elif param == "sortby":
if neg:
sortby.append((value, -1))
else:
sortby.append((value, 1))
elif param in ["open", "filtered", "closed"]:
value = value.replace("_", "/").split(",")
protos = {}
for port in value:
if "/" in port:
proto, port = port.split("/")
else:
proto, port = "tcp", port
protos.setdefault(proto, []).append(int(port))
for proto, ports in protos.iteritems():
flt = db.nmap.flt_and(
flt,
db.nmap.searchport(ports[0], protocol=proto, state=param)
if len(ports) == 1
else db.nmap.searchports(ports, protocol=proto, state=param),
)
elif param == "otheropenport":
flt = db.nmap.flt_and(flt, db.nmap.searchportsother(map(int, value.split(","))))
elif param == "screenshot":
if value is None:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(neg=neg))
elif value.isdigit():
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(port=int(value), neg=neg))
elif value.startswith("tcp/") or value.startswith("udp/"):
value = value.split("/", 1)
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(port=int(value[1]), protocol=value[0], neg=neg))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(service=value, neg=neg))
elif param == "screenwords":
if value is None:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(words=not neg))
else:
params = value.split(":", 1)
words = map(utils.str2regexp, params[0].split(",")) if "," in params[0] else utils.str2regexp(params[0])
if len(params) == 1:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(words=words, neg=neg))
elif params[1].isdigit():
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(port=int(value), neg=neg, words=words))
elif params[1].startswith("tcp/") or params[1].startswith("udp/"):
params[1] = params[1].split("/", 1)
flt = db.nmap.flt_and(
flt,
db.nmap.searchscreenshot(port=int(params[1][1]), protocol=params[1][0], neg=neg, words=words),
)
else:
flt = db.nmap.flt_and(flt, db.nmap.searchscreenshot(service=value, neg=neg, words=words))
elif param == "cpe":
if value:
cpe_kwargs = {}
cpe_fields = ["cpe_type", "vendor", "product", "version"]
for field, cpe_arg in zip(cpe_fields, value.split(":", 3)):
cpe_kwargs[field] = utils.str2regexp(cpe_arg)
flt = db.nmap.flt_and(flt, db.nmap.searchcpe(**cpe_kwargs))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchcpe())
elif param == "display":
# ignore this parameter
pass
elif value is None:
if (
param.startswith("tcp_")
or param.startswith("tcp/")
or param.startswith("udp_")
or param.startswith("udp/")
):
proto, port = param.replace("_", "/").split("/", 1)
port = int(port)
flt = db.nmap.flt_and(flt, db.nmap.searchport(port, protocol=proto, neg=neg))
elif param == "openport":
flt = db.nmap.flt_and(flt, db.nmap.searchopenport(neg=neg))
elif param.isdigit():
flt = db.nmap.flt_and(flt, db.nmap.searchport(int(param), neg=neg))
elif all(x.isdigit() for x in param.split(",")):
flt = db.nmap.flt_and(flt, db.nmap.searchports(map(int, param.split(",")), neg=neg))
elif IPADDR.match(param):
flt = db.nmap.flt_and(flt, db.nmap.searchhost(param, neg=neg))
elif NETADDR.match(param):
flt = db.nmap.flt_and(flt, db.nmap.searchnet(param, neg=neg))
elif get_notepad_pages is not None and param == "notes":
flt = db.nmap.flt_and(flt, db.nmap.searchhosts(get_notepad_pages(), neg=neg))
elif "<" in param:
param = param.split("<", 1)
if param[1] and param[1][0] == "=":
flt = db.nmap.flt_and(flt, db.nmap.searchcmp(param[0], int(param[1][1:]), ">" if neg else "<="))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchcmp(param[0], int(param[1]), ">=" if neg else "<"))
elif ">" in param:
param = param.split(">", 1)
if param[1] and param[1][0] == "=":
flt = db.nmap.flt_and(flt, db.nmap.searchcmp(param[0], int(param[1][1:]), "<" if neg else ">="))
else:
flt = db.nmap.flt_and(flt, db.nmap.searchcmp(param[0], int(param[1]), "<=" if neg else ">"))
else:
add_unused(neg, param, value)
else:
add_unused(neg, param, value)
return flt, archive, sortby, unused, skip, limit
def topvalues(self, field, flt=None, topnbr=10, sort=None, least=False):
"""
This method uses an aggregation to produce top values for a given
field or pseudo-field. Pseudo-fields are:
- category / asnum / country / net[:mask]
- port
- port:open / :closed / :filtered / :<servicename>
- portlist:open / :closed / :filtered
- countports:open / :closed / :filtered
- service / service:<portnbr>
- product / product:<portnbr>
- cpe / cpe.<part> / cpe:<cpe_spec> / cpe.<part>:<cpe_spec>
- devicetype / devicetype:<portnbr>
- script:<scriptid> / script:<port>:<scriptid>
/ script:host:<scriptid>
- cert.* / smb.* / sshkey.* / ike.*
- httphdr / httphdr.{name,value} / httphdr:<name>
- modbus.* / s7.* / enip.*
- mongo.dbs.*
- vulns.*
- screenwords
- file.* / file.*:scriptid
- hop
"""
outputproc = None
nested = None
if flt is None:
flt = self.flt_empty
if field == "asnum":
flt = self.flt_and(flt, Q("exists", field="infos.as_num"))
field = {"field": "infos.as_num"}
elif field == "as":
def outputproc(value):
return tuple(val if i else int(val)
for i, val in enumerate(value.split(', ', 1)))
flt = self.flt_and(flt, Q("exists", field="infos.as_num"))
field = {"script": {
"lang": "painless",
"source":
"doc['infos.as_num'].value + ', ' + "
"doc['infos.as_name'].value",
}}
elif field == "port" or field.startswith("port:"):
def outputproc(value):
return tuple(int(val) if i else val
for i, val in enumerate(value.split('/', 1)))
if field == "port":
flt = self.flt_and(flt,
Q('nested', path='ports',
query=Q('exists', field="ports.port")))
field = {"script": {
"lang": "painless",
"source": """List result = new ArrayList();
for(item in params._source.ports) {
if(item.port != -1) {
result.add(item.protocol + '/' + item.port);
}
}
return result;
""",
}}
else:
info = field[5:]
if info in ['open', 'filtered', 'closed']:
flt = self.flt_and(flt,
Q('nested', path='ports',
query=Q('match',
ports__state_state=info)))
matchfield = "state_state"
else:
flt = self.flt_and(flt,
Q('nested', path='ports',
query=Q('match',
ports__service_name=info)))
matchfield = "service_name"
field = {"script": {
"lang": "painless",
"source": """List result = new ArrayList();
for(item in params._source.ports) {
if(item[params.field] == params.value) {
result.add(item.protocol + '/' + item.port);
}
}
return result;
""",
"params": {
"field": matchfield,
"value": info,
}
}}
elif field == 'httphdr':
def outputproc(value):
return tuple(value.split(':', 1))
flt = self.flt_and(flt, self.searchscript(name="http-headers"))
field = {"script": {
"lang": "painless",
"source": """List result = new ArrayList();
for(item in params._source.ports) {
if (item.containsKey('scripts')) {
for(script in item.scripts) {
if(script.id == 'http-headers' &&
script.containsKey('http-headers')) {
for(hdr in script['http-headers']) {
result.add(hdr.name + ':' + hdr.value);
}
}
}
}
}
return result;
""",
}}
elif field.startswith('httphdr.'):
flt = self.flt_and(flt, self.searchscript(name="http-headers"))
field = {"field": "ports.scripts.http-headers.%s" % field[8:]}
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {"terms": field}},
}},
}
elif field.startswith('httphdr:'):
subfield = field[8:].lower()
flt = self.flt_and(flt,
self.searchscript(name="http-headers",
values={"name": subfield}))
field = {"script": {
"lang": "painless",
"source": """List result = new ArrayList();
for(item in params._source.ports) {
if (item.containsKey('scripts')) {
for(script in item.scripts) {
if(script.id == 'http-headers' &&
script.containsKey('http-headers')) {
for(hdr in script['http-headers']) {
if (hdr.name == params.name) {
result.add(hdr.value);
}
}
}
}
}
}
return result;
""",
"params": {"name": subfield}
}}
elif field == 'useragent' or field.startswith('useragent:'):
terms = {"field": "ports.scripts.http-user-agent"}
if field == 'useragent':
flt = self.flt_and(flt, self.searchuseragent())
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {"terms": terms}},
}},
}
else:
subfield = utils.str2regexp(field[10:])
flt = self.flt_and(flt,
self.searchuseragent(useragent=subfield))
if isinstance(subfield, utils.REGEXP_T):
subfield = self._get_pattern(subfield)
else:
subfield = re.escape(subfield)
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {
"terms": dict(terms, include=subfield),
}},
}},
}
else:
field = {"field": field}
body = {"query": flt.to_dict()}
if nested is None:
body["aggs"] = {"patterns": {"terms": dict(field, size=topnbr)}}
else:
body["aggs"] = {"patterns": nested}
result = self.db_client.search(
body=body,
index=self.indexes[0],
ignore_unavailable=True,
size=0
)
result = result["aggregations"]
while 'patterns' in result:
result = result['patterns']
result = result['buckets']
if outputproc is None:
for res in result:
yield {'_id': res['key'], 'count': res['doc_count']}
else:
for res in result:
yield {'_id': outputproc(res['key']),
'count': res['doc_count']}
def flt_from_query(dbase, query, base_flt=None):
"""Return a tuple (`flt`, `sortby`, `unused`, `skip`, `limit`, `fields`):
- a filter based on the query
- a list of [`key`, `order`] to sort results
- a list of the unused elements of the query (errors)
- an integer for the number of results to skip
- an integer for the maximum number of results to return
"""
unused = []
sortby = []
skip = 0
limit = None
fields = None
flt = get_init_flt(dbase) if base_flt is None else base_flt
def add_unused(neg, param, value):
"""Add to the `unused` list a string representing (neg, param,
value).
"""
unused.append("%s%s" % (
"-" if neg else "",
"%s=%s" % (param, value) if value is not None else param,
))
for (neg, param, value) in query:
if not neg and param == "skip":
skip = int(value)
elif not neg and param == "limit":
limit = int(value)
elif not neg and param == "fields":
fields = value.split(",")
elif param == "id":
flt = dbase.flt_and(
flt,
dbase.searchobjectid(value.replace("-", ",").split(","),
neg=neg))
elif param == "host":
flt = dbase.flt_and(flt, dbase.searchhost(value, neg=neg))
elif param == "net":
flt = dbase.flt_and(flt, dbase.searchnet(value, neg=neg))
elif param == "range":
flt = dbase.flt_and(
flt,
dbase.searchrange(*value.replace("-", ",").split(",", 1),
neg=neg))
elif param == "countports":
vals = [int(val) for val in value.replace("-", ",").split(",", 1)]
if len(vals) == 1:
flt = dbase.flt_and(
flt,
dbase.searchcountopenports(minn=vals[0],
maxn=vals[0],
neg=neg))
else:
flt = dbase.flt_and(
flt,
dbase.searchcountopenports(minn=vals[0],
maxn=vals[1],
neg=neg))
elif param == "hostname":
flt = dbase.flt_and(
flt, dbase.searchhostname(utils.str2regexp(value), neg=neg))
elif param == "domain":
flt = dbase.flt_and(
flt, dbase.searchdomain(utils.str2regexp(value), neg=neg))
elif param == "category":
flt = dbase.flt_and(
flt, dbase.searchcategory(utils.str2regexp(value), neg=neg))
elif param == "country":
flt = dbase.flt_and(
flt, dbase.searchcountry(utils.str2list(value.upper()),
neg=neg))
elif param == "city":
flt = dbase.flt_and(
flt, dbase.searchcity(utils.str2regexp(value), neg=neg))
elif param == "asnum":
flt = dbase.flt_and(
flt, dbase.searchasnum(utils.str2list(value), neg=neg))
elif param == "asname":
flt = dbase.flt_and(
flt, dbase.searchasname(utils.str2regexp(value), neg=neg))
elif param == "source":
flt = dbase.flt_and(
flt, dbase.searchsource(utils.str2regexp(value), neg=neg))
elif param == "timerange":
flt = dbase.flt_and(
flt,
dbase.searchtimerange(
*(float(val)
for val in value.replace("-", ",").split(",")),
neg=neg),
)
elif param == "timeago":
if value and value[-1].isalpha():
unit = {
"s": 1,
"m": 60,
"h": 3600,
"d": 86400,
"y": 31557600,
}[value[-1]]
timeago = int(value[:-1]) * unit
else:
timeago = int(value)
flt = dbase.flt_and(
flt,
dbase.searchtimeago(datetime.timedelta(0, timeago), neg=neg))
elif not neg and param == "service":
if ":" in value:
req, port = value.split(":", 1)
port = int(port)
flt = dbase.flt_and(
flt,
dbase.searchservice(utils.str2regexpnone(req), port=port))
else:
flt = dbase.flt_and(
flt, dbase.searchservice(utils.str2regexpnone(value)))
elif not neg and param == "product" and ":" in value:
product = value.split(":", 2)
if len(product) == 2:
flt = dbase.flt_and(
flt,
dbase.searchproduct(
product=utils.str2regexpnone(product[1]),
service=utils.str2regexpnone(product[0]),
),
)
else:
flt = dbase.flt_and(
flt,
dbase.searchproduct(
product=utils.str2regexpnone(product[1]),
service=utils.str2regexpnone(product[0]),
port=int(product[2]),
),
)
elif not neg and param == "version" and value.count(":") >= 2:
product = value.split(":", 3)
if len(product) == 3:
flt = dbase.flt_and(
flt,
dbase.searchproduct(
product=utils.str2regexpnone(product[1]),
version=utils.str2regexpnone(product[2]),
service=utils.str2regexpnone(product[0]),
),
)
else:
flt = dbase.flt_and(
flt,
dbase.searchproduct(
product=utils.str2regexpnone(product[1]),
version=utils.str2regexpnone(product[2]),
service=utils.str2regexpnone(product[0]),
port=int(product[3]),
),
)
elif param == "script":
value = value.split(":", 1)
if len(value) == 1:
flt = dbase.flt_and(
flt,
dbase.searchscript(name=utils.str2regexp(value[0]),
neg=neg),
)
else:
flt = dbase.flt_and(
flt,
dbase.searchscript(
name=utils.str2regexp(value[0]),
output=utils.str2regexp(value[1]),
neg=neg,
),
)
# results of scripts or version scans
elif not neg and param == "anonftp":
flt = dbase.flt_and(flt, dbase.searchftpanon())
elif not neg and param == "anonldap":
flt = dbase.flt_and(flt, dbase.searchldapanon())
elif not neg and param == "authbypassvnc":
flt = dbase.flt_and(flt, dbase.searchvncauthbypass())
elif not neg and param == "authhttp":
flt = dbase.flt_and(flt, dbase.searchhttpauth())
elif not neg and param == "banner":
flt = dbase.flt_and(flt,
dbase.searchbanner(utils.str2regexp(value)))
elif param == "cookie":
flt = dbase.flt_and(flt, dbase.searchcookie(value))
elif param == "file":
if value is None:
flt = dbase.flt_and(flt, dbase.searchfile())
else:
value = value.split(":", 1)
if len(value) == 1:
flt = dbase.flt_and(
flt,
dbase.searchfile(fname=utils.str2regexp(value[0])))
else:
flt = dbase.flt_and(
flt,
dbase.searchfile(
fname=utils.str2regexp(value[1]),
scripts=value[0].split(","),
),
)
elif param == "vuln":
try:
vulnid, status = value.split(":", 1)
except ValueError:
vulnid = value
status = None
except AttributeError:
vulnid = None
status = None
flt = dbase.flt_and(flt,
dbase.searchvuln(vulnid=vulnid, status=status))
elif not neg and param == "geovision":
flt = dbase.flt_and(flt, dbase.searchgeovision())
elif param == "httptitle":
flt = dbase.flt_and(flt,
dbase.searchhttptitle(utils.str2regexp(value)))
elif not neg and param == "nfs":
flt = dbase.flt_and(flt, dbase.searchnfs())
elif not neg and param in ["nis", "yp"]:
flt = dbase.flt_and(flt, dbase.searchypserv())
elif not neg and param == "mssqlemptypwd":
flt = dbase.flt_and(flt, dbase.searchmssqlemptypwd())
elif not neg and param == "mysqlemptypwd":
flt = dbase.flt_and(flt, dbase.searchmysqlemptypwd())
elif not neg and param == "sshkey":
if value:
flt = dbase.flt_and(
flt, dbase.searchsshkey(output=utils.str2regexp(value)))
else:
flt = dbase.flt_and(flt, dbase.searchsshkey())
elif not neg and param.startswith("sshkey."):
subfield = param.split(".", 1)[1]
if subfield in ["fingerprint", "key", "type", "bits"]:
if subfield == "type":
subfield = "keytype"
elif subfield == "bits":
try:
value = int(value)
except (ValueError, TypeError):
pass
else:
value = utils.str2regexp(value)
flt = dbase.flt_and(flt,
dbase.searchsshkey(**{subfield: value}))
else:
add_unused(neg, param, value)
elif not neg and param == "cert":
flt = dbase.flt_and(flt, dbase.searchcert())
elif param.startswith("cert."):
subfield = param.split(".", 1)[1]
if subfield == "self_signed" and value is None:
flt = dbase.flt_and(flt, dbase.searchcert(self_signed=not neg))
elif not neg:
if subfield in ["md5", "sha1", "sha256", "subject", "issuer"]:
flt = dbase.flt_and(
flt,
dbase.searchcert(**{
subfield: utils.str2regexp(value),
}),
)
elif subfield in [
"pubkey.md5", "pubkey.sha1", "pubkey.sha256"
]:
flt = dbase.flt_and(
flt,
dbase.searchcert(
**{
"pk%s" % subfield[7:]: utils.str2regexp(value),
}),
)
else:
add_unused(neg, param, value)
else:
add_unused(neg, param, value)
elif not neg and param == "httphdr":
if value is None:
flt = dbase.flt_and(flt, dbase.searchhttphdr())
elif ":" in value:
name, value = value.split(":", 1)
name = utils.str2regexp(name.lower())
value = utils.str2regexp(value)
flt = dbase.flt_and(
flt, dbase.searchhttphdr(name=name, value=value))
else:
flt = dbase.flt_and(
flt,
dbase.searchhttphdr(name=utils.str2regexp(value.lower())))
elif not neg and param == "httpapp":
if value is None:
flt = dbase.flt_and(flt, dbase.searchhttpapp())
elif ":" in value:
name, version = (utils.str2regexp(string)
for string in value.split(":", 1))
flt = dbase.flt_and(
flt, dbase.searchhttpapp(name=name, version=version))
else:
flt = dbase.flt_and(
flt, dbase.searchhttpapp(name=utils.str2regexp(value)))
elif not neg and param == "owa":
flt = dbase.flt_and(flt, dbase.searchowa())
elif param == "phpmyadmin":
flt = dbase.flt_and(flt, dbase.searchphpmyadmin())
elif not neg and param.startswith("smb."):
flt = dbase.flt_and(
flt, dbase.searchsmb(**{param[4:]: utils.str2regexp(value)}))
elif not neg and param.startswith("ntlm."):
key = {
"name": "Target_Name",
"server": "NetBIOS_Computer_Name",
"domain": "NetBIOS_Domain_Name",
"workgroup": "Workgroup",
"domain_dns": "DNS_Domain_Name",
"forest": "DNS_Tree_Name",
"fqdn": "DNS_Computer_Name",
"os": "Product_Version",
"version": "NTLM_Version",
}
flt = dbase.flt_and(
flt,
dbase.searchntlm(
**{key.get(param[5:], param[5:]): utils.str2regexp(value)
}),
)
elif not neg and param == "smbshare":
flt = dbase.flt_and(
flt,
dbase.searchsmbshares(access="" if value is None else value),
)
elif param == "torcert":
flt = dbase.flt_and(flt, dbase.searchtorcert())
elif not neg and param == "webfiles":
flt = dbase.flt_and(flt, dbase.searchwebfiles())
elif not neg and param == "webmin":
flt = dbase.flt_and(flt, dbase.searchwebmin())
elif not neg and param == "x11srv":
flt = dbase.flt_and(flt, dbase.searchx11())
elif not neg and param == "x11open":
flt = dbase.flt_and(flt, dbase.searchx11access())
elif not neg and param == "xp445":
flt = dbase.flt_and(flt, dbase.searchxp445())
elif param == "ssl-ja3-client":
flt = dbase.flt_and(
flt,
dbase.searchja3client(
value_or_hash=(None if value is None else
utils.str2regexp(value)),
neg=neg,
),
)
elif param == "ssl-ja3-server":
if value is None:
# There are no additional arguments
flt = dbase.flt_and(flt, dbase.searchja3server(neg=neg))
else:
split = [
utils.str2regexp(v) if v else None
for v in value.split(":", 1)
]
if len(split) == 1:
# Only a JA3 server is given
flt = dbase.flt_and(
flt,
dbase.searchja3server(
value_or_hash=(split[0]),
neg=neg,
),
)
else:
# Both client and server JA3 are specified
flt = dbase.flt_and(
flt,
dbase.searchja3server(
value_or_hash=split[0],
client_value_or_hash=split[1],
neg=neg,
),
)
elif param == "ssl-jarm":
if value is None:
flt = dbase.flt_and(flt, dbase.searchjarm(neg=neg))
else:
flt = dbase.flt_and(
flt,
dbase.searchjarm(value=utils.str2regexp(value), neg=neg))
elif param in {"hassh", "hassh-client", "hassh-server"}:
server = {
"hassh": None,
"hassh-client": False,
"hassh-server": True
}[param]
flt = dbase.flt_and(
flt,
dbase.searchhassh(
value_or_hash=(None if value is None else
utils.str2regexp(value)),
server=server,
),
)
elif param == "useragent":
if value:
flt = dbase.flt_and(
flt,
dbase.searchuseragent(useragent=utils.str2regexp(value)))
else:
flt = dbase.flt_and(flt, dbase.searchuseragent())
# OS fingerprint
elif not neg and param == "os":
flt = dbase.flt_and(flt, dbase.searchos(utils.str2regexp(value)))
# device types
elif param in ["devicetype", "devtype"]:
flt = dbase.flt_and(
flt, dbase.searchdevicetype(utils.str2regexp(value)))
elif param in ["netdev", "networkdevice"]:
flt = dbase.flt_and(flt, dbase.searchnetdev())
elif param == "phonedev":
flt = dbase.flt_and(flt, dbase.searchphonedev())
# traceroute
elif param == "hop":
if ":" in value:
hop, ttl = value.split(":", 1)
flt = dbase.flt_and(
flt, dbase.searchhop(hop, ttl=int(ttl), neg=neg))
else:
flt = dbase.flt_and(flt, dbase.searchhop(value, neg=neg))
elif param == "hopname":
flt = dbase.flt_and(flt, dbase.searchhopname(value, neg=neg))
elif param == "hopdomain":
flt = dbase.flt_and(flt, dbase.searchhopdomain(value, neg=neg))
elif not neg and param in [
"ike.vendor_id.name", "ike.vendor_id.value"
]:
flt = dbase.flt_and(
flt,
dbase.searchscript(
name="ike-info",
values={
"vendor_ids.%s" % param[14:]: utils.str2regexp(value)
},
),
)
elif not neg and param == "ike.notification":
flt = dbase.flt_and(
flt,
dbase.searchscript(
name="ike-info",
values={"notification_type": utils.str2regexp(value)},
),
)
# sort
elif param == "sortby":
if neg:
sortby.append((value, -1))
else:
sortby.append((value, 1))
elif param in ["open", "filtered", "closed"]:
value = value.replace("_", "/").split(",")
protos = {}
for port in value:
if "/" in port:
proto, port = port.split("/")
else:
proto = "tcp"
protos.setdefault(proto, []).append(int(port))
for proto, ports in protos.items():
flt = dbase.flt_and(
flt,
dbase.searchport(ports[0], protocol=proto, state=param)
if len(ports) == 1 else dbase.searchports(
ports, protocol=proto, state=param),
)
elif param == "otheropenport":
flt = dbase.flt_and(
flt,
dbase.searchportsother([int(val) for val in value.split(",")]))
elif param == "screenshot":
if value is None:
flt = dbase.flt_and(flt, dbase.searchscreenshot(neg=neg))
elif value.isdigit():
flt = dbase.flt_and(
flt, dbase.searchscreenshot(port=int(value), neg=neg))
elif value.startswith("tcp/") or value.startswith("udp/"):
value = value.split("/", 1)
flt = dbase.flt_and(
flt,
dbase.searchscreenshot(port=int(value[1]),
protocol=value[0],
neg=neg),
)
else:
flt = dbase.flt_and(
flt, dbase.searchscreenshot(service=value, neg=neg))
elif param == "screenwords":
if value is None:
flt = dbase.flt_and(flt, dbase.searchscreenshot(words=not neg))
else:
params = value.split(":", 1)
words = ([
utils.str2regexp(elt) for elt in params[0].split(",")
] if "," in params[0] else utils.str2regexp(params[0]))
if len(params) == 1:
flt = dbase.flt_and(
flt, dbase.searchscreenshot(words=words, neg=neg))
elif params[1].isdigit():
flt = dbase.flt_and(
flt,
dbase.searchscreenshot(port=int(value),
neg=neg,
words=words),
)
elif params[1].startswith("tcp/") or params[1].startswith(
"udp/"):
params[1] = params[1].split("/", 1)
flt = dbase.flt_and(
flt,
dbase.searchscreenshot(
port=int(params[1][1]),
protocol=params[1][0],
neg=neg,
words=words,
),
)
else:
flt = dbase.flt_and(
flt,
dbase.searchscreenshot(service=value,
neg=neg,
words=words))
elif param == "cpe":
if value:
cpe_kwargs = {}
cpe_fields = ["cpe_type", "vendor", "product", "version"]
for field, cpe_arg in zip(cpe_fields, value.split(":", 3)):
cpe_kwargs[field] = utils.str2regexp(cpe_arg)
flt = dbase.flt_and(flt, dbase.searchcpe(**cpe_kwargs))
else:
flt = dbase.flt_and(flt, dbase.searchcpe())
elif param == "tag":
if value:
if ":" in value:
tag_val, tag_info = value.split(":", 1)
tag = {}
if tag_val:
tag["value"] = utils.str2regexp(tag_val)
if tag_info:
tag["info"] = utils.str2regexp(tag_info)
flt = dbase.flt_and(flt, dbase.searchtag(tag, neg=neg))
else:
flt = dbase.flt_and(
flt,
dbase.searchtag({"value": utils.str2regexp(value)},
neg=neg),
)
else:
flt = dbase.flt_and(flt, dbase.searchtag(neg=neg))
elif param == "search":
try:
flt = dbase.flt_and(flt, dbase.searchtext(value))
except AttributeError: # .searchtext() only exists in MongoDB
add_unused(neg, param, value)
elif param == "display":
# ignore this parameter
pass
elif value is None:
if PORT.search(param) is not None:
# Python >= 3.8: if (match := PORT.search(param)) is not None
proto, port = PORT.search(param).groups()
flt = dbase.flt_and(
flt,
dbase.searchport(int(port),
protocol=proto or "tcp",
neg=neg))
elif param == "openport":
flt = dbase.flt_and(flt, dbase.searchopenport(neg=neg))
elif param == "ipv4":
if neg:
flt = dbase.flt_and(flt, dbase.searchipv6())
else:
flt = dbase.flt_and(flt, dbase.searchipv4())
elif param == "ipv6":
if neg:
flt = dbase.flt_and(flt, dbase.searchipv4())
else:
flt = dbase.flt_and(flt, dbase.searchipv6())
elif all(PORT.search(x) is not None for x in param.split(",")):
proto_ports = {}
for port in param.split(","):
proto, port = PORT.search(port).groups()
proto_ports.setdefault(proto or "tcp",
set()).add(int(port))
for proto, ports in proto_ports.items():
if len(ports) > 1:
flt = dbase.flt_and(
flt,
dbase.searchports(list(ports),
protocol=proto,
neg=neg),
)
else:
flt = dbase.flt_and(
flt,
dbase.searchport(next(iter(ports)),
protocol=proto,
neg=neg),
)
elif utils.IPADDR.search(param):
flt = dbase.flt_and(flt, dbase.searchhost(param, neg=neg))
elif utils.NETADDR.search(param):
flt = dbase.flt_and(flt, dbase.searchnet(param, neg=neg))
elif get_notepad_pages is not None and param == "notes":
flt = dbase.flt_and(
flt, dbase.searchhosts(get_notepad_pages(), neg=neg))
elif "<" in param:
param = param.split("<", 1)
if param[1] and param[1][0] == "=":
flt = dbase.flt_and(
flt,
dbase.searchcmp(param[0], int(param[1][1:]),
">" if neg else "<="),
)
else:
flt = dbase.flt_and(
flt,
dbase.searchcmp(param[0], int(param[1]),
">=" if neg else "<"),
)
elif ">" in param:
param = param.split(">", 1)
if param[1] and param[1][0] == "=":
flt = dbase.flt_and(
flt,
dbase.searchcmp(param[0], int(param[1][1:]),
"<" if neg else ">="),
)
else:
flt = dbase.flt_and(
flt,
dbase.searchcmp(param[0], int(param[1]),
"<=" if neg else ">"),
)
else:
add_unused(neg, param, value)
else:
add_unused(neg, param, value)
return flt, sortby, unused, skip, limit, fields
def topvalues(self, field, flt=None, topnbr=10, sort=None, least=False):
"""
This method uses an aggregation to produce top values for a given
field or pseudo-field. Pseudo-fields are:
- category / asnum / country / net[:mask]
- port
- port:open / :closed / :filtered / :<servicename>
- portlist:open / :closed / :filtered
- countports:open / :closed / :filtered
- service / service:<portnbr>
- product / product:<portnbr>
- cpe / cpe.<part> / cpe:<cpe_spec> / cpe.<part>:<cpe_spec>
- devicetype / devicetype:<portnbr>
- script:<scriptid> / script:<port>:<scriptid>
/ script:host:<scriptid>
- cert.* / smb.* / sshkey.* / ike.*
- httphdr / httphdr.{name,value} / httphdr:<name>
- modbus.* / s7.* / enip.*
- mongo.dbs.*
- vulns.*
- screenwords
- file.* / file.*:scriptid
- hop
"""
baseterms = {"size": topnbr}
if least:
baseterms["order"] = {"_count": "asc"}
outputproc = None
nested = None
if flt is None:
flt = self.flt_empty
if field == "category":
field = {"field": "categories"}
elif field == "asnum":
flt = self.flt_and(flt, Q("exists", field="infos.as_num"))
field = {"field": "infos.as_num"}
elif field == "as":
def outputproc(value):
return tuple(val if i else int(val)
for i, val in enumerate(value.split(',', 1)))
flt = self.flt_and(flt, Q("exists", field="infos.as_num"))
field = {"script": {
"lang": "painless",
"source":
"doc['infos.as_num'].value + ',' + "
"doc['infos.as_name'].value",
}}
elif field == "port" or field.startswith("port:"):
def outputproc(value):
return tuple(int(val) if i else val
for i, val in enumerate(value.rsplit('/', 1)))
if field == "port":
flt = self.flt_and(flt,
Q('nested', path='ports',
query=Q('exists', field="ports.port")))
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"filter": {'bool': {'must_not': [
{'match': {'ports.port': -1}},
]}},
"aggs": {"patterns": {
"terms": dict(
baseterms,
script={
"lang": "painless",
"source":
'doc["ports.protocol"].value + "/" + '
'doc["ports.port"].value',
},
),
}},
}},
}
else:
info = field[5:]
if info in ['open', 'filtered', 'closed']:
flt = self.flt_and(flt,
Q('nested', path='ports',
query=Q('match',
ports__state_state=info)))
matchfield = "state_state"
else:
flt = self.flt_and(flt,
Q('nested', path='ports',
query=Q('match',
ports__service_name=info)))
matchfield = "service_name"
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"filter": {'bool': {
'must': [{'match': {'ports.%s' % matchfield:
info}}],
'must_not': [{'match': {'ports.port': -1}}],
}},
"aggs": {"patterns": {
"terms": dict(
baseterms,
script={
"lang": "painless",
"source":
'doc["ports.protocol"].value + "/" + '
'doc["ports.port"].value',
},
),
}},
}},
}
elif field == 'service':
def outputproc(value):
return value or None
flt = self.flt_and(flt, self.searchopenport())
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"filter": {"match": {"ports.state_state": "open"}},
"aggs": {"patterns": {
"terms": dict(
baseterms,
field="ports.service_name",
missing="",
),
}},
}},
}
elif field.startswith("service:"):
port = int(field[8:])
flt = self.flt_and(flt, self.searchport(port))
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"filter": {"bool": {"must": [
{"match": {"ports.state_state": "open"}},
{"match": {"ports.port": port}},
]}},
"aggs": {"patterns": {
"terms": dict(
baseterms,
field="ports.service_name",
missing="",
),
}},
}},
}
elif field == 'product':
def outputproc(value):
return tuple(v or None for v in value.split('###', 1))
flt = self.flt_and(flt, self.searchopenport())
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"filter": {"match": {"ports.state_state": "open"}},
"aggs": {"patterns": {
"terms": dict(
baseterms,
script="""
String result = "";
if(doc['ports.service_name'].size() > 0) {
result += doc['ports.service_name'].value;
}
result += "###";
if(doc['ports.service_product'].size() > 0) {
result += doc['ports.service_product'].value;
}
return result;
""",
missing="",
),
}},
}},
}
elif field.startswith("product:"):
def outputproc(value):
return tuple(v or None for v in value.split('###', 1))
info = field[8:]
if info.isdigit():
info = int(info)
flt = self.flt_and(flt, self.searchport(info))
matchfield = "port"
else:
flt = self.flt_and(flt, self.searchservice(info))
matchfield = "service_name"
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"filter": {"bool": {"must": [
{"match": {"ports.state_state": "open"}},
{"match": {"ports.%s" % matchfield: info}},
]}},
"aggs": {"patterns": {
"terms": dict(
baseterms,
script="""
String result = "";
if(doc['ports.service_name'].size() > 0) {
result += doc['ports.service_name'].value;
}
result += "###";
if(doc['ports.service_product'].size() > 0) {
result += doc['ports.service_product'].value;
}
return result;
""",
),
}},
}},
}
elif field == 'version':
def outputproc(value):
return tuple(v or None for v in value.split('###', 2))
flt = self.flt_and(flt, self.searchopenport())
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"filter": {"match": {"ports.state_state": "open"}},
"aggs": {"patterns": {
"terms": dict(
baseterms,
script="""
String result = "";
if(doc['ports.service_name'].size() > 0) {
result += doc['ports.service_name'].value;
}
result += "###";
if(doc['ports.service_product'].size() > 0) {
result += doc['ports.service_product'].value;
}
result += "###";
if(doc['ports.service_version'].size() > 0) {
result += doc['ports.service_version'].value;
}
return result;
""",
missing="",
),
}},
}},
}
elif field.startswith('version:'):
def outputproc(value):
return tuple(v or None for v in value.split('###', 2))
info = field[8:]
if info.isdigit():
port = int(info)
flt = self.flt_and(flt, self.searchport(port))
matchflt = Q("match", ports__port=port)
elif ":" in info:
service, product = info.split(':', 1)
flt = self.flt_and(flt, self.searchproduct(
product=product,
service=service,
))
matchflt = (
Q("match", ports__service_name=service) &
Q("match", ports__service_product=product)
)
else:
flt = self.flt_and(flt, self.searchservice(info))
matchflt = Q("match", ports__service_name=info)
matchflt &= Q("match", ports__state_state="open")
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"filter": matchflt.to_dict(),
"aggs": {"patterns": {
"terms": dict(
baseterms,
script="""
String result = "";
if(doc['ports.service_name'].size() > 0) {
result += doc['ports.service_name'].value;
}
result += "###";
if(doc['ports.service_product'].size() > 0) {
result += doc['ports.service_product'].value;
}
result += "###";
if(doc['ports.service_version'].size() > 0) {
result += doc['ports.service_version'].value;
}
return result;
""",
),
}},
}},
}
elif field == 'httphdr':
def outputproc(value):
return tuple(value.split(':', 1))
flt = self.flt_and(flt, self.searchscript(name="http-headers"))
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts.http-headers"},
"aggs": {"patterns": {
"terms": dict(
baseterms,
script={
"lang": "painless",
"source":
"doc['ports.scripts.http-headers.name']."
"value + ':' + doc['ports.scripts.http-"
"headers.value'].value"
},
)
}},
}},
}},
}
elif field.startswith('httphdr.'):
flt = self.flt_and(flt, self.searchscript(name="http-headers"))
field = "ports.scripts.http-headers.%s" % field[8:]
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts.http-headers"},
"aggs": {"patterns": {
"terms": dict(
baseterms,
field=field
),
}},
}},
}},
}
elif field.startswith('httphdr:'):
subfield = field[8:].lower()
flt = self.flt_and(flt,
self.searchscript(name="http-headers",
values={"name": subfield}))
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts.http-headers"},
"aggs": {"patterns": {
"filter": {"match": {
"ports.scripts.http-headers.name": subfield,
}},
"aggs": {"patterns": {
"terms": dict(
baseterms,
field='ports.scripts.http-headers.value',
),
}},
}},
}},
}},
}
elif field == 'useragent' or field.startswith('useragent:'):
if field == 'useragent':
flt = self.flt_and(flt, self.searchuseragent())
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {
"terms": dict(
baseterms,
field="ports.scripts.http-user-agent",
),
}},
}},
}
else:
subfield = utils.str2regexp(field[10:])
flt = self.flt_and(flt,
self.searchuseragent(useragent=subfield))
if isinstance(subfield, utils.REGEXP_T):
subfield = self._get_pattern(subfield)
else:
subfield = re.escape(subfield)
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {
"terms": dict(
baseterms,
field="ports.scripts.http-user-agent",
include=subfield,
),
}},
}},
}
elif field == 'ja3-client' or (
field.startswith('ja3-client') and field[10] in ':.'
):
if ':' in field:
field, value = field.split(':', 1)
subkey, value = self._ja3keyvalue(utils.str2regexp(value))
if isinstance(value, utils.REGEXP_T):
include_value = self._get_pattern(value)
filter_value = {'regexp': {
"ports.scripts.ssl-ja3-client.%s" % subkey:
include_value,
}}
else:
include_value = re.escape(value)
filter_value = {'match': {
"ports.scripts.ssl-ja3-client.%s" % subkey: value,
}}
else:
value = None
subkey = None
if '.' in field:
field, subfield = field.split('.', 1)
else:
subfield = 'md5'
base = {
"terms": dict(
baseterms,
field="ports.scripts.ssl-ja3-client.%s" % subfield,
),
}
if subkey is not None:
if subkey != subfield:
base = {
"filter": filter_value,
"aggs": {"patterns": base},
}
else:
base["terms"]["include"] = include_value
flt = self.flt_and(flt, self.searchja3client(value_or_hash=value))
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts.ssl-ja3-client"},
"aggs": {"patterns": base},
}},
}},
}
elif field == 'ja3-server' or (
field.startswith('ja3-server') and field[10] in ':.'
):
def outputproc(value):
return tuple(value.split('/'))
if ':' in field:
field, values = field.split(':', 1)
if ':' in values:
value1, value2 = values.split(':', 1)
if value1:
subkey1, value1 = self._ja3keyvalue(
utils.str2regexp(value1)
)
if isinstance(value1, utils.REGEXP_T):
filter_value1 = {'regexp': {
"ports.scripts.ssl-ja3-server.%s" % subkey1:
self._get_pattern(value1),
}}
else:
filter_value1 = {'match': {
"ports.scripts.ssl-ja3-server.%s" % subkey1:
value1,
}}
else:
subkey1, value1 = None, None
if value2:
subkey2, value2 = self._ja3keyvalue(
utils.str2regexp(value2)
)
if isinstance(value2, utils.REGEXP_T):
filter_value2 = {'regexp': {
"ports.scripts.ssl-ja3-server.client.%s" %
subkey2:
self._get_pattern(value2),
}}
else:
filter_value2 = {'match': {
"ports.scripts.ssl-ja3-server.client.%s" %
subkey2:
value2,
}}
else:
subkey2, value2 = None, None
else:
subkey1, value1 = self._ja3keyvalue(
utils.str2regexp(values)
)
if isinstance(value1, utils.REGEXP_T):
filter_value1 = {'regexp': {
"ports.scripts.ssl-ja3-server.%s" % subkey1:
self._get_pattern(value1),
}}
else:
filter_value1 = {'match': {
"ports.scripts.ssl-ja3-server.%s" % subkey1:
value1,
}}
subkey2, value2 = None, None
else:
subkey1, value1 = None, None
subkey2, value2 = None, None
if '.' in field:
field, subfield = field.split('.', 1)
else:
subfield = 'md5'
flt = self.flt_and(flt, self.searchja3server(
value_or_hash=value1,
client_value_or_hash=value2,
))
base = {
"terms": dict(
baseterms,
script={
"lang": "painless",
"source":
"doc['ports.scripts.ssl-ja3-server.%s'].value + '/' + "
"doc['ports.scripts.ssl-ja3-server.client.%s'].value" %
(subfield, subfield),
},
),
}
if value1 is not None:
base = {
"filter": filter_value1,
"aggs": {"patterns": base},
}
if value2 is not None:
base = {
"filter": filter_value2,
"aggs": {"patterns": base},
}
flt = self.flt_and(flt, self.searchja3server(
value_or_hash=value1,
client_value_or_hash=value2,
))
nested = {
"nested": {"path": "ports"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts"},
"aggs": {"patterns": {
"nested": {"path": "ports.scripts.ssl-ja3-server"},
"aggs": {"patterns": base},
}},
}},
}
elif field.startswith('s7.'):
flt = self.flt_and(flt, self.searchscript(name="s7-info"))
subfield = field[3:]
field = {'field': 'ports.scripts.s7-info.' + subfield}
else:
field = {"field": field}
body = {"query": flt.to_dict()}
if nested is None:
body["aggs"] = {"patterns": {"terms": dict(baseterms, **field)}}
else:
body["aggs"] = {"patterns": nested}
utils.LOGGER.debug("DB: Elasticsearch aggregation: %r", body)
result = self.db_client.search(
body=body,
index=self.indexes[0],
ignore_unavailable=True,
size=0
)
result = result["aggregations"]
while 'patterns' in result:
result = result['patterns']
result = result['buckets']
if outputproc is None:
for res in result:
yield {'_id': res['key'], 'count': res['doc_count']}
else:
for res in result:
yield {'_id': outputproc(res['key']),
'count': res['doc_count']}
def main():
if USING_ARGPARSE:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('ips_or_macs',
nargs='*',
help='Display results for specified IP (or '
'networks) or MAC addresses (or MAC address '
'regexps).')
else:
parser = optparse.OptionParser(description=__doc__)
parser.parse_args_orig = parser.parse_args
def my_parse_args():
res = parser.parse_args_orig()
res[0].ensure_value('ips_or_macs', res[1])
return res[0]
parser.parse_args = my_parse_args
parser.add_argument = parser.add_option
parser.add_argument('-s', '--sensor')
parser.add_argument('-c', '--count', action="store_true")
parser.add_argument('-r',
'--resolve',
action="store_true",
help="Resolve MAC manufacturer")
args = parser.parse_args()
flts = ([], []) # MAC & IP filters
for arg in args.ips_or_macs:
if arg[:1] in "-!~":
neg = True
arg = arg[1:]
else:
neg = False
match = MAC_ADDR.search(arg)
if match:
flts[0].append(db.passive.searchmac(mac=arg, neg=neg))
elif arg.startswith('/') and '/' in arg[1:]:
flts[0].append(
db.passive.searchmac(mac=utils.str2regexp(arg), neg=neg))
elif '/' in arg:
flts[1].append(db.passive.searchnet(arg, neg=neg))
else:
flts[1].append(db.passive.searchhost(arg, neg=neg))
if not flts[0]:
flts[0].append(db.passive.searchmac())
flt = db.passive.flt_or(*flts[0])
if flts[1]:
flt = db.passive.flt_and(flt, db.passive.flt_or(*flts[1]))
if args.sensor is not None:
flt = db.passive.flt_and(flt, args.sensor)
if args.count:
print(db.passive.count(flt))
return
for rec in db.passive.get(flt,
sort=[('addr', 1), ('value', 1), ('source', 1)]):
rec["times"] = "s" if rec["count"] > 1 else ""
if not rec.get("sensor"):
rec["sensor"] = "-"
if args.resolve:
try:
manuf = utils.mac2manuf(rec['value'])[0]
except (TypeError, ValueError):
pass
else:
rec['value'] = '%s (%s)' % (rec['value'], manuf)
print("%(addr)s at %(value)s on %(sensor)s (%(source)s %(count)s "
"time%(times)s, %(firstseen)s - %(lastseen)s)" % rec)