def service_list_json_route():
"""list services, data endpoint"""
columns = [
ColumnDT(Service.id, mData='id'),
ColumnDT(Host.id, mData='host_id'),
ColumnDT(Host.address, mData='host_address'),
ColumnDT(Host.hostname, mData='host_hostname'),
ColumnDT(Service.proto, mData='proto'),
ColumnDT(Service.port, mData='port'),
ColumnDT(Service.name, mData='name'),
ColumnDT(Service.state, mData='state'),
ColumnDT(Service.info, mData='info'),
ColumnDT(Service.tags, mData='tags'),
ColumnDT(Service.comment, mData='comment'),
ColumnDT(literal_column('1'),
mData='_buttons',
search_method='none',
global_search=False)
]
query = db.session.query().select_from(Service).outerjoin(Host)
if 'filter' in request.values:
query = apply_filters(query,
FILTER_PARSER.parse(
request.values.get('filter')),
do_auto_join=False)
services = DataTables(request.values.to_dict(), query,
columns).output_result()
return jsonify(services)
def note_list_json_route():
"""list notes, data endpoint"""
columns = [
ColumnDT(Note.id, mData='id'),
ColumnDT(Host.id, mData='host_id'),
ColumnDT(Host.address, mData='host_address'),
ColumnDT(Host.hostname, mData='host_hostname'),
# break pylint duplicate-code
ColumnDT(Service.proto, mData='service_proto'),
ColumnDT(Service.port, mData='service_port'),
ColumnDT(func.concat_ws('/', Service.port, Service.proto), mData='service'),
ColumnDT(Note.via_target, mData='via_target'),
ColumnDT(Note.xtype, mData='xtype'),
ColumnDT(Note.data, mData='data'),
ColumnDT(Note.tags, mData='tags'),
ColumnDT(Note.comment, mData='comment'),
ColumnDT(literal_column('1'), mData='_buttons', search_method='none', global_search=False)
]
query = db.session.query().select_from(Note).outerjoin(Host, Note.host_id == Host.id).outerjoin(Service, Note.service_id == Service.id)
if 'filter' in request.values:
query = apply_filters(query, FILTER_PARSER.parse(request.values.get('filter')), do_auto_join=False)
notes = DataTables(request.values.to_dict(), query, columns).output_result()
return jsonify(notes)
def vuln_export(qfilter=None):
"""export all vulns in storage without aggregation"""
host_address_format = case([(func.family(
Host.address) == 6, func.concat('[', func.host(Host.address), ']'))],
else_=func.host(Host.address))
host_ident = case([(func.char_length(Host.hostname) > 0, Host.hostname)],
else_=host_address_format)
endpoint_address = func.concat_ws(':', host_address_format, Service.port)
endpoint_hostname = func.concat_ws(':', host_ident, Service.port)
query = db.session \
.query(
host_ident.label('host_ident'),
Vuln.name.label('vulnerability'),
Vuln.descr.label('description'),
Vuln.data,
func.text(Vuln.severity).label('severity'),
Vuln.tags,
endpoint_address.label('endpoint_address'),
endpoint_hostname.label('endpoint_hostname'),
Vuln.refs.label('references')
) \
.outerjoin(Host, Vuln.host_id == Host.id) \
.outerjoin(Service, Vuln.service_id == Service.id)
if qfilter:
query = apply_filters(query,
FILTER_PARSER.parse(qfilter),
do_auto_join=False)
content_trimmed = False
fieldnames = [
'id', 'host_ident', 'vulnerability', 'severity', 'description', 'data',
'tags', 'endpoint_address', 'endpoint_hostname', 'references'
]
output_buffer = StringIO()
output = DictWriter(output_buffer,
fieldnames,
restval='',
quoting=QUOTE_ALL)
output.writeheader()
for row in query.all():
rdata = row._asdict()
rdata['tags'] = list_to_lines(rdata['tags'])
rdata['references'] = list_to_lines(
map(url_for_ref, rdata['references']))
rdata, trim_trigger = trim_rdata(rdata)
content_trimmed |= trim_trigger
output.writerow(rdata)
if content_trimmed:
output.writerow({'host_ident': 'WARNING: some cells were trimmed'})
return output_buffer.getvalue()
def storage_service_list(**kwargs):
"""service listing; used to feed manymap queues from storage data"""
def get_host(svc, hostnames=False):
"""return address or hostname"""
if hostnames and svc.host.hostname:
return svc.host.hostname
return format_host_address(svc.host.address)
def get_data(svc):
"""return common data as dict"""
return {
'proto': svc.proto,
'port': svc.port,
'name': svc.name,
'state': svc.state,
'info': json.dumps(svc.info)
}
if kwargs['long'] and kwargs['short']:
current_app.logger.error(
'--short and --long are mutualy exclusive options')
sys.exit(1)
query = Service.query
if kwargs['filter']:
query = apply_filters(query,
FILTER_PARSER.parse(kwargs['filter']),
do_auto_join=False)
fmt = '{proto}://{host}:{port}'
if kwargs['short']:
fmt = '{host}'
elif kwargs['simple']:
fmt = '{host} {port}'
elif kwargs['long']:
fmt = '{proto}://{host}:{port} {name} {state} {info}'
for tmp in query.all():
print(
fmt.format(**get_data(tmp),
host=get_host(tmp, kwargs['hostnames'])))
def dnstree_json_route():
"""dns hierarchy tree visualization data generator"""
# from all hostnames we know, create tree structure dict-of-dicts
def to_tree(node, items):
if not items:
return {}
if items[0] not in node:
node[items[0]] = {}
node[items[0]] = to_tree(node[items[0]], items[1:])
return node
# walk through the tree and generate list of nodes and links
def to_graph_data(parentid, treedata, nodes, links):
for node in treedata:
nodeid = len(nodes)
nodes.append({'name': node, 'id': nodeid})
if parentid is not None:
links.append({'source': parentid, 'target': nodeid})
(nodes, links) = to_graph_data(nodeid, treedata[node], nodes,
links)
return (nodes, links)
query = Host.query
if 'filter' in request.values:
query = apply_filters(query,
FILTER_PARSER.parse(
request.values.get('filter')),
do_auto_join=False)
crop = request.values.get('crop', 0, type=int)
hostnames_tree = {}
for ihost in query.all():
if ihost.hostname:
tmp = list(reversed(ihost.hostname.split('.')[crop:]))
if tmp:
hostnames_tree = to_tree(hostnames_tree, ['DOTROOT'] + tmp)
(nodes, links) = to_graph_data(None, hostnames_tree, [], [])
nodes[0].update({'size': 10})
return jsonify({'nodes': nodes, 'links': links})
def vuln_list_json_route():
"""list vulns, data endpoint"""
columns = [
ColumnDT(literal_column('1'),
mData='_select',
search_method='none',
global_search=False),
ColumnDT(Vuln.id, mData='id'),
ColumnDT(Host.id, mData='host_id'),
ColumnDT(Host.address, mData='host_address'),
ColumnDT(Host.hostname, mData='host_hostname'),
ColumnDT(Service.proto, mData='service_proto'),
ColumnDT(Service.port, mData='service_port'),
ColumnDT(func.concat_ws('/', Service.port, Service.proto),
mData='service'),
ColumnDT(Vuln.via_target, mData='via_target'),
ColumnDT(Vuln.name, mData='name'),
ColumnDT(Vuln.xtype, mData='xtype'),
ColumnDT(Vuln.severity, mData='severity'),
ColumnDT(Vuln.refs, mData='refs'),
ColumnDT(Vuln.tags, mData='tags'),
ColumnDT(Vuln.comment, mData='comment'),
ColumnDT(literal_column('1'),
mData='_buttons',
search_method='none',
global_search=False)
]
query = db.session.query().select_from(Vuln).outerjoin(
Host, Vuln.host_id == Host.id).outerjoin(Service,
Vuln.service_id == Service.id)
if 'filter' in request.values:
query = apply_filters(query,
FILTER_PARSER.parse(
request.values.get('filter')),
do_auto_join=False)
vulns = DataTables(request.values.to_dict(), query,
columns).output_result()
return Response(json.dumps(vulns, cls=SnerJSONEncoder),
mimetype='application/json')
def service_grouped_json_route():
"""view grouped services, data endpoint"""
info_column = service_info_column(request.args.get('crop'))
columns = [
ColumnDT(info_column, mData='info'),
ColumnDT(func.count(Service.id),
mData='cnt_services',
global_search=False),
]
# join allows filter over host attrs
query = db.session.query().select_from(Service).join(Host).group_by(
info_column)
if 'filter' in request.values:
query = apply_filters(query,
FILTER_PARSER.parse(
request.values.get('filter')),
do_auto_join=False)
services = DataTables(request.values.to_dict(), query,
columns).output_result()
return jsonify(services)
def vuln_grouped_json_route():
"""view grouped vulns, data endpoint"""
columns = [
ColumnDT(Vuln.name, mData='name'),
ColumnDT(Vuln.severity, mData='severity'),
ColumnDT(Vuln.tags, mData='tags'),
ColumnDT(func.count(Vuln.id), mData='cnt_vulns', global_search=False),
]
# join allows filter over host attrs
query = db.session.query().select_from(Vuln).join(Host).group_by(
Vuln.name, Vuln.severity, Vuln.tags)
if 'filter' in request.values:
query = apply_filters(query,
FILTER_PARSER.parse(
request.values.get('filter')),
do_auto_join=False)
vulns = DataTables(request.values.to_dict(), query,
columns).output_result()
return Response(json.dumps(vulns, cls=SnerJSONEncoder),
mimetype='application/json')
def check(testcase, expected):
"""test helper"""
output = FILTER_PARSER.parse(testcase)
print('testcase: %s outputs %s' % (testcase, output))
assert output == expected
def vuln_report(qfilter=None, group_by_host=False): # pylint: disable=too-many-locals
"""generate report from storage data"""
host_address_format = case([(func.family(
Host.address) == 6, func.concat('[', func.host(Host.address), ']'))],
else_=func.host(Host.address))
host_ident = case([(func.char_length(Host.hostname) > 0, Host.hostname)],
else_=host_address_format)
endpoint_address = func.concat_ws(':', host_address_format, Service.port)
endpoint_hostname = func.concat_ws(':', host_ident, Service.port)
# note1: refs (itself and array) must be unnested in order to be correctly uniq and agg as individual elements by used axis
# note2: unnesting refs should be implemented as
# SELECT vuln.name, array_remove(array_agg(urefs.ref), NULL) FROM vuln
# LEFT OUTER JOIN LATERAL unnest(vuln.refs) as urefs(ref) ON TRUE
# GROUP BY vuln.name;`
# but could not construct appropriate sqla expression `.label('x(y)')` always rendered as string instead of 'table with column'
unnested_refs = db.session.query(Vuln.id,
func.unnest(
Vuln.refs).label('ref')).subquery()
query = db.session \
.query(
Vuln.name.label('vulnerability'),
Vuln.descr.label('description'),
func.text(Vuln.severity).label('severity'),
Vuln.tags,
func.array_agg(func.distinct(host_ident)).label('host_ident'),
func.array_agg(func.distinct(endpoint_address)).label('endpoint_address'),
func.array_agg(func.distinct(endpoint_hostname)).label('endpoint_hostname'),
func.array_remove(func.array_agg(func.distinct(unnested_refs.c.ref)), None).label('references')
) \
.outerjoin(Host, Vuln.host_id == Host.id) \
.outerjoin(Service, Vuln.service_id == Service.id) \
.outerjoin(unnested_refs, Vuln.id == unnested_refs.c.id) \
.group_by(Vuln.name, Vuln.descr, Vuln.severity, Vuln.tags)
if group_by_host:
query = query.group_by(host_ident)
if qfilter:
query = apply_filters(query,
FILTER_PARSER.parse(qfilter),
do_auto_join=False)
content_trimmed = False
fieldnames = [
'id', 'asset', 'vulnerability', 'severity', 'advisory', 'state',
'endpoint_address', 'description', 'tags', 'endpoint_hostname',
'references'
]
output_buffer = StringIO()
output = DictWriter(output_buffer,
fieldnames,
restval='',
extrasaction='ignore',
quoting=QUOTE_ALL)
output.writeheader()
for row in query.all():
rdata = row._asdict()
# must count endpoints, multiple addrs can coline in hostnames
if group_by_host:
rdata['asset'] = rdata['host_ident'][0]
else:
rdata['asset'] = rdata['host_ident'][0] if len(
rdata['endpoint_address']) == 1 else 'misc'
for col in ['endpoint_address', 'endpoint_hostname', 'tags']:
rdata[col] = list_to_lines(rdata[col])
rdata['references'] = list_to_lines(
map(url_for_ref, rdata['references']))
rdata, trim_trigger = trim_rdata(rdata)
content_trimmed |= trim_trigger
output.writerow(rdata)
if content_trimmed:
output.writerow({'asset': 'WARNING: some cells were trimmed'})
return output_buffer.getvalue()