def get_network_events():
"""
get network events
Returns:
an array contain network events
"""
date = fix_date(get_value_from_request("date"))
if date:
try:
return jsonify([
i for i in connector.network_events.find(
{
"date": {
"$gte": date[0],
"$lte": date[1]
}
}, {
"_id": 0
}).skip(fix_skip(get_value_from_request("skip"))).limit(
fix_limit(get_value_from_request("limit")))
]), 200
except Exception as _:
return flask_null_array_response()
else:
try:
return jsonify([
i for i in connector.network_events.find({}, {
"_id": 0
}).skip(fix_skip(get_value_from_request("skip"))).limit(
fix_limit(get_value_from_request("limit")))
]), 200
except Exception as _:
return flask_null_array_response()
def module_events():
"""
Get total number of credential events according to module
Returns:
JSON/Dict of credential events according to module
"""
module_name = get_value_from_request("module_name")
module_query = [
group_by_ip_dest,
{
"$skip": fix_skip(
get_value_from_request("skip")
)
},
{
"$limit": fix_limit(
get_value_from_request("limit")
)
}
]
if module_name:
module_query.insert(0, {"$match": {'module_name': module_name}})
try:
return jsonify(
aggregate_function(connector.credential_events, module_query)
), 200
except Exception as _:
return flask_null_array_response()
def top_honeypot_machine_names():
"""
get top honeypot machine names in honeypot events
Returns:
JSON/Dict top honeypot machine names
"""
date = fix_date(get_value_from_request("date"))
top_machinenames_query = [
top_machine_names_groupby, sort_by_count_and_id, {
"$skip": fix_skip(get_value_from_request("skip"))
}, {
"$limit": fix_limit(get_value_from_request("limit"))
}
]
if date:
match_by_date = {
"$match": {
"date": {
"$gte": date[0],
"$lte": date[1]
}
}
}
top_machinenames_query.insert(0, match_by_date)
try:
return jsonify(
aggregate_function(connector.honeypot_events,
top_machinenames_query)), 200
except Exception as _:
return flask_null_array_response()
def get_network_events():
"""
get network events
Returns:
an array contain network events
"""
try:
return jsonify(
[
i for i in
connector.network_events.find(
{},
{
"_id": 0
}
).skip(
fix_skip(
get_value_from_request("skip")
)
).limit(
fix_limit(
get_value_from_request("limit")
)
)
]
), 200
except Exception as _:
del _
return flask_null_array_response()
def get_events_data(event_type):
"""
get events data
Returns:
an array contain event data
"""
abort(404) if event_type not in event_types else event_type
module_name = get_value_from_request("module_name")
date = get_value_from_request("date")
filter = get_value_from_request("filter")
try:
query = filter_by_date(date) if date else {}
query.update(filter_by_module_name(module_name) if module_name else {})
query.update({
key: filter_by_regex(fix_filter_query(filter)[key])
for key in fix_filter_query(filter)
} if filter else {})
return jsonify({
"total":
event_types[event_type].count(query),
"data": [
i for i in event_types[event_type].find(query, {
"_id": 0
}).skip(fix_skip(get_value_from_request("skip"))).limit(
fix_limit(get_value_from_request("limit")))
]
}), 200
except Exception:
abort(500)
def top_ten_countries_in_network_events():
"""
get top ten repeated countries in network events
Returns:
JSON/Dict top ten repeated countries in network events
"""
date = fix_date(
get_value_from_request("date")
)
top_countries_query = [
top_countries_groupby,
sort_by_count,
{
"$skip": fix_skip(
get_value_from_request("skip")
)
},
{
"$limit": fix_limit(
get_value_from_request("limit")
)
}
]
if date:
match_by_date_and_country = {
"$match":
{
"country_ip_dest": {
"$gt": "-"
},
"date":
{
"$gte": date[0],
"$lte": date[1]
}
}
}
top_countries_query.insert(0, match_by_date_and_country)
else:
match_by_country = {
"$match":
{
"country_ip_dest": {
"$gt": "-"
}
}
}
top_countries_query.insert(0, match_by_country)
try:
return jsonify(
aggregate_function(
connector.network_events,
top_countries_query
)
), 200
except Exception as _:
return flask_null_array_response()
def top_ten_ips_in_network_events_by_date():
"""
get top ten repeated ips in network events by date
Returns:
JSON/Dict top ten repeated ips in network events by date
"""
date = fix_date(get_value_from_request("date"))
if date:
try:
return jsonify(
[
i for i in
connector.network_events.aggregate(
[
{
"$match": {
"date": {
"$gte": date[0],
"$lte": date[1]
}
}
},
{
"$group":
{
"_id":
{
"ip": "$ip",
"country": "$country"
},
"count":
{
"$sum": 1
}
}
},
{
"$sort":
SON(
[
("count", -1),
("_id", -1)]
)
},
{
"$skip": fix_skip(get_value_from_request("skip"))
},
{
"$limit": fix_limit(get_value_from_request("limit"))
}
]
)
]
), 200
except Exception as _:
del _
return flask_null_array_response()
def top_ten_ports_in_honeypot_events():
"""
get top ten repeated ports in honeypot events
Returns:
JSON/Dict top ten repeated ports in honeypot events
"""
date = fix_date(get_value_from_request("date"))
country = get_value_from_request("country")
top_ports_query = [
top_ports_groupby, {
"$skip": fix_skip(get_value_from_request("skip"))
}, {
"$limit": fix_limit(get_value_from_request("limit"))
}
]
if country and date:
match_by_country_and_date = {
"$match": {
"country": country,
"date": {
"$gte": date[0],
"$lte": date[1]
}
}
}
top_ports_query.insert(0, match_by_country_and_date)
top_ports_query.insert(2, sort_by_count_and_id)
elif country:
match_by_country = {
"$match": {
"country": country,
}
}
top_ports_query.insert(0, match_by_country)
top_ports_query.insert(2, sort_by_count_and_id)
elif date:
match_by_date = {
"$match": {
"date": {
"$gte": date[0],
"$lte": date[1]
}
}
}
top_ports_query.insert(0, match_by_date)
top_ports_query.insert(2, sort_by_count)
else:
top_ports_query.insert(1, sort_by_count)
try:
return jsonify(
aggregate_function(connector.honeypot_events,
top_ports_query)), 200
except Exception as _:
return flask_null_array_response()
def top_ten_countries_in_honeypot_events():
"""
get top ten repeated countries in honeypot events
Returns:
JSON/Dict top ten repeated countries honeypot in events
"""
try:
return jsonify(
[
i for i in
connector.honeypot_events.aggregate(
[
{
"$match": {
"country": {
"$gt": "-"
}
}
},
{
"$group":
{
"_id": "$country",
"count":
{
"$sum": 1
}
}
},
{
"$sort": SON(
[
("count", -1)
]
)
},
{
"$skip": fix_skip(get_value_from_request("skip"))
},
{
"$limit": fix_limit(get_value_from_request("limit"))
}
]
)
]
), 200
except Exception as _:
del _
def top_ten_ports_in_network_events():
"""
get top ten repeated ports in network events
Returns:
JSON/Dict top ten repeated ports in network events
"""
try:
return jsonify(
[
i for i in
connector.network_events.aggregate(
[
{
"$group":
{
"_id": "$port",
"count":
{
"$sum": 1
}
}
},
{
"$sort":
SON(
[
("count", -1),
("_id", -1)
]
)
},
{
"$skip": fix_skip(get_value_from_request("skip"))
},
{
"$limit": fix_limit(get_value_from_request("limit"))
}
]
)
]
), 200
except Exception as _:
del _
return flask_null_array_response()
def get_honeypot_events_by_date():
"""
get honeypot events by date
Returns:
an array contain honeypot events by date
"""
date = fix_date(get_value_from_request("date"))
if date:
try:
return jsonify(
[
i for i in
connector.honeypot_events.find(
{
"date":
{
"$gte": date[0],
"$lte": date[1]
}
},
{
"_id": 0
}
).skip(
fix_skip(
get_value_from_request("skip")
)
).limit(
fix_limit(
get_value_from_request("limit")
)
)
]
), 200
except Exception as _:
del _
return flask_null_array_response()
def filter_by_limit(limit):
return {
"$limit": fix_limit(limit)
}
def test_fix_limit_fail(self):
returned_values = fix_limit('0')
self.assertNotEqual(returned_values, 11)
self.assertFalse(isinstance(returned_values, str))
def test_fix_limit_pass(self):
returned_values = fix_limit('10')
self.assertEqual(returned_values, 10)
self.assertTrue(isinstance(returned_values, int))
def get_events_data(event_type):
"""
Get events data
---
parameters:
- in: path
name: event_type
schema:
type: string
required: true
enum:
- honeypot
- network
- credential
- file
- data
- pcap
- in: query
name: date
schema:
type: date
required: false
description: Date to filter records for particular day
- in: query
name: module_name
schema:
type: string
required: false
enum:
- ftp/strong_password
- ftp/weak_password
- http/basic_auth_strong_password
- http/basic_auth_weak_password
- ics/veeder_root_guardian_ast
- smtp/strong_password
- ssh/strong_password
- ssh/weak_password
description: one of the module names supported by the framework
- in: query
name: filter
schema:
type: string
example: ip_dest=192.16.1.1&ip_src=192.168.0.*
description: filter on serverside by query (regex)
required: false
- in: query
name: skip
schema:
type: number
example: 0
description: skip the number of records from start
required: false
- in: query
name: limit
schema:
type: number
example: 100
description: number of records to fetch
required: false
responses:
'200':
description: Ok
examples:
application/json:
{
"data": [
{
"country_ip_dest": "US",
"country_ip_src": "-",
"date": "2021-06-15 16:39:54",
"ip_dest": "142.250.183.131",
"ip_src": "192.168.0.106",
"machine_name": "stockholm_server_1",
"module_name": "http/basic_auth_strong_password",
"port_dest": 80,
"port_src": 59984,
"protocol": "TCP"
}
],
"total": 614
}
'404':
description: Not Found
examples:
application/json: { "msg": "file/path not found!","status": "error"}
"""
abort(404) if event_type not in event_types else event_type
module_name = get_value_from_request("module_name")
date = get_value_from_request("date")
filter_by = get_value_from_request("filter")
try:
query = {
"query": {
"bool": {
"must": [filter_by_date(date)['query']] if date else [],
"filter": []
}
}
}
if module_name:
query['query']['bool']['must'].append(
filter_by_module_name(module_name))
if filter_by:
for key in fix_filter_query(filter_by):
filter_query = filter_by_regex(
key,
fix_filter_query(filter_by)[key])
query['query']['bool']['filter'].append(filter_query)
records = []
if get_value_from_request("limit") == "infinite":
data = elasticsearch_events.search(index=event_types[event_type],
body=query,
scroll="2m",
size=10000)
scroll_id = data['_scroll_id']
scroll_size = len(data['hits']['hits'])
while scroll_size > 0:
for record in data['hits']['hits']:
records.append(record['_source'])
data = elasticsearch_events.scroll(scroll_id=scroll_id,
scroll='2m')
scroll_id = data['_scroll_id']
scroll_size = len(data['hits']['hits'])
else:
records = [
record['_source'] for record in elasticsearch_events.search(
index=event_types[event_type],
body=query,
from_=fix_skip(get_value_from_request("skip")),
size=fix_limit(get_value_from_request("limit")))['hits']
['hits']
]
return jsonify({
"total":
int(
elasticsearch_events.count(index=event_types[event_type],
body=query)['count']),
"data":
records
}), 200
except Exception:
abort(500)
