def clear_old_records():
days_for_keeping = int(minutes_to_keep_feed())
lock.acquire()
current_date = make_date_time(seconds_to_subtract=days_for_keeping * 60)
feeds = get_feeds()
blocked = get_blocked_by_ip()
n_blocked = get_blocked_by_network()
warned = get_warned_by_ip()
n_warned = get_warned_by_network()
to_delete = feeds.find({"time.observation": {"$lt": current_date}})
with open(os.path.join(os.getcwd(), "backup.txt"), "a+",
encoding="utf-8") as f:
tmp = []
for x in to_delete:
del x['_id']
tmp.append(x)
f.write(json.dumps(tmp, ensure_ascii=False))
f.close()
blocked.delete_many({"time.observation": {"$lt": current_date}})
n_blocked.delete_many({"time.observation": {"$lt": current_date}})
warned.delete_many({"time.observation": {"$lt": current_date}})
n_warned.delete_many({"time.observation": {"$lt": current_date}})
feeds.delete_many(
{"time.observation": {
"$lt": parser.parse(current_date).isoformat()
}})
lock.release()
def warn_all(ips=None, networks=None):
"""
:param ips :{as format: [{document}, {document}...]}
:param networks :{as format: [{document}, {document}...]}
:rtype: object
"""
if ips is None:
ips = []
if networks is None:
networks = []
if not ips and not networks:
return
lock.acquire()
blocked_collection = get_blocked_by_ip()
warning_collection = get_warned_by_ip()
n_blocked_collection = get_blocked_by_network()
n_warning_collection = get_warned_by_network()
if ips:
for threat in ips:
ip = threat['source_ip']
network = threat['source_network']
if ip:
if '_id' in threat:
del threat['_id']
b = blocked_collection.find_one({
"source_ip": ip,
"source_network": network
})
if not b:
warning_collection.find_one_and_update(
{
"source_ip": ip,
"source_network": network
}, {"$set": threat},
upsert=True)
if networks:
for threat in networks:
ip = threat['source_ip']
network = threat['source_network']
if ip:
if '_id' in threat:
del threat['_id']
b = n_blocked_collection.find_one({
"source_ip": ip,
"source_network": network
})
if not b:
n_warning_collection.find_one_and_update(
{
"source_ip": ip,
"source_network": network
}, {"$set": threat},
upsert=True)
lock.release()
def blocked_or_warned(category=None, order=None, user_id=None, token=None):
"""
:param token:
:param user_id:
:param category: category of analysis ie blocked/(unblocked/warned)
:param order: order of the category: ie network or ip
:parameter: user_id: the user making the request
:parameter: token: the user token for each request
:rtype: object
"""
f = do_get_auth(user_id, token)
if not isinstance(f, bool):
return f
else:
if f is not True:
return make_response(jsonify({"error": 'Invalid User id.', "data": False}), 200)
if not category or not order:
return make_response(jsonify({"error": False, "token": get_request_token(user_id), "data": False}), 200)
ips = None
if category == "warned" or category == "unblocked":
if order == "ip":
ips = get_warned_by_ip().find({})
elif order == "network":
ips = get_warned_by_network().find({})
elif category == "blocked":
if order == "ip":
ips = get_blocked_by_ip().find({})
elif order == "network":
ips = get_blocked_by_network().find({})
if ips:
ibl = ips.sort([("block_date", DESCENDING), ("attempts", ASCENDING)])
ibl_list = []
for x in ibl:
x['id'] = x['_id']
del x['_id']
ibl_list.append(x)
create_log(find_user(user_id), request, "retrieved analysis data", LOG_USER_TYPE)
return make_response(jsonify({"error": False, "token": get_request_token(user_id),
"data": json.dumps(ibl_list, cls=JSONEncoder)}), 200)
return make_response(jsonify({"error": False, "token": get_request_token(user_id), "data": False}), 200)
def unblock_ip():
user_id = request.json['user_id']
if "data" not in request.json:
return make_response(jsonify({"error": 'no data sent to server', "token": get_request_token(user_id),
"data": False}), 200)
data = request.json['data']
if not isinstance(data, dict):
return make_response(jsonify({"error": 'Data is not json formatted.', "token": get_request_token(user_id),
"data": False}), 200)
keys = ("ip", "network")
if not set(keys).issubset(set(data)):
return make_response(jsonify({"error": 'the data submitted must contain both the `ip` and `network` fields ',
"token": get_request_token(user_id), "data": False}), 200)
ip, network = validate_ips(ip=data['ip'], network=data['network'])
if not ip or not network:
return make_response(jsonify({"error": 'Invalid address has been forwarded. please check your data.',
"token": get_request_token(user_id), "data": False}), 200)
net_data = get_blocked_by_network().find_one({"source_ip": ip, "source_network": network})
feed = []
if net_data and len(net_data) > 0:
feed.clear()
feed.append(net_data)
unblock_all(ips=feed)
else:
ip_data = get_blocked_by_ip().find_one({"source_ip": ip, "source_network": network})
if ip_data and len(ip_data) > 0:
feed.clear()
feed.append(ip_data)
unblock_all(ips=feed)
else:
return make_response(jsonify({"error": False, "token": get_request_token(user_id),
"data": "Unable to take action on the provided data."}), 200)
return make_response(jsonify({"error": False,
"token": get_request_token(user_id), "data": "Address has been white-listed "}), 200)
def auto_unblocking():
lock.acquire()
unblocking_mins = minutes_before_ip_is_unblocked()
blocked = get_blocked_by_ip()
n_blocked = get_blocked_by_network()
current_date = make_date_time(seconds_to_subtract=(unblocking_mins * 60))
records = blocked.find({"block_date": {"$lte": current_date}})
data = []
if records:
for record in records:
data.append({record['source_ip'], record['source_network']})
records_2 = n_blocked.find({"block_date": {"$lte": current_date}})
if records_2:
for record in records_2:
data.append({record['source_ip'], record['source_network']})
lock.release()
unblock_all(ips=data)
def unblock_network():
user_id = request.json['user_id']
if "network" not in request.json:
return make_response(jsonify({"error": 'Incorrect data sent to server. network value is missing',
"token": get_request_token(user_id),
"data": False}), 200)
ip, network = validate_ips(network=request.json['network'])
if not network:
return make_response(jsonify({"error": 'Invalid network address', "token": get_request_token(user_id),
"data": False}), 200)
fn = get_blocked_by_ip().find({"source_network": network})
if fn and fn.count() > 0:
unblock_all(ips=list(fn))
fn_2 = get_blocked_by_network().find({"source_network": network})
if fn_2 and fn_2.count() > 0:
unblock_all(ips=list(fn_2))
return make_response(jsonify({"error": False, "token": get_request_token(user_id),
"data": "Network `{0}` has been white-listed ".format(network)}), 200)
def block_all(ips=None, networks=None, by_country=None):
"""
:param by_country: string eg 'US'
:param ips :{as format: [{document}, {document}...]}
:param networks :{as format: [{document}, {document}...]}
:rtype: object
"""
if ips is None:
ips = []
if networks is None:
networks = []
if not ips and not networks and not by_country:
return
lock.acquire()
blocked_collection = get_blocked_by_ip()
warning_collection = get_warned_by_ip()
n_blocked_collection = get_blocked_by_network()
n_warning_collection = get_warned_by_network()
if ips:
for threat in ips:
ip = threat['source_ip'] or None
network = threat['source_network'] or None
if ip:
if '_id' in threat:
del threat['_id']
threat['block_date'] = make_date_time()
threat['unblock_date'] = ""
blocked_collection.find_one_and_update(
{
"source_ip": ip,
"source_network": network
}, {"$set": threat},
upsert=True)
warning_collection.find_one_and_delete({
"source_ip":
ip,
"source_network":
network
})
threading.Thread(target=__do_the_actual_blocking,
kwargs={
'ip': ip
},
daemon=True).start()
if networks:
for threat in networks:
ip = threat['source_ip']
network = threat['source_network']
if network:
if '_id' in threat:
del threat['_id']
threat['block_date'] = make_date_time()
threat['unblock_date'] = ""
n_blocked_collection.find_one_and_update(
{
"source_ip": ip,
"source_network": network
}, {"$set": threat},
upsert=True)
n_warning_collection.find_one_and_delete({
"source_ip":
ip,
"source_network":
network
})
threading.Thread(target=__do_the_actual_blocking,
kwargs={
'network': network
},
daemon=True).start()
if by_country:
records = blocked_collection.find(
{"source_geolocation_cc": by_country})
if not records:
records_2 = warning_collection.find(
{"source_geolocation_cc": by_country})
if records_2:
for rec in records_2:
if '_id' in rec:
del rec['_id']
rec['block_date'] = make_date_time()
rec['unblock_date'] = ""
blocked_collection.insert_one(rec)
threading.Thread(target=__do_the_actual_blocking,
kwargs={
'ip': rec['source_ip']
},
daemon=True).start()
warning_collection.delete_many(
{"source_geolocation_cc": by_country})
records_1 = n_blocked_collection.find(
{"source_geolocation_cc": by_country})
if not records_1:
records_2 = n_warning_collection.find(
{"source_geolocation_cc": by_country})
if records_2:
for rec in records_2:
if '_id' in rec:
del rec['_id']
rec['block_date'] = make_date_time()
rec['unblock_date'] = ""
n_blocked_collection.insert_one(rec)
threading.Thread(target=__do_the_actual_blocking,
kwargs={
'network': rec['source_network']
},
daemon=True).start()
n_warning_collection.delete_many(
{"source_geolocation_cc": by_country})
lock.release()
def unblock_all(ips=None, by_country=None):
"""
:param by_country: country name/geolocation to unblock eg 'US'
:param ips :{as format: [{source_ip, source_network}, ...]}
:rtype: object
"""
if not ips and not by_country:
return
lock.acquire()
blocked_collection = get_blocked_by_ip()
warning_collection = get_warned_by_ip()
n_blocked_collection = get_blocked_by_network()
n_warning_collection = get_warned_by_network()
if ips:
for ip in ips:
i_p = ip['source_ip']
net = ip['source_network']
record = blocked_collection.find_one_and_delete({
"source_ip":
i_p,
"source_network":
net
})
if record:
if '_id' in record:
del record['_id']
record['block_date'] = ""
record['unblock_date'] = make_date_time()
warning_collection.find_one_and_update({"source_ip": i_p},
{"$set": record},
upsert=True)
threading.Thread(target=__do_the_actual_unblocking,
kwargs={
'ip': i_p
},
daemon=True).start()
else:
record = n_blocked_collection.find_one_and_delete({
"source_ip":
i_p,
"source_network":
net
})
if record:
if '_id' in record:
del record['_id']
record['block_date'] = ""
record['unblock_date'] = make_date_time()
n_warning_collection.find_one_and_update(
{"source_network": net}, {"$set": record}, upsert=True)
threading.Thread(target=__do_the_actual_unblocking,
kwargs={
'network': net
},
daemon=True).start()
if by_country:
records = blocked_collection.find(
{"source_geolocation_cc": by_country})
if records:
for rec in records:
if '_id' in rec:
del rec['_id']
rec['block_date'] = ""
rec['unblock_date'] = make_date_time()
warning_collection.insert_one(rec)
threading.Thread(target=__do_the_actual_unblocking,
kwargs={
'ip': rec['source_ip']
},
daemon=True).start()
blocked_collection.delete_many(
{"source_geolocation_cc": by_country})
records_1 = n_blocked_collection.find(
{"source_geolocation_cc": by_country})
if records_1:
for rec in records_1:
if '_id' in rec:
del rec['_id']
rec['block_date'] = ""
rec['unblock_date'] = make_date_time()
n_warning_collection.insert_one(rec)
threading.Thread(target=__do_the_actual_unblocking,
kwargs={
'network': rec['source_network']
},
daemon=True).start()
n_blocked_collection.delete_many(
{"source_geolocation_cc": by_country})
lock.release()