def get_new_local_alerts(): """ alerts from own local network :return: """ ip_details = my_ip_details() m = int(minutes_a_record_remains_new()) new_date = make_date_time(seconds_to_subtract=m * 60) f = [] if not ip_details: return None data = get_feeds().find({"time.observation": {"$gt": new_date}, "source.network": ip_details['network'], "$or": [{"source.ip": {"$exists": True}, "source.network": {"$exists": True}}]})\ .sort([("time_observation", DESCENDING)]) if data and data.count() > 0: data = normalize_dict(data) for x, value in data.items(): value['id'] = x if '_id' in value: del value['_id'] if 'raw' in value: del value['raw'] f.append(value) create_log(None, None, "{0} Local attacks detected : ".format(len(f)), LOG_SYSTEM_TYPE) return f
def __do_the_actual_unblocking(ip=None, network=None): if not ip and not network: return False try: cmd1 = subprocess.Popen(['echo', SUDO_PASSWORD], stdout=subprocess.PIPE) if ip and not ip == '0.0.0.0': subprocess.Popen(['sudo', '-S'] + IPTABLES_UNBLOCK_RULE.format(ip).split(), stdin=cmd1.stdout, stdout=subprocess.PIPE) if network and not network == '0.0.0.0/0': subprocess.Popen(['sudo', '-S'] + IPTABLES_UNBLOCK_RULE.format(network).split(), stdin=cmd1.stdout, stdout=subprocess.PIPE) subprocess.Popen(['sudo', '-S', 'iptables-save'], stdin=cmd1.stdout, stdout=subprocess.PIPE) except subprocess.CalledProcessError as e: print(e.output) create_log(None, None, "Unblocked : " + str(ip or network), LOG_SYSTEM_TYPE) return True
def get_plot_data(user_id=None, token=None): """ :param: category of analysis ie blocked/(unblocked/warned) :parameter: user_id: the user making the request :parameter: token: the user token for each request :rtype: object """ f = do_get_auth(user_id, token) if not isinstance(f, bool): return f else: if f is not True: return make_response(jsonify({"error": 'Invalid User id.', "data": False}), 200) ips = get_plots().find({}) if ips: ibl_list = [] for x in ips: x['id'] = x['_id'] del x['_id'] ibl_list.append(x) create_log(find_user(user_id), request, "retrieved analysis data", LOG_USER_TYPE) return make_response(jsonify({"error": False, "token": get_request_token(user_id), "data": json.dumps(ibl_list, cls=JSONEncoder)}), 200) return make_response(jsonify({"error": False, "token": get_request_token(user_id), "data": False}), 200)
def delete_logs(user_id, log_type="user", retain=None, token=None): """ :param token: :param user_id :param log_type :param retain :rtype: object """ f = do_get_auth(user_id, token) if not isinstance(f, bool): return f else: if f is not True: return make_response( jsonify({ "error": 'Invalid User id.', "data": False }), 200) logs = get_logs() try: """ To change this so that deleting is limited to certain date to use retain """ data = logs.delete_many({ 'type': log_type, "lastAccessTime": { "$lt": "2018-04-03 19:33:53" } }) if not data or data.deleted_count < 1: return make_response( jsonify({ "error": 'Unable to delete records. No records found ', "token": get_request_token(user_id), "data": False })) user = find_user(_id=user_id) if user: create_log(user, request, "accessed " + log_type + " logs", LOG_SYSTEM_TYPE) except IndexError: return make_response( jsonify({ "error": 'No data found for deletion', "token": get_request_token(user_id), "data": False })) return make_response( jsonify({ "error": False, "token": get_request_token(user_id), "data": data.deleted_count }))
def fetch_logs(user_id=None, log_type="user", token=None): """ :param token: :param user_id :param log_type :return: """ f = do_get_auth(user_id, token) if not isinstance(f, bool): return f else: if f is not True: return make_response( jsonify({ "error": 'Invalid User id.', "data": False }), 200) logs = get_logs() try: user = find_user(_id=user_id) if user: create_log(user, request, "accessed " + log_type + " logs", LOG_SYSTEM_TYPE) data = logs.find({ 'type': log_type }, { '_id': 0 }).sort("lastAccessTime", -1) if not data or data.count() < 1: return make_response( jsonify({ "error": 'No logs found..', "token": get_request_token(user_id), "data": False })) data = [x for x in data] except IndexError: return make_response( jsonify({ "error": 'No data found', "token": get_request_token(user_id), "data": False })) return make_response( jsonify({ "error": False, "token": get_request_token(user_id), "data": data }))
def change_password(): """ :parameter: request request: {user_id, token, old_password, new_password, new_password_verify} :rtype: object """ user_id = request.json['user_id'] cond = ("old_password", "new_password", "new_password_verify") if not all(val in request.json.keys() for val in cond): return make_response( jsonify({ "error": 'Some data was missing in the submitted request. ', "token": get_request_token(user_id), "data": False }), 200) user = find_user(user_id) if not user: return make_response( jsonify({ "error": 'Could not find this user. Please logout and in again', "token": get_request_token(user_id), "data": False }), 200) if not request.json['new_password'] == request.json['new_password_verify']: return make_response( jsonify({ "error": 'Passwords do not match.', "token": get_request_token(user_id), "data": False }), 200) if verify_password(request.json['old_password'], user['password']): get_users().update_one({"email": user['email']}, { "$set": { "password": hash_password(request.json['new_password']) } }) create_log(find_user(user_id), request, "changed his password", LOG_USER_TYPE) return make_response( jsonify({ "error": False, "token": get_request_token(user_id), "data": "Password change has been successful." }), 200) return make_response( jsonify({ "error": "Old password is not valid. please correct it.", "token": get_request_token(user_id), "data": False }), 200)
def fetch_single_user(user_email_id, user_id=None, token=None): """ :parameter: user_id: the user making the request :parameter: token: the user token for each request :param: user_email_id: user id to update :rtype: object """ f = do_get_auth(user_id, token) if not isinstance(f, bool): return f else: if f is not True: return make_response( jsonify({ "error": 'Invalid User id.', "data": False }), 200) if not user_email_id or not len(user_email_id) == 24: return make_response( jsonify({ "error": 'A valid User id is required', "token": get_request_token(user_id), "data": False }), 200) user = find_user(user_email_id) if user: user['id'] = user['_id'] del user['_id'] del user['password'] create_log(find_user(user_id), request, "retrieved user data", LOG_USER_TYPE) return make_response( jsonify({ "error": False, "token": get_request_token(user_id), "data": json.dumps(user, cls=JSONEncoder) }), 200) return make_response( jsonify({ "error": "Could not find the requested user.", "token": get_request_token(user_id), "data": False }), 200)
def deactivate_user(email, user_id=None, token=None): """ :parameter: user_id: the user making the request :parameter: email: the email of user to deactivate :parameter: token: the user token for each request :rtype: object """ f = do_get_auth(user_id, token) if not isinstance(f, bool): return f else: if f is not True: return make_response( jsonify({ "error": 'Invalid User id.', "data": False }), 200) email = email.replace('-', '@') if not validate_email(email): return make_response( jsonify({ "error": "Email is invalid. ex. [email protected]", "token": get_request_token(user_id), "data": False }), 200) if deactivate_user_status(email): create_log(find_user(user_id), request, "deactivated an account of {0}".format(email), LOG_USER_TYPE) return make_response( jsonify({ "error": False, "token": get_request_token(user_id), "data": "User has been deactivated successfully" }), 200) return make_response( jsonify({ "error": "Sorry. Could not deactivate user. Something happened", "token": get_request_token(user_id), "data": False }), 200)
def activate_user(): """ :parameter: user_id: the user making the request :parameter: email: the email of user to activate :parameter: token: the user token for each request :rtype: object """ keys = ("email", "user_id") user_id = request.json['user_id'] email = request.json['email'] if not set(keys).issubset(set(request.json)): return make_response( jsonify({ "error": 'Some fields are missing', "token": get_request_token(user_id), "data": False }), 200) if not validate_email(email): return make_response( jsonify({ "error": "Email is invalid. ex. [email protected]", "token": get_request_token(user_id), "data": False }), 200) if activate_user_status(email): create_log(find_user(user_id), request, "Activated account of {0}".format(email), LOG_USER_TYPE) return make_response( jsonify({ "error": False, "token": get_request_token(user_id), "data": "User has been activated successfully" }), 200) return make_response( jsonify({ "error": "Sorry. Could not activate user. Something happened", "token": get_request_token(user_id), "data": False }), 200)
def blocked_or_warned(category=None, order=None, user_id=None, token=None): """ :param token: :param user_id: :param category: category of analysis ie blocked/(unblocked/warned) :param order: order of the category: ie network or ip :parameter: user_id: the user making the request :parameter: token: the user token for each request :rtype: object """ f = do_get_auth(user_id, token) if not isinstance(f, bool): return f else: if f is not True: return make_response(jsonify({"error": 'Invalid User id.', "data": False}), 200) if not category or not order: return make_response(jsonify({"error": False, "token": get_request_token(user_id), "data": False}), 200) ips = None if category == "warned" or category == "unblocked": if order == "ip": ips = get_warned_by_ip().find({}) elif order == "network": ips = get_warned_by_network().find({}) elif category == "blocked": if order == "ip": ips = get_blocked_by_ip().find({}) elif order == "network": ips = get_blocked_by_network().find({}) if ips: ibl = ips.sort([("block_date", DESCENDING), ("attempts", ASCENDING)]) ibl_list = [] for x in ibl: x['id'] = x['_id'] del x['_id'] ibl_list.append(x) create_log(find_user(user_id), request, "retrieved analysis data", LOG_USER_TYPE) return make_response(jsonify({"error": False, "token": get_request_token(user_id), "data": json.dumps(ibl_list, cls=JSONEncoder)}), 200) return make_response(jsonify({"error": False, "token": get_request_token(user_id), "data": False}), 200)
def fetch_all_users(user_id=None, token=None): """ :parameter: user_id: the user making the request :parameter: token: the user token for each request :rtype: object """ f = do_get_auth(user_id, token) if not isinstance(f, bool): return f else: if f is not True: return make_response( jsonify({ "error": 'Invalid User id.', "data": False }), 200) users = get_users().find({}) if users: users = users.sort([("status", ASCENDING), ("first_name", ASCENDING), ("last_name", ASCENDING)]) user_list = [] for x in users: x['id'] = x['_id'] del x['_id'] del x['password'] user_list.append(x) create_log(find_user(user_id), request, "retrieved user data", LOG_USER_TYPE) return make_response( jsonify({ "error": False, "token": get_request_token(user_id), "data": json.dumps(user_list, cls=JSONEncoder) }), 200) return make_response( jsonify({ "error": False, "token": get_request_token(user_id), "data": False }), 200)
def login(): if not request.is_json or not (request.json.keys() & {"username", "password"}): return make_response( jsonify({ "error": 'request is missing information', "data": False }), 200) username = request.json['username'] password = request.json['password'] if not username or not password: return make_response( jsonify({ "error": 'required fields are not provided', "data": False }), 200) try: users = get_users() user = users.find_one( {"$or": [{ "username": username }, { "email": username }]}) if not user: return make_response( jsonify({ "error": 'Wrong username', "data": False }), 200) if not verify_password(password, user['password']): return make_response( jsonify({ "error": 'Invalid user credentials', "data": False })) if 'status' not in user.keys() or not user['status'] == "active": return make_response( jsonify({ "error": 'Access denied. User was deactivated.', "data": False }), 200) user['id'] = user['_id'] del user['_id'] del user['password'] if user_has_session(user['id']): if is_session_active(user_id=user['id']): return make_response( jsonify({ "error": False, "data": "already logged in", "token": get_request_token(user['id']), "user": json.dumps(user, cls=JSONEncoder) }), 200) return session_expired() session['username'] = user['username'] session['email'] = user['email'] session['is_logged_in'] = True session['user_id'] = user['id'] session['token'] = generate_token() session['expiration'] = make_date_time(SESSION_LIFETIME) update_last_login(user['email']) create_log(user, request, "Logged in", LOG_USER_TYPE) except IndexError: return make_response(jsonify({ "error": 'No data found', "data": False })) return make_response( jsonify({ "error": False, "token": session["token"], "data": json.dumps(user, cls=JSONEncoder) }))
def generate_plots(da=None): if da is None: return None def modify_date(x): if "T" in x: return datetime.strptime(x.split(".")[0], "%Y-%m-%dT%H:%M:%S") return datetime.strptime(x.split(".")[0], "%Y-%m-%d %H:%M:%S") groups = da.groupby(['classification_type', 'classification_taxonomy']).groups bar = {} # name_types = {} for attack_result in groups: name, taxonomy = attack_result bar[str(name)] = groups[attack_result].size # name_types[name] = taxonomy description_groups = da.groupby( ['classification_type', 'event_description_text']).groups class_type_description = {} for description_result in description_groups: event_type, name = description_result if not name: name = "unknown" if not event_type: event_type = "unclassified" name_count = description_groups[description_result].size if event_type not in class_type_description: class_type_description[str(event_type)] = [] ct = {"name": name, "count": name_count} class_type_description[str(event_type)].append(ct) network_groups = da.groupby(['source_network']).groups network_counts = {} for network_result in network_groups: network_counts[str( network_result)] = network_groups[network_result].size ip_groups = da.groupby(['source_ip']).groups ip_counts = {} for ip_result in ip_groups: ip_counts[str(ip_result)] = ip_groups[ip_result].size asn_groups = da.groupby(['source_asn']).groups asn_counts = {} for asn_result in asn_groups: asn_counts[str(asn_result)] = asn_groups[asn_result].size geo_groups = da.groupby(['source_geolocation_cc']).groups geo_counts = {} for geo_result in geo_groups: geo_counts[str(geo_result)] = geo_groups[geo_result].size type_series = da.groupby(['classification_type']) series = {} for group, data in type_series: temp = {} items = dict(Counter(list(data.time_observation.values))) for key, val in items.items(): if isinstance(key, Timestamp): key = str(key) temp[key] = val series[str(group)] = temp hourly = da.groupby(da.time_source.map(lambda t: t.hour)).groups hourly_plots = [] for hour in hourly: tmp = {"hour": hour, "counts": len(hourly[hour])} hourly_plots.append(tmp) da['time_observation'] = da.time_observation.astype(str).apply( lambda x: modify_date(x)) da['time_source'] = da.time_observation.astype(str).apply( lambda x: modify_date(x)) da['time_observation'] = da.time_observation.astype(datetime) da['time_source'] = da.time_source.astype(datetime) today = datetime.now().date() yesterday = today - relativedelta.relativedelta(days=1) this_week = today.isocalendar()[1] last_month = today - relativedelta.relativedelta(months=1) dt = da[da['time_observation'].dt.date == today] dy = da[da['time_observation'].dt.date == yesterday] dtw = da[(da['time_observation'].dt.week == this_week) & (da['time_observation'].dt.year == today.year)] dtm = da[(da['time_observation'].dt.month == today.month) & (da['time_observation'].dt.year == today.year)] dtlm = da[(da['time_observation'].dt.month == last_month.month) & (da['time_observation'].dt.year == last_month.year)] by_dates = { "attacks_today": len(dt), "attacks_yesterday": len(dy), "attacks_this_week": len(dtw), "attacks_this_month": len(dtm), "attacks_last_month": len(dtlm), } plots = { CLASSIFICATION_TABLE: class_type_description, ATTACK_BY_TYPE: bar, ATTACK_BY_IP: ip_counts, ATTACK_BY_NETWORK: network_counts, ATTACK_BY_ASN: asn_counts, ATTACK_BY_LOCATION: geo_counts, ATTACK_BY_DATES: by_dates, ATTACK_BY_TYPE_AGAINST_DATES_SERIES: series, ATTACK_PER_DAY_HOUR: hourly_plots, } save_plots(plots) create_log(None, None, "system generated plots", LOG_SYSTEM_TYPE)
def confirm_password_reset(): """ request: {forgot__token, new_password, new_password_verify} :return: """ if not request or not request.json: return not_json_request() cond = ("forgot_token", "new_password", "new_password_verify") if not all(val in request.json.keys() for val in cond): return make_response( jsonify({ "error": 'Some data was missing in the submitted request. ', "token": "", "data": False }), 200) token = request.json['forgot_token'] password = request.json['new_password'] password_2 = request.json['new_password_verify'] email = confirm_token(token) if not email: return make_response( jsonify({ "error": 'This token has already expired. please request a new token', "token": "", "data": False }), 200) user = find_user(email=email) if not user or ("forgot_token" not in user) or not (user['forgot_token'] == token): return make_response( jsonify({ "error": 'Could not find this user. The token you provided could be corrupt', "token": "", "data": False }), 200) if not password == password_2: return make_response( jsonify({ "error": 'Passwords do not match.', "token": "", "data": False }), 200) update = get_users().find_one_and_update( {"email": email}, {"$set": { "forgot_token": "", "password": hash_password(password) }}) if update: create_log(find_user(email=email), request, "Reset his password", LOG_USER_TYPE) return make_response( jsonify({ "error": False, "token": get_request_token(user['_id']), "data": "Password change has been successful. You can now login." }), 200) return make_response( jsonify({ "error": "Password reset Failed. Consult the admin.", "token": "", "data": False }), 200)
def add_user(): """ :parameter: user_id: the user making the request :parameter: token: the user token for each request :param: request request: POST {first_name, last_name, email, username, password, password_verify, user_id, token} :return: """ keys = ("username", "password", "password_verify", "first_name", "last_name", "email", "user_id") user_id = request.json['user_id'] if not set(keys).issubset(set(request.json)): return make_response( jsonify({ "error": 'Some fields are missing', "token": get_request_token(user_id), "data": False }), 200) username = request.json['username'] password = request.json['password'] password_verify = request.json['password_verify'] email = request.json['email'] first_name = request.json['first_name'] last_name = request.json['last_name'] if not validate(username, 1): return make_response( jsonify({ "error": 'Username should only contain letters', "token": get_request_token(user_id), "data": False }), 200) if not validate_email(email): return make_response( jsonify({ "error": "Email is invalid. ex. [email protected]", "token": get_request_token(user_id), "data": False }), 200) if user_exists(email): return make_response( jsonify({ "error": "Email already in used. Use another email", "token": get_request_token(user_id), "data": False }), 200) if not password == password_verify: return make_response( jsonify({ "error": 'Passwords do not match.', "token": get_request_token(user_id), "data": False }), 200) if not validate_password(password, 6): return make_response( jsonify({ "error": 'Password should only be alphanumeric and min length 6', "token": get_request_token(user_id), "data": False }), 200) """ Need to implement permissions on each user add permissions object/dict """ conf_token = generate_confirmation_token(email) user_object = { "username": username, "first_name": first_name, "last_name": last_name, "email": email, "password": hash_password(password), "status": "pending", "confirmation_token": conf_token, "confirmed_on": "", "confirmed": False } mess = EMAIL_CONFIRM_LINK + conf_token + "/" mess = "<b style='color: blue'>Use this link to activate your account :</b> " \ "<br> <br> <a href='" + mess + "' target='_blank' >" + mess + "</a>" if not send_mail(email, mess): return make_response( jsonify({ "error": "Unable to create user, failed to send an email to this user. " "Cant connect to mail server.", "token": get_request_token(user_id), "data": False }), 200) if create_user(user_object): us = find_user(request.json['user_id']) create_log(us, request, "Added new user", LOG_USER_TYPE) return make_response( jsonify({ "error": False, "token": get_request_token(user_id), "data": "New user has been created. User should confirm his/her" " email by visiting the link sent to him." }), 200) return make_response( jsonify({ "error": "Something went wrong. Unable to create new user.", "token": get_request_token(user_id), "data": False }), 200)