def __parse_bots_data(self): """ This method will send request through API to get more information about the specific User-Agent than parse the information and return it. :return: dict """ try: bots_response = requests.post(self._bots_url, data=json.dumps(self._bots_data), headers=self._bots_header) except Exception as e: log(e, LogLevel.ERROR, self.__parse_bots_data) # We could not get the data return Classification.NoConclusion # ---- Check that the request is succeed ---- # if bots_response.status_code != 200: return Classification.NoConclusion elif type(bots_response.json()) is str: bots_response = json.loads(bots_response.json()) else: bots_response = bots_response.json() log("Bots response: {}".format(bots_response), LogLevel.DEBUG, self.__parse_bots_data) if "parse" not in bots_response: return Classification.NoConclusion # ---- Parse the information ---- # bots_response = bots_response["parse"] return { key: bots_response.get(key, value) for key, value in self.kb["bots_detectors"].items() }
def wrapper(self, *args, **kwargs): if not request.is_json: log("[API][only_json] Could not process the request, is_json: {}".format(request.is_json), LogLevel.INFO, self.post) return {"msg": "Please send json request.", "ip": request.remote_addr, "contact": "*****@*****.**"} if request.get_json() is None: log("[API][only_json] Could not process the request, get_json: None", LogLevel.INFO, self.post) return {"msg": "Please send json with the request.", "ip": request.remote_addr, "contact": "*****@*****.**"} return func(self, *args, **kwargs)
def post(self): all_users = db.get_session().query(Users).all() all_servers = db.get_session().query(Server).all() all_services = db.get_session().query(Services).all() joined_objects = [] for user in all_users: current_object = to_json(user, to_str=True) del current_object["password"] for server in all_servers: if server.user_id == user.item_id: current_object = {**current_object, **to_json(server, to_str=True)} for service in all_services: if service.server_id == server.item_id: current_object = {**current_object, **to_json(service, ignore_list=["server_id", "user_id", "id", "created_on"], to_str=True)} joined_objects.append(current_object) log("[API][GetUsersDataHandler] joined_objects: {}".format(joined_objects), LogLevel.DEBUG, self.post) return joined_objects
def parse(self, data_to_parse, is_user_protection=False): parsed_data = HttpResponse() parsed_data.original = data_to_parse parsed_data.text = data_to_parse.text parsed_data.content = data_to_parse.text parsed_data.headers = data_to_parse.headers parsed_data.status_code = data_to_parse.status_code parsed_data.cookies = data_to_parse.cookies parsed_data.is_redirect = data_to_parse.is_redirect parsed_data.response_url = data_to_parse.url parsed_data.time_stamp = datetime.now() if not is_user_protection: parsed_data.from_server_id = self.__request.to_server_id parsed_data.to_ip = self.__request.from_ip parsed_data.from_dns_name = self.__request.host_name log("The Parsed data is: {}".format(to_json(parsed_data)), LogLevel.DEBUG, self.parse) log("Finish parsing the request.", LogLevel.INFO, self.parse) return parsed_data
def post(self): incoming_json = request.get_json() errors = check_json_object(incoming_json, ["email", "services"], "Could not find {} at the incoming json object.") if len(errors) > 0: log("[API][AddNewWebsiteHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return 0 errors = check_json_object(incoming_json["services"], services_credentials, "Could not find {} at the services json object.") if len(errors) > 0: log("[API][AddNewWebsiteHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return 0 user = db.get_session().query(Users).filter(Users.email == incoming_json["email"]).first() if user is None: log("[API][AddNewWebsiteHandler] Could not find this user at the datABASE: {}".format(incoming_json["email"]), LogLevel.INFO, self.post) return 0 server = Server(user_id=user.item_id, server_ip=incoming_json['services']['ip'], server_dns=incoming_json['services']['website']) try: db.insert(server) except Exception as e: errors = [e] finally: if server.item_id is None: log("[API][RegisterHandler] Could not insert the server into the database, contact the server " "administrator: {} ".format(errors), LogLevel.INFO, self.post) return 0 users_services = create_services_object(user_id=user.item_id, incoming_json=incoming_json, server_id=server.item_id) try: db.insert(users_services) # TODO: Royi this is Duplicate code (just copy past from register user) except Exception as e: errors = [str(e)] finally: if users_services.item_id is None: log("[API][AddNewWebsiteHandler] Could not insert the services into the database, contact the server " "administrator: {} ".format(errors), LogLevel.INFO, self.post) return 0 return 1
def parse(self, data_to_parse): parsed_data = HttpRequest() log("Parse the url {}".format(data_to_parse.url), LogLevel.DEBUG, self.parse) url = urlparse(data_to_parse.url) log("The Parsed url is: {}".format(url), LogLevel.DEBUG, self.parse) parsed_data.method = "{}".format(data_to_parse.method).upper() parsed_data.content = data_to_parse.get_data() parsed_data.headers = data_to_parse.headers parsed_data.query = '{uri.query}'.format(uri=url) parsed_data.path = '{uri.path}'.format(uri=url) parsed_data.host_name = '{uri.netloc}'.format(uri=url) parsed_data.from_ip = data_to_parse.remote_addr parsed_data.time_stamp = datetime.now() parsed_data.args = data_to_parse.args parsed_data.form = data_to_parse.form log("The Parsed data is: {}".format(to_json(parsed_data)), LogLevel.DEBUG, self.parse) log("Finish parsing the request.", LogLevel.INFO, self.parse) return parsed_data
def post(self): incoming_json = request.get_json() errors = check_json_object(incoming_json, ["email", "password"], "Could not find {} at the incoming json object.") if len(errors) > 0: log("[API][LoginHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return 0 user = db.get_session().query(Users).filter(Users.email == incoming_json['email']).first() if user is None: log("[API][LoginHandler] Could not locate the {} user: "******"[API][LoginHandler] Admin login has occurred: {}".format(incoming_json['email']), LogLevel.INFO, self.post) return 2 return 1 log("[API][LoginHandler] Login Failure has occurred:: {} {}".format(request.remote_addr, incoming_json['email']), LogLevel.INFO, self.post) return 0
def post(self): incoming_json = request.get_json() errors = check_json_object(incoming_json, ["website", "update_data"], "Could not find {} at the incoming json object.") if len(errors) > 0: log("[API][UpdateServiceStatusHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return 0 server = db.get_session().query(Server).filter(Server.server_dns == incoming_json['website']).first() if server is None: log("[API][UpdateServiceStatusHandler] Could not find the server at the Database: {}" .format(incoming_json['website']), LogLevel.INFO, self.post) return 0 update_data = incoming_json['update_data'] update_data_final = {k: 1 if v == 'True' else 0 for k, v in update_data.items()} log("[API][UpdateServiceStatusHandler] Services update_data_final: {}".format(update_data_final), LogLevel.INFO, self.post) sess = db.get_session() sess.query(Services).filter(Services.server_id == server.item_id).update(update_data_final) sess.commit() log("[API][UpdateServiceStatusHandler] Services update successfully for: {}".format(server.server_dns), LogLevel.INFO, self.post) return 1
def post(self): incoming_json = request.get_json() errors = check_json_object(incoming_json, ["update_data"], "Could not find {} at the incoming json object.") if len(errors) > 0: log("[API][AdminUpdateServiceStatusHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return 0 log("[API][AdminUpdateServiceStatusHandler] admin update: {}" .format(type(incoming_json['update_data'])), LogLevel.DEBUG, self.post) update_data = incoming_json['update_data'] sess = db.get_session() sess.query(Services).update(update_data) sess.commit() log("[API][AdminUpdateServiceStatusHandler] Date updated: {}".format(incoming_json['update_data']), LogLevel.DEBUG, self.post) return 1
def post(self): incoming_json = request.get_json() errors = check_json_object(incoming_json, ["host_name"], "Could not find {} at the incoming json object.") if len(errors) > 0: log("[API][UserProtectorHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return 0 host_to_detect = urlparse(str(incoming_json['host_name'])) host_to_detect = '{uri.netloc}'.format(uri=host_to_detect).lower() if len(host_to_detect) < 3: log("[API][UserProtectorHandler] Could not detect this host: {}".format(host_to_detect), LogLevel.INFO, self.post) return 0 host_to_detect = "https://{}".format(host_to_detect) log("[API][UserProtectorHandler] getting info with UserProtectionDetector for: {}" .format(host_to_detect), LogLevel.INFO, self.post) try: response = requests.get(host_to_detect) except Exception as e: log("[API][UserProtectorHandler] Could not get response: {}".format(e), LogLevel.ERROR, self.post) return {"alerts": ["We could not process the request, please check that the url is valid."]} parser = HTTPResponseParser(None) parsed_response = parser.parse(response, is_user_protection=True) upc = UserProtectionDetector(parsed_response) resp = upc.detect(255) return {"alerts": resp.detected_alerts}
def request_handler(): # try: log("Is the request is HTTPS? ", LogLevel.INFO, request_handler) if not request.is_secure: return Response(status=301, headers=hsts.enforce(request.url)) log("Start parsing the request", LogLevel.INFO, request_handler) parser = FlaskHTTPRequestParser() parsed_request = parser.parse(request) log("Creating Controller", LogLevel.INFO, request_handler) log("The Controller Detectors are {}".format(detectors), LogLevel.DEBUG, request_handler) controller = ElroController(detectors=detectors) log("Activating controller request handler", LogLevel.INFO, request_handler) response_code, send_to, new_request, parsed_request = controller.request_handler( parsed_request, request) log("Controller response is: {} {}".format(response_code, send_to), LogLevel.DEBUG, request_handler) url = 'https://{}{}?{}'.format(parsed_request.host_name, parsed_request.path, parsed_request.query) if response_code == ControllerResponseCode.NotValid: log("The Request for {} is not valid.".format(request.url), LogLevel.INFO, request_handler) log("Redirecting to {}".format(url), LogLevel.INFO, request_handler) response = Response(status=302, headers={"Location": url}) elif response_code == ControllerResponseCode.Valid: send_headers = {key: value for key, value in new_request.headers} log("The Request for {} valid and OK".format(request.url), LogLevel.INFO, request_handler) resp = requests.request(method=parsed_request.method, url=url, verify=True, json=new_request.get_json(), headers=send_headers, params=new_request.args, data=new_request.form) log("The Response is {}".format(to_json(resp)), LogLevel.DEBUG, request_handler) parser = HTTPResponseParser(parsed_request) log("Parse the response", LogLevel.INFO, request_handler) parsed_response = parser.parse(resp) log("Activating controller response handler", LogLevel.INFO, request_handler) response_code, send_to, new_content = controller.response_handler( parsed_response, resp) log("Controller response is: {} {}".format(response_code, send_to), LogLevel.DEBUG, request_handler) if response_code == ControllerResponseCode.NotValid: send_content = new_content elif response_code == ControllerResponseCode.Valid: send_content = resp.content else: send_content = None excluded_headers = [ 'content-encoding', 'content-length', 'transfer-encoding', 'connection' ] headers = [(name, value) for (name, value) in resp.raw.headers.items() if name.lower() not in excluded_headers] log("Generating response...", LogLevel.INFO, request_handler) response = Response(status=403) if send_content is None else Response( send_content, resp.status_code, headers) else: log( "The Request for {} is not found in the database".format( request.url), LogLevel.INFO, request_handler) response = Response(status=404) abort(404) # Abort the request. log("Sending response", LogLevel.INFO, request_handler) return response
def proxy(path): log( "Request has arrived: {} From {}".format(request.url, request.remote_addr), LogLevel.INFO, request_handler) return request_handler()
def request_handler(self, parsed_request, original_request): self._request_data = DetectorRequestData( from_ip=parsed_request.from_ip) self._request = parsed_request session = db.get_session() # Get The Server id from DB log( "Looking for the server on the DataBase {}".format( parsed_request.host_name), LogLevel.DEBUG, self.request_handler) self._server = session.query(Server).filter_by( server_dns=parsed_request.host_name).first() if self._server is None: log( "Server not found at the DataBase {}".format( parsed_request.host_name), LogLevel.DEBUG, self.request_handler) return ControllerResponseCode.Failed, RedirectAnswerTo.Client, original_request, parsed_request # check if authorized requester. self._request_data.to_server_id = self._server.item_id log("Activate _is_authorized method", LogLevel.DEBUG, self.request_handler) is_authorized = self._is_authorized(parsed_request.from_ip) if is_authorized == IsAuthorized.Yes: log("_is_authorized method results is Yes", LogLevel.DEBUG, self.request_handler) self._request_data.detected = "white_list" return ControllerResponseCode.Valid, RedirectAnswerTo.Server, original_request, parsed_request elif is_authorized == IsAuthorized.No: log("_is_authorized method results is No", LogLevel.DEBUG, self.request_handler) self._request_data.detected = "black_list" return ControllerResponseCode.NotValid, RedirectAnswerTo.Client, original_request, parsed_request # Get list of detectors for the server log("_is_authorized method results is NoConclusions", LogLevel.DEBUG, self.request_handler) parsed_request.to_server_id = self._server.item_id if str(original_request.headers.get('sec-fetch-dest', "")) in self.kb["white_content"]: self._is_white_content = True self._request_data.detected = "none" return ControllerResponseCode.Valid, RedirectAnswerTo.Server, original_request, parsed_request log("Activate _list_of_detectors method", LogLevel.DEBUG, self.request_handler) detectors = self._list_of_detectors(self._server.item_id) log("_list_of_detectors results is {}".format(detectors), LogLevel.DEBUG, self.request_handler) for detector_constructor in detectors: detector = detector_constructor() validate = detector.detect(parsed_request) if detector.name in self.kb["non_blocking_detectors"] and validate: # Detected => Removing cookies. original_request.headers.replace_header("Cookie", "") elif detector.name in self.kb["non_blocking_detectors"]: # Creating new token self.response_cookie = CookiesToken( dns_name=parsed_request.host_name, ip=parsed_request.from_ip, active=True, token=secrets.token_hex(256)) elif validate: log( " ************* Detector {} is detected unusual activity for {} ************" .format(detector.name, original_request.url), LogLevel.INFO, self.request_handler) self._request_data.detected = detector.name log("Insert Information to database".format(detector.name), LogLevel.DEBUG, self.request_handler) return ControllerResponseCode.NotValid, RedirectAnswerTo.Client, original_request, parsed_request log("Nothing unusual detected by the detectors.", LogLevel.INFO, self.request_handler) self._request_data.detected = "none" log("Insert Information to database", LogLevel.DEBUG, self.request_handler) return ControllerResponseCode.Valid, RedirectAnswerTo.Server, original_request, parsed_request
def post(self): incoming_json = request.get_json() errors = check_json_object(incoming_json, ["users", "services"], "Could not find {} at the incoming json object.") if len(errors) > 0: log("[API][RegisterHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return 0 errors += check_json_object(incoming_json['users'], ["email", "password"], "Missing {} for the user credentials") if len(errors) > 0: log("[API][RegisterHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return 0 errors += check_json_object(incoming_json['services'], services_credentials, message="Missing {} for the services credentials") if len(errors) > 0: log("[API][RegisterHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return 0 log("[API][RegisterHandler] Registering new Client: {}".format(incoming_json['users']['email']), LogLevel.INFO, self.post) user = Users(email=incoming_json['users']['email'], password=sha256_crypt.hash(str(incoming_json['users']['password']))) try: db.insert(user) except Exception as e: errors = [str(e)] finally: if user.item_id is None: log("[API][RegisterHandler] Could not insert the user into the database, contact the server " "administrator: {} ".format(errors), LogLevel.INFO, self.post) return 0 server = Server(user_id=user.item_id, server_ip=incoming_json['services']['ip'], server_dns=incoming_json['services']['website']) try: db.insert(server) except Exception as e: errors = [str(e)] finally: if server.item_id is None: log("[API][RegisterHandler] Could not insert the server into the database, contact the server " "administrator: {} ".format(errors), LogLevel.INFO, self.post) return 0 user_services = create_services_object(user_id=user.item_id, incoming_json=incoming_json, server_id=server.item_id) try: db.insert(user_services) except Exception as e: errors = [str(e)] finally: if user_services.item_id is None: log("[API][RegisterHandler] Could not insert the services into the database, contact the server " "administrator: {} ".format(errors), LogLevel.INFO, self.post) return 0 return 1
def post(self): def count_total_records(key): return db.get_session().query(DetectorRequestData).filter_by(detected=key).count() def count_server_records(server_id, key): return db.get_session().query(DetectorRequestData).filter_by(detected=key, to_server_id=server_id).count() incoming_json = request.get_json() errors = check_json_object(incoming_json, ["email"], "Could not find {} at the incoming json object.") if len(errors) > 0: log("[API][GetActiveServicesHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post) return False user = db.get_session().query(Users).filter(Users.email == incoming_json["email"]).first() if user is None: log("[API][GetActiveServicesHandler] Could not find the user in the DB: {}".format(incoming_json["email"]), LogLevel.INFO, self.post) return False try: joined_statuses = [] all_servers = db.get_session().query(Server).filter(Server.user_id == user.item_id).all() log("[API][GetActiveServicesHandler] all_servers: {}".format(all_servers), LogLevel.INFO, self.post) for server in all_servers: log("[API][GetActiveServicesHandler] server: {}".format(server), LogLevel.INFO, self.post) services = db.get_session().query(Services).filter(Services.server_id == server.item_id).first() if services is None: log("[API][GetActiveServicesHandler] Could not find the services in the DB: {}".format( server.server_dns), LogLevel.INFO, self.post) return False jsoned_services = to_json(services, ignore_list=["server_id", "user_id", "id", "created_on"], to_str=True) final_services_json = { key: { "state": value, "count": count_total_records(key) if user.is_admin else count_server_records(server.item_id, key) } for key, value in jsoned_services.items() } log("[API][GetActiveServicesHandler] final_services_json: {}".format(final_services_json), LogLevel.INFO, self.post) joined_object = {**final_services_json, **to_json(server, to_str=True)} joined_object['website'] = joined_object['server_dns'] del joined_object['server_dns'] joined_statuses.append(joined_object) log("[API][GetActiveServicesHandler] return {}".format(joined_statuses), LogLevel.DEBUG, self.post) return joined_statuses except Exception as e: log("[API][GetActiveServicesHandler] Exception: {}".format(e), LogLevel.ERROR, self.post) return False
def detect(self, parsed_data, sensitivity=Sensitivity.Regular, forbidden=None, legitimate=None): """ This method will detect an SQL attack attempt by the regex and data given in the 'SQLInjection' folder under 'Data' folder. :param parsed_data: Parsed Data (from the parser module) of the request / response :param sensitivity: The sensitivity of the detection :param forbidden: list of additional malicious words/regex that we wish to add to the forbidden list on runtime :param legitimate: The legitimate words/regex that we need to automatically approve :return: boolean """ # check the query string first (url params) query_string = parsed_data.headers.__dict__['environ']['QUERY_STRING'] for break_char in self.__break_characters: context_breaks = re.findall(break_char, query_string) if len(context_breaks) > 0: log("INSIDE LEN > 0", LogLevel.INFO, self.detect) final_decision_assurance = self.final_validation( query_string, Sensitivity.VerySensitive) return True if final_decision_assurance >= self.kb[ "threshold"] else False # if no context breaks were found continue to check the request parsed_data_as_str = create_content_as_str( parsed_data) # Copy the parsed data to avoid change the origin if forbidden is not None: self.__forbidden += forbidden if legitimate is not None: self.__forbidden -= legitimate self.__break_characters -= legitimate context_break_list = [] forbidden_word_list = [] # check for context break intentions in the request itself for break_char in self.__break_characters: context_breaks = re.findall(break_char, parsed_data_as_str) if len(context_breaks) > 0: # logger.info("Found Threat of SQL INJECTION ATTACK, " # "CONTEXT BREAK CHAR: " + break_char + " was found in: " + parsed_data) context_break_list.append(context_breaks) if sensitivity == Sensitivity.Sensitive: final_decision_assurance = self.final_validation( parsed_data_as_str) return True if final_decision_assurance >= self.kb[ "threshold"] else False # check for forbidden words for forbidden_word in self.__forbidden: forbidden_words = re.findall(forbidden_word, parsed_data_as_str) if len(forbidden_words) > 0: # logger.info("Found Threat of SQL INJECTION ATTACK, " # "Forbidden word: " + forbidden_word + " was found in: " + parsed_data) forbidden_word_list.append(forbidden_words) # if tries to break context with forbidden words it is probably an attack if len(context_break_list) > 0 and len(forbidden_word_list) > 0: final_decision_assurance = self.final_validation( parsed_data_as_str) return True if final_decision_assurance >= self.kb[ "threshold"] else False return False
def wrapper(self, *args, **kwargs): if request.remote_addr not in authorized_servers: log("[API][required_authentication] Block the request from {}".format(request.remote_addr), LogLevel.INFO, self.post) return {"msg": "Your not authorized to perform this action", "ip": request.remote_addr, "contact": "*****@*****.**"} return func(self, *args, **kwargs)