def __parse_bots_data(self):
"""
This method will send request through API to get more information about the specific User-Agent
than parse the information and return it.
:return: dict
"""
try:
bots_response = requests.post(self._bots_url,
data=json.dumps(self._bots_data),
headers=self._bots_header)
except Exception as e:
log(e, LogLevel.ERROR, self.__parse_bots_data)
# We could not get the data
return Classification.NoConclusion
# ---- Check that the request is succeed ---- #
if bots_response.status_code != 200:
return Classification.NoConclusion
elif type(bots_response.json()) is str:
bots_response = json.loads(bots_response.json())
else:
bots_response = bots_response.json()
log("Bots response: {}".format(bots_response), LogLevel.DEBUG,
self.__parse_bots_data)
if "parse" not in bots_response:
return Classification.NoConclusion
# ---- Parse the information ---- #
bots_response = bots_response["parse"]
return {
key: bots_response.get(key, value)
for key, value in self.kb["bots_detectors"].items()
}
def wrapper(self, *args, **kwargs):
if not request.is_json:
log("[API][only_json] Could not process the request, is_json: {}".format(request.is_json), LogLevel.INFO, self.post)
return {"msg": "Please send json request.", "ip": request.remote_addr, "contact": "*****@*****.**"}
if request.get_json() is None:
log("[API][only_json] Could not process the request, get_json: None", LogLevel.INFO, self.post)
return {"msg": "Please send json with the request.", "ip": request.remote_addr, "contact": "*****@*****.**"}
return func(self, *args, **kwargs)
def post(self):
all_users = db.get_session().query(Users).all()
all_servers = db.get_session().query(Server).all()
all_services = db.get_session().query(Services).all()
joined_objects = []
for user in all_users:
current_object = to_json(user, to_str=True)
del current_object["password"]
for server in all_servers:
if server.user_id == user.item_id:
current_object = {**current_object, **to_json(server, to_str=True)}
for service in all_services:
if service.server_id == server.item_id:
current_object = {**current_object, **to_json(service, ignore_list=["server_id", "user_id", "id", "created_on"], to_str=True)}
joined_objects.append(current_object)
log("[API][GetUsersDataHandler] joined_objects: {}".format(joined_objects), LogLevel.DEBUG, self.post)
return joined_objects
def parse(self, data_to_parse, is_user_protection=False):
parsed_data = HttpResponse()
parsed_data.original = data_to_parse
parsed_data.text = data_to_parse.text
parsed_data.content = data_to_parse.text
parsed_data.headers = data_to_parse.headers
parsed_data.status_code = data_to_parse.status_code
parsed_data.cookies = data_to_parse.cookies
parsed_data.is_redirect = data_to_parse.is_redirect
parsed_data.response_url = data_to_parse.url
parsed_data.time_stamp = datetime.now()
if not is_user_protection:
parsed_data.from_server_id = self.__request.to_server_id
parsed_data.to_ip = self.__request.from_ip
parsed_data.from_dns_name = self.__request.host_name
log("The Parsed data is: {}".format(to_json(parsed_data)), LogLevel.DEBUG, self.parse)
log("Finish parsing the request.", LogLevel.INFO, self.parse)
return parsed_data
def post(self):
incoming_json = request.get_json()
errors = check_json_object(incoming_json, ["email", "services"], "Could not find {} at the incoming json object.")
if len(errors) > 0:
log("[API][AddNewWebsiteHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return 0
errors = check_json_object(incoming_json["services"], services_credentials,
"Could not find {} at the services json object.")
if len(errors) > 0:
log("[API][AddNewWebsiteHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return 0
user = db.get_session().query(Users).filter(Users.email == incoming_json["email"]).first()
if user is None:
log("[API][AddNewWebsiteHandler] Could not find this user at the datABASE: {}".format(incoming_json["email"]), LogLevel.INFO, self.post)
return 0
server = Server(user_id=user.item_id,
server_ip=incoming_json['services']['ip'],
server_dns=incoming_json['services']['website'])
try:
db.insert(server)
except Exception as e:
errors = [e]
finally:
if server.item_id is None:
log("[API][RegisterHandler] Could not insert the server into the database, contact the server "
"administrator: {} ".format(errors), LogLevel.INFO, self.post)
return 0
users_services = create_services_object(user_id=user.item_id, incoming_json=incoming_json, server_id=server.item_id)
try:
db.insert(users_services) # TODO: Royi this is Duplicate code (just copy past from register user)
except Exception as e:
errors = [str(e)]
finally:
if users_services.item_id is None:
log("[API][AddNewWebsiteHandler] Could not insert the services into the database, contact the server "
"administrator: {} ".format(errors), LogLevel.INFO, self.post)
return 0
return 1
def parse(self, data_to_parse):
parsed_data = HttpRequest()
log("Parse the url {}".format(data_to_parse.url), LogLevel.DEBUG, self.parse)
url = urlparse(data_to_parse.url)
log("The Parsed url is: {}".format(url), LogLevel.DEBUG, self.parse)
parsed_data.method = "{}".format(data_to_parse.method).upper()
parsed_data.content = data_to_parse.get_data()
parsed_data.headers = data_to_parse.headers
parsed_data.query = '{uri.query}'.format(uri=url)
parsed_data.path = '{uri.path}'.format(uri=url)
parsed_data.host_name = '{uri.netloc}'.format(uri=url)
parsed_data.from_ip = data_to_parse.remote_addr
parsed_data.time_stamp = datetime.now()
parsed_data.args = data_to_parse.args
parsed_data.form = data_to_parse.form
log("The Parsed data is: {}".format(to_json(parsed_data)), LogLevel.DEBUG, self.parse)
log("Finish parsing the request.", LogLevel.INFO, self.parse)
return parsed_data
def post(self):
incoming_json = request.get_json()
errors = check_json_object(incoming_json, ["email", "password"],
"Could not find {} at the incoming json object.")
if len(errors) > 0:
log("[API][LoginHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return 0
user = db.get_session().query(Users).filter(Users.email == incoming_json['email']).first()
if user is None:
log("[API][LoginHandler] Could not locate the {} user: "******"[API][LoginHandler] Admin login has occurred: {}".format(incoming_json['email']), LogLevel.INFO, self.post)
return 2
return 1
log("[API][LoginHandler] Login Failure has occurred:: {} {}".format(request.remote_addr, incoming_json['email']), LogLevel.INFO, self.post)
return 0
def post(self):
incoming_json = request.get_json()
errors = check_json_object(incoming_json, ["website", "update_data"], "Could not find {} at the incoming json object.")
if len(errors) > 0:
log("[API][UpdateServiceStatusHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return 0
server = db.get_session().query(Server).filter(Server.server_dns == incoming_json['website']).first()
if server is None:
log("[API][UpdateServiceStatusHandler] Could not find the server at the Database: {}"
.format(incoming_json['website']), LogLevel.INFO, self.post)
return 0
update_data = incoming_json['update_data']
update_data_final = {k: 1 if v == 'True' else 0 for k, v in update_data.items()}
log("[API][UpdateServiceStatusHandler] Services update_data_final: {}".format(update_data_final), LogLevel.INFO, self.post)
sess = db.get_session()
sess.query(Services).filter(Services.server_id == server.item_id).update(update_data_final)
sess.commit()
log("[API][UpdateServiceStatusHandler] Services update successfully for: {}".format(server.server_dns), LogLevel.INFO,
self.post)
return 1
def post(self):
incoming_json = request.get_json()
errors = check_json_object(incoming_json, ["update_data"], "Could not find {} at the incoming json object.")
if len(errors) > 0:
log("[API][AdminUpdateServiceStatusHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return 0
log("[API][AdminUpdateServiceStatusHandler] admin update: {}"
.format(type(incoming_json['update_data'])), LogLevel.DEBUG, self.post)
update_data = incoming_json['update_data']
sess = db.get_session()
sess.query(Services).update(update_data)
sess.commit()
log("[API][AdminUpdateServiceStatusHandler] Date updated: {}".format(incoming_json['update_data']),
LogLevel.DEBUG, self.post)
return 1
def post(self):
incoming_json = request.get_json()
errors = check_json_object(incoming_json, ["host_name"], "Could not find {} at the incoming json object.")
if len(errors) > 0:
log("[API][UserProtectorHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return 0
host_to_detect = urlparse(str(incoming_json['host_name']))
host_to_detect = '{uri.netloc}'.format(uri=host_to_detect).lower()
if len(host_to_detect) < 3:
log("[API][UserProtectorHandler] Could not detect this host: {}".format(host_to_detect), LogLevel.INFO, self.post)
return 0
host_to_detect = "https://{}".format(host_to_detect)
log("[API][UserProtectorHandler] getting info with UserProtectionDetector for: {}"
.format(host_to_detect), LogLevel.INFO, self.post)
try:
response = requests.get(host_to_detect)
except Exception as e:
log("[API][UserProtectorHandler] Could not get response: {}".format(e), LogLevel.ERROR, self.post)
return {"alerts": ["We could not process the request, please check that the url is valid."]}
parser = HTTPResponseParser(None)
parsed_response = parser.parse(response, is_user_protection=True)
upc = UserProtectionDetector(parsed_response)
resp = upc.detect(255)
return {"alerts": resp.detected_alerts}
def request_handler():
# try:
log("Is the request is HTTPS? ", LogLevel.INFO, request_handler)
if not request.is_secure:
return Response(status=301, headers=hsts.enforce(request.url))
log("Start parsing the request", LogLevel.INFO, request_handler)
parser = FlaskHTTPRequestParser()
parsed_request = parser.parse(request)
log("Creating Controller", LogLevel.INFO, request_handler)
log("The Controller Detectors are {}".format(detectors), LogLevel.DEBUG,
request_handler)
controller = ElroController(detectors=detectors)
log("Activating controller request handler", LogLevel.INFO,
request_handler)
response_code, send_to, new_request, parsed_request = controller.request_handler(
parsed_request, request)
log("Controller response is: {} {}".format(response_code, send_to),
LogLevel.DEBUG, request_handler)
url = 'https://{}{}?{}'.format(parsed_request.host_name,
parsed_request.path, parsed_request.query)
if response_code == ControllerResponseCode.NotValid:
log("The Request for {} is not valid.".format(request.url),
LogLevel.INFO, request_handler)
log("Redirecting to {}".format(url), LogLevel.INFO, request_handler)
response = Response(status=302, headers={"Location": url})
elif response_code == ControllerResponseCode.Valid:
send_headers = {key: value for key, value in new_request.headers}
log("The Request for {} valid and OK".format(request.url),
LogLevel.INFO, request_handler)
resp = requests.request(method=parsed_request.method,
url=url,
verify=True,
json=new_request.get_json(),
headers=send_headers,
params=new_request.args,
data=new_request.form)
log("The Response is {}".format(to_json(resp)), LogLevel.DEBUG,
request_handler)
parser = HTTPResponseParser(parsed_request)
log("Parse the response", LogLevel.INFO, request_handler)
parsed_response = parser.parse(resp)
log("Activating controller response handler", LogLevel.INFO,
request_handler)
response_code, send_to, new_content = controller.response_handler(
parsed_response, resp)
log("Controller response is: {} {}".format(response_code, send_to),
LogLevel.DEBUG, request_handler)
if response_code == ControllerResponseCode.NotValid:
send_content = new_content
elif response_code == ControllerResponseCode.Valid:
send_content = resp.content
else:
send_content = None
excluded_headers = [
'content-encoding', 'content-length', 'transfer-encoding',
'connection'
]
headers = [(name, value) for (name, value) in resp.raw.headers.items()
if name.lower() not in excluded_headers]
log("Generating response...", LogLevel.INFO, request_handler)
response = Response(status=403) if send_content is None else Response(
send_content, resp.status_code, headers)
else:
log(
"The Request for {} is not found in the database".format(
request.url), LogLevel.INFO, request_handler)
response = Response(status=404)
abort(404) # Abort the request.
log("Sending response", LogLevel.INFO, request_handler)
return response
def proxy(path):
log(
"Request has arrived: {} From {}".format(request.url,
request.remote_addr),
LogLevel.INFO, request_handler)
return request_handler()
def request_handler(self, parsed_request, original_request):
self._request_data = DetectorRequestData(
from_ip=parsed_request.from_ip)
self._request = parsed_request
session = db.get_session()
# Get The Server id from DB
log(
"Looking for the server on the DataBase {}".format(
parsed_request.host_name), LogLevel.DEBUG,
self.request_handler)
self._server = session.query(Server).filter_by(
server_dns=parsed_request.host_name).first()
if self._server is None:
log(
"Server not found at the DataBase {}".format(
parsed_request.host_name), LogLevel.DEBUG,
self.request_handler)
return ControllerResponseCode.Failed, RedirectAnswerTo.Client, original_request, parsed_request
# check if authorized requester.
self._request_data.to_server_id = self._server.item_id
log("Activate _is_authorized method", LogLevel.DEBUG,
self.request_handler)
is_authorized = self._is_authorized(parsed_request.from_ip)
if is_authorized == IsAuthorized.Yes:
log("_is_authorized method results is Yes", LogLevel.DEBUG,
self.request_handler)
self._request_data.detected = "white_list"
return ControllerResponseCode.Valid, RedirectAnswerTo.Server, original_request, parsed_request
elif is_authorized == IsAuthorized.No:
log("_is_authorized method results is No", LogLevel.DEBUG,
self.request_handler)
self._request_data.detected = "black_list"
return ControllerResponseCode.NotValid, RedirectAnswerTo.Client, original_request, parsed_request
# Get list of detectors for the server
log("_is_authorized method results is NoConclusions", LogLevel.DEBUG,
self.request_handler)
parsed_request.to_server_id = self._server.item_id
if str(original_request.headers.get('sec-fetch-dest',
"")) in self.kb["white_content"]:
self._is_white_content = True
self._request_data.detected = "none"
return ControllerResponseCode.Valid, RedirectAnswerTo.Server, original_request, parsed_request
log("Activate _list_of_detectors method", LogLevel.DEBUG,
self.request_handler)
detectors = self._list_of_detectors(self._server.item_id)
log("_list_of_detectors results is {}".format(detectors),
LogLevel.DEBUG, self.request_handler)
for detector_constructor in detectors:
detector = detector_constructor()
validate = detector.detect(parsed_request)
if detector.name in self.kb["non_blocking_detectors"] and validate:
# Detected => Removing cookies.
original_request.headers.replace_header("Cookie", "")
elif detector.name in self.kb["non_blocking_detectors"]:
# Creating new token
self.response_cookie = CookiesToken(
dns_name=parsed_request.host_name,
ip=parsed_request.from_ip,
active=True,
token=secrets.token_hex(256))
elif validate:
log(
" ************* Detector {} is detected unusual activity for {} ************"
.format(detector.name, original_request.url),
LogLevel.INFO, self.request_handler)
self._request_data.detected = detector.name
log("Insert Information to database".format(detector.name),
LogLevel.DEBUG, self.request_handler)
return ControllerResponseCode.NotValid, RedirectAnswerTo.Client, original_request, parsed_request
log("Nothing unusual detected by the detectors.", LogLevel.INFO,
self.request_handler)
self._request_data.detected = "none"
log("Insert Information to database", LogLevel.DEBUG,
self.request_handler)
return ControllerResponseCode.Valid, RedirectAnswerTo.Server, original_request, parsed_request
def post(self):
incoming_json = request.get_json()
errors = check_json_object(incoming_json, ["users", "services"], "Could not find {} at the incoming json object.")
if len(errors) > 0:
log("[API][RegisterHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return 0
errors += check_json_object(incoming_json['users'], ["email", "password"], "Missing {} for the user credentials")
if len(errors) > 0:
log("[API][RegisterHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return 0
errors += check_json_object(incoming_json['services'], services_credentials,
message="Missing {} for the services credentials")
if len(errors) > 0:
log("[API][RegisterHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return 0
log("[API][RegisterHandler] Registering new Client: {}".format(incoming_json['users']['email']), LogLevel.INFO, self.post)
user = Users(email=incoming_json['users']['email'],
password=sha256_crypt.hash(str(incoming_json['users']['password'])))
try:
db.insert(user)
except Exception as e:
errors = [str(e)]
finally:
if user.item_id is None:
log("[API][RegisterHandler] Could not insert the user into the database, contact the server "
"administrator: {} ".format(errors), LogLevel.INFO, self.post)
return 0
server = Server(user_id=user.item_id,
server_ip=incoming_json['services']['ip'],
server_dns=incoming_json['services']['website'])
try:
db.insert(server)
except Exception as e:
errors = [str(e)]
finally:
if server.item_id is None:
log("[API][RegisterHandler] Could not insert the server into the database, contact the server "
"administrator: {} ".format(errors), LogLevel.INFO, self.post)
return 0
user_services = create_services_object(user_id=user.item_id, incoming_json=incoming_json,
server_id=server.item_id)
try:
db.insert(user_services)
except Exception as e:
errors = [str(e)]
finally:
if user_services.item_id is None:
log("[API][RegisterHandler] Could not insert the services into the database, contact the server "
"administrator: {} ".format(errors), LogLevel.INFO, self.post)
return 0
return 1
def post(self):
def count_total_records(key):
return db.get_session().query(DetectorRequestData).filter_by(detected=key).count()
def count_server_records(server_id, key):
return db.get_session().query(DetectorRequestData).filter_by(detected=key, to_server_id=server_id).count()
incoming_json = request.get_json()
errors = check_json_object(incoming_json, ["email"], "Could not find {} at the incoming json object.")
if len(errors) > 0:
log("[API][GetActiveServicesHandler] Could not process the request: {}".format(errors), LogLevel.INFO, self.post)
return False
user = db.get_session().query(Users).filter(Users.email == incoming_json["email"]).first()
if user is None:
log("[API][GetActiveServicesHandler] Could not find the user in the DB: {}".format(incoming_json["email"]), LogLevel.INFO, self.post)
return False
try:
joined_statuses = []
all_servers = db.get_session().query(Server).filter(Server.user_id == user.item_id).all()
log("[API][GetActiveServicesHandler] all_servers: {}".format(all_servers), LogLevel.INFO, self.post)
for server in all_servers:
log("[API][GetActiveServicesHandler] server: {}".format(server), LogLevel.INFO, self.post)
services = db.get_session().query(Services).filter(Services.server_id == server.item_id).first()
if services is None:
log("[API][GetActiveServicesHandler] Could not find the services in the DB: {}".format(
server.server_dns), LogLevel.INFO, self.post)
return False
jsoned_services = to_json(services, ignore_list=["server_id", "user_id", "id", "created_on"], to_str=True)
final_services_json = {
key: {
"state": value,
"count": count_total_records(key) if user.is_admin else count_server_records(server.item_id, key)
}
for key, value in jsoned_services.items()
}
log("[API][GetActiveServicesHandler] final_services_json: {}".format(final_services_json), LogLevel.INFO, self.post)
joined_object = {**final_services_json, **to_json(server, to_str=True)}
joined_object['website'] = joined_object['server_dns']
del joined_object['server_dns']
joined_statuses.append(joined_object)
log("[API][GetActiveServicesHandler] return {}".format(joined_statuses), LogLevel.DEBUG, self.post)
return joined_statuses
except Exception as e:
log("[API][GetActiveServicesHandler] Exception: {}".format(e), LogLevel.ERROR, self.post)
return False
def detect(self,
parsed_data,
sensitivity=Sensitivity.Regular,
forbidden=None,
legitimate=None):
"""
This method will detect an SQL attack attempt by the regex and data given in the 'SQLInjection' folder under
'Data' folder.
:param parsed_data: Parsed Data (from the parser module) of the request / response
:param sensitivity: The sensitivity of the detection
:param forbidden: list of additional malicious words/regex that we wish to add to the forbidden list on runtime
:param legitimate: The legitimate words/regex that we need to automatically approve
:return: boolean
"""
# check the query string first (url params)
query_string = parsed_data.headers.__dict__['environ']['QUERY_STRING']
for break_char in self.__break_characters:
context_breaks = re.findall(break_char, query_string)
if len(context_breaks) > 0:
log("INSIDE LEN > 0", LogLevel.INFO, self.detect)
final_decision_assurance = self.final_validation(
query_string, Sensitivity.VerySensitive)
return True if final_decision_assurance >= self.kb[
"threshold"] else False
# if no context breaks were found continue to check the request
parsed_data_as_str = create_content_as_str(
parsed_data) # Copy the parsed data to avoid change the origin
if forbidden is not None:
self.__forbidden += forbidden
if legitimate is not None:
self.__forbidden -= legitimate
self.__break_characters -= legitimate
context_break_list = []
forbidden_word_list = []
# check for context break intentions in the request itself
for break_char in self.__break_characters:
context_breaks = re.findall(break_char, parsed_data_as_str)
if len(context_breaks) > 0:
# logger.info("Found Threat of SQL INJECTION ATTACK, "
# "CONTEXT BREAK CHAR: " + break_char + " was found in: " + parsed_data)
context_break_list.append(context_breaks)
if sensitivity == Sensitivity.Sensitive:
final_decision_assurance = self.final_validation(
parsed_data_as_str)
return True if final_decision_assurance >= self.kb[
"threshold"] else False
# check for forbidden words
for forbidden_word in self.__forbidden:
forbidden_words = re.findall(forbidden_word, parsed_data_as_str)
if len(forbidden_words) > 0:
# logger.info("Found Threat of SQL INJECTION ATTACK, "
# "Forbidden word: " + forbidden_word + " was found in: " + parsed_data)
forbidden_word_list.append(forbidden_words)
# if tries to break context with forbidden words it is probably an attack
if len(context_break_list) > 0 and len(forbidden_word_list) > 0:
final_decision_assurance = self.final_validation(
parsed_data_as_str)
return True if final_decision_assurance >= self.kb[
"threshold"] else False
return False
def wrapper(self, *args, **kwargs):
if request.remote_addr not in authorized_servers:
log("[API][required_authentication] Block the request from {}".format(request.remote_addr), LogLevel.INFO, self.post)
return {"msg": "Your not authorized to perform this action", "ip": request.remote_addr, "contact": "*****@*****.**"}
return func(self, *args, **kwargs)
