def test_search_domain(self, hyperscan):
"""
Starts with a populated hyperscan database with 5 rules.
Test the search_domain() function.
Expected behaviour:
1. Returns the correct ids corresponding to the expressions
"""
# Import needed functions and classes
from redirectory.libs_int.hyperscan import HsManager, SearchContext
# Check
search_ctx: SearchContext = HsManager().search_domain(
'asd.test.kumina.nl')
assert search_ctx.matched_ids == [1, 3, 5]
search_ctx: SearchContext = HsManager().search_domain(
'123.test.kumina.nl')
assert search_ctx.matched_ids == [3, 4, 5]
search_ctx: SearchContext = HsManager().search_domain(
'123!.test.kumina.nl')
assert search_ctx.matched_ids == [5]
search_ctx: SearchContext = HsManager().search_domain(
'ggg.test.kumina.nl')
assert search_ctx.matched_ids == [2, 3, 5]
search_ctx: SearchContext = HsManager().search_domain('test.kumina.nl')
assert search_ctx.matched_ids == []
def post(self):
url = request.get_json()["request_url"]
host, path = self.parse_url(url)
# Init managers and sessions
hs_manager = HsManager()
db_session = DatabaseManager().get_session()
# Search
try:
start_search_time = time()
search_data = hs_manager.search(domain=host,
path=path,
is_test=True)
end_search_time = time()
except AssertionError as e:
Logger() \
.event(category="hyperscan", action="hyperscan search") \
.error(message=str(e)) \
.out(severity=Severity.ERROR)
DatabaseManager().return_session(db_session)
return api_error(
message="Something went wrong during Hyperscan search!",
errors=str(e),
status_code=400)
# Convert ids into models
domain_rules_data = {}
redirect_rules_data = {}
d_r_map = search_data["domain_rule_map"]
for domain_id in d_r_map.keys():
domain_rule = get_model_by_id(db_session, DomainRule, domain_id)
domain_rules_data[domain_id] = db_encode_model(domain_rule)
for rule_id in d_r_map[domain_id]:
redirect_rule = get_model_by_id(db_session, RedirectRule,
rule_id)
redirect_rules_data[rule_id] = db_encode_model(redirect_rule,
expand=True)
# Get final result
final_redirect_rule, is_ambiguous = HsManager.pick_result(
db_session, list(redirect_rules_data.keys()))
# Fill in test data
search_data[
"final_result_id"] = final_redirect_rule.id if final_redirect_rule is not None else None
search_data["is_ambiguous"] = is_ambiguous
search_data["time"] = str(end_search_time - start_search_time)
DatabaseManager().return_session(db_session)
return make_response(
jsonify({
"domain_rules": domain_rules_data,
"redirect_rules": redirect_rules_data,
"search_data": search_data
}), 200)
def run(self):
HsManager().database.load_database()
# Add the redirect
from redirectory.services import WorkerRedirect
self.api.add_resource(WorkerRedirect, "/", "/<path:content>")
# Get needed namespaces
worker_ns = NamespaceManager().get_namespace("worker")
status_ns = NamespaceManager().get_namespace("status")
# Add namespaces to api
self.api.add_namespace(worker_ns)
self.api.add_namespace(status_ns)
# Init api with application
self.api.init_app(self.application)
# Log
Logger() \
.event(category="runnable", action="service configured") \
.service(name="worker").out(severity=Severity.INFO)
# Run according to configuration
if self.config.deployment == "prod":
self._run_production(is_worker=True)
elif self.config.deployment == "dev":
self._run_development()
elif self.config.deployment == "test":
self._run_test()
def get(self):
hs_db_version = HsManager().database.db_version
if hs_db_version:
return make_response(jsonify({"hs_db_version": hs_db_version}))
else:
return api_error(message="Worker has no Hyperscan Database loaded",
errors=[],
status_code=400)
def test_search_rules(self, hyperscan):
"""
Starts with a populated hyperscan database with 5 rules.
Test the search_rules() function.
Expected behaviour:
1. Returns the correct matched Redirect Rules from the database
"""
# Import needed functions and classes
from redirectory.libs_int.hyperscan import HsManager, SearchContext
# Check
search_ctx: SearchContext = HsManager().search_rule('1/test/path')
assert search_ctx.matched_ids == [1]
search_ctx: SearchContext = HsManager().search_rule('3/test/path/aa')
assert search_ctx.matched_ids == [3]
search_ctx: SearchContext = HsManager().search_rule('4/test/path/aa')
assert search_ctx.matched_ids == [4]
def get(self):
hs_manager = HsManager()
hs_manager.database.reload_database()
new_db_version = hs_manager.database.db_version
return make_response(
jsonify({
"new_hs_db_version": new_db_version,
"status": "done"
}), 200)
def get(self):
config = Configuration().values
is_management = config.node_type == "management"
if HsManager().database.is_loaded or is_management:
return make_response(jsonify({"status": "ready"}), 200)
else:
return api_error(message="Not Ready",
errors="Hyperscan database not loaded yet!",
status_code=400)
def get(self):
old_version, current_version = get_hs_db_version()
hs_db_version = HsManager().database.db_version
return make_response(
jsonify({
"old_version": old_version,
"current_version": current_version,
"loaded_version": hs_db_version,
"status": "done"
}), 200)
def test_search(self, hyperscan):
"""
Starts with a populated hyperscan database with 5 rules.
Test the search() function.
Expected behaviour:
1. Matches the right ids for the right expressions
"""
# Import needed functions and classes
from redirectory.libs_int.hyperscan import HsManager
# Check
result = HsManager().search(domain='asd.test.kumina.nl',
path='/test/path')
assert result == [1, 5]
result = HsManager().search(domain='asd.test.kumina.nl',
path='/test/path/aa')
assert result == [3, 5]
result = HsManager().search(domain='123.test.kumina.nl',
path='/test/path')
assert result == [4, 5]
result = HsManager().search(domain='123!.test.kumina.nl',
path='/test/path')
assert result == [5]
result = HsManager().search(domain='123a.test.kumina.nl',
path='/test/path/a')
assert result == [3, 5]
result = HsManager().search(domain='ggg.test.kumina.nl',
path='/test/path/a')
assert result == [2, 3, 5]
def worker_sync_files(self):
"""
This function gets called when the application is in worker mode.
Stops the readiness checks from succeeding.
Downloads and reloads the databases.
After that it logs a couple of metrics and also logging.
"""
Logger() \
.event(category="synchronizer", action="synchronizer sync started") \
.out(severity=Severity.INFO)
db_manager = DatabaseManager()
hs_manager = HsManager()
# Mark hs database as not loaded which causes ready check to fail
hs_manager.database.is_loaded = False
# Download sync files and write to disc
sync_zip_file = self.management_get_sync_files()
if sync_zip_file is None:
return
self.util_save_sync_zip_file(sync_zip_file)
# Reload sql
db_manager.reload()
# Reload hs
hs_manager.database.reload_database()
new_hs_db_version = hs_manager.database.db_version
# Metrics
HYPERSCAN_DB_RELOADED_TOTAL.labels(self.configuration.node_type).inc()
HYPERSCAN_DB_VERSION.labels(
self.configuration.node_type).set(new_hs_db_version)
# Log
Logger() \
.event(category="synchronizer", action="synchronizer sync complete",
dataset=f"New hs db version: {new_hs_db_version}") \
.out(severity=Severity.INFO)
def run(self):
# Load hyperscan database because of test in UI
HsManager().database.load_database()
# Get needed namespaces
management_ns = NamespaceManager().get_namespace("management")
status_ns = NamespaceManager().get_namespace("status")
# Add the ui
from redirectory.services import ManagementUI
self.api.add_resource(ManagementUI, "/", "/<path:path>")
# Log ui folder
Logger().event(
category="ui",
action="ui loaded",
dataset=self.config.directories.ui).out(severity=Severity.INFO)
# Add namespaces to api
self.api.add_namespace(management_ns)
self.api.add_namespace(status_ns)
# Init api with application
self.api.init_app(self.application)
# Log
Logger() \
.event(category="runnable", action="service configured") \
.service(name="management").out(severity=Severity.INFO)
# Run according to configuration
if self.config.deployment == "prod":
self._run_production()
elif self.config.deployment == "dev":
self._run_development()
elif self.config.deployment == "test":
self._run_test()
def get(self, content=None):
del content
host = request.host.split(":")[0]
path = request.path
# Stop spamming for favicon pls
if path == "/favicon.ico":
self.page_404()
# Init managers and sessions
hs_manager = HsManager()
db_session = DatabaseManager().get_session()
# Search
try:
matching_ids = None
with HYPERSCAN_REQUESTS_REDIRECTED_DURATION_SECONDS.time(
): # Metric
matching_ids = hs_manager.search(domain=host, path=path)
is_404 = matching_ids is None
except AssertionError as e:
Logger() \
.event(category="hyperscan", action="hyperscan search failed") \
.error(message=str(e)) \
.out(severity=Severity.ERROR)
DatabaseManager().return_session(db_session)
is_404 = True
if is_404:
REQUESTS_REDIRECTED_TOTAL.labels("worker", "404",
"not_found").inc()
self.page_404()
# Get final result
with DB_LOOKUP_REQUESTS_REDIRECTED_DURATION_SECONDS.time(): # Metric
final_redirect_rule, is_ambiguous = HsManager.pick_result(
db_session, list(matching_ids))
final_destination_url = final_redirect_rule.destination_rule.destination_url
is_back_ref: bool = final_redirect_rule.destination_rule.is_rewrite
# Do back reference
if is_back_ref:
final_destination_url = self.apply_back_reference(
path, final_redirect_rule)
# Add ambiguous request to db if needed
if is_ambiguous or final_destination_url is None:
from redirectory.libs_int.kubernetes import K8sManager, ManagementPod
try:
management_pod: ManagementPod = K8sManager(
).get_management_pod()
management_pod.add_ambiguous_request(request.url)
except AssertionError as e:
Logger() \
.event(category="sync", action="sync ambiguous request",
dataset="Unable to sync ambiguous request with the management pod") \
.error(message=str(e)) \
.out(severity=Severity.ERROR)
if final_destination_url is None:
self.page_404()
# Sanitize final redirect url
final_destination_url = self.sanitize_outgoing_redirect(
final_destination_url)
DatabaseManager().return_session(db_session)
if is_ambiguous:
REQUESTS_REDIRECTED_TOTAL.labels("worker", "301",
"ambiguous").inc()
elif is_back_ref:
REQUESTS_REDIRECTED_TOTAL.labels("worker", "301", "back_ref").inc()
else:
REQUESTS_REDIRECTED_TOTAL.labels("worker", "301", "normal").inc()
return redirect(final_destination_url, 301)
def test_pick_result(self, hyperscan):
"""
Starts with a populated hyperscan database with 5 rules.
Test the pick_result() function.
Expected behaviour:
1. Picks the correct one with the highest value
"""
# Get session
from redirectory.libs_int.database import DatabaseManager
db_session = DatabaseManager().get_session()
# Import needed functions and classes
from redirectory.libs_int.hyperscan import HsManager
# Check
matched_ids = HsManager().search(domain='asd.test.kumina.nl',
path='/test/path')
assert matched_ids == [1, 5]
result, is_ambiguous = HsManager.pick_result(db_session, matched_ids)
assert is_ambiguous
assert result.id == 1
matched_ids = HsManager().search(domain='asd.test.kumina.nl',
path='/test/path/aa')
assert matched_ids == [3, 5]
result, is_ambiguous = HsManager.pick_result(db_session, matched_ids)
assert is_ambiguous
assert result.id == 3
matched_ids = HsManager().search(domain='123.test.kumina.nl',
path='/test/path')
assert matched_ids == [4, 5]
result, is_ambiguous = HsManager.pick_result(db_session, matched_ids)
assert is_ambiguous
assert result.id == 4
matched_ids = HsManager().search(domain='123!.test.kumina.nl',
path='/test/path')
assert matched_ids == [5]
result, is_ambiguous = HsManager.pick_result(db_session, matched_ids)
assert not is_ambiguous
assert result.id == 5
matched_ids = HsManager().search(domain='123a.test.kumina.nl',
path='/test/path/a')
assert matched_ids == [3, 5]
result, is_ambiguous = HsManager.pick_result(db_session, matched_ids)
assert is_ambiguous
assert result.id == 3
matched_ids = HsManager().search(domain='ggg.test.kumina.nl',
path='/test/path/a')
assert matched_ids == [2, 3, 5]
result, is_ambiguous = HsManager.pick_result(db_session, matched_ids)
assert is_ambiguous
assert result.id == 2
# Return session
DatabaseManager().return_session(db_session)