def test_search_domain(self, hyperscan): """ Starts with a populated hyperscan database with 5 rules. Test the search_domain() function. Expected behaviour: 1. Returns the correct ids corresponding to the expressions """ # Import needed functions and classes from redirectory.libs_int.hyperscan import HsManager, SearchContext # Check search_ctx: SearchContext = HsManager().search_domain( 'asd.test.kumina.nl') assert search_ctx.matched_ids == [1, 3, 5] search_ctx: SearchContext = HsManager().search_domain( '123.test.kumina.nl') assert search_ctx.matched_ids == [3, 4, 5] search_ctx: SearchContext = HsManager().search_domain( '123!.test.kumina.nl') assert search_ctx.matched_ids == [5] search_ctx: SearchContext = HsManager().search_domain( 'ggg.test.kumina.nl') assert search_ctx.matched_ids == [2, 3, 5] search_ctx: SearchContext = HsManager().search_domain('test.kumina.nl') assert search_ctx.matched_ids == []
def post(self): url = request.get_json()["request_url"] host, path = self.parse_url(url) # Init managers and sessions hs_manager = HsManager() db_session = DatabaseManager().get_session() # Search try: start_search_time = time() search_data = hs_manager.search(domain=host, path=path, is_test=True) end_search_time = time() except AssertionError as e: Logger() \ .event(category="hyperscan", action="hyperscan search") \ .error(message=str(e)) \ .out(severity=Severity.ERROR) DatabaseManager().return_session(db_session) return api_error( message="Something went wrong during Hyperscan search!", errors=str(e), status_code=400) # Convert ids into models domain_rules_data = {} redirect_rules_data = {} d_r_map = search_data["domain_rule_map"] for domain_id in d_r_map.keys(): domain_rule = get_model_by_id(db_session, DomainRule, domain_id) domain_rules_data[domain_id] = db_encode_model(domain_rule) for rule_id in d_r_map[domain_id]: redirect_rule = get_model_by_id(db_session, RedirectRule, rule_id) redirect_rules_data[rule_id] = db_encode_model(redirect_rule, expand=True) # Get final result final_redirect_rule, is_ambiguous = HsManager.pick_result( db_session, list(redirect_rules_data.keys())) # Fill in test data search_data[ "final_result_id"] = final_redirect_rule.id if final_redirect_rule is not None else None search_data["is_ambiguous"] = is_ambiguous search_data["time"] = str(end_search_time - start_search_time) DatabaseManager().return_session(db_session) return make_response( jsonify({ "domain_rules": domain_rules_data, "redirect_rules": redirect_rules_data, "search_data": search_data }), 200)
def run(self): HsManager().database.load_database() # Add the redirect from redirectory.services import WorkerRedirect self.api.add_resource(WorkerRedirect, "/", "/<path:content>") # Get needed namespaces worker_ns = NamespaceManager().get_namespace("worker") status_ns = NamespaceManager().get_namespace("status") # Add namespaces to api self.api.add_namespace(worker_ns) self.api.add_namespace(status_ns) # Init api with application self.api.init_app(self.application) # Log Logger() \ .event(category="runnable", action="service configured") \ .service(name="worker").out(severity=Severity.INFO) # Run according to configuration if self.config.deployment == "prod": self._run_production(is_worker=True) elif self.config.deployment == "dev": self._run_development() elif self.config.deployment == "test": self._run_test()
def get(self): hs_db_version = HsManager().database.db_version if hs_db_version: return make_response(jsonify({"hs_db_version": hs_db_version})) else: return api_error(message="Worker has no Hyperscan Database loaded", errors=[], status_code=400)
def test_search_rules(self, hyperscan): """ Starts with a populated hyperscan database with 5 rules. Test the search_rules() function. Expected behaviour: 1. Returns the correct matched Redirect Rules from the database """ # Import needed functions and classes from redirectory.libs_int.hyperscan import HsManager, SearchContext # Check search_ctx: SearchContext = HsManager().search_rule('1/test/path') assert search_ctx.matched_ids == [1] search_ctx: SearchContext = HsManager().search_rule('3/test/path/aa') assert search_ctx.matched_ids == [3] search_ctx: SearchContext = HsManager().search_rule('4/test/path/aa') assert search_ctx.matched_ids == [4]
def get(self): hs_manager = HsManager() hs_manager.database.reload_database() new_db_version = hs_manager.database.db_version return make_response( jsonify({ "new_hs_db_version": new_db_version, "status": "done" }), 200)
def get(self): config = Configuration().values is_management = config.node_type == "management" if HsManager().database.is_loaded or is_management: return make_response(jsonify({"status": "ready"}), 200) else: return api_error(message="Not Ready", errors="Hyperscan database not loaded yet!", status_code=400)
def get(self): old_version, current_version = get_hs_db_version() hs_db_version = HsManager().database.db_version return make_response( jsonify({ "old_version": old_version, "current_version": current_version, "loaded_version": hs_db_version, "status": "done" }), 200)
def test_search(self, hyperscan): """ Starts with a populated hyperscan database with 5 rules. Test the search() function. Expected behaviour: 1. Matches the right ids for the right expressions """ # Import needed functions and classes from redirectory.libs_int.hyperscan import HsManager # Check result = HsManager().search(domain='asd.test.kumina.nl', path='/test/path') assert result == [1, 5] result = HsManager().search(domain='asd.test.kumina.nl', path='/test/path/aa') assert result == [3, 5] result = HsManager().search(domain='123.test.kumina.nl', path='/test/path') assert result == [4, 5] result = HsManager().search(domain='123!.test.kumina.nl', path='/test/path') assert result == [5] result = HsManager().search(domain='123a.test.kumina.nl', path='/test/path/a') assert result == [3, 5] result = HsManager().search(domain='ggg.test.kumina.nl', path='/test/path/a') assert result == [2, 3, 5]
def worker_sync_files(self): """ This function gets called when the application is in worker mode. Stops the readiness checks from succeeding. Downloads and reloads the databases. After that it logs a couple of metrics and also logging. """ Logger() \ .event(category="synchronizer", action="synchronizer sync started") \ .out(severity=Severity.INFO) db_manager = DatabaseManager() hs_manager = HsManager() # Mark hs database as not loaded which causes ready check to fail hs_manager.database.is_loaded = False # Download sync files and write to disc sync_zip_file = self.management_get_sync_files() if sync_zip_file is None: return self.util_save_sync_zip_file(sync_zip_file) # Reload sql db_manager.reload() # Reload hs hs_manager.database.reload_database() new_hs_db_version = hs_manager.database.db_version # Metrics HYPERSCAN_DB_RELOADED_TOTAL.labels(self.configuration.node_type).inc() HYPERSCAN_DB_VERSION.labels( self.configuration.node_type).set(new_hs_db_version) # Log Logger() \ .event(category="synchronizer", action="synchronizer sync complete", dataset=f"New hs db version: {new_hs_db_version}") \ .out(severity=Severity.INFO)
def run(self): # Load hyperscan database because of test in UI HsManager().database.load_database() # Get needed namespaces management_ns = NamespaceManager().get_namespace("management") status_ns = NamespaceManager().get_namespace("status") # Add the ui from redirectory.services import ManagementUI self.api.add_resource(ManagementUI, "/", "/<path:path>") # Log ui folder Logger().event( category="ui", action="ui loaded", dataset=self.config.directories.ui).out(severity=Severity.INFO) # Add namespaces to api self.api.add_namespace(management_ns) self.api.add_namespace(status_ns) # Init api with application self.api.init_app(self.application) # Log Logger() \ .event(category="runnable", action="service configured") \ .service(name="management").out(severity=Severity.INFO) # Run according to configuration if self.config.deployment == "prod": self._run_production() elif self.config.deployment == "dev": self._run_development() elif self.config.deployment == "test": self._run_test()
def get(self, content=None): del content host = request.host.split(":")[0] path = request.path # Stop spamming for favicon pls if path == "/favicon.ico": self.page_404() # Init managers and sessions hs_manager = HsManager() db_session = DatabaseManager().get_session() # Search try: matching_ids = None with HYPERSCAN_REQUESTS_REDIRECTED_DURATION_SECONDS.time( ): # Metric matching_ids = hs_manager.search(domain=host, path=path) is_404 = matching_ids is None except AssertionError as e: Logger() \ .event(category="hyperscan", action="hyperscan search failed") \ .error(message=str(e)) \ .out(severity=Severity.ERROR) DatabaseManager().return_session(db_session) is_404 = True if is_404: REQUESTS_REDIRECTED_TOTAL.labels("worker", "404", "not_found").inc() self.page_404() # Get final result with DB_LOOKUP_REQUESTS_REDIRECTED_DURATION_SECONDS.time(): # Metric final_redirect_rule, is_ambiguous = HsManager.pick_result( db_session, list(matching_ids)) final_destination_url = final_redirect_rule.destination_rule.destination_url is_back_ref: bool = final_redirect_rule.destination_rule.is_rewrite # Do back reference if is_back_ref: final_destination_url = self.apply_back_reference( path, final_redirect_rule) # Add ambiguous request to db if needed if is_ambiguous or final_destination_url is None: from redirectory.libs_int.kubernetes import K8sManager, ManagementPod try: management_pod: ManagementPod = K8sManager( ).get_management_pod() management_pod.add_ambiguous_request(request.url) except AssertionError as e: Logger() \ .event(category="sync", action="sync ambiguous request", dataset="Unable to sync ambiguous request with the management pod") \ .error(message=str(e)) \ .out(severity=Severity.ERROR) if final_destination_url is None: self.page_404() # Sanitize final redirect url final_destination_url = self.sanitize_outgoing_redirect( final_destination_url) DatabaseManager().return_session(db_session) if is_ambiguous: REQUESTS_REDIRECTED_TOTAL.labels("worker", "301", "ambiguous").inc() elif is_back_ref: REQUESTS_REDIRECTED_TOTAL.labels("worker", "301", "back_ref").inc() else: REQUESTS_REDIRECTED_TOTAL.labels("worker", "301", "normal").inc() return redirect(final_destination_url, 301)
def test_pick_result(self, hyperscan): """ Starts with a populated hyperscan database with 5 rules. Test the pick_result() function. Expected behaviour: 1. Picks the correct one with the highest value """ # Get session from redirectory.libs_int.database import DatabaseManager db_session = DatabaseManager().get_session() # Import needed functions and classes from redirectory.libs_int.hyperscan import HsManager # Check matched_ids = HsManager().search(domain='asd.test.kumina.nl', path='/test/path') assert matched_ids == [1, 5] result, is_ambiguous = HsManager.pick_result(db_session, matched_ids) assert is_ambiguous assert result.id == 1 matched_ids = HsManager().search(domain='asd.test.kumina.nl', path='/test/path/aa') assert matched_ids == [3, 5] result, is_ambiguous = HsManager.pick_result(db_session, matched_ids) assert is_ambiguous assert result.id == 3 matched_ids = HsManager().search(domain='123.test.kumina.nl', path='/test/path') assert matched_ids == [4, 5] result, is_ambiguous = HsManager.pick_result(db_session, matched_ids) assert is_ambiguous assert result.id == 4 matched_ids = HsManager().search(domain='123!.test.kumina.nl', path='/test/path') assert matched_ids == [5] result, is_ambiguous = HsManager.pick_result(db_session, matched_ids) assert not is_ambiguous assert result.id == 5 matched_ids = HsManager().search(domain='123a.test.kumina.nl', path='/test/path/a') assert matched_ids == [3, 5] result, is_ambiguous = HsManager.pick_result(db_session, matched_ids) assert is_ambiguous assert result.id == 3 matched_ids = HsManager().search(domain='ggg.test.kumina.nl', path='/test/path/a') assert matched_ids == [2, 3, 5] result, is_ambiguous = HsManager.pick_result(db_session, matched_ids) assert is_ambiguous assert result.id == 2 # Return session DatabaseManager().return_session(db_session)