- 200: The configuration as JSON is returned
The Status Get Node Configuration provides the ability to see the
configuration of the current Node. After the configuration is loaded
(which is one of the first things that the application does) it is in
dictionary form and is easily serializable to JSON and returned.
"""
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.config import Configuration
from redirectory.libs_int.service import NamespaceManager
# Metrics
STATUS_CONFIG_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels("management")
# Api Namespace
api = NamespaceManager().get_namespace("status")
@api.route("/get_node_configuration")
class ServiceGetConfiguration(Resource):
@api.doc(description="Get the configuration of the service")
@STATUS_CONFIG_REQUEST_DURATION_SECONDS.time()
def get(self):
config = Configuration().values
return make_response(jsonify({
"configuration": config
The endpoint also accepts filters (optional) which will be applied and the result
of the filtered query will be paginated after that.
If no items are found with the specified filters then an api_error with error code 404
will be returned.
"""
from time import time
from flask import make_response, jsonify, request
from flask_restplus import Resource, fields
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS, metric_update_rules_total
from redirectory.models import RedirectRule, DomainRule, PathRule, DestinationRule
from redirectory.libs_int.database import DatabaseManager, db_encode_model, paginate, Page, db_sanitize_like_query
from redirectory.libs_int.service import NamespaceManager, api_error
# Metrics
RULES_GET_PAGE_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
# Api Namespace
api = NamespaceManager().get_namespace("management")
# All possible filters to apply to the page selection
filter_args = api.model(
"filter", {
"redirect_rule_id":
fields.Integer(required=False, default=None),
"domain_rule_id":
fields.Integer(required=False, default=None),
"path_rule_id":
fields.Integer(required=False, default=None),
"destination_rule_id":
fields.Integer(required=False, default=None),
are the UI itself. The path is the path to the file which the frontend requires.
If the path is None then the index.html will be served.
The UI static files are located in a folder specified in the Configuration of the node itself.
The endpoint serves files only from the specified folder.
If a file doesn't exists then a 404 is returned.
"""
import os
from flask import send_from_directory
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.config import Configuration
from redirectory.libs_int.service import api_error
# Metrics
UI_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels("management")
class ManagementUI(Resource):
@UI_REQUEST_DURATION_SECONDS.time()
def get(self, path=None):
if path is None:
path = "index.html"
ui_directory = Configuration().values.directories.ui
does_file_exist = os.path.isfile(os.path.join(ui_directory, path))
if does_file_exist:
return send_from_directory(ui_directory, path)
else:
return api_error(message="Something went wrong!",
Method: GET
RESPONSES:
- 200: Service is up and running
A really simple endpoint that just returns a status OK.
Useful for Kubernetes to know if the service has started and it's
running. For more in depth check see the Status Readiness.
The endpoint returns the same no matter the Node Configuration.
"""
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.service import NamespaceManager
# Metrics
STATUS_HEALTH_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
# Api Namespace
api = NamespaceManager().get_namespace("status")
@api.route("/health_check")
class ServiceHealth(Resource):
@api.doc(description="Get the status of the service")
@STATUS_HEALTH_REQUEST_DURATION_SECONDS.time()
def get(self):
return make_response(jsonify({"status": "ok"}))
RESPONSES:
- 200: A list of all ambiguous request entries
- 404: No ambiguous entries in the SQL database
The List Ambiguous endpoint provides the ability to list all
currently stored ambiguous request entries in the SQL database.
"""
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.database import DatabaseManager, list_ambiguous_requests
from redirectory.libs_int.service import NamespaceManager, api_error
AMBIGUOUS_LIST_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
api = NamespaceManager().get_namespace("management")
@api.route("/ambiguous/list")
class ManagementListAmbiguous(Resource):
@api.doc(description="List all ambiguous request entries")
@AMBIGUOUS_LIST_REQUEST_DURATION_SECONDS.time()
def get(self):
db_session = DatabaseManager().get_session()
data = list_ambiguous_requests(db_session)
DatabaseManager().return_session(db_session)
if len(data) > 0:
return make_response(
Before sending an update worker request to the worker the endpoint checks if
the worker actually exists by making a health status request.
If the health status request fails then the worker is considered unreachable and a 400 is returned.
If the health status request succeeds then a second reload worker hs db request is send.
If the reload worker hs db requests returns 200 then the worker has started updating itself.
"""
from flask import make_response, jsonify, request
from flask_restplus import Resource, fields
from kubi_ecs_logger import Logger, Severity
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.sync import Synchronizer
from redirectory.libs_int.kubernetes import WorkerPod
from redirectory.libs_int.service import NamespaceManager, api_error
DATABASE_RELOAD_WORKER_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
api = NamespaceManager().get_namespace("management")
endpoint_args = api.model(
"update_worker", {
"name": fields.String(required=True, example="redirectory-worker"),
"ip": fields.String(required=True, example="127.0.0.1"),
"port": fields.Integer(required=True, example=8001),
})
@api.route("/database/reload_worker")
class ManagementReloadWorkerHsDb(Resource):
@api.expect(endpoint_args, validate=True)
@api.doc(description=
"With the help of kubernetes api update every worker with new db."
from the database. It will not take effect for the Hyperscan database. That
must be recompiled. The endpoint takes one argument which is the id of the Redirect Rule.
If the rule is found it's delete() method will be executed. The delete is custom
and it will delete Domain Rules, Path Rules and Destination Rules if they are not
used by any other Redirect Rule. For more insights on the topic take a look at delete_redirect_rule() function.
If no Redirect Rule with the given id exists then a 404 will be returned.
"""
from flask import make_response, jsonify, request
from flask_restplus import Resource, fields
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS, metric_update_rules_total
from redirectory.libs_int.database import DatabaseManager, delete_redirect_rule
from redirectory.libs_int.service import NamespaceManager, api_error
# Metrics
RULES_DELETE_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels("management")
# Api Namespace
api = NamespaceManager().get_namespace("management")
endpoint_args = api.model("del_rule", {
"rule_id": fields.Integer(required=True, example=1, min=1)
})
@api.route("/rules/delete")
class ManagementDeleteRule(Resource):
@api.expect(endpoint_args, validate=True)
@api.doc(description="Delete an already existing rule from the database")
@RULES_DELETE_REQUEST_DURATION_SECONDS.time()
1. If duplicate Redirect Rules are encountered in the CSV file they will be ignored/skipped.
2. If there is a parsing error somewhere in the file then the whole import process fails
and all of the so far added Redirect Rules to the DB are rolled back like nothing happened.
3. At the moment there is no way of telling if an import is finished.
"""
import threading
from flask import make_response, jsonify
from flask_restplus import Resource, reqparse
from werkzeug.datastructures import FileStorage
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS, metric_update_rules_total
from redirectory.libs_int.importers import CSVImporter
from redirectory.libs_int.service import NamespaceManager, api_error
# Metrics
RULES_BULK_IMPORT_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels("management")
# Api Namespace
api = NamespaceManager().get_namespace("management")
file_upload_args = reqparse.RequestParser()
file_upload_args.add_argument("csv_file", type=FileStorage, location='files', required=True, help="CSV FILE")
@api.route("/rules/bulk_import")
class ManagementBulkImportRules(Resource):
@api.expect(file_upload_args, validate=True)
@api.doc(description="Adds a rule to the database from the given args in the post data")
@RULES_BULK_IMPORT_REQUEST_DURATION_SECONDS.time()
def post(self):
RESPONSES:
- 200: Returns information about the management pod
- 400: Unable to get management pod. Not running in a cluster
This endpoint provides the management pod with the ability to get
information about itself. This is done with the use of the Kubernetes API.
"""
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.kubernetes import K8sManager
from redirectory.libs_int.service import NamespaceManager, api_error
K8S_GET_MANAGEMENT_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
api = NamespaceManager().get_namespace("management")
@api.route("/kubernetes/get_management")
class ManagementKubernetesGetManagement(Resource):
@api.doc(description="Get data about the management pod!")
@K8S_GET_MANAGEMENT_REQUEST_DURATION_SECONDS.time()
def get(self):
try:
management_pod = K8sManager().get_management_pod()
except AssertionError as e:
return api_error(message="Unable to get management pod",
errors=str(e),
status_code=400)
- 200: Returns the current Hyperscan DB version that the worker is using
- 400: The worker has not Hyperscan DB loaded at the moment
The Get Hyperscan DB Version endpoint provides with the ability to retrieve
the current Hyperscan DB Version that the HsManager() has loaded and is using to run queries.
If the worker still has not Hyperscan DB loaded then a 400 is returned.
"""
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_TOTAL, REQUESTS_DURATION_SECONDS
from redirectory.libs_int.hyperscan import HsManager
from redirectory.libs_int.service import NamespaceManager, api_error
# Metrics
WORKER_GET_HS_DB_VERSION_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"worker")
# Api Namespace
api = NamespaceManager().get_namespace("worker")
@api.route("/get_hs_db_version")
class WorkerGetHsDbVersion(Resource):
@api.doc(
description=
"Get the current hyperscan database version that the worker is using")
@WORKER_GET_HS_DB_VERSION_REQUEST_DURATION_SECONDS.time()
def get(self):
hs_db_version = HsManager().database.db_version
if hs_db_version:
return make_response(jsonify({"hs_db_version": hs_db_version}))
Files in zip:
1. sqlite database
2. hyperscan domain database
3. hyperscan rule database
"""
from flask import send_file
from flask_restplus import Resource
from kubi_ecs_logger import Logger, Severity
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.sync import Synchronizer
from redirectory.libs_int.service import NamespaceManager, api_error
# Metrics
SYNC_DOWNLOAD_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
# Api Namespace
api = NamespaceManager().get_namespace("management")
@api.route("/sync/download")
class ManagementSyncDownload(Resource):
@api.doc(description="Download all of the needed files in one as a zip")
@SYNC_DOWNLOAD_REQUEST_DURATION_SECONDS.time()
def get(self):
sync = Synchronizer()
try:
zip_sync_file = sync.util_get_sync_files_as_zip()
except FileNotFoundError as e:
also the SQL database is reloaded as well. When the thread starts it find the management
pod with the help of the Kubernetes API and downloads a zip file from it containing all
the needed file to reload itself. The zip file is then extracted and first the SQL manager
is reloaded and after that the Hyperscan database
"""
from threading import Thread
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.sync import Synchronizer
from redirectory.libs_int.service import NamespaceManager
# Metrics
WORKER_RELOAD_HS_DB_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"worker")
# Api Namespace
api = NamespaceManager().get_namespace("worker")
@api.route("/reload_hs_db")
class WorkerReloadHsDb(Resource):
@api.doc(
description=
"Tells the current worker to update itself. Runs in a separate thread")
@WORKER_RELOAD_HS_DB_REQUEST_DURATION_SECONDS.time()
def get(self):
sync = Synchronizer()
sync_thread = Thread(name="sync worker thread",
Method: GET
RESPONSES:
- 200: The Hyperscan Database has been reloaded
This endpoint provides the management pod with the ability to reload it's
hyperscan database that it uses for testing purposes.
"""
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.hyperscan import HsManager
from redirectory.libs_int.service import NamespaceManager
DATABASE_RELOAD_MANAGEMENT_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
api = NamespaceManager().get_namespace("management")
@api.route("/database/reload_management")
class ManagementReloadManagementHsDb(Resource):
@api.doc(description="Reload Management Hyperscan Test Database")
@DATABASE_RELOAD_MANAGEMENT_REQUEST_DURATION_SECONDS.time()
def get(self):
hs_manager = HsManager()
hs_manager.database.reload_database()
new_db_version = hs_manager.database.db_version
return make_response(
jsonify({
"new_hs_db_version": new_db_version,
configured correctly then a 404 page will be returned and the request will be also
added to the Ambiguous Table for later checking by a person.
"""
from typing import Optional
from flask import request, redirect
from flask_restplus import Resource
from kubi_ecs_logger import Logger, Severity
from redirectory.libs_int.hyperscan import HsManager
from redirectory.libs_int.database import DatabaseManager
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS, REQUESTS_REDIRECTED_DURATION_SECONDS, \
REQUESTS_REDIRECTED_TOTAL
# Metrics
REDIRECT_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels("worker")
TOTAL_REQUESTS_REDIRECTED_DURATION_SECONDS = REQUESTS_REDIRECTED_DURATION_SECONDS.labels(
"worker", "total")
HYPERSCAN_REQUESTS_REDIRECTED_DURATION_SECONDS = REQUESTS_REDIRECTED_DURATION_SECONDS.labels(
"worker", "hyperscan")
DB_LOOKUP_REQUESTS_REDIRECTED_DURATION_SECONDS = REQUESTS_REDIRECTED_DURATION_SECONDS.labels(
"worker", "db_lookup")
class WorkerRedirect(Resource):
@REDIRECT_REQUEST_DURATION_SECONDS.time()
@TOTAL_REQUESTS_REDIRECTED_DURATION_SECONDS.time()
def get(self, content=None):
del content
host = request.host.split(":")[0]
path = request.path
the Redirect Rules.
When you call this endpoint only the management pod is reloaded with the new DB in order to test!
All of the workers are untouched and will continue running the old version!
"""
from threading import Thread
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.models import RedirectRule
from redirectory.runnables import CompilerJob
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS, metric_update_rules_total
from redirectory.libs_int.database import DatabaseManager, db_get_table_row_count
from redirectory.libs_int.sync import Synchronizer
from redirectory.libs_int.service import NamespaceManager, api_error
DATABASE_COMPILE_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
api = NamespaceManager().get_namespace("management")
@api.route("/database/compile_test")
class ManagementCompileNewDbTest(Resource):
@api.doc(
description=
"Compile a new version of HS databases based on the sqlite DB only for testing"
)
@DATABASE_COMPILE_REQUEST_DURATION_SECONDS.time()
def get(self):
db_session = DatabaseManager().get_session()
sync = Synchronizer()
# Check if sqlite DB is empty.
- 200: Returns a list of worker pods information
- 400: Unable to get workers. Not running in a cluster
This endpoint provides the management pod with the ability to get
information about all of the worker pods. This is done with the use
of the Kubernetes API. It returns an array with every worker as an object.
If the application is not running in a Kubernetes environment then a 400 will be returned.
"""
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.kubernetes import K8sManager
from redirectory.libs_int.service import NamespaceManager, api_error
K8S_GET_WORKERS_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
api = NamespaceManager().get_namespace("management")
@api.route("/kubernetes/get_workers")
class ManagementKubernetesGetWorkers(Resource):
@api.doc(
description="Get a list of all worker nodes currently up and running!")
@K8S_GET_WORKERS_REQUEST_DURATION_SECONDS.time()
def get(self):
try:
workers = K8sManager().get_worker_pods()
except AssertionError as e:
return api_error(message="Unable to get worker pods",
errors=str(e),
status_code=400)
RESPONSES:
- 200: Returns the old_version and the current_version. If not versions are yet available then None
The Get Hyperscan DB Version endpoint provides with the ability to retrieve
the previous and the current Hyperscan DB Version which are stored in the database.
It will return None for both if there is still no entry about versions in the database.
"""
from flask import make_response, jsonify
from flask_restplus import Resource
from redirectory.libs_int.metrics import REQUESTS_DURATION_SECONDS
from redirectory.libs_int.hyperscan import get_hs_db_version, HsManager
from redirectory.libs_int.service import NamespaceManager
DATABASE_GET_VERSION_REQUEST_DURATION_SECONDS = REQUESTS_DURATION_SECONDS.labels(
"management")
api = NamespaceManager().get_namespace("management")
@api.route("/database/version")
class ManagementGetDBVersion(Resource):
@api.doc(
description="Returns the previous and current Hyperscan database version"
)
@DATABASE_GET_VERSION_REQUEST_DURATION_SECONDS.time()
def get(self):
old_version, current_version = get_hs_db_version()
hs_db_version = HsManager().database.db_version
return make_response(
jsonify({