def launch() -> None: # pragma: no cover """Launch the RAPyDo-based HTTP API server""" mywait() if initializing(): print_and_exit( "Please wait few more seconds: initialization is still in progress" ) current_app = Env.get("FLASK_APP", "").strip() if not current_app: os.environ["FLASK_APP"] = f"{current_package}.__main__" args = [ "run", "--host", BIND_INTERFACE, "--port", Env.get("FLASK_PORT", "8080"), "--reload", "--no-debugger", "--eager-loading", "--with-threads", ] # Call to untyped function "FlaskGroup" in typed context fg_cli = FlaskGroup() # type: ignore # Call to untyped function "main" in typed context fg_cli.main(prog_name="restapi", args=args) # type: ignore log.warning("Server shutdown")
def load_default_user() -> None: BaseAuthentication.default_user = Env.get("AUTH_DEFAULT_USERNAME", "") BaseAuthentication.default_password = Env.get("AUTH_DEFAULT_PASSWORD", "") if (not BaseAuthentication.default_user or not BaseAuthentication.default_password): # pragma: no cover print_and_exit("Default credentials are unavailable!")
def send_notification( subject: str, template: str, # if None will be sent to the administrator to_address: Optional[str] = None, data: Optional[Dict[str, Any]] = None, user: Optional[User] = None, send_async: bool = False, ) -> bool: # Always enabled during tests if not Connector.check_availability("smtp"): # pragma: no cover return False title = get_project_configuration("project.title", default="Unkown title") reply_to = Env.get("SMTP_NOREPLY", Env.get("SMTP_ADMIN", "")) if data is None: data = {} data.setdefault("project", title) data.setdefault("reply_to", reply_to) if user: data.setdefault("username", user.email) data.setdefault("name", user.name) data.setdefault("surname", user.surname) html_body, plain_body = get_html_template(template, data) if not html_body: # pragma: no cover log.error("Can't load {}", template) return False subject = f"{title}: {subject}" if send_async: Mail.send_async( subject=subject, body=html_body, to_address=to_address, plain_body=plain_body, ) return False smtp_client = smtp.get_instance() return smtp_client.send( subject=subject, body=html_body, to_address=to_address, plain_body=plain_body, )
def get_frontend_url() -> str: FRONTEND_URL = Env.get("FRONTEND_URL", "") if FRONTEND_URL: return FRONTEND_URL FRONTEND_PREFIX = Env.get("FRONTEND_PREFIX", "").strip("/") if FRONTEND_PREFIX: FRONTEND_PREFIX = f"/{FRONTEND_PREFIX}" protocol = "https" if PRODUCTION else "http" return f"{protocol}://{DOMAIN}{FRONTEND_PREFIX}"
def api( path: str, method: str, base: str = "api", payload: Optional[Dict[str, Any]] = None, ) -> Any: host = BotApiClient.variables.get("backend_host") port = Env.get("FLASK_PORT", "8080") url = f"http://{host}:{port}/{base}/{path}" log.debug("Calling {} on {}", method, url) try: data: Optional[str] = None if payload: data = orjson.dumps(payload).decode("UTF8") response = requests.request(method, url=url, data=data, timeout=10) out = response.json() # Never raised during tests: how to test it? except Exception as e: # pragma: no cover log.error(f"API call failed: {e}") raise ServerError(str(e)) if response.status_code >= 300: raise RestApiException(out, status_code=response.status_code) return out
def get_backend_url() -> str: BACKEND_URL = Env.get("BACKEND_URL", "") if BACKEND_URL: return BACKEND_URL BACKEND_PREFIX = Env.get("BACKEND_PREFIX", "").strip("/") if BACKEND_PREFIX: BACKEND_PREFIX = f"/{BACKEND_PREFIX}" if PRODUCTION: return f"https://{DOMAIN}{BACKEND_PREFIX}" port = Env.get("FLASK_PORT", "8080") return f"http://{DOMAIN}{BACKEND_PREFIX}:{port}"
def send_activation_link(user: User, url: str) -> bool: return send_notification( subject=Env.get("EMAIL_ACTIVATION_SUBJECT", "Account activation"), template="activate_account.html", to_address=user.email, data={"url": url}, user=user, )
def post(self, reset_email: str) -> Response: reset_email = reset_email.lower() self.auth.verify_blocked_username(reset_email) user = self.auth.get_user(username=reset_email) if user is None: raise Forbidden( f"Sorry, {reset_email} is not recognized as a valid username", ) self.auth.verify_user_status(user) title = get_project_configuration("project.title", default="Unkown title") reset_token, payload = self.auth.create_temporary_token( user, self.auth.PWD_RESET) server_url = get_frontend_url() rt = reset_token.replace(".", "+") uri = Env.get("RESET_PASSWORD_URI", "/public/reset") complete_uri = f"{server_url}{uri}/{rt}" smtp_client = smtp.get_instance() send_password_reset_link(smtp_client, complete_uri, title, reset_email) ################## # Completing the reset task self.auth.save_token(user, reset_token, payload, token_type=self.auth.PWD_RESET) msg = "We'll send instructions to the email provided if it's associated " msg += "with an account. Please check your spam/junk folder." self.log_event(self.events.reset_password_request, user=user) return self.response(msg)
def post(self, reset_email: str) -> Response: reset_email = reset_email.lower() self.auth.verify_blocked_username(reset_email) user = self.auth.get_user(username=reset_email) if user is None: raise Forbidden( f"Sorry, {reset_email} is not recognized as a valid username", ) self.auth.verify_user_status(user) reset_token, payload = self.auth.create_temporary_token( user, self.auth.PWD_RESET) server_url = get_frontend_url() rt = reset_token.replace(".", "+") uri = Env.get("RESET_PASSWORD_URI", "/public/reset") complete_uri = f"{server_url}{uri}/{rt}" sent = send_password_reset_link(user, complete_uri, reset_email) if not sent: # pragma: no cover raise ServiceUnavailable("Error sending email, please retry") ################## # Completing the reset task self.auth.save_token(user, reset_token, payload, token_type=self.auth.PWD_RESET) msg = "We'll send instructions to the email provided if it's associated " msg += "with an account. Please check your spam/junk folder." self.log_event(self.events.reset_password_request, user=user) return self.response(msg)
def api(path, method, base="api", payload=None): host = BotApiClient.variables.get("backend_host") port = Env.get("FLASK_PORT") url = f"http://{host}:{port}/{base}/{path}" log.debug("Calling {} on {}", method, url) try: response = requests.request(method, url=url, data=payload) out = response.json() # Never raised during tests: how to test it? except Exception as e: # pragma: no cover log.error(f"API call failed: {e}") raise RestApiException(str(e), status_code=500) if response.status_code >= 300: raise RestApiException(out, status_code=response.status_code) return out
def test_bot() -> None: if not Env.get_bool("TELEGRAM_ENABLE"): log.warning("Skipping BOT tests: service not available") return runner = CliRunner() start_timeout(3) try: runner.invoke(cli.bot, []) except Timeout: # pragma: no cover pass stop_timeout() from restapi.services.telegram import bot # Your API ID, hash and session string here # How to generate StringSessions: # https://docs.telethon.dev/en/latest/concepts/sessions.html#string-sessions api_id = Env.get_int("TELEGRAM_APP_ID") api_hash = Env.get("TELEGRAM_APP_HASH", "") or None session_str = Env.get("TELETHON_SESSION", "") or None botname = Env.get("TELEGRAM_BOTNAME", "") or None # use TelegramClient as a type once released the typed version 2 (issue #1195) async def send_command(client: Any, command: str) -> str: await client.send_message(botname, command) sleep(1) messages = await client.get_messages(botname) # TelegramClient is not typed => message is Any return messages[0].message # type: ignore async def test() -> None: client = TelegramClient(StringSession(session_str), api_id, api_hash) await client.start() message = await send_command(client, "/me") assert re.match(r"^Hello .*, your Telegram ID is [0-9]+", message) message = await send_command(client, "/invalid") assert message == "Invalid command, ask for /help" message = await send_command(client, "/help") assert "Available Commands:" in message assert "- /help print this help" in message assert "- /me info about yourself" in message assert "- /status get server status" in message assert "- /monitor get server monitoring stats" in message # commands requiring APIs can only be tested in PRODUCTION MODE if PRODUCTION: message = await send_command(client, "/status") assert message == "Server is alive" message = await send_command(client, "/monitor") assert message == "Please select the type of monitor" message = await send_command(client, "/monitor x") assert message == "Please select the type of monitor" message = await send_command(client, "/monitor disk") error = "Missing credentials in headers, e.g. Authorization: 'Bearer TOKEN'" assert error in message message = await send_command(client, "/monitor disk 2") assert message == "Too many inputs" # # ############################# # # # TEST USER # # # ############################# # bot.users = bot.admins bot.admins = [] message = await send_command(client, "/me") assert message == PERMISSION_DENIED message = await send_command(client, "/invalid") assert message == "Invalid command, ask for /help" message = await send_command(client, "/help") assert "Available Commands:" in message assert "- /help print this help" in message assert "- /me info about yourself" in message assert "- /status get server status" in message assert "- /monitor get server monitoring stats" in message # commands requiring APIs can only be tested in PRODUCTION MODE if PRODUCTION: message = await send_command(client, "/status") assert message == "Server is alive" message = await send_command(client, "/monitor") assert message == PERMISSION_DENIED # # ############################# # # # TEST UNAUTHORIZED # # # ############################# # bot.admins = [] bot.users = [] message = await send_command(client, "/me") assert message == PERMISSION_DENIED message = await send_command(client, "/invalid") assert message == PERMISSION_DENIED message = await send_command(client, "/help") assert message == PERMISSION_DENIED message = await send_command(client, "/status") assert message == PERMISSION_DENIED message = await send_command(client, "/monitor") assert message == PERMISSION_DENIED asyncio.run(test()) bot.shutdown()
""" Configuration variables set for the server instance """ from functools import lru_cache from pathlib import Path from glom import glom from restapi.env import Env from restapi.utilities.globals import mem # ENDPOINTS bases API_URL = "/api" AUTH_URL = "/auth" APP_MODE: str = Env.get("APP_MODE", "development") FORCE_PRODUCTION_TESTS: bool = Env.get_bool("FORCE_PRODUCTION_TESTS") TESTING: bool = APP_MODE == "test" or FORCE_PRODUCTION_TESTS PRODUCTION: bool = APP_MODE == "production" STACKTRACE: bool = False REMOVE_DATA_AT_INIT_TIME: bool = False HOSTNAME: str = Env.get("HOSTNAME", "backend-server") # hostnames as defined in backend.yml MAIN_SERVER_NAME = "REST_API" BACKEND_HOSTNAME = "backend-server" FLOWER_HOSTNAME = "flower" CELERYBEAT_HOSTNAME = "celery-beat" BOT_HOSTNAME = "telegram-bot" CELERY_HOSTNAME = "celery"
def get_totp_secret(self, user: User) -> str: if TESTING: # pragma: no cover if (p := Env.get("AUTH_TESTING_TOTP_HASH")) is not None: return p
class Connector(metaclass=abc.ABCMeta): authentication_service: str = Env.get("AUTH_SERVICE", NO_AUTH) # Available services with associated env variables services: Services = {} # Assigned by init_app app: Flask = None # Used by get_authentication_module _authentication_module: Optional[ModuleType] = None # Returned by __getattr__ in neo4j and sqlalchemy connectors _models: Dict[str, Type] = {} # Used by set_object and get_object _instances: InstancesCache = {} def __init__(self) -> None: # This is the lower-cased class name (neomodel, celeryext) self.name = self.__class__.__name__.lower() # This is the folder name corresponding to the connector name (neo4j, celery, ) # self.__class__.__module__ == restapi.connectors.sqlalchemy # .split(".") == ['restapi', 'connectors', 'sqlalchemy'] # [-1] == 'sqlalchemy' self.name = self.__class__.__module__.split(".")[-1] # Will be modified by self.disconnect() self.disconnected = False # Added to convince mypy that self.app cannot be None if self.app is None: # pragma: no cover # This should never happen because app is # assigned during init_services from flask import current_app self.app = current_app @staticmethod def init() -> None: if Connector.authentication_service == NO_AUTH: log.info("No Authentication service configured") else: log.debug("Authentication service: {}", Connector.authentication_service) Connector.services = Connector.load_connectors( ABS_RESTAPI_PATH, BACKEND_PACKAGE, Connector.services ) if EXTENDED_PACKAGE != EXTENDED_PROJECT_DISABLED: Connector.services = Connector.load_connectors( Path(EXTENDED_PACKAGE), EXTENDED_PACKAGE, Connector.services, ) Connector.services = Connector.load_connectors( Path(CUSTOM_PACKAGE), CUSTOM_PACKAGE, Connector.services ) Connector.load_models(Connector.services.keys()) def __del__(self) -> None: if not self.disconnected: self.disconnect() def __enter__(self: T) -> T: return self def __exit__( self, exctype: Optional[Type[Exception]], excinst: Optional[Exception], exctb: Optional[TracebackType], ) -> bool: if not self.disconnected: self.disconnect() if excinst: raise excinst return True @staticmethod @abc.abstractmethod def get_connection_exception() -> ExceptionsList: # pragma: no cover return None @abc.abstractmethod def connect(self: T, **kwargs: Any) -> Generic[T]: # pragma: no cover return self @abc.abstractmethod def disconnect(self) -> None: # pragma: no cover return @abc.abstractmethod def is_connected(instance: T) -> bool: # pragma: no cover return True def destroy(self) -> None: # pragma: no cover print_and_exit("Missing destroy method in {}", self.__class__.__name__) def initialize(self) -> None: # pragma: no cover print_and_exit("Missing initialize method in {}", self.__class__.__name__) @property def variables(self) -> Dict[str, str]: return self.services.get(self.name) or {} @classmethod def load_connectors(cls, path: Path, module: str, services: Services) -> Services: main_folder = path.joinpath(CONNECTORS_FOLDER) if not main_folder.is_dir(): log.debug("Connectors folder not found: {}", main_folder) return services for connector in main_folder.iterdir(): if not connector.is_dir(): continue connector_name = connector.name if connector_name.startswith("_"): continue # This is the only exception... we should rename sqlalchemy as alchemy if connector_name == "sqlalchemy": variables = Env.load_variables_group(prefix="alchemy") else: variables = Env.load_variables_group(prefix=connector_name) if not Env.to_bool( variables.get("enable_connector", True) ): # pragma: no cover log.debug("{} connector is disabled", connector_name) continue external = False if "host" in variables: if host := variables.get("host"): external = cls.is_external(host) # HOST found in variables but empty... never happens during tests else: # pragma: no cover variables["enable"] = "0" enabled = Env.to_bool(variables.get("enable")) # Celery is always enabled, if connector is enabled # No further check is needed on host/external available = enabled or external or connector_name == "celery" if not available: continue connector_module = Connector.get_module(connector_name, module) connector_class = Connector.get_class(connector_module) # Can't test connector misconfiguration... if not connector_class: # pragma: no cover log.error("No connector class found in {}/{}", main_folder, connector) continue try: # This is to test the Connector compliance, # i.e. to verify instance and get_instance in the connector module # and verify that the Connector can be instanced connector_module.instance connector_module.get_instance connector_class() except AttributeError as e: # pragma: no cover print_and_exit(e) services[connector_name] = variables log.debug("Got class definition for {}", connector_class) return services
def get_totp_secret(self, user: User) -> str: if TESTING: # pragma: no cover # TESTING_TOTP_HASH is set by setup-cypress github action if p := Env.get("AUTH_TESTING_TOTP_HASH", ""): return p
def is_mysql() -> bool: # could be based on self.variables but this version Env based # can be used as static method and be used before creating instances return (Env.get("AUTH_SERVICE", "NO_AUTHENTICATION") == "sqlalchemy" and Env.get("ALCHEMY_DBTYPE", "postgresql") == "mysql+pymysql")
import os import re import sys import urllib.parse from enum import Enum from pathlib import Path from typing import Any, Dict, Optional, Tuple import orjson from loguru import logger as log from restapi.config import HOST_TYPE, PRODUCTION from restapi.env import Env log_level = Env.get("LOGURU_LEVEL", "DEBUG") LOG_RETENTION = Env.get("LOG_RETENTION", "180") FILE_LOGLEVEL = Env.get("FILE_LOGLEVEL", "WARNING") # FILE_LOGLEVEL = "WARNING" if not TESTING else "INFO" LOGS_FOLDER = Path("/logs") LOGS_PATH: Optional[str] = LOGS_FOLDER.joinpath(f"{HOST_TYPE}.log") EVENTS_PATH: Optional[str] = LOGS_FOLDER.joinpath("security-events.log") if Path(LOGS_PATH).exists() and not os.access(LOGS_PATH, os.W_OK): # pragma: no cover print( f"\nCan't initialize logging because {LOGS_PATH} is not writeable, " "backend server cannot start\n" ) sys.exit(1) if Path(EVENTS_PATH).exists() and not os.access(