def launch() -> None: # pragma: no cover
"""Launch the RAPyDo-based HTTP API server"""
mywait()
if initializing():
print_and_exit(
"Please wait few more seconds: initialization is still in progress"
)
current_app = Env.get("FLASK_APP", "").strip()
if not current_app:
os.environ["FLASK_APP"] = f"{current_package}.__main__"
args = [
"run",
"--host",
BIND_INTERFACE,
"--port",
Env.get("FLASK_PORT", "8080"),
"--reload",
"--no-debugger",
"--eager-loading",
"--with-threads",
]
# Call to untyped function "FlaskGroup" in typed context
fg_cli = FlaskGroup() # type: ignore
# Call to untyped function "main" in typed context
fg_cli.main(prog_name="restapi", args=args) # type: ignore
log.warning("Server shutdown")
def load_default_user() -> None:
BaseAuthentication.default_user = Env.get("AUTH_DEFAULT_USERNAME", "")
BaseAuthentication.default_password = Env.get("AUTH_DEFAULT_PASSWORD",
"")
if (not BaseAuthentication.default_user or
not BaseAuthentication.default_password): # pragma: no cover
print_and_exit("Default credentials are unavailable!")
def send_notification(
subject: str,
template: str,
# if None will be sent to the administrator
to_address: Optional[str] = None,
data: Optional[Dict[str, Any]] = None,
user: Optional[User] = None,
send_async: bool = False,
) -> bool:
# Always enabled during tests
if not Connector.check_availability("smtp"): # pragma: no cover
return False
title = get_project_configuration("project.title", default="Unkown title")
reply_to = Env.get("SMTP_NOREPLY", Env.get("SMTP_ADMIN", ""))
if data is None:
data = {}
data.setdefault("project", title)
data.setdefault("reply_to", reply_to)
if user:
data.setdefault("username", user.email)
data.setdefault("name", user.name)
data.setdefault("surname", user.surname)
html_body, plain_body = get_html_template(template, data)
if not html_body: # pragma: no cover
log.error("Can't load {}", template)
return False
subject = f"{title}: {subject}"
if send_async:
Mail.send_async(
subject=subject,
body=html_body,
to_address=to_address,
plain_body=plain_body,
)
return False
smtp_client = smtp.get_instance()
return smtp_client.send(
subject=subject,
body=html_body,
to_address=to_address,
plain_body=plain_body,
)
def get_frontend_url() -> str:
FRONTEND_URL = Env.get("FRONTEND_URL", "")
if FRONTEND_URL:
return FRONTEND_URL
FRONTEND_PREFIX = Env.get("FRONTEND_PREFIX", "").strip("/")
if FRONTEND_PREFIX:
FRONTEND_PREFIX = f"/{FRONTEND_PREFIX}"
protocol = "https" if PRODUCTION else "http"
return f"{protocol}://{DOMAIN}{FRONTEND_PREFIX}"
def api(
path: str,
method: str,
base: str = "api",
payload: Optional[Dict[str, Any]] = None,
) -> Any:
host = BotApiClient.variables.get("backend_host")
port = Env.get("FLASK_PORT", "8080")
url = f"http://{host}:{port}/{base}/{path}"
log.debug("Calling {} on {}", method, url)
try:
data: Optional[str] = None
if payload:
data = orjson.dumps(payload).decode("UTF8")
response = requests.request(method, url=url, data=data, timeout=10)
out = response.json()
# Never raised during tests: how to test it?
except Exception as e: # pragma: no cover
log.error(f"API call failed: {e}")
raise ServerError(str(e))
if response.status_code >= 300:
raise RestApiException(out, status_code=response.status_code)
return out
def get_backend_url() -> str:
BACKEND_URL = Env.get("BACKEND_URL", "")
if BACKEND_URL:
return BACKEND_URL
BACKEND_PREFIX = Env.get("BACKEND_PREFIX", "").strip("/")
if BACKEND_PREFIX:
BACKEND_PREFIX = f"/{BACKEND_PREFIX}"
if PRODUCTION:
return f"https://{DOMAIN}{BACKEND_PREFIX}"
port = Env.get("FLASK_PORT", "8080")
return f"http://{DOMAIN}{BACKEND_PREFIX}:{port}"
def send_activation_link(user: User, url: str) -> bool:
return send_notification(
subject=Env.get("EMAIL_ACTIVATION_SUBJECT", "Account activation"),
template="activate_account.html",
to_address=user.email,
data={"url": url},
user=user,
)
def post(self, reset_email: str) -> Response:
reset_email = reset_email.lower()
self.auth.verify_blocked_username(reset_email)
user = self.auth.get_user(username=reset_email)
if user is None:
raise Forbidden(
f"Sorry, {reset_email} is not recognized as a valid username",
)
self.auth.verify_user_status(user)
title = get_project_configuration("project.title",
default="Unkown title")
reset_token, payload = self.auth.create_temporary_token(
user, self.auth.PWD_RESET)
server_url = get_frontend_url()
rt = reset_token.replace(".", "+")
uri = Env.get("RESET_PASSWORD_URI", "/public/reset")
complete_uri = f"{server_url}{uri}/{rt}"
smtp_client = smtp.get_instance()
send_password_reset_link(smtp_client, complete_uri, title,
reset_email)
##################
# Completing the reset task
self.auth.save_token(user,
reset_token,
payload,
token_type=self.auth.PWD_RESET)
msg = "We'll send instructions to the email provided if it's associated "
msg += "with an account. Please check your spam/junk folder."
self.log_event(self.events.reset_password_request, user=user)
return self.response(msg)
def post(self, reset_email: str) -> Response:
reset_email = reset_email.lower()
self.auth.verify_blocked_username(reset_email)
user = self.auth.get_user(username=reset_email)
if user is None:
raise Forbidden(
f"Sorry, {reset_email} is not recognized as a valid username",
)
self.auth.verify_user_status(user)
reset_token, payload = self.auth.create_temporary_token(
user, self.auth.PWD_RESET)
server_url = get_frontend_url()
rt = reset_token.replace(".", "+")
uri = Env.get("RESET_PASSWORD_URI", "/public/reset")
complete_uri = f"{server_url}{uri}/{rt}"
sent = send_password_reset_link(user, complete_uri, reset_email)
if not sent: # pragma: no cover
raise ServiceUnavailable("Error sending email, please retry")
##################
# Completing the reset task
self.auth.save_token(user,
reset_token,
payload,
token_type=self.auth.PWD_RESET)
msg = "We'll send instructions to the email provided if it's associated "
msg += "with an account. Please check your spam/junk folder."
self.log_event(self.events.reset_password_request, user=user)
return self.response(msg)
def api(path, method, base="api", payload=None):
host = BotApiClient.variables.get("backend_host")
port = Env.get("FLASK_PORT")
url = f"http://{host}:{port}/{base}/{path}"
log.debug("Calling {} on {}", method, url)
try:
response = requests.request(method, url=url, data=payload)
out = response.json()
# Never raised during tests: how to test it?
except Exception as e: # pragma: no cover
log.error(f"API call failed: {e}")
raise RestApiException(str(e), status_code=500)
if response.status_code >= 300:
raise RestApiException(out, status_code=response.status_code)
return out
def test_bot() -> None:
if not Env.get_bool("TELEGRAM_ENABLE"):
log.warning("Skipping BOT tests: service not available")
return
runner = CliRunner()
start_timeout(3)
try:
runner.invoke(cli.bot, [])
except Timeout: # pragma: no cover
pass
stop_timeout()
from restapi.services.telegram import bot
# Your API ID, hash and session string here
# How to generate StringSessions:
# https://docs.telethon.dev/en/latest/concepts/sessions.html#string-sessions
api_id = Env.get_int("TELEGRAM_APP_ID")
api_hash = Env.get("TELEGRAM_APP_HASH", "") or None
session_str = Env.get("TELETHON_SESSION", "") or None
botname = Env.get("TELEGRAM_BOTNAME", "") or None
# use TelegramClient as a type once released the typed version 2 (issue #1195)
async def send_command(client: Any, command: str) -> str:
await client.send_message(botname, command)
sleep(1)
messages = await client.get_messages(botname)
# TelegramClient is not typed => message is Any
return messages[0].message # type: ignore
async def test() -> None:
client = TelegramClient(StringSession(session_str), api_id, api_hash)
await client.start()
message = await send_command(client, "/me")
assert re.match(r"^Hello .*, your Telegram ID is [0-9]+", message)
message = await send_command(client, "/invalid")
assert message == "Invalid command, ask for /help"
message = await send_command(client, "/help")
assert "Available Commands:" in message
assert "- /help print this help" in message
assert "- /me info about yourself" in message
assert "- /status get server status" in message
assert "- /monitor get server monitoring stats" in message
# commands requiring APIs can only be tested in PRODUCTION MODE
if PRODUCTION:
message = await send_command(client, "/status")
assert message == "Server is alive"
message = await send_command(client, "/monitor")
assert message == "Please select the type of monitor"
message = await send_command(client, "/monitor x")
assert message == "Please select the type of monitor"
message = await send_command(client, "/monitor disk")
error = "Missing credentials in headers, e.g. Authorization: 'Bearer TOKEN'"
assert error in message
message = await send_command(client, "/monitor disk 2")
assert message == "Too many inputs"
# # ############################# #
# # TEST USER #
# # ############################# #
bot.users = bot.admins
bot.admins = []
message = await send_command(client, "/me")
assert message == PERMISSION_DENIED
message = await send_command(client, "/invalid")
assert message == "Invalid command, ask for /help"
message = await send_command(client, "/help")
assert "Available Commands:" in message
assert "- /help print this help" in message
assert "- /me info about yourself" in message
assert "- /status get server status" in message
assert "- /monitor get server monitoring stats" in message
# commands requiring APIs can only be tested in PRODUCTION MODE
if PRODUCTION:
message = await send_command(client, "/status")
assert message == "Server is alive"
message = await send_command(client, "/monitor")
assert message == PERMISSION_DENIED
# # ############################# #
# # TEST UNAUTHORIZED #
# # ############################# #
bot.admins = []
bot.users = []
message = await send_command(client, "/me")
assert message == PERMISSION_DENIED
message = await send_command(client, "/invalid")
assert message == PERMISSION_DENIED
message = await send_command(client, "/help")
assert message == PERMISSION_DENIED
message = await send_command(client, "/status")
assert message == PERMISSION_DENIED
message = await send_command(client, "/monitor")
assert message == PERMISSION_DENIED
asyncio.run(test())
bot.shutdown()
"""
Configuration variables set for the server instance
"""
from functools import lru_cache
from pathlib import Path
from glom import glom
from restapi.env import Env
from restapi.utilities.globals import mem
# ENDPOINTS bases
API_URL = "/api"
AUTH_URL = "/auth"
APP_MODE: str = Env.get("APP_MODE", "development")
FORCE_PRODUCTION_TESTS: bool = Env.get_bool("FORCE_PRODUCTION_TESTS")
TESTING: bool = APP_MODE == "test" or FORCE_PRODUCTION_TESTS
PRODUCTION: bool = APP_MODE == "production"
STACKTRACE: bool = False
REMOVE_DATA_AT_INIT_TIME: bool = False
HOSTNAME: str = Env.get("HOSTNAME", "backend-server")
# hostnames as defined in backend.yml
MAIN_SERVER_NAME = "REST_API"
BACKEND_HOSTNAME = "backend-server"
FLOWER_HOSTNAME = "flower"
CELERYBEAT_HOSTNAME = "celery-beat"
BOT_HOSTNAME = "telegram-bot"
CELERY_HOSTNAME = "celery"
def get_totp_secret(self, user: User) -> str:
if TESTING: # pragma: no cover
if (p := Env.get("AUTH_TESTING_TOTP_HASH")) is not None:
return p
class Connector(metaclass=abc.ABCMeta):
authentication_service: str = Env.get("AUTH_SERVICE", NO_AUTH)
# Available services with associated env variables
services: Services = {}
# Assigned by init_app
app: Flask = None
# Used by get_authentication_module
_authentication_module: Optional[ModuleType] = None
# Returned by __getattr__ in neo4j and sqlalchemy connectors
_models: Dict[str, Type] = {}
# Used by set_object and get_object
_instances: InstancesCache = {}
def __init__(self) -> None:
# This is the lower-cased class name (neomodel, celeryext)
self.name = self.__class__.__name__.lower()
# This is the folder name corresponding to the connector name (neo4j, celery, )
# self.__class__.__module__ == restapi.connectors.sqlalchemy
# .split(".") == ['restapi', 'connectors', 'sqlalchemy']
# [-1] == 'sqlalchemy'
self.name = self.__class__.__module__.split(".")[-1]
# Will be modified by self.disconnect()
self.disconnected = False
# Added to convince mypy that self.app cannot be None
if self.app is None: # pragma: no cover
# This should never happen because app is
# assigned during init_services
from flask import current_app
self.app = current_app
@staticmethod
def init() -> None:
if Connector.authentication_service == NO_AUTH:
log.info("No Authentication service configured")
else:
log.debug("Authentication service: {}", Connector.authentication_service)
Connector.services = Connector.load_connectors(
ABS_RESTAPI_PATH, BACKEND_PACKAGE, Connector.services
)
if EXTENDED_PACKAGE != EXTENDED_PROJECT_DISABLED:
Connector.services = Connector.load_connectors(
Path(EXTENDED_PACKAGE),
EXTENDED_PACKAGE,
Connector.services,
)
Connector.services = Connector.load_connectors(
Path(CUSTOM_PACKAGE), CUSTOM_PACKAGE, Connector.services
)
Connector.load_models(Connector.services.keys())
def __del__(self) -> None:
if not self.disconnected:
self.disconnect()
def __enter__(self: T) -> T:
return self
def __exit__(
self,
exctype: Optional[Type[Exception]],
excinst: Optional[Exception],
exctb: Optional[TracebackType],
) -> bool:
if not self.disconnected:
self.disconnect()
if excinst:
raise excinst
return True
@staticmethod
@abc.abstractmethod
def get_connection_exception() -> ExceptionsList: # pragma: no cover
return None
@abc.abstractmethod
def connect(self: T, **kwargs: Any) -> Generic[T]: # pragma: no cover
return self
@abc.abstractmethod
def disconnect(self) -> None: # pragma: no cover
return
@abc.abstractmethod
def is_connected(instance: T) -> bool: # pragma: no cover
return True
def destroy(self) -> None: # pragma: no cover
print_and_exit("Missing destroy method in {}", self.__class__.__name__)
def initialize(self) -> None: # pragma: no cover
print_and_exit("Missing initialize method in {}", self.__class__.__name__)
@property
def variables(self) -> Dict[str, str]:
return self.services.get(self.name) or {}
@classmethod
def load_connectors(cls, path: Path, module: str, services: Services) -> Services:
main_folder = path.joinpath(CONNECTORS_FOLDER)
if not main_folder.is_dir():
log.debug("Connectors folder not found: {}", main_folder)
return services
for connector in main_folder.iterdir():
if not connector.is_dir():
continue
connector_name = connector.name
if connector_name.startswith("_"):
continue
# This is the only exception... we should rename sqlalchemy as alchemy
if connector_name == "sqlalchemy":
variables = Env.load_variables_group(prefix="alchemy")
else:
variables = Env.load_variables_group(prefix=connector_name)
if not Env.to_bool(
variables.get("enable_connector", True)
): # pragma: no cover
log.debug("{} connector is disabled", connector_name)
continue
external = False
if "host" in variables:
if host := variables.get("host"):
external = cls.is_external(host)
# HOST found in variables but empty... never happens during tests
else: # pragma: no cover
variables["enable"] = "0"
enabled = Env.to_bool(variables.get("enable"))
# Celery is always enabled, if connector is enabled
# No further check is needed on host/external
available = enabled or external or connector_name == "celery"
if not available:
continue
connector_module = Connector.get_module(connector_name, module)
connector_class = Connector.get_class(connector_module)
# Can't test connector misconfiguration...
if not connector_class: # pragma: no cover
log.error("No connector class found in {}/{}", main_folder, connector)
continue
try:
# This is to test the Connector compliance,
# i.e. to verify instance and get_instance in the connector module
# and verify that the Connector can be instanced
connector_module.instance
connector_module.get_instance
connector_class()
except AttributeError as e: # pragma: no cover
print_and_exit(e)
services[connector_name] = variables
log.debug("Got class definition for {}", connector_class)
return services
def get_totp_secret(self, user: User) -> str:
if TESTING: # pragma: no cover
# TESTING_TOTP_HASH is set by setup-cypress github action
if p := Env.get("AUTH_TESTING_TOTP_HASH", ""):
return p
def is_mysql() -> bool:
# could be based on self.variables but this version Env based
# can be used as static method and be used before creating instances
return (Env.get("AUTH_SERVICE", "NO_AUTHENTICATION") == "sqlalchemy"
and Env.get("ALCHEMY_DBTYPE", "postgresql") == "mysql+pymysql")
import os
import re
import sys
import urllib.parse
from enum import Enum
from pathlib import Path
from typing import Any, Dict, Optional, Tuple
import orjson
from loguru import logger as log
from restapi.config import HOST_TYPE, PRODUCTION
from restapi.env import Env
log_level = Env.get("LOGURU_LEVEL", "DEBUG")
LOG_RETENTION = Env.get("LOG_RETENTION", "180")
FILE_LOGLEVEL = Env.get("FILE_LOGLEVEL", "WARNING")
# FILE_LOGLEVEL = "WARNING" if not TESTING else "INFO"
LOGS_FOLDER = Path("/logs")
LOGS_PATH: Optional[str] = LOGS_FOLDER.joinpath(f"{HOST_TYPE}.log")
EVENTS_PATH: Optional[str] = LOGS_FOLDER.joinpath("security-events.log")
if Path(LOGS_PATH).exists() and not os.access(LOGS_PATH, os.W_OK): # pragma: no cover
print(
f"\nCan't initialize logging because {LOGS_PATH} is not writeable, "
"backend server cannot start\n"
)
sys.exit(1)
if Path(EVENTS_PATH).exists() and not os.access(