def extract_endpoints(self, base_dir):
endpoints_classes = []
# get last item of the path
# normpath is required to strip final / if any
base_module = os.path.basename(os.path.normpath(base_dir))
apis_dir = os.path.join(base_dir, "endpoints")
apiclass_module = f"{base_module}.endpoints"
for epfiles in glob.glob(f"{apis_dir}/*.py"):
# get module name (es: endpoints.filename)
module_file = os.path.basename(os.path.splitext(epfiles)[0])
module_name = f"{apiclass_module}.{module_file}"
# Convert module name into a module
log.debug("Importing {}", module_name)
module = Meta.get_module_from_string(
module_name,
exit_on_fail=True,
)
# Extract classes from the module
# module can't be none because of exit_on_fail=True...
# but my-py can't understand this
classes = Meta.get_new_classes_from_module(module) # type: ignore
for class_name, epclss in classes.items():
# Filtering out classes without expected data
if not hasattr(epclss, "methods") or epclss.methods is None:
continue
log.debug("Importing {} from {}.{}", class_name, apis_dir,
module_file)
skip, dependency = self.skip_endpoint(epclss.depends_on)
if skip:
log.debug(
"Skipping '{} {}' due to unmet dependency: {}",
module_name,
class_name,
dependency,
)
continue
endpoints_classes.append(epclss)
return endpoints_classes
def load_models(connectors: List[str]) -> None:
for connector in connectors:
# Models are strictly core-dependent. If you need to enable models starting
# from a custom connector this function has to be refactored:
# 1) now is checked the existence of models.py in ABS_RESTAPI_PATH/connector
# 2) Core model is mandatory
# 3) Connector class, used to inject models is taken from BACKEND_PACKAGE
models_path = ABS_RESTAPI_PATH.joinpath(
CONNECTORS_FOLDER, connector, "models.py"
)
if not models_path.is_file():
log.debug("No model found for {}", connector)
continue
log.debug("Loading models from {}", connector)
base_models = Meta.import_models(connector, BACKEND_PACKAGE, mandatory=True)
if EXTENDED_PACKAGE == EXTENDED_PROJECT_DISABLED:
extended_models = {}
else:
extended_models = Meta.import_models(connector, EXTENDED_PACKAGE)
custom_models = Meta.import_models(connector, CUSTOM_PACKAGE)
log.debug(
"Models loaded from {}: core {}, extended {}, custom {}",
connector,
len(base_models),
len(extended_models),
len(custom_models),
)
connector_module = Connector.get_module(connector, BACKEND_PACKAGE)
connector_class = Connector.get_class(connector_module)
if connector_class:
connector_class.set_models(base_models, extended_models, custom_models)
else: # pragma: no cover
log.error("Connector class not found for {}", connector)
def load_connectors(cls, path: str, module: str, services: Services) -> Services:
main_folder = os.path.join(path, CONNECTORS_FOLDER)
if not os.path.isdir(main_folder):
log.debug("Connectors folder not found: {}", main_folder)
return services
for connector in os.listdir(main_folder):
connector_path = os.path.join(path, CONNECTORS_FOLDER, connector)
if not os.path.isdir(connector_path):
continue
if connector.startswith("_"):
continue
# This is the only exception... we should rename sqlalchemy as alchemy
if connector == "sqlalchemy":
variables = Env.load_variables_group(prefix="alchemy")
else:
variables = Env.load_variables_group(prefix=connector)
if not Env.to_bool(
variables.get("enable_connector", True)
): # pragma: no cover
log.info("{} connector is disabled", connector)
continue
# if host is not in variables (like for Celery) do not consider it
external = False
if "host" in variables:
if host := variables.get("host"):
external = cls.is_external(host)
# HOST found in variables but empty... never happens during tests
else: # pragma: no cover
variables["enable"] = "0"
enabled = Env.to_bool(variables.get("enable"))
available = enabled or external
if not available:
continue
connector_module = Connector.get_module(connector, module)
connector_class = Connector.get_class(connector_module)
# Can't test connector misconfiguration...
if not connector_class: # pragma: no cover
log.error("No connector class found in {}/{}", main_folder, connector)
continue
try:
# This is to test the Connector compliance,
# i.e. to verify instance and get_instance in the connector module
# and verify that the Connector can be instanced
connector_module.instance
connector_module.get_instance
connector_class()
except AttributeError as e: # pragma: no cover
print_and_exit(e)
services[connector] = variables
log.debug("Got class definition for {}", connector_class)
def wrapper(*args: Any, **kwargs: Any) -> Any:
# Recover the auth object
auth_type, token = HTTPTokenAuth.get_authorization_token(
allow_access_token_parameter=allow_access_token_parameter)
# Internal API 'self' reference
caller = Meta.get_self_reference_from_args(*args)
if caller is None: # pragma: no cover
# An exit here is really really dangerous, but even if
# get_self_reference_from_args can return None, this case is quite
# impossible... however with None the server can't continue!
print_and_exit(
"Server misconfiguration, self reference can't be None!"
)
if auth_type is None or auth_type != HTTPAUTH_SCHEME:
# Wrong authentication string
msg = (
"Missing credentials in headers"
f", e.g. {HTTPAUTH_AUTH_FIELD}: '{HTTPAUTH_SCHEME} TOKEN'"
)
log.debug("Unauthorized request: missing credentials")
return caller.response(msg,
code=401,
headers=HTTPAUTH_ERR_HEADER,
allow_html=True)
# Handling OPTIONS forwarded to our application:
# ignore headers and let go, avoid unwanted interactions with CORS
if request.method != "OPTIONS":
# valid, token, jti, user
valid, token, _, user = caller.auth.verify_token(token)
# Check authentication
if not valid:
# Clear TCP receive buffer of any pending data
_ = request.data
# Mimic the response from a normal endpoint
# To use the same standards
# log.info("Invalid token received '{}'", token)
log.debug("Invalid token received")
return caller.response(
"Invalid token received",
headers=HTTPAUTH_ERR_HEADER,
code=401,
allow_html=True,
)
request.environ[TOKEN_VALIDATED_KEY] = True
# Check roles
if not caller.auth.verify_roles(
user, roles, required_roles=required_roles):
log.info("Unauthorized request: missing privileges.")
return caller.response(
"You are not authorized: missing privileges",
code=401,
allow_html=True,
)
caller.authorized_user = user.uuid
kwargs["user"] = user
return func(*args, **kwargs)
from restapi.config import SENTRY_URL
from restapi.exceptions import (
BadRequest,
Conflict,
DatabaseDuplicatedEntry,
RestApiException,
)
from restapi.models import PartialSchema, fields, validate
from restapi.rest.annotations import inject_apispec_docs
from restapi.rest.bearer import TOKEN_VALIDATED_KEY
from restapi.rest.bearer import HTTPTokenAuth as auth # imported as alias for endpoints
from restapi.utilities.globals import mem
from restapi.utilities.logs import log
log.debug("Auth loaded {}", auth)
log.debug("Marshal loaded {}", marshal_with)
SYSTEM_EXCEPTIONS = ["AttributeError", "ValueError", "KeyError", "SystemError"]
F = TypeVar("F", bound=Callable[..., Any])
# same definition as in:
# https://github.com/jmcarp/flask-apispec/blob/master/flask_apispec/annotations.py
def use_kwargs(args, location=None, inherit=None, apply=None, **kwargs):
# this use_kwargs is used override the default location (json)
# with a more extensive default location (body)
# This trick will prevent to add location='body' to mostly all models
# Please note that body is also used as "in" parameter in swagger specs and well
def test_01_x(self, client):
log.debug("Executing tests from {}", self.__class__.__module__)
endpoint = API_URI + '/tests/1'
r = client.get(endpoint)
assert r.status_code == 200
def save_token(
self, user: User, token: str, payload: Payload, token_type: Optional[str] = None
) -> None:
log.debug("Tokens is not saved in base authentication")
def get(self) -> Response:
specs = mem.docs.spec.to_dict()
if self.get_user():
# Set security requirements for endpoint
for key, data in specs.items():
# Find endpoint mapping flagged as private
if key == "paths":
for uri, endpoint in data.items():
u = uri.replace("{", "<").replace("}", ">")
for method, definition in endpoint.items():
auth_required = glom(
mem.authenticated_endpoints,
f"{u}.{method}",
default=False,
)
if auth_required:
definition["security"] = [{"Bearer": []}]
return self.response(specs)
log.info("Unauthenticated request, filtering out private endpoints")
# Remove sensible data
filtered_specs: Dict[str, Dict[str, Dict[str, Any]]] = {}
# schemaName => True|False (private|public)
privatedefs: Dict[str, bool] = {}
# schemaName => [list of definitions including this]
parentdefs: Dict[str, List[Any]] = {}
for key, data in specs.items():
# Find endpoint mapping flagged as private
if key == "paths":
for uri, endpoint in data.items():
u = uri.replace("{", "<").replace("}", ">")
for method, definition in endpoint.items():
is_private = glom(
mem.private_endpoints,
f"{u}.{method}",
default=False,
)
defs = definition.get("parameters", [])[:]
for p in defs:
if "schema" not in p:
continue
if "$ref" not in p["schema"]:
continue
ref = p["schema"]["$ref"]
def_name = ref.replace("#/definitions/", "")
privatedefs.setdefault(def_name, True)
# Will be True if all occurrences are private
privatedefs[def_name] = privatedefs[
def_name] and is_private
if is_private:
log.debug("Skipping {} {}", method, uri)
continue
auth_required = glom(
mem.authenticated_endpoints,
f"{u}.{method}",
default=False,
)
if auth_required:
definition["security"] = [{"Bearer": []}]
filtered_specs.setdefault(key, {})
filtered_specs[key].setdefault(uri, {})
filtered_specs[key][uri].setdefault(method, definition)
# definitions
elif key == "definitions":
# Saving definition inclusion, will be used later to determine
# if a definition is private or not
# If a definition is referenced by an endpoint, the definition
# visibility matches the endpoint visibility
# If a definition is referenced by other definitions, its visibility
# will be calculated as AND(parent definitions)
# Verification postponed
for schema, definition in data.items():
# parentdefs
for d in definition.get("properties", {}).values():
if "$ref" in d:
ref = d["$ref"]
def_name = ref.replace("#/definitions/", "")
parentdefs.setdefault(def_name, [])
parentdefs[def_name].append(schema)
# Generated by Nested?
# 'allOf': [{'$ref': '#/definitions/MyReferenceName'}]}
if "allOf" in d:
for nested in d["allOf"]:
ref = nested["$ref"]
def_name = ref.replace("#/definitions/", "")
parentdefs.setdefault(def_name, [])
parentdefs[def_name].append(schema)
else:
filtered_specs.setdefault(key, data)
if "definitions" in specs:
filtered_specs.setdefault("definitions", {})
for schema, definition in specs["definitions"].items():
if self.is_definition_private(schema, privatedefs, parentdefs):
log.debug("Skipping private definition {}", schema)
continue
filtered_specs["definitions"].setdefault(schema, definition)
return self.response(filtered_specs)
def save_user(self, user):
log.debug("User is not saved in base authentication")
def save_token(self, user, token, jti, token_type=None):
log.debug("Token is not saved in base authentication")
from restapi.services.detect import detector
from restapi.utilities.logs import log
################################################
# Reload Flask app code also for the worker
# This is necessary to have the app context available
# app = create_app(worker_mode=True)
app = Flask("beat")
app.extensions = detector.init_services(
app=app, worker_mode=True, project_init=False, project_clean=False
)
celery_app = app.extensions.get('celery').celery_app
celery_app.app = app
def get_service(service, **kwargs):
ext = celery_app.app.extensions.get(service)
if ext is None:
log.error("{} is not enabled", service)
return None
return ext.get_instance(**kwargs)
celery_app.get_service = get_service
log.debug("Celery beat is ready {}", celery_app)
def pre_connection(self, **kwargs):
session = kwargs.get('user_session')
external = self.variables.get('external')
# Retrieve authentication schema
self.authscheme = kwargs.get('authscheme')
# Authentication scheme fallback to default from project_configuration
if self.authscheme is None or self.authscheme.strip() == '':
self.authscheme = self.variables.get('authscheme')
# Authentication scheme fallback to default (credentials)
if self.authscheme is None or self.authscheme.strip() == '':
self.authscheme = NORMAL_AUTH_SCHEME
if session is not None:
user = session.email
else:
user = kwargs.get('user')
self.password = kwargs.get('password')
gss = kwargs.get('gss', False)
myproxy_host = self.variables.get("myproxy_host")
admin = kwargs.get('be_admin', False)
if user is None:
user_key = 'default_admin_user' if admin else 'user'
user = self.variables.get(user_key)
if self.password is None:
if self.authscheme == NORMAL_AUTH_SCHEME:
self.password = self.variables.get('password')
elif self.authscheme == PAM_AUTH_SCHEME:
self.password = self.variables.get('password')
log.verbose(
"Check connection parameters:" +
"\nexternal[{}], auth[{}], user[{}], admin[{}]",
external,
self.authscheme,
user,
admin,
)
# Check if the user requested for GSI explicitely
if self.authscheme == GSI_AUTH_SCHEME:
# if self.variables.get('external'):
gss = True
if user is None:
raise AttributeError("No user is defined")
else:
self.user = user
log.debug("Irods user: {}", self.user)
######################
# Irods/b2safe direct credentials
if session is not None:
return True
######################
# Identity with GSI
elif gss:
if self.authscheme != GSI_AUTH_SCHEME:
log.debug("Forcing {} authscheme", GSI_AUTH_SCHEME)
self.authscheme = GSI_AUTH_SCHEME
proxy_cert_name = "{}{}".format(
self.variables.get('certificates_prefix', ""),
kwargs.get("proxy_cert_name"),
)
valid_cert = Certificates.globus_proxy(
proxy_file=kwargs.get('proxy_file'),
user_proxy=self.user,
cert_dir=self.variables.get("x509_cert_dir"),
myproxy_host=myproxy_host,
cert_name=proxy_cert_name,
cert_pwd=kwargs.get("proxy_pass"),
)
if not valid_cert:
return False
elif self.authscheme == PAM_AUTH_SCHEME:
pass
elif self.password is not None:
self.authscheme = NORMAL_AUTH_SCHEME
else:
raise NotImplementedError(
"Unable to create session: invalid iRODS-auth scheme")
return True
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
RESTful API Python 3 Flask server
"""
import os
import pretty_errors
from restapi.confs import PRODUCTION
from restapi.server import create_app
from restapi.utilities.logs import log
# Connection internal to containers, proxy handle all HTTPS calls
# We may safely disable HTTPS on OAUTHLIB requests
if PRODUCTION:
# http://stackoverflow.com/a/27785830/2114395
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
BIND_INTERFACE = "0.0.0.0"
#############################
# BE FLASK
app = create_app(name='REST_API')
if __name__ == "__main__":
log.debug("Server running (w/ {})", pretty_errors.__name__)
app.run(host=BIND_INTERFACE, threaded=True)
def send(self, body, routing_key="", exchange="", headers=None):
"""
Send a message to the RabbitMQ queue
:param body: the data to be send.
If this message should be json-encoded please use .send_json()
:param exchange: RabbitMQ exchange where the message should be sent.
Empty for default exchange.
:param queue: RabbitMQ routing key.
"""
# Settings for the message:
permanent_delivery = 2 # make message persistent
if headers is None:
headers = {}
props = pika.BasicProperties(
delivery_mode=permanent_delivery,
headers=headers,
# This should be the same used by the connect method, i.e.:
# self.variables + kwargs
# Otherwise it will fail with error:
# Failed to write message, channel is dead (
# (406, "PRECONDITION_FAILED - user_id property
# set to 'CUSTOM' but authenticated user was 'BASE'
# "
# )
# )
user_id=self.variables.get("user"),
)
try:
channel = self.get_channel()
channel.basic_publish(
exchange=exchange,
routing_key=routing_key,
body=body,
properties=props,
mandatory=True,
)
log.debug("Message sent to RabbitMQ")
return True
except UnroutableError as e:
log.error(e)
except ConnectionClosed as e:
# TODO: This happens often. Check if heartbeat solves problem.
log.error("Failed to write message, connection is dead ({})", e)
except AMQPConnectionError as e:
log.error("Failed to write message, connection failed ({})", e)
except AMQPChannelError as e:
log.error("Failed to write message, channel is dead ({})", e)
self.channel = None
except AttributeError as e: # pragma: no cover
log.error("Failed to write message:, {}", e)
return False
def delete_exchange(self, exchange: str) -> None:
channel = self.get_channel()
out = channel.exchange_delete(exchange, if_unused=False)
log.debug(out)
from restapi.connectors import celery
from restapi.server import ServerModes, create_app
from restapi.utilities.logs import log
instance = celery.get_instance()
# Used by Celery to run the instance (--app app)
celery_app = instance.celery_app
# Reload Flask app code for the worker (needed to have the app context available)
celery.CeleryExt.app = create_app(mode=ServerModes.WORKER)
log.debug("Celery worker is ready {}", celery_app)
def modify_user_password(self, user, password):
log.debug("Changing {} password", user)
return self.prc.users.modify(user, 'password', password)
def send(
self,
body: str,
subject: str,
to_address: Optional[str] = None,
from_address: Optional[str] = None,
cc: Union[None, str, List[str]] = None,
bcc: Union[None, str, List[str]] = None,
plain_body: Optional[str] = None,
) -> bool:
if not to_address:
to_address = self.instance_variables.get("admin")
if not to_address:
log.error("Skipping send email: destination address not configured")
return False
if not from_address:
from_address = self.instance_variables.get("noreply")
if not from_address:
from_address = self.instance_variables.get("admin")
if not from_address:
log.error("Skipping send email: from address not configured")
return False
try:
if plain_body is not None:
msg = MIMEMultipart("alternative")
else:
msg = MIMEText(body)
msg["Subject"] = subject
msg["From"] = from_address
msg["To"] = to_address
dest_addresses = [to_address]
if cc is None:
pass
elif isinstance(cc, str):
msg["Cc"] = cc
dest_addresses.append(cc.split(","))
elif isinstance(cc, list):
msg["Cc"] = ",".join(cc)
dest_addresses.append(cc)
else:
log.warning("Invalid CC value: {}", cc)
cc = None
if bcc is None:
pass
elif isinstance(bcc, str):
msg["Bcc"] = bcc
dest_addresses.append(bcc.split(","))
elif isinstance(bcc, list):
msg["Bcc"] = ",".join(bcc)
dest_addresses.append(bcc)
else:
log.warning("Invalid BCC value: {}", bcc)
bcc = None
msg["Date"] = formatdate()
if plain_body is not None:
part1 = MIMEText(plain_body, "plain")
part2 = MIMEText(body, "html")
msg.attach(part1)
msg.attach(part2)
log.debug("Sending email to {}", to_address)
self.smtp.sendmail(from_address, dest_addresses, msg.as_string())
log.info(
"Successfully sent email to {} [cc={}], [bcc={}]",
to_address,
cc,
bcc,
)
return True
except SMTPException as e:
log.error("Unable to send email to {} ({})", to_address, e)
# Force the invalidation of this client
self.disconnect()
return False
except Exception as e:
log.error(str(e))
return False
# Set up secure data connection
ftp_tls_conn.prot_p()
self.connection = ftp_tls_conn
else:
ftp_conn = FTP(timeout=10)
# ftp_conn.debugging = 1
ftp_conn.connect(host, port)
ftp_conn.login(user, password)
ftp_conn.set_pasv(True)
self.connection = ftp_conn
self.initialized = True
log.debug("Current directory: {}", self.connection.pwd())
return self
def disconnect(self) -> None:
self.disconnected = True
if self.connection and self.initialized and self.connection.sock:
self.connection.quit()
self.initialized = False
def is_connected(self) -> bool:
# Can't happen because connection is not Optional[]
if not self.connection: # pragma: no cover
return False
if self.disconnected:
return False
def swaggerish(self):
"""
Go through all endpoints configured by the current development.
Provide the minimum required data according to swagger specs.
"""
# Better chosen dinamically from endpoint.py
schemes = ['http']
if PRODUCTION:
schemes = ['https']
# A template base
output = {
# TODO: update to 3.0.1? Replace bravado with something else?
# https://github.com/Yelp/bravado/issues/306
"swagger": "2.0",
"info": {
"version": "0.0.1",
"title": "Your application name"
},
"schemes": schemes,
# "host": "localhost" # chosen dinamically
"basePath": "/",
"securityDefinitions": {
"Bearer": {
"type": "apiKey",
"name": "Authorization",
"in": "header"
}
},
"security": [{
"Bearer": []
}],
}
###################
# Set existing values
proj = self._customizer._configurations['project']
if 'version' in proj:
output['info']['version'] = proj['version']
if 'title' in proj:
output['info']['title'] = proj['title']
###################
models = self.get_models()
self._fdp = models.pop('FormDataParameters', {})
for k in ["definitions", "parameters", "responses"]:
if k in models:
output[k] = models.get(k, {})
output['consumes'] = [
JSON_APPLICATION,
# required for parameters with "in: formData"
"application/x-www-form-urlencoded",
# required for parameters of "type: file"
"multipart/form-data"
]
output['produces'] = [JSON_APPLICATION]
###################
# Read endpoints swagger files
for key, endpoint in enumerate(self._endpoints):
endpoint.custom['methods'] = {}
endpoint.custom['params'] = {}
for method, mapping in endpoint.methods.items():
# add the custom part to the endpoint
self._endpoints[key] = self.read_my_swagger(
method, endpoint, mapping)
###################
# Save query parameters globally
self._customizer._query_params = self._qparams
self._customizer._parameter_schemas = self._parameter_schemas
output['paths'] = self._paths
###################
tags = []
for tag, desc in self._customizer._configurations['tags'].items():
if tag not in self._used_swagger_tags:
log.debug("Skipping unsed tag: {}", tag)
continue
tags.append({'name': tag, 'description': desc})
output['tags'] = tags
self._customizer._original_paths = self._original_paths
return output
def get(self, task_id=None):
data = []
# Inspect all worker nodes
celery = self.get_service_instance('celery')
if task_id is not None:
task_result = celery.AsyncResult(task_id)
res = task_result.result
if not isinstance(res, dict):
res = str(res)
return self.force_response({
'status': task_result.status,
# 'info': task_result.info,
'output': res,
})
#############################
# FAST WAY
stats = celery.control.inspect().stats()
workers = list(stats.keys())
active_tasks = {}
revoked_tasks = {}
scheduled_tasks = {}
reserved_tasks = {}
for worker in workers:
i = celery.control.inspect([worker])
log.debug('checked worker: {}', worker)
for key, value in i.active().items():
active_tasks[key] = value
for key, value in i.revoked().items():
revoked_tasks[key] = value
for key, value in i.reserved().items():
reserved_tasks[key] = value
for key, value in i.scheduled().items():
scheduled_tasks[key] = value
#############################
# workers = celery.control.inspect()
# SLOW WAY
# active_tasks = workers.active()
# revoked_tasks = workers.revoked()
# reserved_tasks = workers.reserved()
# scheduled_tasks = workers.scheduled()
# SLOW WAY
# if active_tasks is None:
# active_tasks = []
# if revoked_tasks is None:
# revoked_tasks = []
# if scheduled_tasks is None:
# scheduled_tasks = []
# if reserved_tasks is None:
# reserved_tasks = []
log.verbose('listing items')
for worker, tasks in active_tasks.items():
for task in tasks:
if task_id is not None and task["id"] != task_id:
continue
row = {}
row['status'] = 'ACTIVE'
row['worker'] = worker
row['ETA'] = task["time_start"]
row['task_id'] = task["id"]
row['task'] = task["name"]
row['args'] = task["args"]
if task_id is not None:
task_result = celery.AsyncResult(task_id)
row['task_status'] = task_result.status
row['info'] = task_result.info
data.append(row)
for worker, tasks in revoked_tasks.items():
for task in tasks:
if task_id is not None and task != task_id:
continue
row = {}
row['status'] = 'REVOKED'
row['task_id'] = task
data.append(row)
for worker, tasks in scheduled_tasks.items():
for task in tasks:
if task_id is not None and task["request"]["id"] != task_id:
continue
row = {}
row['status'] = 'SCHEDULED'
row['worker'] = worker
row['ETA'] = task["eta"]
row['task_id'] = task["request"]["id"]
row['priority'] = task["priority"]
row['task'] = task["request"]["name"]
row['args'] = task["request"]["args"]
data.append(row)
for worker, tasks in reserved_tasks.items():
for task in tasks:
if task_id is not None and task["id"] != task_id:
continue
data.append(
{
'status': 'SCHEDULED',
'worker': worker,
'ETA': task['time_start'],
'task_id': task["id"],
'priority': task['delivery_info']["priority"],
'task': task["name"],
'args': task["args"],
}
)
# from celery.task.control import inspect
# tasks = inspect()
log.verbose('listing completed')
return self.force_response(data)
def upload(self, subfolder=None, force=False):
if 'file' not in request.files:
# # the PUT problem for uploading?
# tmp = request.stream.read()
# print("TEST", len(tmp))
# with open('uploaded_image.jpg', 'w') as f:
# f.write(request.stream.read())
# # print("TEST", request.data)
return self.force_response(
errors={"Missing file": "No files specified"},
code=hcodes.HTTP_BAD_METHOD_NOT_ALLOWED,
)
myfile = request.files['file']
# Check file extension?
if not self.allowed_file(myfile.filename):
return self.force_response(
errors={"Wrong extension": "File extension not allowed"})
# Check file name
filename = secure_filename(myfile.filename)
abs_file = self.absolute_upload_file(filename, subfolder)
log.info("File request for [{}]({})", myfile, abs_file)
# ## IMPORTANT NOTE TO SELF:
# If you are going to receive chunks here there could be problems.
# In fact a chunk will truncate the connection
# and make a second request.
# You will end up with having already the file
# But corrupted...
if os.path.exists(abs_file):
log.warning("Already exists")
if force:
os.remove(abs_file)
log.debug("Forced removal")
else:
return self.force_response(
errors={
"File '" + filename + "' already exists.":
"Change file name or use the force parameter"
},
code=hcodes.HTTP_BAD_REQUEST,
)
# Save the file
try:
myfile.save(abs_file)
log.debug("Absolute file path should be '{}'", abs_file)
except Exception:
return self.force_response(
errors={"Permissions": "Failed to write uploaded file"},
code=hcodes.HTTP_DEFAULT_SERVICE_FAIL,
)
# Check exists
if not os.path.exists(abs_file):
return self.force_response(
errors={
"Server file system": "Unable to recover the uploaded file"
},
code=hcodes.HTTP_DEFAULT_SERVICE_FAIL,
)
# Extra info
ftype = None
fcharset = None
try:
# Check the type
from plumbum.cmd import file
out = file["-ib", abs_file]()
tmp = out.split(';')
ftype = tmp[0].strip()
fcharset = tmp[1].split('=')[1].strip()
except Exception:
log.warning("Unknown type for '{}'", abs_file)
########################
# ## Final response
# Default redirect is to 302 state, which makes client
# think that response was unauthorized....
# see http://dotnet.dzone.com/articles/getting-know-cross-origin
return self.force_response(
{
'filename': filename,
'meta': {
'type': ftype,
'charset': fcharset
}
},
code=hcodes.HTTP_OK_BASIC,
)
class BaseAuthentication(metaclass=abc.ABCMeta):
"""
An almost abstract class with methods
to be implemented with a new service
that aims to store credentials of users and roles.
"""
# Secret loaded from secret.key file
JWT_SECRET: Optional[bytes] = None
# JWT_ALGO = 'HS256'
# Should be faster on 64bit machines
JWT_ALGO = "HS512"
# 1 month in seconds
DEFAULT_TOKEN_TTL = Env.get_int("AUTH_JWT_TOKEN_TTL", 2_592_000)
# Grace period before starting to evaluate IP address on token validation
GRACE_PERIOD = timedelta(seconds=Env.get_int("AUTH_TOKEN_IP_GRACE_PERIOD", 7200))
SAVE_LAST_ACCESS_EVERY = timedelta(
seconds=Env.get_int("AUTH_TOKEN_SAVE_FREQUENCY", 60)
)
FULL_TOKEN = "f"
PWD_RESET = "r"
ACTIVATE_ACCOUNT = "a"
TOTP = "TOTP"
MIN_PASSWORD_LENGTH = Env.get_int("AUTH_MIN_PASSWORD_LENGTH", 8)
SECOND_FACTOR_AUTHENTICATION = Env.get_bool(
"AUTH_SECOND_FACTOR_AUTHENTICATION", False
)
# enabled if explicitly set or for 2FA is enabled
FORCE_FIRST_PASSWORD_CHANGE = SECOND_FACTOR_AUTHENTICATION or Env.get_bool(
"AUTH_FORCE_FIRST_PASSWORD_CHANGE", False
)
# enabled if explicitly set or for 2FA is enabled
VERIFY_PASSWORD_STRENGTH = SECOND_FACTOR_AUTHENTICATION or Env.get_bool(
"AUTH_VERIFY_PASSWORD_STRENGTH", False
)
MAX_PASSWORD_VALIDITY: Optional[timedelta] = get_timedelta(
Env.get_int("AUTH_MAX_PASSWORD_VALIDITY", 0),
MAX_PASSWORD_VALIDITY_MIN_TESTNIG_VALUE,
)
DISABLE_UNUSED_CREDENTIALS_AFTER: Optional[timedelta] = get_timedelta(
Env.get_int("AUTH_DISABLE_UNUSED_CREDENTIALS_AFTER", 0),
# min 60 seconds are required when testing
DISABLE_UNUSED_CREDENTIALS_AFTER_MIN_TESTNIG_VALUE,
)
MAX_LOGIN_ATTEMPTS = get_max_login_attempts(
Env.get_int("AUTH_MAX_LOGIN_ATTEMPTS", 0)
)
FAILED_LOGINS_EXPIRATION: timedelta = timedelta(
seconds=Env.get_int("AUTH_LOGIN_BAN_TIME", 3600)
)
default_user: Optional[str] = None
default_password: Optional[str] = None
roles: List[str] = []
roles_data: Dict[str, str] = {}
default_role: str = Role.USER.value
# To be stored on DB
failed_logins: Dict[str, List[FailedLogin]] = {}
# Executed once by Connector in init_app
@classmethod
def module_initialization(cls) -> None:
cls.load_default_user()
cls.load_roles()
cls.import_secret(SECRET_KEY_FILE)
@staticmethod
def load_default_user() -> None:
BaseAuthentication.default_user = Env.get("AUTH_DEFAULT_USERNAME")
BaseAuthentication.default_password = Env.get("AUTH_DEFAULT_PASSWORD")
if (
BaseAuthentication.default_user is None
or BaseAuthentication.default_password is None
): # pragma: no cover
print_and_exit("Default credentials are unavailable!")
@staticmethod
def load_roles() -> None:
BaseAuthentication.roles_data = get_project_configuration(
"variables.roles"
).copy()
if not BaseAuthentication.roles_data: # pragma: no cover
print_and_exit("No roles configured")
BaseAuthentication.default_role = BaseAuthentication.roles_data.pop("default")
BaseAuthentication.roles = []
for role, description in BaseAuthentication.roles_data.items():
if description != ROLE_DISABLED:
BaseAuthentication.roles.append(role)
if not BaseAuthentication.default_role: # pragma: no cover
print_and_exit(
"Default role {} not available!", BaseAuthentication.default_role
)
def make_login(self, username: str, password: str) -> Tuple[str, Payload, User]:
""" The method which will check if credentials are good to go """
try:
user = self.get_user(username=username)
except ValueError as e: # pragma: no cover
# SqlAlchemy can raise the following error:
# A string literal cannot contain NUL (0x00) characters.
log.error(e)
raise BadRequest("Invalid input received")
except BaseException as e: # pragma: no cover
log.error("Unable to connect to auth backend\n[{}] {}", type(e), e)
raise ServiceUnavailable("Unable to connect to auth backend")
if user is None:
self.register_failed_login(username)
self.log_event(
Events.failed_login,
payload={"username": username},
user=user,
)
raise Unauthorized("Invalid access credentials", is_warning=True)
# Check if Oauth2 is enabled
if user.authmethod != "credentials": # pragma: no cover
raise BadRequest("Invalid authentication method")
# New hashing algorithm, based on bcrypt
if self.verify_password(password, user.password):
# Token expiration is capped by the user expiration date, if set
payload, full_payload = self.fill_payload(user, expiration=user.expiration)
token = self.create_token(payload)
self.log_event(Events.login, user=user)
return token, full_payload, user
self.log_event(
Events.failed_login,
payload={"username": username},
user=user,
)
self.register_failed_login(username)
raise Unauthorized("Invalid access credentials", is_warning=True)
@classmethod
def import_secret(cls, abs_filename: str) -> None:
try:
cls.JWT_SECRET = open(abs_filename, "rb").read()
except OSError: # pragma: no cover
print_and_exit("Jwt secret file {} not found", abs_filename)
# #####################
# # Password handling #
####################
@staticmethod
def verify_password(plain_password: str, hashed_password: str) -> bool:
try:
return cast(bool, pwd_context.verify(plain_password, hashed_password))
except ValueError as e: # pragma: no cover
log.error(e)
return False
@staticmethod
def get_password_hash(password):
if not password:
raise Unauthorized("Invalid password")
return pwd_context.hash(password)
@staticmethod
def get_remote_ip() -> str:
try:
if forwarded_ips := request.headers.getlist("X-Forwarded-For"):
# it can be something like: ['IP1, IP2']
return str(forwarded_ips[-1].split(",")[0].strip())
if PRODUCTION and not TESTING: # pragma: no cover
log.warning(
"Production mode is enabled, but X-Forwarded-For header is missing"
)
if request.remote_addr:
return request.remote_addr
# Raised when get_remote_ip is executed outside request context
# For example when creating tokens in initialize_testing_environment
except RuntimeError as e:
log.debug(e)
def upload_data(self, filename, subfolder=None, force=False):
filename = secure_filename(filename)
# Check file extension?
if not self.allowed_file(filename):
return self.force_response(
errors=["Wrong extension, file extension not allowed"])
content = request.data
abs_file = self.absolute_upload_file(filename, subfolder)
log.info("File request for {}", abs_file)
if os.path.exists(abs_file):
log.warning("File already exists")
if force:
os.remove(abs_file)
log.debug("Forced removal")
else:
return self.force_response(
errors=["File '" + filename + "' already exists"],
code=hcodes.HTTP_BAD_REQUEST,
)
with open(abs_file, "ab") as f:
f.write(content)
f.close()
# Check exists
if not os.path.exists(abs_file):
return self.force_response(
errors=["Server error: unable to recover the uploaded file"],
code=hcodes.HTTP_DEFAULT_SERVICE_FAIL,
)
# Extra info
ftype = None
fcharset = None
try:
# Check the type
from plumbum.cmd import file
out = file["-ib", abs_file]()
tmp = out.split(';')
ftype = tmp[0].strip()
fcharset = tmp[1].split('=')[1].strip()
except Exception:
log.warning("Unknown type for '{}'", abs_file)
########################
# ## Final response
# Default redirect is to 302 state, which makes client
# think that response was unauthorized....
# see http://dotnet.dzone.com/articles/getting-know-cross-origin
return self.force_response(
{
'filename': filename,
'meta': {
'type': ftype,
'charset': fcharset
}
},
code=hcodes.HTTP_OK_BASIC,
)
def close_connection(self):
# TODO: This must be called!
if self.__connection.is_closed or self.__connection.is_closing:
log.debug('Connection already closed or closing.')
else:
self.__connection.close()
def load_connectors(cls, path: Path, module: str, services: Services) -> Services:
main_folder = path.joinpath(CONNECTORS_FOLDER)
if not main_folder.is_dir():
log.debug("Connectors folder not found: {}", main_folder)
return services
for connector in main_folder.iterdir():
if not connector.is_dir():
continue
connector_name = connector.name
if connector_name.startswith("_"):
continue
# This is the only exception... we should rename sqlalchemy as alchemy
if connector_name == "sqlalchemy":
variables = Env.load_variables_group(prefix="alchemy")
else:
variables = Env.load_variables_group(prefix=connector_name)
if not Env.to_bool(
variables.get("enable_connector", True)
): # pragma: no cover
log.debug("{} connector is disabled", connector_name)
continue
external = False
if "host" in variables:
if host := variables.get("host"):
external = cls.is_external(host)
# HOST found in variables but empty... never happens during tests
else: # pragma: no cover
variables["enable"] = "0"
enabled = Env.to_bool(variables.get("enable"))
# Celery is always enabled, if connector is enabled
# No further check is needed on host/external
available = enabled or external or connector_name == "celery"
if not available:
continue
connector_module = Connector.get_module(connector_name, module)
connector_class = Connector.get_class(connector_module)
# Can't test connector misconfiguration...
if not connector_class: # pragma: no cover
log.error("No connector class found in {}/{}", main_folder, connector)
continue
try:
# This is to test the Connector compliance,
# i.e. to verify instance and get_instance in the connector module
# and verify that the Connector can be instanced
connector_module.instance
connector_module.get_instance
connector_class()
except AttributeError as e: # pragma: no cover
print_and_exit(e)
services[connector_name] = variables
log.debug("Got class definition for {}", connector_class)
try:
if request.mimetype in [
"application/octet-stream", "multipart/form-data"
]:
return " STREAM_UPLOAD"
if request.data:
if data := handle_log_output(request.data):
return f" {data}"
if request.form:
if data := obfuscate_dict(request.form):
return f" {data}"
except Exception as e: # pragma: no cover
log.debug(e)
return ""
def handle_response(response: FlaskResponse) -> FlaskResponse:
response.headers["_RV"] = str(version)
PROJECT_VERSION = get_project_configuration("project.version", default="0")
if PROJECT_VERSION is not None:
response.headers["Version"] = str(PROJECT_VERSION)
data_string = get_data_from_request()
url = obfuscate_query_parameters(request.url)
def get_instance(
self: T,
verification: Optional[int] = None,
expiration: Optional[int] = None,
**kwargs: str,
) -> T:
if not Connector.check_availability(self.name):
raise ServiceUnavailable(f"Service {self.name} is not available")
if verification is None:
# this should be the default value for this connector
verification = Env.to_int(self.variables.get("verification_time"))
if expiration is None:
# this should be the default value for this connector
expiration = Env.to_int(self.variables.get("expiration_time"))
# When context is empty this is a connection at loading time
# Do not save it
if stack.top is None:
log.debug("First connection for {}", self.name)
# can raise ServiceUnavailable exception
obj = self.initialize_connection(expiration, verification, **kwargs)
return obj
unique_hash = str(sorted(kwargs.items()))
obj = self.get_object(name=self.name, key=unique_hash)
# if an expiration time is set, verify the instance age
if obj and obj.connection_expiration_time:
# the instance is invalidated if older than the expiration time
if datetime.now() >= obj.connection_expiration_time:
log.info("{} connection is expired", self.name)
obj.disconnect()
obj = None
# If a verification time is set, verify the instance age
if obj and obj.connection_verification_time:
now = datetime.now()
# the instance is verified if older than the verification time
if now >= obj.connection_verification_time:
# if the connection is still valid, set a new verification time
if obj.is_connected():
# Set the new verification time
ver = timedelta(seconds=verification)
obj.connection_verification_time = now + ver
# if the connection is no longer valid, invalidate the instance
else: # pragma: no cover
log.info(
"{} is no longer connected, connector invalidated", self.name
)
obj.disconnected = True
# return the instance only if still connected
# (and not invalidated by the verification check)
if obj and not obj.disconnected:
return obj
# can raise ServiceUnavailable exception
obj = self.initialize_connection(expiration, verification, **kwargs)
self.set_object(name=self.name, obj=obj, key=unique_hash)
return obj
def get(self, test_num):
sql = self.get_service_instance('sqlalchemy')
log.debug(sql)
return self.response("1")
def callback(self, result, message):
if result:
log.debug('Message successfully published on pushpin')
else:
log.error('Publish failed on pushpin: {}', message)
