def __init__(self, testing=False, init=False): # Input self._testing = testing # Some initialization self._endpoints = [] self._definitions = {} self._configurations = {} self._query_params = {} self._schemas_map = {} # Reading configuration confs_path = os.path.join(os.curdir, CONF_PATH) defaults_path = CONF_FOLDERS.get('defaults_path', confs_path) base_path = CONF_FOLDERS.get('base_path', confs_path) projects_path = CONF_FOLDERS.get('projects_path', confs_path) submodules_path = CONF_FOLDERS.get('submodules_path', confs_path) try: self._configurations, self._extended_project, self._extended_path = \ read_configuration( default_file_path=defaults_path, base_project_path=base_path, projects_path=projects_path, submodules_path=submodules_path ) except AttributeError as e: log.exit(e) if not init: self.do_schema() self.find_endpoints() self.do_swagger()
def custom_init(self, pinit=False, pdestroy=False, abackend=None, **kwargs): # NOTE: we ignore args here # if pinit and not self.variables.get('external'): # log.debug("waiting for internal certificates") # # should actually connect with user and password # # and verify if GSI is already registered with admin rodsminer # import time # time.sleep(5) # recover instance with the parent method session = super().custom_init() # IF variable 'IRODS_ANONYMOUS? is set THEN # Check if external iRODS / B2SAFE has the 'anonymous' user available user = '******' if self.variables.get('external') and self.variables.get(user): if not session.query_user_exists(user): log.exit( "Cannot find '{}' inside " + "the currently connected iRODS instance", user, ) return session
def retry(self, retry_interval=3, max_retries=-1): retry_count = 0 # Get the exception which will signal a missing connection exceptions = self.set_connection_exception() if exceptions is None: exceptions = (BaseException, ) while max_retries != 0 or retry_count < max_retries: retry_count += 1 if retry_count > 1: log.verbose("testing again in {} secs", retry_interval) try: obj = self.custom_connection() except exceptions as e: log.error("Catched: {}({})", e.__class__.__name__, e) log.exit("Service '{}' not available", self.name) else: break # Increment sleeps time if doing a lot of retries if retry_count % 3 == 0: log.debug("Incrementing interval") retry_interval += retry_interval return obj
def get_models(self): """ Read models from base/custom yaml files """ # BASE definitions path = os.path.join(ABS_RESTAPI_PATH, MODELS_DIR) try: data = load_yaml_file('swagger.yaml', path=path) except AttributeError as e: log.exit(e) # EXTENDED definitions, if any extended_models = None if EXTENDED_PACKAGE != EXTENDED_PROJECT_DISABLED: path = os.path.join(os.curdir, EXTENDED_PACKAGE, MODELS_DIR) # NOTE: with logger=False I skip the warning if this file doesn't exist try: extended_models = load_yaml_file('swagger.yaml', path=path) except AttributeError as e: log.verbose(e) # CUSTOM definitions path = os.path.join(os.curdir, CUSTOM_PACKAGE, MODELS_DIR) try: custom_models = load_yaml_file('swagger.yaml', path=path) except AttributeError as e: log.verbose(e) custom_models = {} if extended_models is None: return mix(data, custom_models) m1 = mix(data, extended_models) return mix(m1, custom_models)
def check_configuration(self): try: self.services_configuration = load_yaml_file( file='services.yaml', path=ABS_RESTAPI_CONFSPATH) except AttributeError as e: log.exit(e) for service in self.services_configuration: name, prefix = self.prefix_name(service) # Was this service enabled from the developer? enable_var = str(prefix + 'enable').upper() self.available_services[name] = self.get_bool_from_os(enable_var) if self.available_services[name]: # read variables variables = self.load_variables(service, enable_var, prefix) service['variables'] = variables # set auth service if name == self.authentication_name: self.authentication_service = variables.get('service') if self.authentication_service is None: log.info("No service defined for authentication") else: log.info( "Authentication based on '{}' service", self.authentication_service )
def custom_init(self, pinit=False, pdestroy=False, abackend=None, **kwargs): """ Note: we ignore args here """ # recover instance with the parent method graph = super().custom_init() # db.init_app(self.app) with self.app.app_context(): if pdestroy: log.critical("Destroy current Neo4j data") from neomodel import clear_neo4j_database clear_neo4j_database(graph.db) if pinit: auto_index = self.variables.get("autoindexing", 'True') == 'True' if auto_index: try: from neomodel import remove_all_labels, install_all_labels remove_all_labels() install_all_labels() except BaseException as e: log.exit(str(e)) return graph
def import_secret(self, abs_filename): """ Load the jwt secret from a file """ try: self.JWT_SECRET = open(abs_filename, 'rb').read() return self.JWT_SECRET except IOError: log.exit("Jwt secret file {} not found", abs_filename)
def do_swagger(self): # SWAGGER read endpoints definition swag = BeSwagger(self._endpoints, self) swag_dict = swag.swaggerish() # TODO: update internal endpoints from swagger self._endpoints = swag._endpoints[:] # SWAGGER validation if not swag.validation(swag_dict): log.exit("Current swagger definition is invalid") self._definitions = swag_dict
def load_classes(self): for service in self.services_configuration: name, _ = self.prefix_name(service) if not self.available_services.get(name): continue log.verbose("Looking for class {}", name) variables = service.get('variables') ext_name = service.get('class') # Get the existing class try: MyClass = self.load_class_from_module(ext_name, service=service) # Passing variables MyClass.set_variables(variables) if service.get('load_models'): base_models = self.meta.import_models( name, BACKEND_PACKAGE, exit_on_fail=True ) if EXTENDED_PACKAGE == EXTENDED_PROJECT_DISABLED: extended_models = {} else: extended_models = self.meta.import_models( name, EXTENDED_PACKAGE, exit_on_fail=False ) custom_models = self.meta.import_models( name, CUSTOM_PACKAGE, exit_on_fail=False ) MyClass.set_models(base_models, extended_models, custom_models) except AttributeError as e: log.error(str(e)) log.exit('Invalid Extension class: {}', ext_name) # Save self.services_classes[name] = MyClass log.debug("Got class definition for {}", MyClass) if len(self.services_classes) < 1: raise KeyError("No classes were recovered!") return self.services_classes
def myinit(cls): credentials = get_project_configuration( "variables.backend.credentials") if credentials.get('username') is not None: log.exit("Obsolete use of variables.backend.credentials.username") if credentials.get('password') is not None: log.exit("Obsolete use of variables.backend.credentials.password") # cls.default_user = credentials.get('username', None) # cls.default_password = credentials.get('password', None) cls.default_user = Detector.get_global_var('AUTH_DEFAULT_USERNAME') cls.default_password = Detector.get_global_var('AUTH_DEFAULT_PASSWORD') if cls.default_user is None or cls.default_password is None: log.exit("Default credentials are unavailable!") roles = credentials.get('roles', {}) cls.default_role = roles.get('default') cls.role_admin = roles.get('admin') cls.default_roles = [ roles.get('user'), roles.get('internal'), cls.role_admin ] if cls.default_role is None or None in cls.default_roles: log.exit("Default roles are not available!")
def get_service_address(variables, host_var, port_var, service): host = variables.get(host_var) if host is None: log.exit("Cannot find any variable matching {} for {}", host_var, service) port = variables.get(port_var) if port is None: log.exit("Cannot find any variable matching {} for {}", port_var, service) log.debug("Checking address: {}:{}", host, port) return host, int(port)
def load_class_from_module(self, classname, service=None): if service is None: flaskext = '' else: flaskext = '.' + service.get('extension') # Try inside our extensions module = Meta.get_module_from_string( modulestring=BACKEND_PACKAGE + '.flask_ext' + flaskext, exit_on_fail=True ) if module is None: log.exit("Missing {} for {}", flaskext, service) return getattr(module, classname)
def mywait(): """ Wait for a service on his host:port configuration basing the check on a socket connection. """ from restapi.services.detect import detector for name, myclass in detector.services_classes.items(): if name == 'authentication': continue if name == 'celery': broker = myclass.variables.get('broker') if broker == 'RABBIT': service_vars = detector.load_variables({'prefix': 'rabbitmq'}) elif broker == 'REDIS': service_vars = detector.load_variables({'prefix': 'redis'}) else: log.exit("Invalid celery broker: {}", broker) host, port = get_service_address(service_vars, 'host', 'port', broker) wait_socket(host, port, broker) backend = myclass.variables.get('backend') if backend == 'RABBIT': service_vars = detector.load_variables({'prefix': 'rabbitmq'}) elif backend == 'REDIS': service_vars = detector.load_variables({'prefix': 'redis'}) elif backend == 'MONGODB': service_vars = detector.load_variables({'prefix': 'mongo'}) else: log.exit("Invalid celery backend: {}", backend) host, port = get_service_address(service_vars, 'host', 'port', backend) wait_socket(host, port, backend) else: host, port = get_service_address(myclass.variables, 'host', 'port', name) wait_socket(host, port, name)
def tests(wait, core, file, folder): """Compute tests and coverage""" if wait: while starting_up(): log.debug('Waiting service startup') time.sleep(5) mywait() log.debug("Starting unit tests: {}", pretty_errors) # launch unittests and also compute coverage log.warning("Running all tests and computing coverage.\n" + "This may take some minutes.") parameters = [] if core: parameters.append(current_package) elif file is not None: if not os.path.isfile(os.path.join("tests", file)): log.exit("File not found: {}", file) else: parameters.append("default") parameters.append(file) elif folder is not None: if not os.path.isdir(os.path.join("tests", folder)): log.exit("Folder not found: {}", folder) else: parameters.append("default") parameters.append(folder) try: # TODO: convert the `pyunittests` script from the docker image into python # Pattern in plumbum library for executing a shell command from plumbum import local command = local["pyunittests"] log.verbose("Executing command pyunittests {}", parameters) output = command(parameters) except Exception as e: log.error(str(e)) raise e log.info("Completed:\n{}", output)
def verify(services): """Verify connected service""" from restapi.services.detect import detector if len(services) == 0: log.warning("Empty list of services, nothing to be verified.") log.info("Provide list of services by using --services option") for service in services: myclass = detector.services_classes.get(service) if myclass is None: log.exit("Service \"{}\" was NOT detected", service) log.info("Verifying service: {}", service) host, port = get_service_address(myclass.variables, 'host', 'port', service) wait_socket(host, port, service) log.info("Completed successfully")
def launch(): """Launch the RAPyDo-based HTTP API server""" args = [ 'run', '--host', BIND_INTERFACE, '--port', os.environ.get('FLASK_PORT'), '--reload', '--no-debugger', '--eager-loading', '--with-threads', ] if starting_up(): log.exit( "Please wait few more seconds: resources are still starting up") else: main(args) log.warning("Server shutdown")
def get_module_from_string(modulestring, prefix_package=False, exit_if_not_found=False, exit_on_fail=False): """ Getting a module import when your module is stored as a string in a variable """ module = None if prefix_package: modulestring = BACKEND_PACKAGE + '.' + modulestring.lstrip('.') # which version of python is this? # Retrocompatibility for Python < 3.6 try: import_exceptions = (ModuleNotFoundError, ImportError) except NameError: import_exceptions = ImportError try: # Meta language for dinamically import module = import_module(modulestring) except import_exceptions as e: # pylint:disable=catching-non-exception if exit_on_fail: raise e elif exit_if_not_found: log.exit("Failed to load module:\n{}", e) # else: # log.warning("Failed to load module:\n{}", e) except BaseException as e: if exit_on_fail: raise e else: log.error("Module {} not found.\nError: {}", modulestring, e) return module
""" Sql handling authentication process """ import pytz import sqlalchemy from datetime import datetime, timedelta from restapi.services.authentication import BaseAuthentication from restapi.services.detect import detector from restapi.exceptions import RestApiException from restapi.utilities.htmlcodes import hcodes from restapi.utilities.uuid import getUUID from restapi.utilities.logs import log if not detector.check_availability(__name__): log.exit("No sqlalchemy service available for authentication") class Authentication(BaseAuthentication): # Also used by POST user def create_user(self, userdata, roles): if "authmethod" not in userdata: userdata["authmethod"] = "credentials" if "password" in userdata: userdata["password"] = self.get_password_hash(userdata["password"]) if "uuid" not in userdata: userdata["uuid"] = getUUID()
# -*- coding: utf-8 -*- """ Mongodb based implementation """ from pytz import utc from datetime import datetime, timedelta from restapi.services.authentication import BaseAuthentication from restapi.flask_ext.flask_mongo import AUTH_DB from restapi.utilities.uuid import getUUID from restapi.services.detect import detector from restapi.utilities.logs import log if not detector.check_availability(__name__): log.exit("No mongodb service available for authentication") class Authentication(BaseAuthentication): def __init__(self): # Read init credentials and configuration super().__init__() # Get the instance for mongodb name = __name__.split('.')[::-1][0] # returns 'mongo' extension = detector.services_classes.get(name) self.db = extension().get_instance(dbname=AUTH_DB) def custom_user_properties(self, userdata): new_userdata = super(Authentication,
def init_services( self, app, worker_mode=False, project_init=False, project_clean=False ): instances = {} auth_backend = None for service in self.services_configuration: name, _ = self.prefix_name(service) if not self.available_services.get(name): continue if name == self.authentication_name and auth_backend is None: if self.authentication_service is None: log.warning("No authentication") continue else: log.exit( "Auth service '{}' is unreachable".format( self.authentication_service) ) args = {} if name == self.task_service_name: args['worker_mode'] = worker_mode # Get extension class and build the extension object ExtClass = self.services_classes.get(name) try: ext_instance = ExtClass(app, **args) except TypeError as e: log.exit('Your class {} is not compliant:\n{}', name, e) else: self.extensions_instances[name] = ext_instance if not project_init: do_init = False elif name == self.authentication_service: do_init = True elif name == self.authentication_name: do_init = True else: do_init = False # Initialize the real service getting the first service object log.debug("Initializing {} (pinit={})", name, do_init) service_instance = ext_instance.custom_init( pinit=do_init, pdestroy=project_clean, abackend=auth_backend ) instances[name] = service_instance if name == self.authentication_service: auth_backend = service_instance # NOTE: commented, looks like a duplicate from try/expect above # self.extensions_instances[name] = ext_instance # Injecting into the Celery Extension Class # all celery tasks found in *vanilla_package/tasks* if name == self.task_service_name: do_init = True task_package = "{}.tasks".format(CUSTOM_PACKAGE) submodules = self.meta.import_submodules_from_package( task_package, exit_on_fail=True ) for submodule in submodules: tasks = Meta.get_celery_tasks_from_module(submodule) for func_name, funct in tasks.items(): setattr(ExtClass, func_name, funct) if len(self.extensions_instances) < 1: raise KeyError("No instances available for modules") # Only once in a lifetime if project_init: self.project_initialization(instances, app=app) return self.extensions_instances
# -*- coding: utf-8 -*- import sys import os import yaml import json from restapi.utilities.configuration import mix from restapi.utilities.logs import log if len(sys.argv) <= 1: log.exit("Usage: {} project_name", sys.argv[0]) PROJECT = sys.argv[1] PROJECT_DIR = "projects/{}/backend/swagger".format(PROJECT) if not os.path.exists(PROJECT_DIR): log.exit("{} folder does not exist", PROJECT_DIR) if not os.path.isdir(PROJECT_DIR): log.exit("{} is not a folder", PROJECT_DIR) yamls = ["specs", "get", "post", "put", "patch", "delete", "head"] for swagger_folder in os.listdir(PROJECT_DIR): conf_output = "" decorators_output = "" pfile = None pclass = None mappings = None for y in yamls:
from restapi.confs import get_project_configuration if Detector.get_global_var("AUTH_SECOND_FACTOR_AUTHENTICATION", '') == 'TOTP': try: import pyotp import pyqrcode # import base64 from io import BytesIO # FIXME: cannot use the proper exception (available in python 3.6+) # because we are stuck on python 3.5 con IMC # except ModuleNotFoundError: except BaseException: log.exit( "Missing libraries for TOTP 2FA authentication" ) class Authenticator(BaseExtension): """ The generic authentication Flask extension """ def custom_connection(self, **kwargs): # What service will hold authentication? auth_service = self.variables.get('service') auth_module = Meta.get_authentication_module(auth_service) custom_auth = auth_module.Authentication()
def read_configuration(default_file_path, base_project_path, projects_path, submodules_path): """ Read default configuration """ custom_configuration = load_yaml_file(PROJECT_CONF_FILENAME, path=base_project_path, keep_order=True) # Verify custom project configuration project = custom_configuration.get('project') if project is None: raise AttributeError("Missing project configuration") variables = ['title', 'description', 'version', 'rapydo'] for key in variables: if project.get(key) is None: log.exit( "Project not configured, missing key '{}' in file {}/{}", key, base_project_path, PROJECT_CONF_FILENAME, ) if default_file_path is None: base_configuration = {} else: base_configuration = load_yaml_file(file=PROJECTS_DEFAULTS_FILE, path=default_file_path, keep_order=True) extended_project = project.get('extends') if extended_project is None: # Mix default and custom configuration return mix(base_configuration, custom_configuration), None, None extends_from = project.get('extends-from', 'projects') if extends_from == "projects": extend_path = projects_path elif extends_from.startswith("submodules/"): repository_name = (extends_from.split("/")[1]).strip() if repository_name == '': log.exit('Invalid repository name in extends-from, name is empty') extend_path = submodules_path else: suggest = "Expected values: 'projects' or 'submodules/${REPOSITORY_NAME}'" log.exit("Invalid extends-from parameter: {}.\n{}", extends_from, suggest) if not os.path.exists(extend_path): log.exit("From project not found: {}", extend_path) extend_file = "extended_{}".format(PROJECT_CONF_FILENAME) extended_configuration = load_yaml_file(file=extend_file, path=extend_path, keep_order=True) m1 = mix(base_configuration, extended_configuration) return mix(m1, custom_configuration), extended_project, extend_path
import os import pwd import re from datetime import datetime, timedelta import pytz from restapi.utilities.logs import log try: from OpenSSL import crypto from plumbum import local import dateutil.parser except ImportError as e: log.exit("\nThis module requires an extra package:\n{}", e) class Certificates: _dir = os.environ.get('CERTDIR') _proxyfile = 'userproxy.crt' def __init__(self): log.warning( "All methods of this class are static, no need to create an instance" ) @classmethod def get_dn_from_cert(cls, certdir, certfilename, ext='pem'):
def find_endpoints(self): ################## # Walk folders looking for endpoints endpoints_folders = [] # base swagger dir (rapydo/http-ap) endpoints_folders.append({'path': ABS_RESTAPI_PATH, 'iscore': True}) # swagger dir from extended project, if any if self._extended_project is not None: endpoints_folders.append({ 'path': os.path.join(os.curdir, self._extended_project), 'iscore': False }) # custom swagger dir endpoints_folders.append({ 'path': os.path.join(os.curdir, CUSTOM_PACKAGE), 'iscore': False }) # already_loaded = {} for folder in endpoints_folders: base_dir = folder.get('path') iscore = folder.get('iscore') # get last item of the path # normapath is required to strip final / is any base_module = os.path.basename(os.path.normpath(base_dir)) if iscore: apis_dir = os.path.join(base_dir, 'resources') apiclass_module = '{}.resources'.format(base_module) else: apis_dir = os.path.join(base_dir, 'apis') apiclass_module = '{}.apis'.format(base_module) # Looking for all file in apis folder for epfiles in os.listdir(apis_dir): # get module name (es: apis.filename) module_file = os.path.splitext(epfiles)[0] module_name = "{}.{}".format(apiclass_module, module_file) # Convert module name into a module log.debug("Importing {}", module_name) try: module = Meta.get_module_from_string( module_name, exit_on_fail=True, exit_if_not_found=True) except BaseException as e: log.exit("Cannot import {}\nError: {}", module_name, e) # Extract classes from the module # classes = meta.get_classes_from_module(module) classes = meta.get_new_classes_from_module(module) for class_name in classes: ep_class = classes.get(class_name) # Filtering out classes without required data if not hasattr(ep_class, "methods"): continue if ep_class.methods is None: continue # if class_name in already_loaded: # log.warning( # "Skipping import of {} from {}.{}, already loded from {}", # class_name, # apis_dir, # module_file, # already_loaded[class_name], # ) # continue # already_loaded[class_name] = "{}.{}".format(apis_dir, module_file) log.debug("Importing {} from {}.{}", class_name, apis_dir, module_file) if not self._testing: skip = False for var in ep_class.depends_on: pieces = var.strip().split(' ') pieces_num = len(pieces) if pieces_num == 1: dependency = pieces.pop() negate = False elif pieces_num == 2: negate, dependency = pieces negate = negate.lower() == 'not' else: log.exit('Wrong parameter: {}', var) check = detector.get_bool_from_os(dependency) if negate: check = not check # Skip if not meeting the requirements of the dependency if not check: skip = True break if skip: log.debug( "Skipping '{} {}' due to unmet dependency: {}", module_name, class_name, dependency) continue # Building endpoint endpoint = EndpointElements(custom={}) endpoint.cls = ep_class endpoint.exists = True endpoint.iscore = iscore # Global tags to be applied to all methods endpoint.tags = ep_class.labels # base URI base = ep_class.baseuri if base not in BASE_URLS: log.warning("Invalid base {}", base) base = API_URL base = base.strip('/') endpoint.base_uri = base endpoint.uris = {} # attrs python lib bug? endpoint.custom['schema'] = { 'expose': ep_class.expose_schema, 'publish': {}, } endpoint.methods = {} mapping_lists = [] for m in ep_class.methods: method_name = "_{}".format(m) if not hasattr(ep_class, method_name): method_name = m if not hasattr(ep_class, method_name): log.warning("{} configuration not found in {}", m, class_name) continue # Enable this warning to start conversions GET -> _GET # Find other warning like this by searching: # **FASTAPI** # else: # log.warning( # "Obsolete dict {} in {}", m, class_name # ) conf = getattr(ep_class, method_name) kk = conf.keys() mapping_lists.extend(kk) endpoint.methods[m.lower()] = copy.deepcopy(conf) if endpoint.custom['schema']['expose']: for uri in mapping_lists: total_uri = '/{}{}'.format(endpoint.base_uri, uri) schema_uri = '{}/schemas{}'.format(API_URL, uri) p = hex(id(endpoint.cls)) self._schema_endpoint.uris[uri + p] = schema_uri self._schemas_map[schema_uri] = total_uri self._endpoints.append(endpoint)
MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r Remove tokens: MATCH (a:Token) WHERE NOT (a)<-[]-() DELETE a """ from datetime import datetime, timedelta import pytz from restapi.utilities.uuid import getUUID from restapi.services.authentication import BaseAuthentication from restapi.services.detect import detector from restapi.utilities.logs import log if not detector.check_availability(__name__): log.exit("No neo4j GraphDB service found for authentication") class Authentication(BaseAuthentication): def get_user_object(self, username=None, payload=None): from neomodel.exceptions import DeflateError from neo4j.exceptions import ServiceUnavailable user = None try: if username is not None: user = self.db.User.nodes.get(email=username) if payload is not None and 'user_id' in payload: user = self.db.User.nodes.get(uuid=payload['user_id']) except ServiceUnavailable as e:
def custom_connection(self, **kwargs): if len(kwargs) > 0: print("TODO: use args for connection?", kwargs) uri = '{}://{}:{}@{}:{}/{}'.format( self.variables.get('dbtype', 'postgresql'), self.variables.get('user'), self.variables.get('password'), self.variables.get('host'), self.variables.get('port'), self.variables.get('db'), ) # TODO: in case we need different connection binds # (multiple connections with sql) then: # SQLALCHEMY_BINDS = { # 'users': 'mysqldb://localhost/users', # 'appmeta': 'sqlite:////path/to/appmeta.db' # } self.app.config['SQLALCHEMY_DATABASE_URI'] = uri # self.app.config['SQLALCHEMY_POOL_TIMEOUT'] = 3 self.app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # pool_size = self.variables.get('poolsize') # if pool_size is not None: # # sqlalchemy docs: http://j.mp/2xT0GOc # # defaults: overflow=10, pool_size=5 # # self.app.config['SQLALCHEMY_MAX_OVERFLOW'] = 0 # self.app.config['SQLALCHEMY_POOL_SIZE'] = int(pool_size) # log.debug("Setting SQLALCHEMY_POOL_SIZE = {}", pool_size) obj_name = 'db' # search the original sqlalchemy object into models db = Meta.obj_from_models(obj_name, self.name, CUSTOM_PACKAGE) try: from flask_migrate import Migrate # The Alembic package, which handles the migration work, does not recognize # type changes in columns by default. If you want that fine level of # detection you need to enable the compare_type option Migrate(self.app, db, compare_type=True) except BaseException as e: log.warning("Flask Migrate not enabled") log.error(str(e)) # no 'db' set in CUSTOM_PACKAGE, looking for EXTENDED PACKAGE, if any if db is None and EXTENDED_PACKAGE != EXTENDED_PROJECT_DISABLED: db = Meta.obj_from_models(obj_name, self.name, EXTENDED_PACKAGE) if db is None: log.warning("No sqlalchemy db imported in custom package") db = Meta.obj_from_models(obj_name, self.name, BACKEND_PACKAGE) if db is None: log.exit("Could not get {} within {} models", obj_name, self.name) # Overwrite db.session created by flask_alchemy due to errors # with transaction when concurrent requests... from sqlalchemy import create_engine from sqlalchemy.orm import scoped_session from sqlalchemy.orm import sessionmaker db.engine_bis = create_engine(uri) db.session = scoped_session(sessionmaker(bind=db.engine_bis)) return db
def create_app( name=__name__, init_mode=False, destroy_mode=False, worker_mode=False, testing_mode=False, skip_endpoint_mapping=False, **kwargs, ): """ Create the server istance for Flask application """ if PRODUCTION and testing_mode: log.exit("Unable to execute tests in production") # Initialize reading of all files mem.customizer = Customizer(testing_mode, init_mode) mem.geo_reader = geolite2.reader() # when to close?? # geolite2.close() # Add template dir for output in HTML kwargs['template_folder'] = os.path.join(ABS_RESTAPI_PATH, 'templates') # Flask app instance microservice = Flask(name, **kwargs) # Add commands to 'flask' binary if init_mode: microservice.config['INIT_MODE'] = init_mode skip_endpoint_mapping = True elif destroy_mode: microservice.config['DESTROY_MODE'] = destroy_mode skip_endpoint_mapping = True elif testing_mode: microservice.config['TESTING'] = testing_mode init_mode = True elif worker_mode: skip_endpoint_mapping = True # Fix proxy wsgi for production calls microservice.wsgi_app = ProxyFix(microservice.wsgi_app) # CORS if not PRODUCTION: cors = CORS( allow_headers=[ 'Content-Type', 'Authorization', 'X-Requested-With', 'x-upload-content-length', 'x-upload-content-type', 'content-range' ], supports_credentials=['true'], methods=['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], ) cors.init_app(microservice) log.verbose("FLASKING! Injected CORS") # Enabling our internal Flask customized response microservice.response_class = InternalResponse # Flask configuration from config file microservice.config.from_object(config) log.debug("Flask app configured") if PRODUCTION: log.info("Production server mode is ON") # Find services and try to connect to the ones available extensions = detector.init_services( app=microservice, worker_mode=worker_mode, project_init=init_mode, project_clean=destroy_mode, ) if worker_mode: microservice.extensions = extensions # Restful plugin if not skip_endpoint_mapping: # Triggering automatic mapping of REST endpoints rest_api = Api(catch_all_404s=True) # Basic configuration (simple): from example class if len(mem.customizer._endpoints) < 1: log.error("No endpoints found!") raise AttributeError("Follow the docs and define your endpoints") for resource in mem.customizer._endpoints: # urls = [uri for _, uri in resource.uris.items()] urls = list(resource.uris.values()) # Create the restful resource with it; # this method is from RESTful plugin rest_api.add_resource(resource.cls, *urls) log.verbose("Map '{}' to {}", resource.cls.__name__, urls) # Enable all schema endpoints to be mapped with this extra step if len(mem.customizer._schema_endpoint.uris) > 0: log.debug("Found one or more schema to expose") urls = [ uri for _, uri in mem.customizer._schema_endpoint.uris.items() ] rest_api.add_resource(mem.customizer._schema_endpoint.cls, *urls) # HERE all endpoints will be registered by using FlaskRestful rest_api.init_app(microservice) microservice.services_instances = {} for m in detector.services_classes: ExtClass = detector.services_classes.get(m) microservice.services_instances[m] = ExtClass(microservice) # FlaskApiSpec experimentation from apispec import APISpec from flask_apispec import FlaskApiSpec from apispec.ext.marshmallow import MarshmallowPlugin # from apispec_webframeworks.flask import FlaskPlugin microservice.config.update({ 'APISPEC_SPEC': APISpec( title=glom(mem.customizer._configurations, 'project.title', default='0.0.1'), version=glom(mem.customizer._configurations, 'project.version', default='Your application name'), openapi_version="2.0", # OpenApi 3 not working with FlaskApiSpec # -> Duplicate parameter with name body and location body # https://github.com/jmcarp/flask-apispec/issues/170 # Find other warning like this by searching: # **FASTAPI** # openapi_version="3.0.2", plugins=[MarshmallowPlugin()], ), 'APISPEC_SWAGGER_URL': '/api/swagger', # 'APISPEC_SWAGGER_UI_URL': '/api/swagger-ui', # Disable Swagger-UI 'APISPEC_SWAGGER_UI_URL': None, }) docs = FlaskApiSpec(microservice) with microservice.app_context(): for resource in mem.customizer._endpoints: urls = list(resource.uris.values()) try: docs.register(resource.cls) except TypeError as e: # log.warning("{} on {}", type(e), resource.cls) # Enable this warning to start conversion to FlaskFastApi # Find other warning like this by searching: # **FASTAPI** log.verbose("{} on {}", type(e), resource.cls) # Clean app routes ignore_verbs = {"HEAD", "OPTIONS"} for rule in microservice.url_map.iter_rules(): rulename = str(rule) # Skip rules that are only exposing schemas if '/schemas/' in rulename: continue endpoint = microservice.view_functions[rule.endpoint] if not hasattr(endpoint, 'view_class'): continue newmethods = ignore_verbs.copy() for verb in rule.methods - ignore_verbs: method = verb.lower() if method in mem.customizer._original_paths[rulename]: # remove from flask mapping # to allow 405 response newmethods.add(verb) else: log.verbose("Removed method {}.{} from mapping", rulename, verb) rule.methods = newmethods # Logging responses @microservice.after_request def log_response(response): response.headers["_RV"] = str(__version__) PROJECT_VERSION = get_project_configuration("project.version", default=None) if PROJECT_VERSION is not None: response.headers["Version"] = str(PROJECT_VERSION) # NOTE: if it is an upload, # I must NOT consume request.data or request.json, # otherwise the content gets lost do_not_log_types = ['application/octet-stream', 'multipart/form-data'] if request.mimetype in do_not_log_types: data = 'STREAM_UPLOAD' else: try: data = handle_log_output(request.data) # Limit the parameters string size, sometimes it's too big for k in data: try: if isinstance(data[k], dict): for kk in data[k]: v = str(data[k][kk]) if len(v) > MAX_CHAR_LEN: v = v[:MAX_CHAR_LEN] + "..." data[k][kk] = v continue if not isinstance(data[k], str): data[k] = str(data[k]) if len(data[k]) > MAX_CHAR_LEN: data[k] = data[k][:MAX_CHAR_LEN] + "..." except IndexError: pass except Exception: data = 'OTHER_UPLOAD' # Obfuscating query parameters url = urllib_parse.urlparse(request.url) try: params = urllib_parse.unquote( urllib_parse.urlencode(handle_log_output(url.query))) url = url._replace(query=params) except TypeError: log.error("Unable to url encode the following parameters:") print(url.query) url = urllib_parse.urlunparse(url) log.info("{} {} {} {}", request.method, url, data, response) return response if send_mail_is_active(): if not test_smtp_client(): log.critical("Bad SMTP configuration, unable to create a client") else: log.info("SMTP configuration verified") # and the flask App is ready now: log.info("Boot completed") if SENTRY_URL is not None: if not PRODUCTION: log.info("Skipping Sentry, only enabled in PRODUCTION mode") else: import sentry_sdk from sentry_sdk.integrations.flask import FlaskIntegration sentry_sdk.init(dsn=SENTRY_URL, integrations=[FlaskIntegration()]) log.info("Enabled Sentry {}", SENTRY_URL) # return our flask app return microservice
def read_my_swagger(self, method, endpoint, mapping=None): # content has to be a dictionary if not isinstance(mapping, dict): raise TypeError("Wrong type: {}".format(type(mapping))) # read common commons = mapping.pop('common', {}) if commons: # Deprecated since 0.7.0 log.warning("Commons specs are deprecated") # Check if there is at least one except for common if len(mapping) < 1: raise ValueError("No definition found in: {}".format(mapping)) ################################ # Using 'attrs': a way to save external attributes # Instance extra = ExtraAttributes() ################################ # Specs should contain only labels written in spec before pattern = re.compile(r'\<([^\>]+)\>') for label, specs in mapping.items(): uri = '/{}{}'.format(endpoint.base_uri, label) # This will be used by server.py.add if uri not in endpoint.uris: endpoint.uris[uri] = uri ################################ # add common elements to all specs for key, value in commons.items(): if key not in specs: specs[key] = value ################################ # Separate external definitions # Find any custom part which is not swagger definition custom = specs.pop('custom', {}) # Publish the specs on the final Swagger JSON # Default is to do it if not otherwise specified extra.publish = custom.get('publish', True) if not extra.publish: # Deprecated since 0.7.0 log.warning("Publish setting is deprecated") # extra.auth = None ########################### # Strip the uri of the parameter # and add it to 'parameters' newuri = uri[:] # create a copy if 'parameters' not in specs: specs['parameters'] = [] ########################### # Read Form Data Custom parameters cparam = specs.pop('custom_parameters', None) if cparam is not None: for fdp in cparam: params = self._fdp.get(fdp) if params is None: log.exit("No custom form data '{}'", fdp) else: # Unable to extend with list by using extends() because # it add references to the original object and do not # create copies. Without copying, the same objects will # be modified several times leading to errors for p in params: specs['parameters'].append(p.copy()) ########################### # Read normal parameters for parameter in pattern.findall(uri): # create parameters x = parameter.split(':') xlen = len(x) paramtype = 'string' if xlen == 1: paramname = x[0] elif xlen == 2: paramtype = x[0] paramname = x[1] # FIXME: complete for all types # http://swagger.io/specification/#data-types-12 if paramtype == 'int': paramtype = 'number' if paramtype == 'path': paramtype = 'string' path_parameter = { 'name': paramname, 'type': paramtype, 'in': 'path', 'required': True, } specs['parameters'].append(path_parameter) # replace in a new uri # <param> -> {param} newuri = newuri.replace('<{}>'.format(parameter), '{{{}}}'.format(paramname)) # cycle parameters and add them to the endpoint class query_params = [] for param in specs['parameters']: if param["in"] != 'path': if uri not in self._parameter_schemas: self._parameter_schemas[uri] = {} if method not in self._parameter_schemas[uri]: self._parameter_schemas[uri][method] = [] self._parameter_schemas[uri][method].append(param.copy()) extrainfo = param.pop('custom', {}) if len(extrainfo) and endpoint.custom['schema']['expose']: # TODO: read a 'custom.publish' in every yaml # to decide if the /schema uri should be in swagger if uri not in endpoint.custom['params']: endpoint.custom['params'][uri] = {} endpoint.custom['params'][uri][method] = extrainfo enum = param.pop("enum", None) if enum is not None: param["enum"] = [] for option in enum: if isinstance(option, str): param["enum"].append(option) else: # enum [{key1: value1}, {key2: value2}] # became enum [key1, ke2] for k in option: param["enum"].append(k) # handle parameters in URI for Flask if param['in'] == 'query': query_params.append(param) if len(query_params) > 0: self.query_parameters(endpoint.cls, method=method, uri=uri, params=query_params) # Swagger does not like empty arrays if len(specs['parameters']) < 1: specs.pop('parameters') ################## # Save definition for checking if uri not in self._original_paths: self._original_paths[uri] = {} self._original_paths[uri][method] = specs ################## # Skip what the developers does not want to be public in swagger # NOTE: do not skip if in testing mode if not extra.publish and not self._customizer._testing: continue # Handle global tags if 'tags' not in specs and len(endpoint.tags) > 0: specs['tags'] = [] for tag in endpoint.tags: self._used_swagger_tags[tag] = True if tag not in specs['tags']: specs['tags'].append(tag) ################## # NOTE: whatever is left inside 'specs' will be # passed later on to Swagger Validator... # Save definition for publishing if newuri not in self._paths: self._paths[newuri] = {} self._paths[newuri][method] = specs log.verbose("Built definition '{}:{}'", method.upper(), newuri) endpoint.custom['methods'][method] = extra return endpoint
def custom_connection(self, **kwargs): # set here to avoid warnings like 'Possible hardcoded password' EMPTY = "" broker = self.variables.get("broker") if broker is None: log.exit("Unable to start Celery, missing broker service") # celery_app = None # return celery_app # Do not import before loading the ext! from restapi.services.detect import Detector if broker == 'RABBIT': service_vars = Detector.load_variables({'prefix': 'rabbitmq'}) BROKER_HOST = service_vars.get("host") BROKER_PORT = int(service_vars.get("port")) BROKER_USER = service_vars.get("user", "") BROKER_PASSWORD = service_vars.get("password", "") BROKER_VHOST = service_vars.get("vhost", "") BROKER_USE_SSL = Detector.get_bool_envvar( service_vars.get("ssl_enabled", False) ) elif broker == 'REDIS': service_vars = Detector.load_variables({'prefix': 'redis'}) BROKER_HOST = service_vars.get("host") BROKER_PORT = int(service_vars.get("port")) BROKER_USER = None BROKER_PASSWORD = None BROKER_VHOST = "" BROKER_USE_SSL = False else: log.exit("Invalid celery broker: {}", broker) if BROKER_USER == "": BROKER_USER = None if BROKER_PASSWORD == EMPTY: BROKER_PASSWORD = None if BROKER_VHOST != "": BROKER_VHOST = "/{}".format(BROKER_VHOST) if BROKER_USER is not None and BROKER_PASSWORD is not None: BROKER_CREDENTIALS = '{}:{}@'.format(BROKER_USER, BROKER_PASSWORD) else: BROKER_CREDENTIALS = "" if broker == 'RABBIT': BROKER_URL = 'amqp://{}{}:{}{}'.format( BROKER_CREDENTIALS, BROKER_HOST, BROKER_PORT, BROKER_VHOST, ) log.info( "Configured RabbitMQ as Celery broker {}", obfuscate_url(BROKER_URL)) elif broker == 'REDIS': BROKER_URL = 'redis://{}{}:{}/0'.format( BROKER_CREDENTIALS, BROKER_HOST, BROKER_PORT, ) log.info( "Configured Redis as Celery broker {}", obfuscate_url(BROKER_URL)) else: log.error("Unable to start Celery unknown broker service: {}", broker) celery_app = None return celery_app backend = self.variables.get("backend", broker) if backend == 'RABBIT': service_vars = Detector.load_variables({'prefix': 'rabbitmq'}) BACKEND_HOST = service_vars.get("host") BACKEND_PORT = int(service_vars.get("port")) BACKEND_USER = service_vars.get("user", "") BACKEND_PASSWORD = service_vars.get("password", "") elif backend == 'REDIS': service_vars = Detector.load_variables({'prefix': 'redis'}) BACKEND_HOST = service_vars.get("host") BACKEND_PORT = int(service_vars.get("port")) BACKEND_USER = "" BACKEND_PASSWORD = None elif backend == 'MONGODB': service_vars = Detector.load_variables({'prefix': 'mongo'}) BACKEND_HOST = service_vars.get("host") BACKEND_PORT = int(service_vars.get("port")) BACKEND_USER = service_vars.get("user", "") BACKEND_PASSWORD = service_vars.get("password", "") else: log.exit("Invalid celery backend: {}", backend) if BACKEND_USER == EMPTY: BACKEND_USER = None if BACKEND_PASSWORD == EMPTY: BACKEND_PASSWORD = None if BACKEND_USER is not None and BACKEND_PASSWORD is not None: BACKEND_CREDENTIALS = '{}:{}@'.format(BACKEND_USER, BACKEND_PASSWORD) else: BACKEND_CREDENTIALS = "" if backend == 'RABBIT': BACKEND_URL = 'rpc://{}{}:{}/0'.format( BACKEND_CREDENTIALS, BACKEND_HOST, BACKEND_PORT, ) log.info( "Configured RabbitMQ as Celery backend {}", obfuscate_url(BACKEND_URL)) elif backend == 'REDIS': BACKEND_URL = 'redis://{}{}:{}/0'.format( BACKEND_CREDENTIALS, BACKEND_HOST, BACKEND_PORT, ) log.info( "Configured Redis as Celery backend {}", obfuscate_url(BACKEND_URL)) elif backend == 'MONGODB': BACKEND_URL = 'mongodb://{}{}:{}'.format( BACKEND_CREDENTIALS, BACKEND_HOST, BACKEND_PORT, ) log.info( "Configured MongoDB as Celery backend {}", obfuscate_url(BACKEND_URL)) else: log.exit("Unable to start Celery unknown backend service: {}", backend) # celery_app = None # return celery_app celery_app = Celery('RestApiQueue', broker=BROKER_URL, backend=BACKEND_URL) celery_app.conf['broker_use_ssl'] = BROKER_USE_SSL # if not worker_mode: # from celery.task.control import inspect # insp = inspect() # if not insp.stats(): # log.warning("No running Celery workers were found") # Skip initial warnings, avoiding pickle format (deprecated) celery_app.conf.accept_content = ['json'] celery_app.conf.task_serializer = 'json' celery_app.conf.result_serializer = 'json' # Max priority default value for all queues # Required to be able to set priority parameter on apply_async calls # celery_app.conf.task_queue_max_priority = 10 # default_exchange = Exchange('default', type='direct') # priority_exchange = Exchange('priority', type='direct') # celery_app.conf.task_queues = [ # Queue( # 'priority', # priority_exchange, # routing_key='priority', # queue_arguments={ # 'x-max-priority': 10 # } # ) # ] # If you want to apply a more strict priority to items # probably prefetching should also be disabled: # CELERY_ACKS_LATE = True # CELERYD_PREFETCH_MULTIPLIER = 1 # celery_app.conf.broker_pool_limit = None if Detector.get_bool_from_os('CELERY_BEAT_ENABLED'): CeleryExt.CELERY_BEAT_SCHEDULER = backend if backend == 'MONGODB': SCHEDULER_DB = 'celery' celery_app.conf['CELERY_MONGODB_SCHEDULER_DB'] = SCHEDULER_DB celery_app.conf['CELERY_MONGODB_SCHEDULER_COLLECTION'] = "schedules" celery_app.conf['CELERY_MONGODB_SCHEDULER_URL'] = BACKEND_URL import mongoengine m = mongoengine.connect(SCHEDULER_DB, host=BACKEND_URL) log.info("Celery-beat connected to MongoDB: {}", m) elif backend == 'REDIS': BEAT_BACKEND_URL = 'redis://{}{}:{}/1'.format( BACKEND_CREDENTIALS, BACKEND_HOST, BACKEND_PORT, ) celery_app.conf['REDBEAT_REDIS_URL'] = BEAT_BACKEND_URL celery_app.conf['REDBEAT_KEY_PREFIX'] = CeleryExt.REDBEAT_KEY_PREFIX log.info("Celery-beat connected to Redis: {}", BEAT_BACKEND_URL) else: log.warning("Cannot configure celery beat scheduler") if CeleryExt.celery_app is None: CeleryExt.celery_app = celery_app return celery_app