def project_initialization(self, instances): """ Custom initialization of your project Please define your class Initializer in vanilla/project/initialization.py """ try: meta = Meta() module_path = "%s.%s.%s" % \ (CUSTOM_PACKAGE, 'initialization', 'initialization') module = meta.get_module_from_string( module_path, debug_on_fail=False, ) Initializer = meta.get_class_from_string('Initializer', module, skip_error=True) if Initializer is None: log.debug("No custom init available") else: try: Initializer(instances) except BaseException as e: log.error("Errors during custom initialization: %s", e) else: log.info("Vanilla project has been initialized") except BaseException: log.debug("No custom init available")
def test(): meta = Meta() cls_name = "Meta" import utilities modules = meta.get_submodules_from_package(utilities) assert "meta" in modules sub_modules = meta.import_submodules_from_package("utilities") assert sub_modules is not None assert isinstance(sub_modules, list) assert len(sub_modules) > 0 module = meta.get_module_from_string("utilities.meta") assert module is not None assert hasattr(module, cls_name) classes = meta.get_classes_from_module(module) assert classes is not None assert isinstance(classes, dict) assert cls_name in classes assert classes[cls_name].__name__ == cls_name new_classes = meta.get_new_classes_from_module(module) assert new_classes is not None assert isinstance(new_classes, dict) assert cls_name in new_classes assert new_classes[cls_name].__name__ == cls_name meta_class = meta.get_class_from_string(cls_name, module) assert meta_class is not None assert meta_class.__name__ == cls_name methods = meta.get_methods_inside_instance(meta, private_methods=True) assert methods is not None assert isinstance(methods, dict) # it is a static method assert "get_methods_inside_instance" not in methods assert "get_authentication_module" in methods metacls = meta.metaclassing(Meta, "Test") assert metacls is not None assert metacls.__name__ == "Test" self_ref = meta.get_self_reference_from_args(meta, "test", 1) assert self_ref == meta
def custom_post_handle_user_input(self, user_node, input_data): meta = Meta() module_path = "%s.%s.%s" % \ (CUSTOM_PACKAGE, 'initialization', 'initialization') module = meta.get_module_from_string( module_path, debug_on_fail=False, ) Customizer = meta.get_class_from_string('Customizer', module, skip_error=True) if Customizer is None: log.debug("No user properties customizer available") else: try: Customizer().custom_post_handle_user_input( self, user_node, input_data) except BaseException as e: log.error("Unable to customize user properties: %s", e)
def custom_user_properties(self, userdata): meta = Meta() module_path = "%s.%s.%s" % \ (CUSTOM_PACKAGE, 'initialization', 'initialization') module = meta.get_module_from_string( module_path, debug_on_fail=False, ) Customizer = meta.get_class_from_string('Customizer', module, skip_error=True) if Customizer is None: log.debug("No user properties customizer available") else: try: userdata = Customizer().custom_user_properties(userdata) except BaseException as e: log.error("Unable to customize user properties: %s", e) if "email" in userdata: userdata["email"] = userdata["email"].lower() return userdata
class Customizer(object): """ Customize your BACKEND: Read all of available configurations and definitions. """ def __init__(self, testing=False, production=False, init=False): # Input self._testing = testing self._production = production self._initiliazing = init # Some initialization self._endpoints = [] self._definitions = {} self._configurations = {} self._query_params = {} self._schemas_map = {} self._meta = Meta() # Do things self.read_configuration() if not self._initiliazing: self.do_schema() self.find_endpoints() self.do_swagger() def read_configuration(self): ################## # Reading configuration default_file_path = helpers.current_dir(CONF_PATH) project_file_path = helpers.current_dir(CONF_PATH) self._configurations = configuration.read( default_file_path, project_path=project_file_path, ) def do_schema(self): """ Schemas exposing, if requested """ name = '%s.%s.%s' % (BACKEND_PACKAGE, 'rest', 'schema') module = self._meta.get_module_from_string( name, exit_if_not_found=True ) schema_class = getattr(module, 'RecoverSchema') self._schema_endpoint = EndpointElements( cls=schema_class, exists=True, custom={ 'methods': { 'get': ExtraAttributes(auth=None), # WHY DOES POST REQUEST AUTHENTICATION # 'post': ExtraAttributes(auth=None) } }, methods={} ) # TODO: find a way to map authentication # as in the original endpoint for the schema 'get' method # TODO: find a way to publish on swagger the schema # if endpoint is enabled to publish and the developer asks for it def find_endpoints(self): ################## # Walk swagger directories looking for endpoints # FIXME: how to do this? # from utilities import helpers # custom_dir = helpers.current_dir(CUSTOM_PACKAGE) # base_dir = helpers.script_abspath(__file__) base_swagger_confdir = helpers.script_abspath(__file__) custom_swagger_confdir = helpers.current_dir(CUSTOM_PACKAGE) # for base_dir in [BACKEND_PACKAGE, CUSTOM_PACKAGE]: for base_dir in [base_swagger_confdir, custom_swagger_confdir]: swagger_dir = os.path.join(base_dir, 'swagger') log.verbose("Swagger dir: %s" % swagger_dir) for ep in os.listdir(swagger_dir): swagger_endpoint_dir = os.path.join(swagger_dir, ep) if os.path.isfile(swagger_endpoint_dir): exception = '%s.yaml' % SWAGGER_MODELS_FILE if not swagger_endpoint_dir.endswith('/' + exception): log.debug( "Found a file instead of a folder: %s", swagger_endpoint_dir ) continue # isbase = base_dir == BACKEND_PACKAGE isbase = base_dir.startswith('/usr/local') base_module = helpers.last_dir(base_dir) from utilities import ENDPOINTS_CODE_DIR if isbase: apiclass_module = '%s.%s' % (base_module, 'resources') else: apiclass_module = '%s.%s' % ( base_module, ENDPOINTS_CODE_DIR) current = self.lookup( ep, apiclass_module, swagger_endpoint_dir, isbase) if current is not None and current.exists: # Add endpoint to REST mapping self._endpoints.append(current) def do_swagger(self): # SWAGGER read endpoints definition swag = BeSwagger(self._endpoints, self) swag_dict = swag.swaggerish() # TODO: update internal endpoints from swagger self._endpoints = swag._endpoints[:] # SWAGGER validation if not swag.validation(swag_dict): log.critical_exit("Current swagger definition is invalid") self._definitions = swag_dict def read_frameworks(self): file = os.path.join("config", "frameworks.yaml") self._frameworks = load_yaml_file(file) def lookup(self, endpoint, apiclass_module, swagger_endpoint_dir, isbase): log.verbose("Found endpoint dir: '%s'" % endpoint) if os.path.exists(os.path.join(swagger_endpoint_dir, 'SKIP')): log.info("Skipping: %s", endpoint) return None # Find yaml files conf = None yaml_files = {} yaml_listing = os.path.join(swagger_endpoint_dir, "*.%s" % YAML_EXT) for file in glob.glob(yaml_listing): if file.endswith('specs.%s' % YAML_EXT): # load configuration and find file and class conf = load_yaml_file(file) else: # add file to be loaded from swagger extension p = re.compile(r'\/([^\.\/]+)\.' + YAML_EXT + '$') match = p.search(file) method = match.groups()[0] yaml_files[method] = file if len(yaml_files) < 1: raise Exception("%s: no methods defined in any YAML" % endpoint) if conf is None or 'class' not in conf: raise ValueError("No 'class' defined for '%s'" % endpoint) current = self.load_endpoint(endpoint, apiclass_module, conf, isbase) current.methods = yaml_files return current # def read_complex_config(self, configfile): # """ A more complex configuration is available in JSON format """ # content = {} # with open(configfile) as fp: # content = json.load(fp) # return content def load_endpoint(self, default_uri, apiclass_module, conf, isbase): endpoint = EndpointElements(custom={}) # Load the endpoint class defined in the YAML file file_name = conf.pop('file', default_uri) class_name = conf.pop('class') name = '%s.%s' % (apiclass_module, file_name) module = self._meta.get_module_from_string(name, exit_on_fail=False) # Error if unable to find the module in python if module is None: log.critical_exit( "Could not find module %s (in %s)" % (name, file_name)) # Check for dependecies and skip if missing from restapi.services.detect import detector for var in conf.pop('depends_on', []): negate = '' pieces = var.strip().split(' ') pieces_num = len(pieces) if pieces_num == 1: dependency = pieces.pop() elif pieces_num == 2: negate, dependency = pieces else: log.exit('Wrong parameter: %s', var) check = detector.get_bool_from_os(dependency) # Enable the possibility to depend on not having a variable if negate.lower() == 'not': check = not check # Skip if not meeting the requirements of the dependency if not check: if not self._testing: log.warning("Skip '%s': unmet %s", default_uri, dependency) return endpoint # Get the class from the module endpoint.cls = self._meta.get_class_from_string(class_name, module) if endpoint.cls is None: log.critical("Could not extract python class '%s'", class_name) return endpoint else: endpoint.exists = True # Is this a base or a custom class? endpoint.isbase = isbase # DEPRECATED # endpoint.instance = endpoint.cls() # Global tags # to be applied to all methods endpoint.tags = conf.pop('labels', []) # base URI base = conf.pop('baseuri', API_URL) if base not in BASE_URLS: log.warning("Invalid base %s", base) base = API_URL base = base.strip('/') ##################### # MAPPING schema = conf.pop('schema', {}) mappings = conf.pop('mapping', []) if len(mappings) < 1: raise KeyError("Missing 'mapping' section") endpoint.uris = {} # attrs python lib bug? endpoint.custom['schema'] = { 'expose': schema.get('expose', False), 'publish': {} } for label, uri in mappings.items(): # BUILD URI total_uri = '/%s%s' % (base, uri) endpoint.uris[label] = total_uri # If SCHEMA requested create if endpoint.custom['schema']['expose']: schema_uri = '%s%s%s' % (API_URL, '/schemas', uri) p = hex(id(endpoint.cls)) self._schema_endpoint.uris[label + p] = schema_uri endpoint.custom['schema']['publish'][label] = \ schema.get('publish', False) self._schemas_map[schema_uri] = total_uri # Description for path parameters endpoint.ids = conf.pop('ids', {}) # Check if something strange is still in configuration if len(conf) > 0: raise KeyError("Unwanted keys: %s" % list(conf.keys())) return endpoint
def test_failures(): meta = Meta() try: meta.import_submodules_from_package("utilities_bla") except AttributeError: pass else: pytest.fail("This call should fail and raise and AttributeError") try: meta.import_submodules_from_package("utilities_bla", exit_if_not_found=True) except AttributeError: pass else: pytest.fail("This call should fail and raise and AttributeError") try: meta.import_submodules_from_package("utilities_bla", exit_if_not_found=True, exit_on_fail=True) except AttributeError: pass else: pytest.fail("This call should fail and raise and AttributeError") try: meta.import_submodules_from_package("utilities_bla", exit_on_fail=True) except AttributeError: pass else: pytest.fail("This call should fail and raise and AttributeError") module = meta.get_module_from_string("utilities.metabla") assert module is None try: module = meta.get_module_from_string("utilities.metabla", exit_if_not_found=True) except SystemExit: pass else: pytest.fail("This call should fail and exit") try: module = meta.get_module_from_string("utilities.metabla", exit_if_not_found=True, exit_on_fail=True) except SystemExit: pass else: pytest.fail("This call should fail and exit") try: module = meta.get_module_from_string("utilities.metabla", exit_if_not_found=True, exit_on_fail=True, debug_on_fail=True) except SystemExit: pass else: pytest.fail("This call should fail and exit") # FIXME: unable to test exit_on_fail... we need a moule with import errors? module = meta.get_module_from_string("utilities.metabla", exit_on_fail=True) assert module is None module = meta.get_module_from_string("utilities.metabla", debug_on_fail=True) assert module is None
class Detector(object): def __init__(self, config_file_name='services'): self.authentication_service = None self.authentication_name = 'authentication' self.task_service_name = 'celery' self.modules = [] self.services_configuration = [] self.services = {} self.services_classes = {} self.extensions_instances = {} self.available_services = {} self.meta = Meta() self.check_configuration(config_file_name) self.load_classes() @staticmethod def get_global_var(key, default=None): return os.environ.get(key, default) @staticmethod @lru_cache(maxsize=None) # avoid calling it twice for the same var def get_bool_from_os(name): bool_var = os.environ.get(name, False) if isinstance(bool_var, bool): return bool_var # if not directly a bool, try an interpretation # INTEGERS try: tmp = int(bool_var) return bool(tmp) except ValueError: pass # STRINGS # any non empty string with a least one char # has to be considered True if isinstance(bool_var, str) and len(bool_var) > 0: return True return False @staticmethod # @lru_cache(maxsize=None) def prefix_name(service): return \ service.get('name'), \ service.get('prefix').lower() + '_' def check_configuration(self, config_file_name): self.services_configuration = load_yaml_file( file=config_file_name, path=os.path.join(helpers.script_abspath(__file__), '..', '..', CORE_CONFIG_PATH), logger=True) for service in self.services_configuration: name, prefix = self.prefix_name(service) # Was this service enabled from the developer? enable_var = str(prefix + 'enable').upper() self.available_services[name] = self.get_bool_from_os(enable_var) if self.available_services[name]: # read variables variables = self.load_variables(service, enable_var, prefix) service['variables'] = variables # set auth service if name == self.authentication_name: self.authentication_service = variables.get('service') # log.pp(self.services_configuration) if self.authentication_service is None: raise AttributeError("no service defined behind authentication") else: log.info("Authentication based on '%s' service" % self.authentication_service) def load_group(self, label): variables = {} for var, value in os.environ.items(): var = var.lower() if var.startswith(label): key = var[len(label):].strip('_') value = value.strip('"').strip("'") variables[key] = value return variables def output_service_variables(self, service_name): service_class = self.services_classes.get(service_name, {}) try: return service_class.variables except BaseException: return {} @staticmethod def load_group(label): from utilities.basher import detect_vargroup return detect_vargroup(label) def output_service_variables(self, service_name): service_class = self.services_classes.get(service_name, {}) try: return service_class.variables except BaseException: return {} @staticmethod def load_variables(service, enable_var=None, prefix=None): variables = {} host = None if prefix is None: _, prefix = Detector.prefix_name(service) for var, value in os.environ.items(): if enable_var is not None and var == enable_var: continue var = var.lower() # This is the case when a variable belongs to a service 'prefix' if var.startswith(prefix): # Fix key and value before saving key = var[len(prefix):] # One thing that we must avoid is any quote around our value value = value.strip('"').strip("'") # save variables[key] = value if key == 'host': host = value # Verify if service is EXTERNAL variables['external'] = False if isinstance(host, str): # and host.count('.') > 2: if not host.endswith('dockerized.io'): variables['external'] = True log.very_verbose("Service %s detected as external:\n%s" % (service, host)) return variables def load_class_from_module(self, classname='BaseInjector', service=None): if service is None: flaskext = '' else: flaskext = '.' + service.get('extension') # Try inside our extensions module = self.meta.get_module_from_string( modulestring=BACKEND_PACKAGE + '.flask_ext' + flaskext, exit_on_fail=True) if module is None: log.critical_exit("Missing %s for %s" % (flaskext, service)) return getattr(module, classname) def load_classes(self): for service in self.services_configuration: name, _ = self.prefix_name(service) if not self.available_services.get(name): continue log.very_verbose("Looking for class %s" % name) variables = service.get('variables') ext_name = service.get('class') # Get the existing class MyClass = self.load_class_from_module(ext_name, service=service) try: # Passing variables MyClass.set_variables(variables) # Passing models if service.get('load_models'): MyClass.set_models( self.meta.import_models(name, custom=False), self.meta.import_models(name, custom=True, exit_on_fail=False)) else: log.very_verbose("Skipping models") except AttributeError: log.critical_exit('Extension class %s ' % ext_name + 'not compliant: missing method(s)' + 'Did you extend "%s"?' % 'BaseExtension') # Save self.services_classes[name] = MyClass log.debug("Got class definition for %s", MyClass) if len(self.services_classes) < 1: raise KeyError("No classes were recovered!") return self.services_classes def init_services(self, app, worker_mode=False, project_init=False, project_clean=False): instances = {} auth_backend = None for service in self.services_configuration: name, _ = self.prefix_name(service) if not self.available_services.get(name): continue if name == self.authentication_name and auth_backend is None: if self.authentication_service is None: log.warning("No authentication") continue else: log.exit("Auth service '%s' seems unreachable" % self.authentication_service) args = {} if name == self.task_service_name: args['worker_mode'] = worker_mode # Get extension class and build the extension object ExtClass = self.services_classes.get(name) try: ext_instance = ExtClass(app, **args) except TypeError as e: log.critical_exit('Your class %s is not compliant:\n%s' % (name, e)) else: self.extensions_instances[name] = ext_instance # Initialize the real service getting the first service object log.debug("Initializing %s", name) service_instance = ext_instance.custom_init(pinit=project_init, pdestroy=project_clean, abackend=auth_backend) instances[name] = service_instance if name == self.authentication_service: auth_backend = service_instance self.extensions_instances[name] = ext_instance # Injecting into the Celery Extension Class # all celery tasks found in *vanilla_package/tasks* if name == self.task_service_name: task_package = "%s.tasks" % CUSTOM_PACKAGE submodules = self.meta.import_submodules_from_package( task_package, exit_on_fail=True) for submodule in submodules: tasks = self.meta.get_celery_tasks_from_module(submodule) for func_name, funct in tasks.items(): setattr(ExtClass, func_name, funct) if len(self.extensions_instances) < 1: raise KeyError("No instances available for modules") # Only once in a lifetime if project_init: self.project_initialization(instances) return self.extensions_instances def load_injector_modules(self): for service in self.services_configuration: name, _ = self.prefix_name(service) if not self.available_services.get(name): continue # Module for injection ModuleBaseClass = self.load_class_from_module() # Create modules programmatically 8) MyModule = self.meta.metaclassing(ModuleBaseClass, service.get('injector')) # Recover class MyClass = self.services_classes.get(name) if MyClass is None: raise AttributeError("No class found for %s" % name) MyModule.set_extension_class(MyClass) self.modules.append(MyModule) return self.modules def check_availability(self, name): if '.' in name: # In this case we are receiving a module name # e.g. restapi.services.mongodb name = name.split('.')[::-1][0] return self.available_services.get(name) @classmethod def project_initialization(self, instances): """ Custom initialization of your project Please define your class Initializer in vanilla/project/initialization.py """ try: meta = Meta() module_path = "%s.%s.%s" % \ (CUSTOM_PACKAGE, 'initialization', 'initialization') module = meta.get_module_from_string( module_path, debug_on_fail=False, ) Initializer = meta.get_class_from_string('Initializer', module, skip_error=True) if Initializer is None: log.debug("No custom init available") else: try: Initializer(instances) except BaseException as e: log.error("Errors during custom initialization: %s", e) else: log.info("Vanilla project has been initialized") except BaseException: log.debug("No custom init available")