def project_initialization(self, instances):
""" Custom initialization of your project
Please define your class Initializer in
vanilla/project/initialization.py
"""
try:
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Initializer = meta.get_class_from_string('Initializer',
module,
skip_error=True)
if Initializer is None:
log.debug("No custom init available")
else:
try:
Initializer(instances)
except BaseException as e:
log.error("Errors during custom initialization: %s", e)
else:
log.info("Vanilla project has been initialized")
except BaseException:
log.debug("No custom init available")
def test():
meta = Meta()
cls_name = "Meta"
import utilities
modules = meta.get_submodules_from_package(utilities)
assert "meta" in modules
sub_modules = meta.import_submodules_from_package("utilities")
assert sub_modules is not None
assert isinstance(sub_modules, list)
assert len(sub_modules) > 0
module = meta.get_module_from_string("utilities.meta")
assert module is not None
assert hasattr(module, cls_name)
classes = meta.get_classes_from_module(module)
assert classes is not None
assert isinstance(classes, dict)
assert cls_name in classes
assert classes[cls_name].__name__ == cls_name
new_classes = meta.get_new_classes_from_module(module)
assert new_classes is not None
assert isinstance(new_classes, dict)
assert cls_name in new_classes
assert new_classes[cls_name].__name__ == cls_name
meta_class = meta.get_class_from_string(cls_name, module)
assert meta_class is not None
assert meta_class.__name__ == cls_name
methods = meta.get_methods_inside_instance(meta, private_methods=True)
assert methods is not None
assert isinstance(methods, dict)
# it is a static method
assert "get_methods_inside_instance" not in methods
assert "get_authentication_module" in methods
metacls = meta.metaclassing(Meta, "Test")
assert metacls is not None
assert metacls.__name__ == "Test"
self_ref = meta.get_self_reference_from_args(meta, "test", 1)
assert self_ref == meta
def custom_post_handle_user_input(self, user_node, input_data):
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Customizer = meta.get_class_from_string('Customizer',
module,
skip_error=True)
if Customizer is None:
log.debug("No user properties customizer available")
else:
try:
Customizer().custom_post_handle_user_input(
self, user_node, input_data)
except BaseException as e:
log.error("Unable to customize user properties: %s", e)
def custom_user_properties(self, userdata):
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Customizer = meta.get_class_from_string('Customizer',
module,
skip_error=True)
if Customizer is None:
log.debug("No user properties customizer available")
else:
try:
userdata = Customizer().custom_user_properties(userdata)
except BaseException as e:
log.error("Unable to customize user properties: %s", e)
if "email" in userdata:
userdata["email"] = userdata["email"].lower()
return userdata
class Customizer(object):
"""
Customize your BACKEND:
Read all of available configurations and definitions.
"""
def __init__(self, testing=False, production=False, init=False):
# Input
self._testing = testing
self._production = production
self._initiliazing = init
# Some initialization
self._endpoints = []
self._definitions = {}
self._configurations = {}
self._query_params = {}
self._schemas_map = {}
self._meta = Meta()
# Do things
self.read_configuration()
if not self._initiliazing:
self.do_schema()
self.find_endpoints()
self.do_swagger()
def read_configuration(self):
##################
# Reading configuration
default_file_path = helpers.current_dir(CONF_PATH)
project_file_path = helpers.current_dir(CONF_PATH)
self._configurations = configuration.read(
default_file_path,
project_path=project_file_path,
)
def do_schema(self):
""" Schemas exposing, if requested """
name = '%s.%s.%s' % (BACKEND_PACKAGE, 'rest', 'schema')
module = self._meta.get_module_from_string(
name,
exit_if_not_found=True
)
schema_class = getattr(module, 'RecoverSchema')
self._schema_endpoint = EndpointElements(
cls=schema_class,
exists=True,
custom={
'methods': {
'get': ExtraAttributes(auth=None),
# WHY DOES POST REQUEST AUTHENTICATION
# 'post': ExtraAttributes(auth=None)
}
},
methods={}
)
# TODO: find a way to map authentication
# as in the original endpoint for the schema 'get' method
# TODO: find a way to publish on swagger the schema
# if endpoint is enabled to publish and the developer asks for it
def find_endpoints(self):
##################
# Walk swagger directories looking for endpoints
# FIXME: how to do this?
# from utilities import helpers
# custom_dir = helpers.current_dir(CUSTOM_PACKAGE)
# base_dir = helpers.script_abspath(__file__)
base_swagger_confdir = helpers.script_abspath(__file__)
custom_swagger_confdir = helpers.current_dir(CUSTOM_PACKAGE)
# for base_dir in [BACKEND_PACKAGE, CUSTOM_PACKAGE]:
for base_dir in [base_swagger_confdir, custom_swagger_confdir]:
swagger_dir = os.path.join(base_dir, 'swagger')
log.verbose("Swagger dir: %s" % swagger_dir)
for ep in os.listdir(swagger_dir):
swagger_endpoint_dir = os.path.join(swagger_dir, ep)
if os.path.isfile(swagger_endpoint_dir):
exception = '%s.yaml' % SWAGGER_MODELS_FILE
if not swagger_endpoint_dir.endswith('/' + exception):
log.debug(
"Found a file instead of a folder: %s",
swagger_endpoint_dir
)
continue
# isbase = base_dir == BACKEND_PACKAGE
isbase = base_dir.startswith('/usr/local')
base_module = helpers.last_dir(base_dir)
from utilities import ENDPOINTS_CODE_DIR
if isbase:
apiclass_module = '%s.%s' % (base_module, 'resources')
else:
apiclass_module = '%s.%s' % (
base_module, ENDPOINTS_CODE_DIR)
current = self.lookup(
ep, apiclass_module, swagger_endpoint_dir, isbase)
if current is not None and current.exists:
# Add endpoint to REST mapping
self._endpoints.append(current)
def do_swagger(self):
# SWAGGER read endpoints definition
swag = BeSwagger(self._endpoints, self)
swag_dict = swag.swaggerish()
# TODO: update internal endpoints from swagger
self._endpoints = swag._endpoints[:]
# SWAGGER validation
if not swag.validation(swag_dict):
log.critical_exit("Current swagger definition is invalid")
self._definitions = swag_dict
def read_frameworks(self):
file = os.path.join("config", "frameworks.yaml")
self._frameworks = load_yaml_file(file)
def lookup(self, endpoint, apiclass_module, swagger_endpoint_dir, isbase):
log.verbose("Found endpoint dir: '%s'" % endpoint)
if os.path.exists(os.path.join(swagger_endpoint_dir, 'SKIP')):
log.info("Skipping: %s", endpoint)
return None
# Find yaml files
conf = None
yaml_files = {}
yaml_listing = os.path.join(swagger_endpoint_dir, "*.%s" % YAML_EXT)
for file in glob.glob(yaml_listing):
if file.endswith('specs.%s' % YAML_EXT):
# load configuration and find file and class
conf = load_yaml_file(file)
else:
# add file to be loaded from swagger extension
p = re.compile(r'\/([^\.\/]+)\.' + YAML_EXT + '$')
match = p.search(file)
method = match.groups()[0]
yaml_files[method] = file
if len(yaml_files) < 1:
raise Exception("%s: no methods defined in any YAML" % endpoint)
if conf is None or 'class' not in conf:
raise ValueError("No 'class' defined for '%s'" % endpoint)
current = self.load_endpoint(endpoint, apiclass_module, conf, isbase)
current.methods = yaml_files
return current
# def read_complex_config(self, configfile):
# """ A more complex configuration is available in JSON format """
# content = {}
# with open(configfile) as fp:
# content = json.load(fp)
# return content
def load_endpoint(self, default_uri, apiclass_module, conf, isbase):
endpoint = EndpointElements(custom={})
# Load the endpoint class defined in the YAML file
file_name = conf.pop('file', default_uri)
class_name = conf.pop('class')
name = '%s.%s' % (apiclass_module, file_name)
module = self._meta.get_module_from_string(name, exit_on_fail=False)
# Error if unable to find the module in python
if module is None:
log.critical_exit(
"Could not find module %s (in %s)" % (name, file_name))
# Check for dependecies and skip if missing
from restapi.services.detect import detector
for var in conf.pop('depends_on', []):
negate = ''
pieces = var.strip().split(' ')
pieces_num = len(pieces)
if pieces_num == 1:
dependency = pieces.pop()
elif pieces_num == 2:
negate, dependency = pieces
else:
log.exit('Wrong parameter: %s', var)
check = detector.get_bool_from_os(dependency)
# Enable the possibility to depend on not having a variable
if negate.lower() == 'not':
check = not check
# Skip if not meeting the requirements of the dependency
if not check:
if not self._testing:
log.warning("Skip '%s': unmet %s", default_uri, dependency)
return endpoint
# Get the class from the module
endpoint.cls = self._meta.get_class_from_string(class_name, module)
if endpoint.cls is None:
log.critical("Could not extract python class '%s'", class_name)
return endpoint
else:
endpoint.exists = True
# Is this a base or a custom class?
endpoint.isbase = isbase
# DEPRECATED
# endpoint.instance = endpoint.cls()
# Global tags
# to be applied to all methods
endpoint.tags = conf.pop('labels', [])
# base URI
base = conf.pop('baseuri', API_URL)
if base not in BASE_URLS:
log.warning("Invalid base %s", base)
base = API_URL
base = base.strip('/')
#####################
# MAPPING
schema = conf.pop('schema', {})
mappings = conf.pop('mapping', [])
if len(mappings) < 1:
raise KeyError("Missing 'mapping' section")
endpoint.uris = {} # attrs python lib bug?
endpoint.custom['schema'] = {
'expose': schema.get('expose', False),
'publish': {}
}
for label, uri in mappings.items():
# BUILD URI
total_uri = '/%s%s' % (base, uri)
endpoint.uris[label] = total_uri
# If SCHEMA requested create
if endpoint.custom['schema']['expose']:
schema_uri = '%s%s%s' % (API_URL, '/schemas', uri)
p = hex(id(endpoint.cls))
self._schema_endpoint.uris[label + p] = schema_uri
endpoint.custom['schema']['publish'][label] = \
schema.get('publish', False)
self._schemas_map[schema_uri] = total_uri
# Description for path parameters
endpoint.ids = conf.pop('ids', {})
# Check if something strange is still in configuration
if len(conf) > 0:
raise KeyError("Unwanted keys: %s" % list(conf.keys()))
return endpoint
def test_failures():
meta = Meta()
try:
meta.import_submodules_from_package("utilities_bla")
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
try:
meta.import_submodules_from_package("utilities_bla",
exit_if_not_found=True)
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
try:
meta.import_submodules_from_package("utilities_bla",
exit_if_not_found=True,
exit_on_fail=True)
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
try:
meta.import_submodules_from_package("utilities_bla", exit_on_fail=True)
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
module = meta.get_module_from_string("utilities.metabla")
assert module is None
try:
module = meta.get_module_from_string("utilities.metabla",
exit_if_not_found=True)
except SystemExit:
pass
else:
pytest.fail("This call should fail and exit")
try:
module = meta.get_module_from_string("utilities.metabla",
exit_if_not_found=True,
exit_on_fail=True)
except SystemExit:
pass
else:
pytest.fail("This call should fail and exit")
try:
module = meta.get_module_from_string("utilities.metabla",
exit_if_not_found=True,
exit_on_fail=True,
debug_on_fail=True)
except SystemExit:
pass
else:
pytest.fail("This call should fail and exit")
# FIXME: unable to test exit_on_fail... we need a moule with import errors?
module = meta.get_module_from_string("utilities.metabla",
exit_on_fail=True)
assert module is None
module = meta.get_module_from_string("utilities.metabla",
debug_on_fail=True)
assert module is None
class Detector(object):
def __init__(self, config_file_name='services'):
self.authentication_service = None
self.authentication_name = 'authentication'
self.task_service_name = 'celery'
self.modules = []
self.services_configuration = []
self.services = {}
self.services_classes = {}
self.extensions_instances = {}
self.available_services = {}
self.meta = Meta()
self.check_configuration(config_file_name)
self.load_classes()
@staticmethod
def get_global_var(key, default=None):
return os.environ.get(key, default)
@staticmethod
@lru_cache(maxsize=None) # avoid calling it twice for the same var
def get_bool_from_os(name):
bool_var = os.environ.get(name, False)
if isinstance(bool_var, bool):
return bool_var
# if not directly a bool, try an interpretation
# INTEGERS
try:
tmp = int(bool_var)
return bool(tmp)
except ValueError:
pass
# STRINGS
# any non empty string with a least one char
# has to be considered True
if isinstance(bool_var, str) and len(bool_var) > 0:
return True
return False
@staticmethod
# @lru_cache(maxsize=None)
def prefix_name(service):
return \
service.get('name'), \
service.get('prefix').lower() + '_'
def check_configuration(self, config_file_name):
self.services_configuration = load_yaml_file(
file=config_file_name,
path=os.path.join(helpers.script_abspath(__file__), '..', '..',
CORE_CONFIG_PATH),
logger=True)
for service in self.services_configuration:
name, prefix = self.prefix_name(service)
# Was this service enabled from the developer?
enable_var = str(prefix + 'enable').upper()
self.available_services[name] = self.get_bool_from_os(enable_var)
if self.available_services[name]:
# read variables
variables = self.load_variables(service, enable_var, prefix)
service['variables'] = variables
# set auth service
if name == self.authentication_name:
self.authentication_service = variables.get('service')
# log.pp(self.services_configuration)
if self.authentication_service is None:
raise AttributeError("no service defined behind authentication")
else:
log.info("Authentication based on '%s' service" %
self.authentication_service)
def load_group(self, label):
variables = {}
for var, value in os.environ.items():
var = var.lower()
if var.startswith(label):
key = var[len(label):].strip('_')
value = value.strip('"').strip("'")
variables[key] = value
return variables
def output_service_variables(self, service_name):
service_class = self.services_classes.get(service_name, {})
try:
return service_class.variables
except BaseException:
return {}
@staticmethod
def load_group(label):
from utilities.basher import detect_vargroup
return detect_vargroup(label)
def output_service_variables(self, service_name):
service_class = self.services_classes.get(service_name, {})
try:
return service_class.variables
except BaseException:
return {}
@staticmethod
def load_variables(service, enable_var=None, prefix=None):
variables = {}
host = None
if prefix is None:
_, prefix = Detector.prefix_name(service)
for var, value in os.environ.items():
if enable_var is not None and var == enable_var:
continue
var = var.lower()
# This is the case when a variable belongs to a service 'prefix'
if var.startswith(prefix):
# Fix key and value before saving
key = var[len(prefix):]
# One thing that we must avoid is any quote around our value
value = value.strip('"').strip("'")
# save
variables[key] = value
if key == 'host':
host = value
# Verify if service is EXTERNAL
variables['external'] = False
if isinstance(host, str): # and host.count('.') > 2:
if not host.endswith('dockerized.io'):
variables['external'] = True
log.very_verbose("Service %s detected as external:\n%s" %
(service, host))
return variables
def load_class_from_module(self, classname='BaseInjector', service=None):
if service is None:
flaskext = ''
else:
flaskext = '.' + service.get('extension')
# Try inside our extensions
module = self.meta.get_module_from_string(
modulestring=BACKEND_PACKAGE + '.flask_ext' + flaskext,
exit_on_fail=True)
if module is None:
log.critical_exit("Missing %s for %s" % (flaskext, service))
return getattr(module, classname)
def load_classes(self):
for service in self.services_configuration:
name, _ = self.prefix_name(service)
if not self.available_services.get(name):
continue
log.very_verbose("Looking for class %s" % name)
variables = service.get('variables')
ext_name = service.get('class')
# Get the existing class
MyClass = self.load_class_from_module(ext_name, service=service)
try:
# Passing variables
MyClass.set_variables(variables)
# Passing models
if service.get('load_models'):
MyClass.set_models(
self.meta.import_models(name, custom=False),
self.meta.import_models(name,
custom=True,
exit_on_fail=False))
else:
log.very_verbose("Skipping models")
except AttributeError:
log.critical_exit('Extension class %s ' % ext_name +
'not compliant: missing method(s)' +
'Did you extend "%s"?' % 'BaseExtension')
# Save
self.services_classes[name] = MyClass
log.debug("Got class definition for %s", MyClass)
if len(self.services_classes) < 1:
raise KeyError("No classes were recovered!")
return self.services_classes
def init_services(self,
app,
worker_mode=False,
project_init=False,
project_clean=False):
instances = {}
auth_backend = None
for service in self.services_configuration:
name, _ = self.prefix_name(service)
if not self.available_services.get(name):
continue
if name == self.authentication_name and auth_backend is None:
if self.authentication_service is None:
log.warning("No authentication")
continue
else:
log.exit("Auth service '%s' seems unreachable" %
self.authentication_service)
args = {}
if name == self.task_service_name:
args['worker_mode'] = worker_mode
# Get extension class and build the extension object
ExtClass = self.services_classes.get(name)
try:
ext_instance = ExtClass(app, **args)
except TypeError as e:
log.critical_exit('Your class %s is not compliant:\n%s' %
(name, e))
else:
self.extensions_instances[name] = ext_instance
# Initialize the real service getting the first service object
log.debug("Initializing %s", name)
service_instance = ext_instance.custom_init(pinit=project_init,
pdestroy=project_clean,
abackend=auth_backend)
instances[name] = service_instance
if name == self.authentication_service:
auth_backend = service_instance
self.extensions_instances[name] = ext_instance
# Injecting into the Celery Extension Class
# all celery tasks found in *vanilla_package/tasks*
if name == self.task_service_name:
task_package = "%s.tasks" % CUSTOM_PACKAGE
submodules = self.meta.import_submodules_from_package(
task_package, exit_on_fail=True)
for submodule in submodules:
tasks = self.meta.get_celery_tasks_from_module(submodule)
for func_name, funct in tasks.items():
setattr(ExtClass, func_name, funct)
if len(self.extensions_instances) < 1:
raise KeyError("No instances available for modules")
# Only once in a lifetime
if project_init:
self.project_initialization(instances)
return self.extensions_instances
def load_injector_modules(self):
for service in self.services_configuration:
name, _ = self.prefix_name(service)
if not self.available_services.get(name):
continue
# Module for injection
ModuleBaseClass = self.load_class_from_module()
# Create modules programmatically 8)
MyModule = self.meta.metaclassing(ModuleBaseClass,
service.get('injector'))
# Recover class
MyClass = self.services_classes.get(name)
if MyClass is None:
raise AttributeError("No class found for %s" % name)
MyModule.set_extension_class(MyClass)
self.modules.append(MyModule)
return self.modules
def check_availability(self, name):
if '.' in name:
# In this case we are receiving a module name
# e.g. restapi.services.mongodb
name = name.split('.')[::-1][0]
return self.available_services.get(name)
@classmethod
def project_initialization(self, instances):
""" Custom initialization of your project
Please define your class Initializer in
vanilla/project/initialization.py
"""
try:
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Initializer = meta.get_class_from_string('Initializer',
module,
skip_error=True)
if Initializer is None:
log.debug("No custom init available")
else:
try:
Initializer(instances)
except BaseException as e:
log.error("Errors during custom initialization: %s", e)
else:
log.info("Vanilla project has been initialized")
except BaseException:
log.debug("No custom init available")