def project_initialization(self, instances):
""" Custom initialization of your project
Please define your class Initializer in
vanilla/project/initialization.py
"""
try:
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Initializer = meta.get_class_from_string('Initializer',
module,
skip_error=True)
if Initializer is None:
log.debug("No custom init available")
else:
try:
Initializer(instances)
except BaseException as e:
log.error("Errors during custom initialization: %s", e)
else:
log.info("Vanilla project has been initialized")
except BaseException:
log.debug("No custom init available")
def get(self, test_num, task_id=None):
celery = self.get_service_instance('celery')
meta = Meta()
methods = meta.get_methods_inside_instance(self)
method_name = "test_%s" % test_num
if method_name not in methods:
raise RestApiException("Test %d not found" % test_num)
method = methods[method_name]
out = method(celery, task_id)
return self.force_response(out)
def __init__(self, config_file_name='services'):
self.authentication_service = None
self.authentication_name = 'authentication'
self.task_service_name = 'celery'
self.modules = []
self.services_configuration = []
self.services = {}
self.services_classes = {}
self.extensions_instances = {}
self.available_services = {}
self.meta = Meta()
self.check_configuration(config_file_name)
self.load_classes()
def test():
meta = Meta()
cls_name = "Meta"
import utilities
modules = meta.get_submodules_from_package(utilities)
assert "meta" in modules
sub_modules = meta.import_submodules_from_package("utilities")
assert sub_modules is not None
assert isinstance(sub_modules, list)
assert len(sub_modules) > 0
module = meta.get_module_from_string("utilities.meta")
assert module is not None
assert hasattr(module, cls_name)
classes = meta.get_classes_from_module(module)
assert classes is not None
assert isinstance(classes, dict)
assert cls_name in classes
assert classes[cls_name].__name__ == cls_name
new_classes = meta.get_new_classes_from_module(module)
assert new_classes is not None
assert isinstance(new_classes, dict)
assert cls_name in new_classes
assert new_classes[cls_name].__name__ == cls_name
meta_class = meta.get_class_from_string(cls_name, module)
assert meta_class is not None
assert meta_class.__name__ == cls_name
methods = meta.get_methods_inside_instance(meta, private_methods=True)
assert methods is not None
assert isinstance(methods, dict)
# it is a static method
assert "get_methods_inside_instance" not in methods
assert "get_authentication_module" in methods
metacls = meta.metaclassing(Meta, "Test")
assert metacls is not None
assert metacls.__name__ == "Test"
self_ref = meta.get_self_reference_from_args(meta, "test", 1)
assert self_ref == meta
def custom_connection(self, **kwargs):
if len(kwargs) > 0:
print("TODO: use args for connection?", kwargs)
uri = 'postgresql://%s:%s@%s:%s/%s' % (
self.variables.get('user'), self.variables.get('password'),
self.variables.get('host'), self.variables.get('port'),
self.variables.get('db'))
log.very_verbose("URI IS %s" % re_obscure_pattern(uri))
# TODO: in case we need different connection binds
# (multiple connections with sql) then:
# SQLALCHEMY_BINDS = {
# 'users': 'mysqldb://localhost/users',
# 'appmeta': 'sqlite:////path/to/appmeta.db'
# }
self.app.config['SQLALCHEMY_POOL_TIMEOUT'] = 3
self.app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
self.app.config['SQLALCHEMY_DATABASE_URI'] = uri
pool_size = self.variables.get('poolsize')
if pool_size is not None:
# sqlalchemy docs: http://j.mp/2xT0GOc
# defaults: overflow=10, pool_size=5
self.app.config['SQLALCHEMY_MAX_OVERFLOW'] = 0
self.app.config['SQLALCHEMY_POOL_SIZE'] = int(pool_size)
obj_name = 'db'
m = Meta()
# search the original sqlalchemy object into models
db = m.obj_from_models(obj_name, self.name, CUSTOM_PACKAGE)
if db is None:
log.warning("No sqlalchemy db imported in custom package")
db = m.obj_from_models(obj_name, self.name, BACKEND_PACKAGE)
if db is None:
log.critical_exit("Could not get %s within %s models" %
(obj_name, self.name))
return db
def custom_post_handle_user_input(self, user_node, input_data):
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Customizer = meta.get_class_from_string('Customizer',
module,
skip_error=True)
if Customizer is None:
log.debug("No user properties customizer available")
else:
try:
Customizer().custom_post_handle_user_input(
self, user_node, input_data)
except BaseException as e:
log.error("Unable to customize user properties: %s", e)
def __init__(self, testing=False, production=False, init=False):
# Input
self._testing = testing
self._production = production
self._initiliazing = init
# Some initialization
self._endpoints = []
self._definitions = {}
self._configurations = {}
self._query_params = {}
self._schemas_map = {}
self._meta = Meta()
# Do things
self.read_configuration()
if not self._initiliazing:
self.do_schema()
self.find_endpoints()
self.do_swagger()
def custom_user_properties(self, userdata):
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Customizer = meta.get_class_from_string('Customizer',
module,
skip_error=True)
if Customizer is None:
log.debug("No user properties customizer available")
else:
try:
userdata = Customizer().custom_user_properties(userdata)
except BaseException as e:
log.error("Unable to customize user properties: %s", e)
if "email" in userdata:
userdata["email"] = userdata["email"].lower()
return userdata
def test_failures():
meta = Meta()
try:
meta.import_submodules_from_package("utilities_bla")
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
try:
meta.import_submodules_from_package("utilities_bla",
exit_if_not_found=True)
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
try:
meta.import_submodules_from_package("utilities_bla",
exit_if_not_found=True,
exit_on_fail=True)
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
try:
meta.import_submodules_from_package("utilities_bla", exit_on_fail=True)
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
module = meta.get_module_from_string("utilities.metabla")
assert module is None
try:
module = meta.get_module_from_string("utilities.metabla",
exit_if_not_found=True)
except SystemExit:
pass
else:
pytest.fail("This call should fail and exit")
try:
module = meta.get_module_from_string("utilities.metabla",
exit_if_not_found=True,
exit_on_fail=True)
except SystemExit:
pass
else:
pytest.fail("This call should fail and exit")
try:
module = meta.get_module_from_string("utilities.metabla",
exit_if_not_found=True,
exit_on_fail=True,
debug_on_fail=True)
except SystemExit:
pass
else:
pytest.fail("This call should fail and exit")
# FIXME: unable to test exit_on_fail... we need a moule with import errors?
module = meta.get_module_from_string("utilities.metabla",
exit_on_fail=True)
assert module is None
module = meta.get_module_from_string("utilities.metabla",
debug_on_fail=True)
assert module is None
from utilities.logs import get_logger
log = get_logger(__name__)
################################################
# Reload Flask app code also for the worker
# This is necessary to have the app context available
app = create_app(worker_mode=True)
celery_app = app.extensions.get('celery').celery_app
celery_app.app = app
def get_service(service):
return celery_app.app.extensions.get(service).get_instance()
celery_app.get_service = get_service
################################################
# Import tasks modules to make sure all tasks are available
meta = Meta()
# main_package = "commons.tasks."
# # Base tasks
# submodules = meta.import_submodules_from_package(main_package + "base")
# # Custom tasks
submodules = meta.import_submodules_from_package("%s.tasks" % CUSTOM_PACKAGE)
log.debug("Celery worker is ready %s", celery_app)
class BaseExtension(metaclass=abc.ABCMeta):
models = {} # I get models on a cls level, instead of instances
meta = Meta()
def __init__(self, app=None, **kwargs):
self.objs = {}
self.set_name()
self.args = kwargs
self.app = app
if app is not None:
self.init_app(app)
def set_name(self):
""" a different name for each extended object """
self.name = self.__class__.__name__.lower()
log.very_verbose("Opening service instance of %s" % self.name)
@classmethod
def set_models(cls, base_models, custom_models):
# Join models as described by issue #16
cls.models = base_models
for key, model in custom_models.items():
# Verify if overriding
if key in base_models.keys():
original_model = base_models[key]
# Override
if issubclass(model, original_model):
log.very_verbose("Overriding model %s" % key)
cls.models[key] = model
continue
# Otherwise just append
cls.models[key] = model
if len(cls.models) > 0:
log.verbose("Loaded models")
@classmethod
def set_variables(cls, envvars):
cls.variables = envvars
def init_app(self, app):
app.teardown_appcontext(self.teardown)
def pre_object(self, ref, key):
""" Make sure reference and key are strings """
if ref is None:
ref = self.__class__.__name__
elif isinstance(ref, object):
ref = ref.__class__.__name__
elif not isinstance(ref, str):
ref = str(ref)
if not isinstance(key, str):
key = str(key)
return ref + key
def set_object(self, obj, key='[]', ref=None):
""" set object into internal array """
h = self.pre_object(ref, key)
self.objs[h] = obj
return obj
def get_object(self, key='[]', ref=None):
""" recover object if any """
h = self.pre_object(ref, key)
obj = self.objs.get(h, None)
return obj
def connect(self, **kwargs):
obj = None
# BEFORE
ok = self.pre_connection(**kwargs)
if not ok:
log.critical("Unable to make preconnection for %s", self.name)
return obj
# Try until it's connected
if len(kwargs) > 0:
obj = self.custom_connection(**kwargs)
else:
obj = self.retry()
log.info("Connected! %s", self.name)
# AFTER
self.post_connection(obj, **kwargs)
obj.connection_time = datetime.now()
return obj
def set_models_to_service(self, obj):
if len(self.models) < 1 and self.__class__.__name__ == 'NeoModel':
raise Exception()
for name, model in self.models.items():
# Save attribute inside class with the same name
log.very_verbose("Injecting model '%s'" % name)
setattr(obj, name, model)
obj.models = self.models
return obj
def set_connection_exception(self):
return None
def retry(self, retry_interval=3, max_retries=-1):
retry_count = 0
# Get the exception which will signal a missing connection
exceptions = self.set_connection_exception()
if exceptions is None:
exceptions = (BaseException, )
while max_retries != 0 or retry_count < max_retries:
retry_count += 1
if retry_count > 1:
log.verbose("testing again in %s secs", retry_interval)
try:
obj = self.custom_connection()
except exceptions as e:
log.error("Catched: %s(%s)", e.__class__.__name__, e)
# NOTE: if you critical_exit uwsgi will not show this line
log.critical("Service '%s' not available", self.name)
log.exit()
# raise e
else:
break
# Increment sleeps time if doing a lot of retries
if retry_count % 3 == 0:
log.debug("Incrementing interval")
retry_interval += retry_interval
return obj
def teardown(self, exception):
ctx = stack.top
if self.get_object(ref=ctx) is not None:
self.close_connection(ctx)
def get_instance(self, **kwargs):
# Parameters
global_instance = kwargs.pop('global_instance', False)
isauth = kwargs.pop('authenticator', False)
cache_expiration = kwargs.pop('cache_expiration', None)
# pinit = kwargs('project_initialization', False)
# Variables
obj = None
ctx = stack.top
ref = self
unique_hash = str(sorted(kwargs.items()))
log.very_verbose("instance hash: %s" % unique_hash)
# When not using the context, this is the first connection
if ctx is None:
# First connection, before any request
obj = self.connect()
if obj is None:
return None
# self.initialization(obj=obj)
self.set_object(obj=obj, ref=ref)
log.verbose("First connection for %s" % self.name)
else:
# isauth = 'Authenticator' == self.__class__.__name__
if not isauth:
if not global_instance:
ref = ctx
obj = self.get_object(ref=ref, key=unique_hash)
if obj is not None and cache_expiration is not None:
now = datetime.now()
exp = timedelta(seconds=cache_expiration)
if now < obj.connection_time + exp:
log.very_verbose("Cache still fresh for %s" % (self))
else:
log.warning("Cache expired for %s", (self))
obj = None
if obj is None:
obj = self.connect(**kwargs)
if obj is None:
return None
self.set_object(obj=obj, ref=ref, key=unique_hash)
else:
pass
log.very_verbose("Instance %s(%s)" % (ref.__class__.__name__, obj))
obj = self.set_models_to_service(obj)
return obj
############################
# OPTIONALLY
# to be executed only at init time?
def pre_connection(self, **kwargs):
return True
def post_connection(self, obj=None, **kwargs):
return True
def close_connection(self, ctx):
""" override this method if you must close
your connection after each request"""
# obj = self.get_object(ref=ctx)
# obj.close()
self.set_object(obj=None, ref=ctx) # it could be overidden
############################
# To be overridden
@abc.abstractmethod
def custom_connection(self, **kwargs):
return
############################
# Already has default
def custom_init(self, **kwargs):
"""
The real signature:
def custom_init(self, pinit=False, pdestroy=False, abackend=None):
- A backend is needed for non-standalone services
e.g. authentication module
- Project initialization/removal could be used here
or carried on to low levels;
they get activated by specific flask cli commands
"""
return self.get_instance()