def project_initialization(self, instances):
""" Custom initialization of your project
Please define your class Initializer in
vanilla/project/initialization.py
"""
try:
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Initializer = meta.get_class_from_string('Initializer',
module,
skip_error=True)
if Initializer is None:
log.debug("No custom init available")
else:
try:
Initializer(instances)
except BaseException as e:
log.error("Errors during custom initialization: %s", e)
else:
log.info("Vanilla project has been initialized")
except BaseException:
log.debug("No custom init available")
def get(self, test_num, task_id=None):
celery = self.get_service_instance('celery')
meta = Meta()
methods = meta.get_methods_inside_instance(self)
method_name = "test_%s" % test_num
if method_name not in methods:
raise RestApiException("Test %d not found" % test_num)
method = methods[method_name]
out = method(celery, task_id)
return self.force_response(out)
def __init__(self, config_file_name='services'):
self.authentication_service = None
self.authentication_name = 'authentication'
self.task_service_name = 'celery'
self.modules = []
self.services_configuration = []
self.services = {}
self.services_classes = {}
self.extensions_instances = {}
self.available_services = {}
self.meta = Meta()
self.check_configuration(config_file_name)
self.load_classes()
def custom_connection(self, **kwargs):
if len(kwargs) > 0:
print("TODO: use args for connection?", kwargs)
uri = 'postgresql://%s:%s@%s:%s/%s' % (
self.variables.get('user'), self.variables.get('password'),
self.variables.get('host'), self.variables.get('port'),
self.variables.get('db'))
log.very_verbose("URI IS %s" % re_obscure_pattern(uri))
# TODO: in case we need different connection binds
# (multiple connections with sql) then:
# SQLALCHEMY_BINDS = {
# 'users': 'mysqldb://localhost/users',
# 'appmeta': 'sqlite:////path/to/appmeta.db'
# }
self.app.config['SQLALCHEMY_POOL_TIMEOUT'] = 3
self.app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
self.app.config['SQLALCHEMY_DATABASE_URI'] = uri
pool_size = self.variables.get('poolsize')
if pool_size is not None:
# sqlalchemy docs: http://j.mp/2xT0GOc
# defaults: overflow=10, pool_size=5
self.app.config['SQLALCHEMY_MAX_OVERFLOW'] = 0
self.app.config['SQLALCHEMY_POOL_SIZE'] = int(pool_size)
obj_name = 'db'
m = Meta()
# search the original sqlalchemy object into models
db = m.obj_from_models(obj_name, self.name, CUSTOM_PACKAGE)
if db is None:
log.warning("No sqlalchemy db imported in custom package")
db = m.obj_from_models(obj_name, self.name, BACKEND_PACKAGE)
if db is None:
log.critical_exit("Could not get %s within %s models" %
(obj_name, self.name))
return db
def custom_post_handle_user_input(self, user_node, input_data):
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Customizer = meta.get_class_from_string('Customizer',
module,
skip_error=True)
if Customizer is None:
log.debug("No user properties customizer available")
else:
try:
Customizer().custom_post_handle_user_input(
self, user_node, input_data)
except BaseException as e:
log.error("Unable to customize user properties: %s", e)
def __init__(self, testing=False, production=False, init=False):
# Input
self._testing = testing
self._production = production
self._initiliazing = init
# Some initialization
self._endpoints = []
self._definitions = {}
self._configurations = {}
self._query_params = {}
self._schemas_map = {}
self._meta = Meta()
# Do things
self.read_configuration()
if not self._initiliazing:
self.do_schema()
self.find_endpoints()
self.do_swagger()
def custom_user_properties(self, userdata):
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Customizer = meta.get_class_from_string('Customizer',
module,
skip_error=True)
if Customizer is None:
log.debug("No user properties customizer available")
else:
try:
userdata = Customizer().custom_user_properties(userdata)
except BaseException as e:
log.error("Unable to customize user properties: %s", e)
if "email" in userdata:
userdata["email"] = userdata["email"].lower()
return userdata
def decorated(*args, **kwargs):
# Recover the auth object
auth_type, token = self.get_auth_from_header()
# Base header for errors
headers = {HTTPAUTH_AUTH_HEADER: self.authenticate_header()}
# Internal API 'self' reference
decorated_self = Meta.get_self_reference_from_args(*args)
if auth_type is None or auth_type.lower() != self._scheme.lower():
# Wrong authentication string
msg = "Valid credentials have to be provided " + \
"inside Headers, e.g. %s: '%s %s'" % \
(HTTPAUTH_AUTH_FIELD, HTTPAUTH_DEFAULT_SCHEME, 'TOKEN')
#
return decorated_self.send_errors(
# label="No authentication schema",
message=msg,
headers=headers,
code=hcodes.HTTP_BAD_UNAUTHORIZED)
# Handling OPTIONS forwarded to our application:
# ignore headers and let go, avoid unwanted interactions with CORS
if request.method != 'OPTIONS':
# Check authentication
token_fn = decorated_self.auth.verify_token
if not self.authenticate(token_fn, token):
# Clear TCP receive buffer of any pending data
request.data
# Mimic the response from a normal endpoint
# To use the same standards
return decorated_self.send_errors(
message="Invalid token received '%s'" % token,
headers=headers,
code=hcodes.HTTP_BAD_UNAUTHORIZED)
# Check roles
if len(roles) > 0:
roles_fn = decorated_self.auth.verify_roles
if not self.authenticate_roles(roles_fn, roles,
required_roles):
return decorated_self.send_errors(
message="You are not authorized: missing privileges",
code=hcodes.HTTP_BAD_UNAUTHORIZED)
return f(*args, **kwargs)
class Customizer(object):
"""
Customize your BACKEND:
Read all of available configurations and definitions.
"""
def __init__(self, testing=False, production=False, init=False):
# Input
self._testing = testing
self._production = production
self._initiliazing = init
# Some initialization
self._endpoints = []
self._definitions = {}
self._configurations = {}
self._query_params = {}
self._schemas_map = {}
self._meta = Meta()
# Do things
self.read_configuration()
if not self._initiliazing:
self.do_schema()
self.find_endpoints()
self.do_swagger()
def read_configuration(self):
##################
# Reading configuration
default_file_path = helpers.current_dir(CONF_PATH)
project_file_path = helpers.current_dir(CONF_PATH)
self._configurations = configuration.read(
default_file_path,
project_path=project_file_path,
)
def do_schema(self):
""" Schemas exposing, if requested """
name = '%s.%s.%s' % (BACKEND_PACKAGE, 'rest', 'schema')
module = self._meta.get_module_from_string(
name,
exit_if_not_found=True
)
schema_class = getattr(module, 'RecoverSchema')
self._schema_endpoint = EndpointElements(
cls=schema_class,
exists=True,
custom={
'methods': {
'get': ExtraAttributes(auth=None),
# WHY DOES POST REQUEST AUTHENTICATION
# 'post': ExtraAttributes(auth=None)
}
},
methods={}
)
# TODO: find a way to map authentication
# as in the original endpoint for the schema 'get' method
# TODO: find a way to publish on swagger the schema
# if endpoint is enabled to publish and the developer asks for it
def find_endpoints(self):
##################
# Walk swagger directories looking for endpoints
# FIXME: how to do this?
# from utilities import helpers
# custom_dir = helpers.current_dir(CUSTOM_PACKAGE)
# base_dir = helpers.script_abspath(__file__)
base_swagger_confdir = helpers.script_abspath(__file__)
custom_swagger_confdir = helpers.current_dir(CUSTOM_PACKAGE)
# for base_dir in [BACKEND_PACKAGE, CUSTOM_PACKAGE]:
for base_dir in [base_swagger_confdir, custom_swagger_confdir]:
swagger_dir = os.path.join(base_dir, 'swagger')
log.verbose("Swagger dir: %s" % swagger_dir)
for ep in os.listdir(swagger_dir):
swagger_endpoint_dir = os.path.join(swagger_dir, ep)
if os.path.isfile(swagger_endpoint_dir):
exception = '%s.yaml' % SWAGGER_MODELS_FILE
if not swagger_endpoint_dir.endswith('/' + exception):
log.debug(
"Found a file instead of a folder: %s",
swagger_endpoint_dir
)
continue
# isbase = base_dir == BACKEND_PACKAGE
isbase = base_dir.startswith('/usr/local')
base_module = helpers.last_dir(base_dir)
from utilities import ENDPOINTS_CODE_DIR
if isbase:
apiclass_module = '%s.%s' % (base_module, 'resources')
else:
apiclass_module = '%s.%s' % (
base_module, ENDPOINTS_CODE_DIR)
current = self.lookup(
ep, apiclass_module, swagger_endpoint_dir, isbase)
if current is not None and current.exists:
# Add endpoint to REST mapping
self._endpoints.append(current)
def do_swagger(self):
# SWAGGER read endpoints definition
swag = BeSwagger(self._endpoints, self)
swag_dict = swag.swaggerish()
# TODO: update internal endpoints from swagger
self._endpoints = swag._endpoints[:]
# SWAGGER validation
if not swag.validation(swag_dict):
log.critical_exit("Current swagger definition is invalid")
self._definitions = swag_dict
def read_frameworks(self):
file = os.path.join("config", "frameworks.yaml")
self._frameworks = load_yaml_file(file)
def lookup(self, endpoint, apiclass_module, swagger_endpoint_dir, isbase):
log.verbose("Found endpoint dir: '%s'" % endpoint)
if os.path.exists(os.path.join(swagger_endpoint_dir, 'SKIP')):
log.info("Skipping: %s", endpoint)
return None
# Find yaml files
conf = None
yaml_files = {}
yaml_listing = os.path.join(swagger_endpoint_dir, "*.%s" % YAML_EXT)
for file in glob.glob(yaml_listing):
if file.endswith('specs.%s' % YAML_EXT):
# load configuration and find file and class
conf = load_yaml_file(file)
else:
# add file to be loaded from swagger extension
p = re.compile(r'\/([^\.\/]+)\.' + YAML_EXT + '$')
match = p.search(file)
method = match.groups()[0]
yaml_files[method] = file
if len(yaml_files) < 1:
raise Exception("%s: no methods defined in any YAML" % endpoint)
if conf is None or 'class' not in conf:
raise ValueError("No 'class' defined for '%s'" % endpoint)
current = self.load_endpoint(endpoint, apiclass_module, conf, isbase)
current.methods = yaml_files
return current
# def read_complex_config(self, configfile):
# """ A more complex configuration is available in JSON format """
# content = {}
# with open(configfile) as fp:
# content = json.load(fp)
# return content
def load_endpoint(self, default_uri, apiclass_module, conf, isbase):
endpoint = EndpointElements(custom={})
# Load the endpoint class defined in the YAML file
file_name = conf.pop('file', default_uri)
class_name = conf.pop('class')
name = '%s.%s' % (apiclass_module, file_name)
module = self._meta.get_module_from_string(name, exit_on_fail=False)
# Error if unable to find the module in python
if module is None:
log.critical_exit(
"Could not find module %s (in %s)" % (name, file_name))
# Check for dependecies and skip if missing
from restapi.services.detect import detector
for var in conf.pop('depends_on', []):
negate = ''
pieces = var.strip().split(' ')
pieces_num = len(pieces)
if pieces_num == 1:
dependency = pieces.pop()
elif pieces_num == 2:
negate, dependency = pieces
else:
log.exit('Wrong parameter: %s', var)
check = detector.get_bool_from_os(dependency)
# Enable the possibility to depend on not having a variable
if negate.lower() == 'not':
check = not check
# Skip if not meeting the requirements of the dependency
if not check:
if not self._testing:
log.warning("Skip '%s': unmet %s", default_uri, dependency)
return endpoint
# Get the class from the module
endpoint.cls = self._meta.get_class_from_string(class_name, module)
if endpoint.cls is None:
log.critical("Could not extract python class '%s'", class_name)
return endpoint
else:
endpoint.exists = True
# Is this a base or a custom class?
endpoint.isbase = isbase
# DEPRECATED
# endpoint.instance = endpoint.cls()
# Global tags
# to be applied to all methods
endpoint.tags = conf.pop('labels', [])
# base URI
base = conf.pop('baseuri', API_URL)
if base not in BASE_URLS:
log.warning("Invalid base %s", base)
base = API_URL
base = base.strip('/')
#####################
# MAPPING
schema = conf.pop('schema', {})
mappings = conf.pop('mapping', [])
if len(mappings) < 1:
raise KeyError("Missing 'mapping' section")
endpoint.uris = {} # attrs python lib bug?
endpoint.custom['schema'] = {
'expose': schema.get('expose', False),
'publish': {}
}
for label, uri in mappings.items():
# BUILD URI
total_uri = '/%s%s' % (base, uri)
endpoint.uris[label] = total_uri
# If SCHEMA requested create
if endpoint.custom['schema']['expose']:
schema_uri = '%s%s%s' % (API_URL, '/schemas', uri)
p = hex(id(endpoint.cls))
self._schema_endpoint.uris[label + p] = schema_uri
endpoint.custom['schema']['publish'][label] = \
schema.get('publish', False)
self._schemas_map[schema_uri] = total_uri
# Description for path parameters
endpoint.ids = conf.pop('ids', {})
# Check if something strange is still in configuration
if len(conf) > 0:
raise KeyError("Unwanted keys: %s" % list(conf.keys()))
return endpoint
def test_failures():
meta = Meta()
try:
meta.import_submodules_from_package("utilities_bla")
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
try:
meta.import_submodules_from_package("utilities_bla",
exit_if_not_found=True)
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
try:
meta.import_submodules_from_package("utilities_bla",
exit_if_not_found=True,
exit_on_fail=True)
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
try:
meta.import_submodules_from_package("utilities_bla", exit_on_fail=True)
except AttributeError:
pass
else:
pytest.fail("This call should fail and raise and AttributeError")
module = meta.get_module_from_string("utilities.metabla")
assert module is None
try:
module = meta.get_module_from_string("utilities.metabla",
exit_if_not_found=True)
except SystemExit:
pass
else:
pytest.fail("This call should fail and exit")
try:
module = meta.get_module_from_string("utilities.metabla",
exit_if_not_found=True,
exit_on_fail=True)
except SystemExit:
pass
else:
pytest.fail("This call should fail and exit")
try:
module = meta.get_module_from_string("utilities.metabla",
exit_if_not_found=True,
exit_on_fail=True,
debug_on_fail=True)
except SystemExit:
pass
else:
pytest.fail("This call should fail and exit")
# FIXME: unable to test exit_on_fail... we need a moule with import errors?
module = meta.get_module_from_string("utilities.metabla",
exit_on_fail=True)
assert module is None
module = meta.get_module_from_string("utilities.metabla",
debug_on_fail=True)
assert module is None
def test():
meta = Meta()
cls_name = "Meta"
import utilities
modules = meta.get_submodules_from_package(utilities)
assert "meta" in modules
sub_modules = meta.import_submodules_from_package("utilities")
assert sub_modules is not None
assert isinstance(sub_modules, list)
assert len(sub_modules) > 0
module = meta.get_module_from_string("utilities.meta")
assert module is not None
assert hasattr(module, cls_name)
classes = meta.get_classes_from_module(module)
assert classes is not None
assert isinstance(classes, dict)
assert cls_name in classes
assert classes[cls_name].__name__ == cls_name
new_classes = meta.get_new_classes_from_module(module)
assert new_classes is not None
assert isinstance(new_classes, dict)
assert cls_name in new_classes
assert new_classes[cls_name].__name__ == cls_name
meta_class = meta.get_class_from_string(cls_name, module)
assert meta_class is not None
assert meta_class.__name__ == cls_name
methods = meta.get_methods_inside_instance(meta, private_methods=True)
assert methods is not None
assert isinstance(methods, dict)
# it is a static method
assert "get_methods_inside_instance" not in methods
assert "get_authentication_module" in methods
metacls = meta.metaclassing(Meta, "Test")
assert metacls is not None
assert metacls.__name__ == "Test"
self_ref = meta.get_self_reference_from_args(meta, "test", 1)
assert self_ref == meta
class Detector(object):
def __init__(self, config_file_name='services'):
self.authentication_service = None
self.authentication_name = 'authentication'
self.task_service_name = 'celery'
self.modules = []
self.services_configuration = []
self.services = {}
self.services_classes = {}
self.extensions_instances = {}
self.available_services = {}
self.meta = Meta()
self.check_configuration(config_file_name)
self.load_classes()
@staticmethod
def get_global_var(key, default=None):
return os.environ.get(key, default)
@staticmethod
@lru_cache(maxsize=None) # avoid calling it twice for the same var
def get_bool_from_os(name):
bool_var = os.environ.get(name, False)
if isinstance(bool_var, bool):
return bool_var
# if not directly a bool, try an interpretation
# INTEGERS
try:
tmp = int(bool_var)
return bool(tmp)
except ValueError:
pass
# STRINGS
# any non empty string with a least one char
# has to be considered True
if isinstance(bool_var, str) and len(bool_var) > 0:
return True
return False
@staticmethod
# @lru_cache(maxsize=None)
def prefix_name(service):
return \
service.get('name'), \
service.get('prefix').lower() + '_'
def check_configuration(self, config_file_name):
self.services_configuration = load_yaml_file(
file=config_file_name,
path=os.path.join(helpers.script_abspath(__file__), '..', '..',
CORE_CONFIG_PATH),
logger=True)
for service in self.services_configuration:
name, prefix = self.prefix_name(service)
# Was this service enabled from the developer?
enable_var = str(prefix + 'enable').upper()
self.available_services[name] = self.get_bool_from_os(enable_var)
if self.available_services[name]:
# read variables
variables = self.load_variables(service, enable_var, prefix)
service['variables'] = variables
# set auth service
if name == self.authentication_name:
self.authentication_service = variables.get('service')
# log.pp(self.services_configuration)
if self.authentication_service is None:
raise AttributeError("no service defined behind authentication")
else:
log.info("Authentication based on '%s' service" %
self.authentication_service)
def load_group(self, label):
variables = {}
for var, value in os.environ.items():
var = var.lower()
if var.startswith(label):
key = var[len(label):].strip('_')
value = value.strip('"').strip("'")
variables[key] = value
return variables
def output_service_variables(self, service_name):
service_class = self.services_classes.get(service_name, {})
try:
return service_class.variables
except BaseException:
return {}
@staticmethod
def load_group(label):
from utilities.basher import detect_vargroup
return detect_vargroup(label)
def output_service_variables(self, service_name):
service_class = self.services_classes.get(service_name, {})
try:
return service_class.variables
except BaseException:
return {}
@staticmethod
def load_variables(service, enable_var=None, prefix=None):
variables = {}
host = None
if prefix is None:
_, prefix = Detector.prefix_name(service)
for var, value in os.environ.items():
if enable_var is not None and var == enable_var:
continue
var = var.lower()
# This is the case when a variable belongs to a service 'prefix'
if var.startswith(prefix):
# Fix key and value before saving
key = var[len(prefix):]
# One thing that we must avoid is any quote around our value
value = value.strip('"').strip("'")
# save
variables[key] = value
if key == 'host':
host = value
# Verify if service is EXTERNAL
variables['external'] = False
if isinstance(host, str): # and host.count('.') > 2:
if not host.endswith('dockerized.io'):
variables['external'] = True
log.very_verbose("Service %s detected as external:\n%s" %
(service, host))
return variables
def load_class_from_module(self, classname='BaseInjector', service=None):
if service is None:
flaskext = ''
else:
flaskext = '.' + service.get('extension')
# Try inside our extensions
module = self.meta.get_module_from_string(
modulestring=BACKEND_PACKAGE + '.flask_ext' + flaskext,
exit_on_fail=True)
if module is None:
log.critical_exit("Missing %s for %s" % (flaskext, service))
return getattr(module, classname)
def load_classes(self):
for service in self.services_configuration:
name, _ = self.prefix_name(service)
if not self.available_services.get(name):
continue
log.very_verbose("Looking for class %s" % name)
variables = service.get('variables')
ext_name = service.get('class')
# Get the existing class
MyClass = self.load_class_from_module(ext_name, service=service)
try:
# Passing variables
MyClass.set_variables(variables)
# Passing models
if service.get('load_models'):
MyClass.set_models(
self.meta.import_models(name, custom=False),
self.meta.import_models(name,
custom=True,
exit_on_fail=False))
else:
log.very_verbose("Skipping models")
except AttributeError:
log.critical_exit('Extension class %s ' % ext_name +
'not compliant: missing method(s)' +
'Did you extend "%s"?' % 'BaseExtension')
# Save
self.services_classes[name] = MyClass
log.debug("Got class definition for %s", MyClass)
if len(self.services_classes) < 1:
raise KeyError("No classes were recovered!")
return self.services_classes
def init_services(self,
app,
worker_mode=False,
project_init=False,
project_clean=False):
instances = {}
auth_backend = None
for service in self.services_configuration:
name, _ = self.prefix_name(service)
if not self.available_services.get(name):
continue
if name == self.authentication_name and auth_backend is None:
if self.authentication_service is None:
log.warning("No authentication")
continue
else:
log.exit("Auth service '%s' seems unreachable" %
self.authentication_service)
args = {}
if name == self.task_service_name:
args['worker_mode'] = worker_mode
# Get extension class and build the extension object
ExtClass = self.services_classes.get(name)
try:
ext_instance = ExtClass(app, **args)
except TypeError as e:
log.critical_exit('Your class %s is not compliant:\n%s' %
(name, e))
else:
self.extensions_instances[name] = ext_instance
# Initialize the real service getting the first service object
log.debug("Initializing %s", name)
service_instance = ext_instance.custom_init(pinit=project_init,
pdestroy=project_clean,
abackend=auth_backend)
instances[name] = service_instance
if name == self.authentication_service:
auth_backend = service_instance
self.extensions_instances[name] = ext_instance
# Injecting into the Celery Extension Class
# all celery tasks found in *vanilla_package/tasks*
if name == self.task_service_name:
task_package = "%s.tasks" % CUSTOM_PACKAGE
submodules = self.meta.import_submodules_from_package(
task_package, exit_on_fail=True)
for submodule in submodules:
tasks = self.meta.get_celery_tasks_from_module(submodule)
for func_name, funct in tasks.items():
setattr(ExtClass, func_name, funct)
if len(self.extensions_instances) < 1:
raise KeyError("No instances available for modules")
# Only once in a lifetime
if project_init:
self.project_initialization(instances)
return self.extensions_instances
def load_injector_modules(self):
for service in self.services_configuration:
name, _ = self.prefix_name(service)
if not self.available_services.get(name):
continue
# Module for injection
ModuleBaseClass = self.load_class_from_module()
# Create modules programmatically 8)
MyModule = self.meta.metaclassing(ModuleBaseClass,
service.get('injector'))
# Recover class
MyClass = self.services_classes.get(name)
if MyClass is None:
raise AttributeError("No class found for %s" % name)
MyModule.set_extension_class(MyClass)
self.modules.append(MyModule)
return self.modules
def check_availability(self, name):
if '.' in name:
# In this case we are receiving a module name
# e.g. restapi.services.mongodb
name = name.split('.')[::-1][0]
return self.available_services.get(name)
@classmethod
def project_initialization(self, instances):
""" Custom initialization of your project
Please define your class Initializer in
vanilla/project/initialization.py
"""
try:
meta = Meta()
module_path = "%s.%s.%s" % \
(CUSTOM_PACKAGE, 'initialization', 'initialization')
module = meta.get_module_from_string(
module_path,
debug_on_fail=False,
)
Initializer = meta.get_class_from_string('Initializer',
module,
skip_error=True)
if Initializer is None:
log.debug("No custom init available")
else:
try:
Initializer(instances)
except BaseException as e:
log.error("Errors during custom initialization: %s", e)
else:
log.info("Vanilla project has been initialized")
except BaseException:
log.debug("No custom init available")
from utilities.logs import get_logger
log = get_logger(__name__)
################################################
# Reload Flask app code also for the worker
# This is necessary to have the app context available
app = create_app(worker_mode=True)
celery_app = app.extensions.get('celery').celery_app
celery_app.app = app
def get_service(service):
return celery_app.app.extensions.get(service).get_instance()
celery_app.get_service = get_service
################################################
# Import tasks modules to make sure all tasks are available
meta = Meta()
# main_package = "commons.tasks."
# # Base tasks
# submodules = meta.import_submodules_from_package(main_package + "base")
# # Custom tasks
submodules = meta.import_submodules_from_package("%s.tasks" % CUSTOM_PACKAGE)
log.debug("Celery worker is ready %s", celery_app)
class BaseExtension(metaclass=abc.ABCMeta):
models = {} # I get models on a cls level, instead of instances
meta = Meta()
def __init__(self, app=None, **kwargs):
self.objs = {}
self.set_name()
self.args = kwargs
self.app = app
if app is not None:
self.init_app(app)
def set_name(self):
""" a different name for each extended object """
self.name = self.__class__.__name__.lower()
log.very_verbose("Opening service instance of %s" % self.name)
@classmethod
def set_models(cls, base_models, custom_models):
# Join models as described by issue #16
cls.models = base_models
for key, model in custom_models.items():
# Verify if overriding
if key in base_models.keys():
original_model = base_models[key]
# Override
if issubclass(model, original_model):
log.very_verbose("Overriding model %s" % key)
cls.models[key] = model
continue
# Otherwise just append
cls.models[key] = model
if len(cls.models) > 0:
log.verbose("Loaded models")
@classmethod
def set_variables(cls, envvars):
cls.variables = envvars
def init_app(self, app):
app.teardown_appcontext(self.teardown)
def pre_object(self, ref, key):
""" Make sure reference and key are strings """
if ref is None:
ref = self.__class__.__name__
elif isinstance(ref, object):
ref = ref.__class__.__name__
elif not isinstance(ref, str):
ref = str(ref)
if not isinstance(key, str):
key = str(key)
return ref + key
def set_object(self, obj, key='[]', ref=None):
""" set object into internal array """
h = self.pre_object(ref, key)
self.objs[h] = obj
return obj
def get_object(self, key='[]', ref=None):
""" recover object if any """
h = self.pre_object(ref, key)
obj = self.objs.get(h, None)
return obj
def connect(self, **kwargs):
obj = None
# BEFORE
ok = self.pre_connection(**kwargs)
if not ok:
log.critical("Unable to make preconnection for %s", self.name)
return obj
# Try until it's connected
if len(kwargs) > 0:
obj = self.custom_connection(**kwargs)
else:
obj = self.retry()
log.info("Connected! %s", self.name)
# AFTER
self.post_connection(obj, **kwargs)
obj.connection_time = datetime.now()
return obj
def set_models_to_service(self, obj):
if len(self.models) < 1 and self.__class__.__name__ == 'NeoModel':
raise Exception()
for name, model in self.models.items():
# Save attribute inside class with the same name
log.very_verbose("Injecting model '%s'" % name)
setattr(obj, name, model)
obj.models = self.models
return obj
def set_connection_exception(self):
return None
def retry(self, retry_interval=3, max_retries=-1):
retry_count = 0
# Get the exception which will signal a missing connection
exceptions = self.set_connection_exception()
if exceptions is None:
exceptions = (BaseException, )
while max_retries != 0 or retry_count < max_retries:
retry_count += 1
if retry_count > 1:
log.verbose("testing again in %s secs", retry_interval)
try:
obj = self.custom_connection()
except exceptions as e:
log.error("Catched: %s(%s)", e.__class__.__name__, e)
# NOTE: if you critical_exit uwsgi will not show this line
log.critical("Service '%s' not available", self.name)
log.exit()
# raise e
else:
break
# Increment sleeps time if doing a lot of retries
if retry_count % 3 == 0:
log.debug("Incrementing interval")
retry_interval += retry_interval
return obj
def teardown(self, exception):
ctx = stack.top
if self.get_object(ref=ctx) is not None:
self.close_connection(ctx)
def get_instance(self, **kwargs):
# Parameters
global_instance = kwargs.pop('global_instance', False)
isauth = kwargs.pop('authenticator', False)
cache_expiration = kwargs.pop('cache_expiration', None)
# pinit = kwargs('project_initialization', False)
# Variables
obj = None
ctx = stack.top
ref = self
unique_hash = str(sorted(kwargs.items()))
log.very_verbose("instance hash: %s" % unique_hash)
# When not using the context, this is the first connection
if ctx is None:
# First connection, before any request
obj = self.connect()
if obj is None:
return None
# self.initialization(obj=obj)
self.set_object(obj=obj, ref=ref)
log.verbose("First connection for %s" % self.name)
else:
# isauth = 'Authenticator' == self.__class__.__name__
if not isauth:
if not global_instance:
ref = ctx
obj = self.get_object(ref=ref, key=unique_hash)
if obj is not None and cache_expiration is not None:
now = datetime.now()
exp = timedelta(seconds=cache_expiration)
if now < obj.connection_time + exp:
log.very_verbose("Cache still fresh for %s" % (self))
else:
log.warning("Cache expired for %s", (self))
obj = None
if obj is None:
obj = self.connect(**kwargs)
if obj is None:
return None
self.set_object(obj=obj, ref=ref, key=unique_hash)
else:
pass
log.very_verbose("Instance %s(%s)" % (ref.__class__.__name__, obj))
obj = self.set_models_to_service(obj)
return obj
############################
# OPTIONALLY
# to be executed only at init time?
def pre_connection(self, **kwargs):
return True
def post_connection(self, obj=None, **kwargs):
return True
def close_connection(self, ctx):
""" override this method if you must close
your connection after each request"""
# obj = self.get_object(ref=ctx)
# obj.close()
self.set_object(obj=None, ref=ctx) # it could be overidden
############################
# To be overridden
@abc.abstractmethod
def custom_connection(self, **kwargs):
return
############################
# Already has default
def custom_init(self, **kwargs):
"""
The real signature:
def custom_init(self, pinit=False, pdestroy=False, abackend=None):
- A backend is needed for non-standalone services
e.g. authentication module
- Project initialization/removal could be used here
or carried on to low levels;
they get activated by specific flask cli commands
"""
return self.get_instance()
"""
Take care of authenticatin with External Service with Oauth2 protocol.
Testend against GitHub, then worked off B2ACCESS (EUDAT oauth service)
"""
import os
from base64 import b64encode
from restapi.protocols.oauth import oauth
# from restapi.confs import PRODUCTION
from utilities.globals import mem
from utilities.meta import Meta
from utilities.logs import get_logger
log = get_logger(__name__)
meta = Meta()
B2ACCESS_MAIN_PORT = 8443
B2ACCESS_CA_PORT = 8445
B2ACCESS_URLS = {
'development': 'unity.eudat-aai.fz-juelich.de',
'staging': 'b2access-integration.fz-juelich.de',
'production': 'b2access.eudat.eu',
}
class ExternalLogins(object):
_available_services = {}
def __init__(self):