def test_update_query(self):
registry = RegistryManager.get(Configuration.get('db_name'))
registry.upgrade(install=('test-me-blok2',))
registry.commit()
self.assertTrue(registry.System.Blok.query().all_name())
self.assertTrue(registry.System.Blok.query().all())
registry.close()
registry = RegistryManager.get(Configuration.get('db_name'))
self.assertTrue(registry.System.Blok.query().all_name())
self.assertTrue(registry.System.Blok.query().all())
def anyblok_createdb():
"""Create a database and install blok from config"""
load_init_function_from_entry_points()
Configuration.load('createdb')
configuration_post_load()
BlokManager.load()
db_name = get_db_name()
db_template_name = Configuration.get('db_template_name', None)
url = Configuration.get('get_url')(db_name=db_name)
create_database(url, template=db_template_name)
registry = RegistryManager.get(db_name)
if registry is None:
return
if Configuration.get('install_all_bloks'):
bloks = registry.System.Blok.list_by_state('uninstalled')
else:
install_bloks = Configuration.get('install_bloks') or []
iou_bloks = Configuration.get('install_or_update_bloks') or []
bloks = list(set(install_bloks + iou_bloks))
registry.upgrade(install=bloks)
registry.commit()
registry.close()
def anyblok_createdb():
"""Create a database and install blok from config"""
load_init_function_from_entry_points()
Configuration.load('createdb')
configuration_post_load()
BlokManager.load()
db_name = get_db_name()
db_template_name = Configuration.get('db_template_name', None)
url = Configuration.get('get_url')(db_name=db_name)
create_database(url, template=db_template_name)
anyblok_registry = RegistryManager.get(db_name)
if anyblok_registry is None:
return
anyblok_registry.System.Parameter.set(
"with-demo", Configuration.get('with_demo', False))
if Configuration.get('install_all_bloks'):
bloks = anyblok_registry.System.Blok.list_by_state('uninstalled')
else:
install_bloks = Configuration.get('install_bloks') or []
iou_bloks = Configuration.get('install_or_update_bloks') or []
bloks = list(set(install_bloks + iou_bloks))
anyblok_registry.upgrade(install=bloks)
anyblok_registry.commit()
anyblok_registry.close()
def createdb(application, configuration_groups, **kwargs):
""" Create a database and install blok from config
:param application: name of the application
:param configuration_groups: list configuration groupe to load
:param \**kwargs: ArgumentParser named arguments
"""
format_configuration(configuration_groups,
'create_db', 'install-bloks',
'install-or-update-bloks')
load_init_function_from_entry_points()
Configuration.load(application, configuration_groups=configuration_groups,
**kwargs)
BlokManager.load()
db_name = Configuration.get('db_name')
db_template_name = Configuration.get('db_template_name', None)
url = Configuration.get('get_url')(db_name=db_name)
create_database(url, template=db_template_name)
registry = RegistryManager.get(db_name)
if registry is None:
return
if Configuration.get('install_all_bloks'):
bloks = registry.System.Blok.list_by_state('uninstalled')
else:
install_bloks = Configuration.get('install_bloks') or []
iou_bloks = Configuration.get('install_or_update_bloks') or []
bloks = list(set(install_bloks + iou_bloks))
registry.upgrade(install=bloks)
registry.commit()
registry.close()
def init_registry_with_bloks(bloks, function, **kwargs):
if bloks is None:
bloks = []
if isinstance(bloks, tuple):
bloks = list(bloks)
if isinstance(bloks, str):
bloks = [bloks]
anyblok_test_name = 'anyblok-test'
if anyblok_test_name not in bloks:
bloks.append(anyblok_test_name)
loaded_bloks = deepcopy(RegistryManager.loaded_bloks)
if function is not None:
EnvironmentManager.set('current_blok', anyblok_test_name)
try:
function(**kwargs)
finally:
EnvironmentManager.set('current_blok', None)
try:
registry = RegistryManager.get(Configuration.get('db_name'),
unittest=True)
# update required blok
registry_bloks = registry.get_bloks_by_states('installed', 'toinstall')
toinstall = [x for x in bloks if x not in registry_bloks]
toupdate = [x for x in bloks if x in registry_bloks]
registry.upgrade(install=toinstall, update=toupdate)
finally:
RegistryManager.loaded_bloks = loaded_bloks
return registry
def load_client(request):
""" Return the client main page """
database = request.session.get('database')
login = request.session.get('login')
password = request.session.get('password')
state = request.session.get('state')
if not(database and login and password and state == "connected"):
logout(request)
return HTTPFound(location=request.route_url('homepage'))
try:
registry = RegistryManager.get(database)
assert registry.Web.Login.check_authentification(login, password)
except:
logout(request)
return HTTPFound(location=request.route_url('homepage'))
css = registry.Web.get_css()
js = registry.Web.get_js()
js_babel = registry.Web.get_js_babel()
# TODO see in system.parmeter if they are no data for title
title = Configuration.get('app_name', 'ERPBlok')
templates = registry.Web.get_templates()
return render_to_response('erpblok:client.mak',
{'title': title,
'css': css,
'js': js,
'js_babel': js_babel,
'templates': templates,
}, request=request)
def before_process_message(self, broker, message):
"""Called before process message
Invalid the cache, this is mean that if a cache have to be invalidated
then it will be invalidated else nothing is done
If the message is in the ``Model.Dramatiq.Message`` then
the status will be change to **running**
:param broker: the broker used
:param message: the message send in the broker
"""
registry = RegistryManager.get(Configuration.get('db_name'))
logger.debug("[before_process_message] %s: update message(%s) status ",
id(registry.session), message.message_id)
# cache may have change, then we do the invalidation in the dramatiq
# side
registry.System.Cache.clear_invalidate_cache()
M = registry.Dramatiq.Message
m = M.get_instance_of(message)
if m:
m.update_status(M.STATUS_RUNNING)
registry.commit()
EnvironmentManager.set('is_called_by_dramatiq_actor', True)
def load_registry(self):
if self.enabled and self.registryLoaded is False:
# Load the registry here not in configuration,
# because the configuration are not load in order of score
self.registryLoaded = True
BlokManager.load()
load_init_function_from_entry_points(unittest=True)
Configuration.load_config_for_test()
Configuration.parse_options(self.AnyBlokOptions, ('bloks',))
db_name = Configuration.get('db_name')
if not db_name:
raise Exception("No database defined to run Test")
registry = RegistryManager.get(db_name)
if registry:
installed_bloks = registry.System.Blok.list_by_state(
"installed")
selected_bloks = Configuration.get('selected_bloks')
if not selected_bloks:
selected_bloks = installed_bloks
unwanted_bloks = Configuration.get('unwanted_bloks')
if unwanted_bloks is None:
unwanted_bloks = []
self.bloks_path = [BlokManager.getPath(x)
for x in BlokManager.ordered_bloks]
self.authoried_bloks_test_files = [
path for blok in installed_bloks
if blok in selected_bloks and blok not in unwanted_bloks
for path in [join(BlokManager.getPath(blok), 'tests')]
if exists(path)]
registry.close() # free the registry to force create it again
def get_registry_for(dbname, loadwithoutmigration=True, log_repeat=False):
settings = {
'anyblok.session.event': [register],
}
return RegistryManager.get(
dbname, loadwithoutmigration=loadwithoutmigration,
log_repeat=log_repeat, **settings)
def createdb(cls, keep_existing=False, with_demo=False):
"""Create the database specified in configuration.
::
cls.init_configuration_manager()
cls.createdb()
:param keep_existing: If false drop the previous db before create it
"""
url = Configuration.get('get_url')()
db_template_name = Configuration.get('db_template_name', None)
if database_exists(url):
if keep_existing:
return False
drop_database(url)
create_database(url, template=db_template_name)
registry = RegistryManager.get(Configuration.get('db_name', None))
if registry is None:
return
registry.System.Parameter.set(
"with-demo", Configuration.get('with_demo', with_demo))
return True
def get_registry_for(dbname, loadwithoutmigration=True, log_repeat=False):
settings = {
'anyblok.session.event': [register],
}
return RegistryManager.get(dbname,
loadwithoutmigration=loadwithoutmigration,
log_repeat=log_repeat,
**settings)
def webserver(request, configuration_loaded, init_session):
with TestClient(
create_app(
RegistryManager.get(
Configuration.get("db_name"),
loadwithoutmigration=True,
))) as client:
yield client
def setUp(self):
super(TestBlok, self).setUp()
self.__class__.init_configuration_manager()
self.__class__.createdb(keep_existing=False)
BlokManager.load(entry_points=('bloks', 'test_bloks'))
registry = RegistryManager.get(Configuration.get('db_name'))
registry.commit()
registry.close()
def before_worker_thread_shutdown(self, *args, **kwargs):
"""Called before worker thread shutdown
remove the session instance to clean the Session pool
"""
registry = RegistryManager.get(Configuration.get('db_name'))
logger.debug("[before_worker_thread_shutdown] %s: %r %r",
id(registry.session), args, kwargs)
registry.Session.remove()
def tearDown(self):
""" Clear the registry, unload the blok manager and drop the database
"""
registry = RegistryManager.get(Configuration.get('db_name'))
registry.close()
RegistryManager.clear()
BlokManager.unload()
clear_mappers()
self.__class__.dropdb()
super(TestBlok, self).tearDown()
def after_worker_shutdown(self, *args, **kwargs):
"""Called before worker shutdown
Close the AnyBlok registry
"""
registry = RegistryManager.get(Configuration.get('db_name'))
logger.debug("[after_worker_shutdown] %s: %r %r", id(registry.session),
args, kwargs)
registry.close()
def setUpClass(cls):
""" Initialize the registry.
"""
super(BlokTestCase, cls).setUpClass()
additional_setting = cls.additional_setting()
if cls.registry is None:
cls.registry = RegistryManager.get(Configuration.get('db_name'),
**additional_setting)
cls.registry.commit()
def get_registry_for(
dbname: str, loadwithoutmigration: bool = True, log_repeat: bool = False
):
# Methode copied from anyblok_pyramid.common"""
settings: Dict[str, Any] = {}
return RegistryManager.get(
dbname,
loadwithoutmigration=loadwithoutmigration,
log_repeat=log_repeat,
**settings,
)
def drop_database(database):
""" Close the registry instance of the database and drop the database
:param: database's name
"""
url = Configuration.get('get_url')(db_name=database)
if not database_exists(url):
raise Exception("Database %r does not already exist")
registry = RegistryManager.get(database)
registry.close()
SU_drop_database(url)
def getRegistry(cls):
"""Return the registry for the test database.
This assumes the database is created, and the registry has already
been initialized::
registry = self.getRegistry()
:rtype: registry instance
"""
additional_setting = cls.additional_setting()
return RegistryManager.get(Configuration.get('db_name'),
**additional_setting)
def setUpClass(cls):
""" Initialize the registry.
"""
super(BlokTestCase, cls).setUpClass()
additional_setting = cls.additional_setting()
if cls.registry is None:
if len(BlokManager.list()) == 0:
BlokManager.load()
cls.registry = RegistryManager.get(Configuration.get('db_name'),
**additional_setting)
cls.registry.commit()
def getRegistry(cls):
"""Return the registry for the test database.
This assumes the database is created, and the registry has already
been initialized::
registry = self.getRegistry()
:rtype: registry instance
"""
additional_setting = cls.additional_setting()
return RegistryManager.get(Configuration.get('db_name'),
**additional_setting)
def create_database(database):
""" Create a new database, initialize it and return an AnyBlok registry
:param: database's name
:rtype: AnyBlok registry instance
"""
url = Configuration.get('get_url')(db_name=database)
if database_exists(url):
raise Exception("Database %r already exist")
db_template_name = Configuration.get('db_template_name', None)
SU_create_database(url, template=db_template_name)
registry = RegistryManager.get(database)
return registry
def after_process_message(self,
broker,
message,
*,
result=None,
exception=None):
"""Called after process message
If the message is in the ``Model.Dramatiq.Message`` then
the status will be change to **done** or **failed**.
.. note::
the status is failed if an exception is passed or a rollback
is need
Before the end, the session is expired to release the Session pool
thread
:param broker: the broker used
:param message: the message send in the broker
:param result: return by the process
:param exception: any ``Exception`` raised by the process
"""
registry = RegistryManager.get(Configuration.get('db_name'))
logger.debug(
"[after_process_message] %s: update message(%s) status "
"with result %r and exception %r", id(registry.session),
message.message_id, result, exception)
EnvironmentManager.set('is_called_by_dramatiq_actor', False)
M = registry.Dramatiq.Message
STATUS_DONE = M.STATUS_DONE
STATUS_FAILED = M.STATUS_FAILED
try:
m = M.get_instance_of(message)
if m:
m.update_status(
STATUS_FAILED if exception is not None else STATUS_DONE,
error=str(exception) if exception else None)
registry.commit()
except Exception as e:
registry.rollback()
m = M.get_instance_of(message)
if m:
m.update_status(M.STATUS_FAILED, error=str(e))
registry.commit()
raise e
finally:
registry.expire_all()
def worker_process(worker_id, logging_fd):
"""consume worker to process messages and execute the actor"""
# TODO preload registries
db_name = Configuration.get('db_name')
try:
logging_pipe = os.fdopen(logging_fd, "w")
broker = prepare_broker(withmiddleware=True)
broker.emit_after("process_boot")
BlokManager.load()
registry = RegistryManager.get(db_name, loadwithoutmigration=True)
if registry is None:
logger.critical("No registry found for %s", db_name)
return os._exit(4)
worker = Worker(
broker, worker_threads=Configuration.get('dramatiq_threads', 1))
worker.start()
print('worker started')
except ImportError as e:
logger.critical(e)
return os._exit(2)
except ConnectionError as e:
logger.critical("Broker connection failed. %s", e)
return os._exit(3)
def termhandler(signum, frame):
nonlocal running
BlokManager.unload()
if running:
logger.info("Stopping worker process...")
running = False
else:
logger.warning("Killing worker process...")
return os._exit(1)
logger.info("Worker process is ready for action.")
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, termhandler)
signal.signal(signal.SIGHUP, termhandler)
running = True
while running:
time.sleep(1)
worker.stop()
broker.close()
logging_pipe.close()
def before_enqueue(self, broker, message, delay):
"""Called when a message is delayed or enqueued
If the message is in the ``Model.Dramatiq.Message`` then
the status will be change to **delayed** or **enqueued**
:param broker: the broker used
:param message: the message send in the broker
:param delay: delay in milliseconds
"""
registry = RegistryManager.get(Configuration.get('db_name'))
logger.debug("[before_enqueue] %s: update message(%s) status ",
id(registry.session), message.message_id)
M = registry.Dramatiq.Message
m = M.get_instance_of(message)
if m:
m.update_status(M.STATUS_DELAYED if delay else M.STATUS_ENQUEUED)
registry.commit()
def update_project():
anyblok.load_init_function_from_entry_points()
Configuration.load('update-project')
anyblok.configuration_post_load()
BlokManager.load()
db_name = Configuration.get('db_name')
logger.info("update project: db_name=%r", db_name)
to_install = []
to_update = []
to_uninstall = []
url = get_url()
options = {}
if not database_exists(url):
db_template_name = Configuration.get('db_template_name', None)
create_database(url, template=db_template_name)
to_install.append('team')
version = None
else:
options.update(dict(loadwithoutmigration=True))
registry = RegistryManager.get(db_name, **options)
registry.update_blok_list() # case, new blok added
version = registry.System.Blok.query().filter_by(
name='team').one().installed_version
if version is None:
pass
else:
to_update.append('team')
registry.upgrade(install=to_install,
update=to_update,
uninstall=to_uninstall)
if version is None:
registry.Pyramid.User.insert(login='******')
registry.Pyramid.CredentialStore.insert(login='******',
password='******')
registry.commit()
registry.close()
logger.info("Project updated: db_name=%r", db_name)
def load_registry(self):
if not self.enabled or self.registryLoaded:
return
from anyblok.config import Configuration, get_db_name
from anyblok import (
load_init_function_from_entry_points,
configuration_post_load,
)
from anyblok.blok import BlokManager
from anyblok.registry import RegistryManager, return_list
# Load the registry here not in configuration,
# because the configurations are not loaded in order of score
self.registryLoaded = True
load_init_function_from_entry_points(unittest=True)
Configuration.load_config_for_test()
Configuration.parse_options(self.AnyBlokOptions)
configuration_post_load()
BlokManager.load()
db_name = get_db_name()
registry = RegistryManager.get(db_name)
if not registry:
return
installed_bloks = registry.System.Blok.list_by_state("installed")
selected_bloks = return_list(
Configuration.get('selected_bloks')) or installed_bloks
unwanted_bloks = return_list(Configuration.get('unwanted_bloks')) or []
self.authorized_blok_paths = set(
BlokManager.getPath(b) for b in BlokManager.list()
if b in selected_bloks and b not in unwanted_bloks)
test_dirs = self.authorized_blok_test_dirs = set()
for startpath in self.authorized_blok_paths:
for root, dirs, _ in walk(startpath):
if 'tests' in dirs:
test_dirs.add(join(root, 'tests'))
registry.close() # free the registry to force create it again
def after_skip_message(self, broker, message):
"""Called after skip message
If the message is in the ``Model.Dramatiq.Message`` then
the status will be change to **skip**
Before the end, the session is expired to release the Session pool
thread
:param broker: the broker used
:param message: the message send in the broker
"""
registry = RegistryManager.get(Configuration.get('db_name'))
logger.debug("[after_skip_message] %s: update message(%s) status ",
id(registry.session), message.message_id)
registry.rollback()
M = registry.Dramatiq.Message
m = M.get_instance_of(message)
m.update_status(M.STATUS_SKIPED)
registry.commit()
def load_main(request):
""" Return the client main page
"""
dbname = Configuration.get('get_db_name')(request)
state = request.session.get('state')
if state == 'connected':
registry = RegistryManager.get(dbname)
css = registry.Web.get_css()
js = registry.Web.get_js()
else:
css = get_static('global_css')
js = get_static('global_js')
title = Configuration.get('app_name', 'AnyBlok / FuretUI')
return render_to_response('anyblok_furetui:client.mak', {
'title': title,
'css': css,
'js': js,
},
request=request)
def init_session(request, configuration_loaded):
# Init registry
additional_setting = {'unittest': True}
if len(BlokManager.list()) == 0:
BlokManager.load()
registry = RegistryManager.get(Configuration.get('db_name'),
**additional_setting)
registry.commit()
# Add savepoint
registry.begin_nested()
@event.listens_for(registry.session, 'after_transaction_end')
def restart_savepoint(session, transaction):
if transaction.nested and not transaction._parent.nested:
session.expire_all()
session.begin_nested()
logger.info('Creating registry')
yield registry
request.addfinalizer(registry.session.close)
def bus_worker_process(logging_fd, consumers):
"""consume worker to process messages and execute the actor"""
# TODO preload registries
db_name = Configuration.get('db_name')
profile = Configuration.get('bus_profile')
try:
logging_pipe = os.fdopen(logging_fd, "w")
registry = RegistryManager.get(db_name, loadwithoutmigration=True)
worker = ReconnectingWorker(registry, profile, consumers)
worker.start()
except ImportError as e:
logger.critical(e)
return os._exit(2)
except ConnectionError as e:
logger.critical("Broker connection failed. %s", e)
return os._exit(3)
def termhandler(signum, frame):
nonlocal running
BlokManager.unload()
if running:
logger.info("Stopping worker process...")
running = False
else:
logger.warning("Killing worker process...")
return os._exit(1)
logger.info("Worker process is ready for action.")
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, termhandler)
signal.signal(signal.SIGHUP, termhandler)
running = True
while running:
time.sleep(1)
worker.stop()
logging_pipe.close()
def load_registry(self):
if self.enabled and self.registryLoaded is False:
# Load the registry here not in configuration,
# because the configuration are not load in order of score
self.registryLoaded = True
load_init_function_from_entry_points(unittest=True)
Configuration.load_config_for_test()
Configuration.parse_options(self.AnyBlokOptions)
configuration_post_load()
BlokManager.load()
db_name = get_db_name()
registry = RegistryManager.get(db_name)
if registry:
installed_bloks = registry.System.Blok.list_by_state(
"installed")
selected_bloks = return_list(
Configuration.get('selected_bloks')) or installed_bloks
unwanted_bloks = return_list(
Configuration.get('unwanted_bloks')) or []
self.bloks_path = [BlokManager.getPath(x)
for x in BlokManager.ordered_bloks]
self.authoried_bloks_test_files = []
for blok in installed_bloks:
if blok not in selected_bloks or blok in unwanted_bloks:
continue
startpath = BlokManager.getPath(blok)
for root, dirs, _ in walk(startpath):
if 'tests' in dirs:
self.authoried_bloks_test_files.append(
join(root, 'tests'))
registry.close() # free the registry to force create it again
def test_get_the_same_registry(self):
registry = self.init_registry(None)
registry2 = RegistryManager.get(
Configuration.get('db_name'),
unittest=True)
assert registry is registry2
def get_registry_for(dbname):
settings = {
'sa.session.extension': AnyBlokZopeTransactionExtension,
}
return RegistryManager.get(dbname, **settings)
