def test_update_query(self): registry = RegistryManager.get(Configuration.get('db_name')) registry.upgrade(install=('test-me-blok2',)) registry.commit() self.assertTrue(registry.System.Blok.query().all_name()) self.assertTrue(registry.System.Blok.query().all()) registry.close() registry = RegistryManager.get(Configuration.get('db_name')) self.assertTrue(registry.System.Blok.query().all_name()) self.assertTrue(registry.System.Blok.query().all())
def anyblok_createdb(): """Create a database and install blok from config""" load_init_function_from_entry_points() Configuration.load('createdb') configuration_post_load() BlokManager.load() db_name = get_db_name() db_template_name = Configuration.get('db_template_name', None) url = Configuration.get('get_url')(db_name=db_name) create_database(url, template=db_template_name) registry = RegistryManager.get(db_name) if registry is None: return if Configuration.get('install_all_bloks'): bloks = registry.System.Blok.list_by_state('uninstalled') else: install_bloks = Configuration.get('install_bloks') or [] iou_bloks = Configuration.get('install_or_update_bloks') or [] bloks = list(set(install_bloks + iou_bloks)) registry.upgrade(install=bloks) registry.commit() registry.close()
def anyblok_createdb(): """Create a database and install blok from config""" load_init_function_from_entry_points() Configuration.load('createdb') configuration_post_load() BlokManager.load() db_name = get_db_name() db_template_name = Configuration.get('db_template_name', None) url = Configuration.get('get_url')(db_name=db_name) create_database(url, template=db_template_name) anyblok_registry = RegistryManager.get(db_name) if anyblok_registry is None: return anyblok_registry.System.Parameter.set( "with-demo", Configuration.get('with_demo', False)) if Configuration.get('install_all_bloks'): bloks = anyblok_registry.System.Blok.list_by_state('uninstalled') else: install_bloks = Configuration.get('install_bloks') or [] iou_bloks = Configuration.get('install_or_update_bloks') or [] bloks = list(set(install_bloks + iou_bloks)) anyblok_registry.upgrade(install=bloks) anyblok_registry.commit() anyblok_registry.close()
def createdb(application, configuration_groups, **kwargs): """ Create a database and install blok from config :param application: name of the application :param configuration_groups: list configuration groupe to load :param \**kwargs: ArgumentParser named arguments """ format_configuration(configuration_groups, 'create_db', 'install-bloks', 'install-or-update-bloks') load_init_function_from_entry_points() Configuration.load(application, configuration_groups=configuration_groups, **kwargs) BlokManager.load() db_name = Configuration.get('db_name') db_template_name = Configuration.get('db_template_name', None) url = Configuration.get('get_url')(db_name=db_name) create_database(url, template=db_template_name) registry = RegistryManager.get(db_name) if registry is None: return if Configuration.get('install_all_bloks'): bloks = registry.System.Blok.list_by_state('uninstalled') else: install_bloks = Configuration.get('install_bloks') or [] iou_bloks = Configuration.get('install_or_update_bloks') or [] bloks = list(set(install_bloks + iou_bloks)) registry.upgrade(install=bloks) registry.commit() registry.close()
def init_registry_with_bloks(bloks, function, **kwargs): if bloks is None: bloks = [] if isinstance(bloks, tuple): bloks = list(bloks) if isinstance(bloks, str): bloks = [bloks] anyblok_test_name = 'anyblok-test' if anyblok_test_name not in bloks: bloks.append(anyblok_test_name) loaded_bloks = deepcopy(RegistryManager.loaded_bloks) if function is not None: EnvironmentManager.set('current_blok', anyblok_test_name) try: function(**kwargs) finally: EnvironmentManager.set('current_blok', None) try: registry = RegistryManager.get(Configuration.get('db_name'), unittest=True) # update required blok registry_bloks = registry.get_bloks_by_states('installed', 'toinstall') toinstall = [x for x in bloks if x not in registry_bloks] toupdate = [x for x in bloks if x in registry_bloks] registry.upgrade(install=toinstall, update=toupdate) finally: RegistryManager.loaded_bloks = loaded_bloks return registry
def load_client(request): """ Return the client main page """ database = request.session.get('database') login = request.session.get('login') password = request.session.get('password') state = request.session.get('state') if not(database and login and password and state == "connected"): logout(request) return HTTPFound(location=request.route_url('homepage')) try: registry = RegistryManager.get(database) assert registry.Web.Login.check_authentification(login, password) except: logout(request) return HTTPFound(location=request.route_url('homepage')) css = registry.Web.get_css() js = registry.Web.get_js() js_babel = registry.Web.get_js_babel() # TODO see in system.parmeter if they are no data for title title = Configuration.get('app_name', 'ERPBlok') templates = registry.Web.get_templates() return render_to_response('erpblok:client.mak', {'title': title, 'css': css, 'js': js, 'js_babel': js_babel, 'templates': templates, }, request=request)
def before_process_message(self, broker, message): """Called before process message Invalid the cache, this is mean that if a cache have to be invalidated then it will be invalidated else nothing is done If the message is in the ``Model.Dramatiq.Message`` then the status will be change to **running** :param broker: the broker used :param message: the message send in the broker """ registry = RegistryManager.get(Configuration.get('db_name')) logger.debug("[before_process_message] %s: update message(%s) status ", id(registry.session), message.message_id) # cache may have change, then we do the invalidation in the dramatiq # side registry.System.Cache.clear_invalidate_cache() M = registry.Dramatiq.Message m = M.get_instance_of(message) if m: m.update_status(M.STATUS_RUNNING) registry.commit() EnvironmentManager.set('is_called_by_dramatiq_actor', True)
def load_registry(self): if self.enabled and self.registryLoaded is False: # Load the registry here not in configuration, # because the configuration are not load in order of score self.registryLoaded = True BlokManager.load() load_init_function_from_entry_points(unittest=True) Configuration.load_config_for_test() Configuration.parse_options(self.AnyBlokOptions, ('bloks',)) db_name = Configuration.get('db_name') if not db_name: raise Exception("No database defined to run Test") registry = RegistryManager.get(db_name) if registry: installed_bloks = registry.System.Blok.list_by_state( "installed") selected_bloks = Configuration.get('selected_bloks') if not selected_bloks: selected_bloks = installed_bloks unwanted_bloks = Configuration.get('unwanted_bloks') if unwanted_bloks is None: unwanted_bloks = [] self.bloks_path = [BlokManager.getPath(x) for x in BlokManager.ordered_bloks] self.authoried_bloks_test_files = [ path for blok in installed_bloks if blok in selected_bloks and blok not in unwanted_bloks for path in [join(BlokManager.getPath(blok), 'tests')] if exists(path)] registry.close() # free the registry to force create it again
def get_registry_for(dbname, loadwithoutmigration=True, log_repeat=False): settings = { 'anyblok.session.event': [register], } return RegistryManager.get( dbname, loadwithoutmigration=loadwithoutmigration, log_repeat=log_repeat, **settings)
def createdb(cls, keep_existing=False, with_demo=False): """Create the database specified in configuration. :: cls.init_configuration_manager() cls.createdb() :param keep_existing: If false drop the previous db before create it """ url = Configuration.get('get_url')() db_template_name = Configuration.get('db_template_name', None) if database_exists(url): if keep_existing: return False drop_database(url) create_database(url, template=db_template_name) registry = RegistryManager.get(Configuration.get('db_name', None)) if registry is None: return registry.System.Parameter.set( "with-demo", Configuration.get('with_demo', with_demo)) return True
def get_registry_for(dbname, loadwithoutmigration=True, log_repeat=False): settings = { 'anyblok.session.event': [register], } return RegistryManager.get(dbname, loadwithoutmigration=loadwithoutmigration, log_repeat=log_repeat, **settings)
def webserver(request, configuration_loaded, init_session): with TestClient( create_app( RegistryManager.get( Configuration.get("db_name"), loadwithoutmigration=True, ))) as client: yield client
def setUp(self): super(TestBlok, self).setUp() self.__class__.init_configuration_manager() self.__class__.createdb(keep_existing=False) BlokManager.load(entry_points=('bloks', 'test_bloks')) registry = RegistryManager.get(Configuration.get('db_name')) registry.commit() registry.close()
def before_worker_thread_shutdown(self, *args, **kwargs): """Called before worker thread shutdown remove the session instance to clean the Session pool """ registry = RegistryManager.get(Configuration.get('db_name')) logger.debug("[before_worker_thread_shutdown] %s: %r %r", id(registry.session), args, kwargs) registry.Session.remove()
def tearDown(self): """ Clear the registry, unload the blok manager and drop the database """ registry = RegistryManager.get(Configuration.get('db_name')) registry.close() RegistryManager.clear() BlokManager.unload() clear_mappers() self.__class__.dropdb() super(TestBlok, self).tearDown()
def after_worker_shutdown(self, *args, **kwargs): """Called before worker shutdown Close the AnyBlok registry """ registry = RegistryManager.get(Configuration.get('db_name')) logger.debug("[after_worker_shutdown] %s: %r %r", id(registry.session), args, kwargs) registry.close()
def setUpClass(cls): """ Initialize the registry. """ super(BlokTestCase, cls).setUpClass() additional_setting = cls.additional_setting() if cls.registry is None: cls.registry = RegistryManager.get(Configuration.get('db_name'), **additional_setting) cls.registry.commit()
def get_registry_for( dbname: str, loadwithoutmigration: bool = True, log_repeat: bool = False ): # Methode copied from anyblok_pyramid.common""" settings: Dict[str, Any] = {} return RegistryManager.get( dbname, loadwithoutmigration=loadwithoutmigration, log_repeat=log_repeat, **settings, )
def drop_database(database): """ Close the registry instance of the database and drop the database :param: database's name """ url = Configuration.get('get_url')(db_name=database) if not database_exists(url): raise Exception("Database %r does not already exist") registry = RegistryManager.get(database) registry.close() SU_drop_database(url)
def getRegistry(cls): """Return the registry for the test database. This assumes the database is created, and the registry has already been initialized:: registry = self.getRegistry() :rtype: registry instance """ additional_setting = cls.additional_setting() return RegistryManager.get(Configuration.get('db_name'), **additional_setting)
def setUpClass(cls): """ Initialize the registry. """ super(BlokTestCase, cls).setUpClass() additional_setting = cls.additional_setting() if cls.registry is None: if len(BlokManager.list()) == 0: BlokManager.load() cls.registry = RegistryManager.get(Configuration.get('db_name'), **additional_setting) cls.registry.commit()
def create_database(database): """ Create a new database, initialize it and return an AnyBlok registry :param: database's name :rtype: AnyBlok registry instance """ url = Configuration.get('get_url')(db_name=database) if database_exists(url): raise Exception("Database %r already exist") db_template_name = Configuration.get('db_template_name', None) SU_create_database(url, template=db_template_name) registry = RegistryManager.get(database) return registry
def after_process_message(self, broker, message, *, result=None, exception=None): """Called after process message If the message is in the ``Model.Dramatiq.Message`` then the status will be change to **done** or **failed**. .. note:: the status is failed if an exception is passed or a rollback is need Before the end, the session is expired to release the Session pool thread :param broker: the broker used :param message: the message send in the broker :param result: return by the process :param exception: any ``Exception`` raised by the process """ registry = RegistryManager.get(Configuration.get('db_name')) logger.debug( "[after_process_message] %s: update message(%s) status " "with result %r and exception %r", id(registry.session), message.message_id, result, exception) EnvironmentManager.set('is_called_by_dramatiq_actor', False) M = registry.Dramatiq.Message STATUS_DONE = M.STATUS_DONE STATUS_FAILED = M.STATUS_FAILED try: m = M.get_instance_of(message) if m: m.update_status( STATUS_FAILED if exception is not None else STATUS_DONE, error=str(exception) if exception else None) registry.commit() except Exception as e: registry.rollback() m = M.get_instance_of(message) if m: m.update_status(M.STATUS_FAILED, error=str(e)) registry.commit() raise e finally: registry.expire_all()
def worker_process(worker_id, logging_fd): """consume worker to process messages and execute the actor""" # TODO preload registries db_name = Configuration.get('db_name') try: logging_pipe = os.fdopen(logging_fd, "w") broker = prepare_broker(withmiddleware=True) broker.emit_after("process_boot") BlokManager.load() registry = RegistryManager.get(db_name, loadwithoutmigration=True) if registry is None: logger.critical("No registry found for %s", db_name) return os._exit(4) worker = Worker( broker, worker_threads=Configuration.get('dramatiq_threads', 1)) worker.start() print('worker started') except ImportError as e: logger.critical(e) return os._exit(2) except ConnectionError as e: logger.critical("Broker connection failed. %s", e) return os._exit(3) def termhandler(signum, frame): nonlocal running BlokManager.unload() if running: logger.info("Stopping worker process...") running = False else: logger.warning("Killing worker process...") return os._exit(1) logger.info("Worker process is ready for action.") signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, termhandler) signal.signal(signal.SIGHUP, termhandler) running = True while running: time.sleep(1) worker.stop() broker.close() logging_pipe.close()
def before_enqueue(self, broker, message, delay): """Called when a message is delayed or enqueued If the message is in the ``Model.Dramatiq.Message`` then the status will be change to **delayed** or **enqueued** :param broker: the broker used :param message: the message send in the broker :param delay: delay in milliseconds """ registry = RegistryManager.get(Configuration.get('db_name')) logger.debug("[before_enqueue] %s: update message(%s) status ", id(registry.session), message.message_id) M = registry.Dramatiq.Message m = M.get_instance_of(message) if m: m.update_status(M.STATUS_DELAYED if delay else M.STATUS_ENQUEUED) registry.commit()
def update_project(): anyblok.load_init_function_from_entry_points() Configuration.load('update-project') anyblok.configuration_post_load() BlokManager.load() db_name = Configuration.get('db_name') logger.info("update project: db_name=%r", db_name) to_install = [] to_update = [] to_uninstall = [] url = get_url() options = {} if not database_exists(url): db_template_name = Configuration.get('db_template_name', None) create_database(url, template=db_template_name) to_install.append('team') version = None else: options.update(dict(loadwithoutmigration=True)) registry = RegistryManager.get(db_name, **options) registry.update_blok_list() # case, new blok added version = registry.System.Blok.query().filter_by( name='team').one().installed_version if version is None: pass else: to_update.append('team') registry.upgrade(install=to_install, update=to_update, uninstall=to_uninstall) if version is None: registry.Pyramid.User.insert(login='******') registry.Pyramid.CredentialStore.insert(login='******', password='******') registry.commit() registry.close() logger.info("Project updated: db_name=%r", db_name)
def load_registry(self): if not self.enabled or self.registryLoaded: return from anyblok.config import Configuration, get_db_name from anyblok import ( load_init_function_from_entry_points, configuration_post_load, ) from anyblok.blok import BlokManager from anyblok.registry import RegistryManager, return_list # Load the registry here not in configuration, # because the configurations are not loaded in order of score self.registryLoaded = True load_init_function_from_entry_points(unittest=True) Configuration.load_config_for_test() Configuration.parse_options(self.AnyBlokOptions) configuration_post_load() BlokManager.load() db_name = get_db_name() registry = RegistryManager.get(db_name) if not registry: return installed_bloks = registry.System.Blok.list_by_state("installed") selected_bloks = return_list( Configuration.get('selected_bloks')) or installed_bloks unwanted_bloks = return_list(Configuration.get('unwanted_bloks')) or [] self.authorized_blok_paths = set( BlokManager.getPath(b) for b in BlokManager.list() if b in selected_bloks and b not in unwanted_bloks) test_dirs = self.authorized_blok_test_dirs = set() for startpath in self.authorized_blok_paths: for root, dirs, _ in walk(startpath): if 'tests' in dirs: test_dirs.add(join(root, 'tests')) registry.close() # free the registry to force create it again
def after_skip_message(self, broker, message): """Called after skip message If the message is in the ``Model.Dramatiq.Message`` then the status will be change to **skip** Before the end, the session is expired to release the Session pool thread :param broker: the broker used :param message: the message send in the broker """ registry = RegistryManager.get(Configuration.get('db_name')) logger.debug("[after_skip_message] %s: update message(%s) status ", id(registry.session), message.message_id) registry.rollback() M = registry.Dramatiq.Message m = M.get_instance_of(message) m.update_status(M.STATUS_SKIPED) registry.commit()
def load_main(request): """ Return the client main page """ dbname = Configuration.get('get_db_name')(request) state = request.session.get('state') if state == 'connected': registry = RegistryManager.get(dbname) css = registry.Web.get_css() js = registry.Web.get_js() else: css = get_static('global_css') js = get_static('global_js') title = Configuration.get('app_name', 'AnyBlok / FuretUI') return render_to_response('anyblok_furetui:client.mak', { 'title': title, 'css': css, 'js': js, }, request=request)
def init_session(request, configuration_loaded): # Init registry additional_setting = {'unittest': True} if len(BlokManager.list()) == 0: BlokManager.load() registry = RegistryManager.get(Configuration.get('db_name'), **additional_setting) registry.commit() # Add savepoint registry.begin_nested() @event.listens_for(registry.session, 'after_transaction_end') def restart_savepoint(session, transaction): if transaction.nested and not transaction._parent.nested: session.expire_all() session.begin_nested() logger.info('Creating registry') yield registry request.addfinalizer(registry.session.close)
def bus_worker_process(logging_fd, consumers): """consume worker to process messages and execute the actor""" # TODO preload registries db_name = Configuration.get('db_name') profile = Configuration.get('bus_profile') try: logging_pipe = os.fdopen(logging_fd, "w") registry = RegistryManager.get(db_name, loadwithoutmigration=True) worker = ReconnectingWorker(registry, profile, consumers) worker.start() except ImportError as e: logger.critical(e) return os._exit(2) except ConnectionError as e: logger.critical("Broker connection failed. %s", e) return os._exit(3) def termhandler(signum, frame): nonlocal running BlokManager.unload() if running: logger.info("Stopping worker process...") running = False else: logger.warning("Killing worker process...") return os._exit(1) logger.info("Worker process is ready for action.") signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, termhandler) signal.signal(signal.SIGHUP, termhandler) running = True while running: time.sleep(1) worker.stop() logging_pipe.close()
def load_registry(self): if self.enabled and self.registryLoaded is False: # Load the registry here not in configuration, # because the configuration are not load in order of score self.registryLoaded = True load_init_function_from_entry_points(unittest=True) Configuration.load_config_for_test() Configuration.parse_options(self.AnyBlokOptions) configuration_post_load() BlokManager.load() db_name = get_db_name() registry = RegistryManager.get(db_name) if registry: installed_bloks = registry.System.Blok.list_by_state( "installed") selected_bloks = return_list( Configuration.get('selected_bloks')) or installed_bloks unwanted_bloks = return_list( Configuration.get('unwanted_bloks')) or [] self.bloks_path = [BlokManager.getPath(x) for x in BlokManager.ordered_bloks] self.authoried_bloks_test_files = [] for blok in installed_bloks: if blok not in selected_bloks or blok in unwanted_bloks: continue startpath = BlokManager.getPath(blok) for root, dirs, _ in walk(startpath): if 'tests' in dirs: self.authoried_bloks_test_files.append( join(root, 'tests')) registry.close() # free the registry to force create it again
def test_get_the_same_registry(self): registry = self.init_registry(None) registry2 = RegistryManager.get( Configuration.get('db_name'), unittest=True) assert registry is registry2
def get_registry_for(dbname): settings = { 'sa.session.extension': AnyBlokZopeTransactionExtension, } return RegistryManager.get(dbname, **settings)