def continuous(wtype, arguments, isolation_level=DEFAULT_ISOLATION, cleanup=False): """Start a continuous worker. :param bool cleanup: if ``True`` remove all existing records of the same worker type. They are considered stale from previous runs. """ registry = anyblok.start('basic', configuration_groups=[], loadwithoutmigration=True, isolation_level=DEFAULT_ISOLATION) if registry is None: logging.critical("continuous worker(type=%s): couldn't init registry", wtype) sys.exit(1) Worker = getattr(registry.Wms.Worker, wtype) if cleanup: logger.info("Cleaning up any stale %s record due to previous runs", Worker.__registry_name__) Worker.query().delete() registry.commit() process = Worker.insert(pid=os.getpid()) registry.commit() while not process.should_proceed(): logger.info("Regular workers not yet running. Waiting a bit") time.sleep(0.1) registry.rollback() process.run()
def registry2doc(application, configuration_groups, **kwargs): """Return auto documentation for the registry :param application: name of the application :param configuration_groups: list configuration groupe to load :param \**kwargs: ArgumentParser named arguments """ format_configuration(configuration_groups, 'doc', 'schema') registry = anyblok.start(application, configuration_groups=configuration_groups, **kwargs) if registry: registry.commit() doc = registry.Documentation() doc.auto_doc() if Configuration.get('doc_format') == 'RST': with open(Configuration.get('doc_output'), 'w') as fp: doc.toRST(fp) elif Configuration.get('doc_format') == 'UML': format_ = Configuration.get('schema_format') name_ = Configuration.get('schema_output') dot = ModelSchema(name_, format=format_) doc.toUML(dot) dot.save() elif Configuration.get('doc_format') == 'SQL': format_ = Configuration.get('schema_format') name_ = Configuration.get('schema_output') dot = SQLSchema(name_, format=format_) doc.toSQL(dot) dot.save()
def translate_en(): """To switch the examples to the english version. Once this has run, add_data() can't work anymore. This is obviously suboptimal: in a real application, one would probably refactor stuff, but we don't want to change the commands that were run during the presentations, which makes more constraints. """ registry = anyblok.start('basic', configuration_groups=[], loadwithoutmigration=True, isolation_level='SERIALIZABLE') if registry is None: logging.critical("Couldn't initialize registry") sys.exit(1) init = Initializer(registry) try: init.translate_en() except: import pdb pdb.post_mortem(sys.exc_info()[2]) registry.rollback() raise registry.commit() registry.close()
def anyblok_nose(): """Run nose unit test for the registry """ registry = anyblok.start('nose', useseparator=True, unittest=True) defaultTest = [] if registry: installed_bloks = registry.System.Blok.list_by_state("installed") selected_bloks = return_list( Configuration.get('selected_bloks')) or installed_bloks unwanted_bloks = return_list( Configuration.get('unwanted_bloks')) or [] defaultTest = [] for blok in installed_bloks: if blok not in selected_bloks or blok in unwanted_bloks: continue startpath = BlokManager.getPath(blok) for root, dirs, _ in walk(startpath): if 'tests' in dirs: defaultTest.append(join(root, 'tests')) registry.close() # free the registry to force create it again sys.exit(main(defaultTest=defaultTest))
def anyblok_nose(): """Run nose unit test for the registry """ warnings.simplefilter('default') registry = anyblok.start('nose', useseparator=True, unittest=True) defaultTest = [] if registry: installed_bloks = registry.System.Blok.list_by_state("installed") selected_bloks = return_list( Configuration.get('selected_bloks')) or installed_bloks unwanted_bloks = return_list( Configuration.get('unwanted_bloks')) or [] defaultTest = [] for blok in installed_bloks: if blok not in selected_bloks or blok in unwanted_bloks: continue startpath = BlokManager.getPath(blok) for root, dirs, _ in walk(startpath): if 'tests' in dirs: defaultTest.append(join(root, 'tests')) registry.close() # free the registry to force create it again sys.exit(main(defaultTest=defaultTest))
def regular_worker(arguments): registry = anyblok.start('basic', configuration_groups=[], loadwithoutmigration=True, isolation_level=DEFAULT_ISOLATION) if registry is None: logging.critical("regular_worker: couldn't init registry") sys.exit(1) Worker = registry.Wms.Worker.Regular previous_run_timeslice = Worker.query(func.max( Worker.done_timeslice)).first()[0] if previous_run_timeslice is None: previous_run_timeslice = 0 timeslices = arguments.timeslices process = Worker.insert( pid=os.getpid(), active=True, done_timeslice=previous_run_timeslice, max_timeslice=previous_run_timeslice + timeslices, ) for i in range(1, 1 + timeslices): registry.commit() try: process.run_timeslice() except KeyboardInterrupt: process.stop() break process.wait_others(i) registry.commit()
def run_exit(application, configuration_groups, **kwargs): """Run nose unit test for the registry :param application: name of the application :param configuration_groups: list configuration groupe to load :param \**kwargs: ArgumentParser named arguments """ format_configuration(configuration_groups, 'unittest') registry = anyblok.start(application, configuration_groups=configuration_groups, useseparator=True, unittest=True, **kwargs) defaultTest = [] if registry: installed_bloks = registry.System.Blok.list_by_state("installed") selected_bloks = Configuration.get('selected_bloks') if not selected_bloks: selected_bloks = installed_bloks unwanted_bloks = Configuration.get('unwanted_bloks') or [] defaultTest = [path for blok in installed_bloks if blok in selected_bloks and blok not in unwanted_bloks for path in [join(BlokManager.getPath(blok), 'tests')] if exists(path)] registry.close() # free the registry to force create it again sys.exit(main(defaultTest=defaultTest))
def anyblok_updatedb(): """Update an existing database""" registry = anyblok.start('updatedb', loadwithoutmigration=True) installed_bloks = registry.System.Blok.list_by_state('installed') toupdate_bloks = registry.System.Blok.list_by_state('toupdate') required_install_bloks = [] required_update_bloks = [] for blok in (Configuration.get('install_or_update_bloks') or []): if blok in installed_bloks: required_update_bloks.append(blok) elif blok not in toupdate_bloks: required_install_bloks.append(blok) if Configuration.get('install_all_bloks'): install_bloks = registry.System.Blok.list_by_state('uninstalled') else: install_bloks = Configuration.get('install_bloks') or [] install_bloks = list(set(install_bloks + required_install_bloks)) if Configuration.get('update_all_bloks'): update_bloks = registry.System.Blok.list_by_state('installed') else: update_bloks = Configuration.get('update_bloks') or [] update_bloks = list(set(update_bloks + required_update_bloks)) uninstall_bloks = Configuration.get('uninstall_bloks') if registry: registry.update_blok_list() # case, new blok added registry.upgrade(install=install_bloks, update=update_bloks, uninstall=uninstall_bloks) registry.commit() registry.close()
def anyblok_updatedb(): """Update an existing database""" anyblok_registry = anyblok.start('updatedb', loadwithoutmigration=True) installed_bloks = anyblok_registry.System.Blok.list_by_state('installed') toupdate_bloks = anyblok_registry.System.Blok.list_by_state('toupdate') required_install_bloks = [] required_update_bloks = [] for blok in (Configuration.get('install_or_update_bloks') or []): if blok in installed_bloks: required_update_bloks.append(blok) elif blok not in toupdate_bloks: required_install_bloks.append(blok) if Configuration.get('install_all_bloks'): install_bloks = anyblok_registry.System.Blok.list_by_state( 'uninstalled') else: install_bloks = Configuration.get('install_bloks') or [] install_bloks = list(set(install_bloks + required_install_bloks)) if Configuration.get('update_all_bloks'): update_bloks = anyblok_registry.System.Blok.list_by_state('installed') else: update_bloks = Configuration.get('update_bloks') or [] update_bloks = list(set(update_bloks + required_update_bloks)) uninstall_bloks = Configuration.get('uninstall_bloks') if anyblok_registry: anyblok_registry.update_blok_list() # case, new blok added anyblok_registry.upgrade(install=install_bloks, update=update_bloks, uninstall=uninstall_bloks) anyblok_registry.commit() anyblok_registry.close()
def furetui_user(): """Update an existing database""" registry = anyblok.start( 'furetui-user', loadwithoutmigration=True) login = Configuration.get('furetui_user_login') password = Configuration.get('furetui_user_password') roles = Configuration.get('furetui_user_roles') or [] registry.FuretUI.user_management(login, password, roles)
def test_start_function(self): BlokManager.unload() db_name = Configuration.get('db_name') or 'test_anyblok' db_driver_name = Configuration.get('db_driver_name') or 'postgresql' testargs = ['default', '--db-name', db_name, '--db-driver-name', db_driver_name] with patch.object(sys, 'argv', testargs): registry = start('default') assert registry is not None
def test_start_function(self): BlokManager.unload() db_name = Configuration.get('db_name') or 'test_anyblok' db_driver_name = Configuration.get('db_driver_name') or 'postgresql' testargs = ['default', '--db-name', db_name, '--db-driver-name', db_driver_name] with patch.object(sys, 'argv', testargs): registry = start('default') self.assertIsNotNone(registry)
def cron_worker(application, configuration_groups, **kwargs): """Execute a cron worker :param application: name of the application :param configuration_groups: list configuration groupe to load :param \**kwargs: ArgumentParser named arguments """ registry = anyblok.start(application, configuration_groups=configuration_groups, **kwargs) if registry: registry.commit() registry.System.Cron.run()
def populate_stations(): """This is the script called by using anyblok_populate_stations -c app.dev.cfg This script can be used to perform a complete population of Model.Station table using data/stations.csv file. This is mainly intented for development purposes, in case of update of CSV parsing methods, in order to be able to run a full population with having to update completely database. [NOTE] This can be ran on production server to update Model.Station table.""" registry = anyblok.start("populate_stations", argparse_groups=["logging"]) file_path = Configuration.get("stations_data") stations_populate(registry=registry, path=file_path)
def populate_reduction_cards(): """This is the script called by using anyblok_populate_reduction_cards -c app.dev.cfg This script can be used to perform a complete population of Model.ReductionCard This is mainly intented for development purposes, in case of update of model Model.ReductionCard, in order to be able to run a full population with having to update completely database. [NOTE] This can be ran on production server to update Model.ReductionCard table.""" registry = anyblok.start("populate_reduction_cards", argparse_groups=["logging"]) reduction_cards_populate(registry=registry)
def update_labels_status(): """Execute a script or open an interpreter """ registry = anyblok.start('update_labels_status') if registry: Shipment = registry.Delivery.Shipment query = Shipment.query() query = query.filter(Shipment.status.in_(status)) for ship in query.all(): try: ship.get_label_status() except Exception: logger.exception('failed to get label') registry.commit() registry.close()
def anyblok_interpreter(): """Execute a script or open an interpreter """ registry = anyblok.start('interpreter') if registry: registry.commit() python_script = Configuration.get('python_script') if python_script: with open(python_script, "r") as fh: exec(fh.read(), None, locals()) else: try: from IPython import embed embed() except ImportError: import code code.interact(local=locals())
def updatedb(application, configuration_groups, **kwargs): """ Update an existing database :param application: name of the application :param configuration_groups: list configuration groupe to load :param \**kwargs: ArgumentParser named arguments """ format_configuration(configuration_groups, 'install-bloks', 'uninstall-bloks', 'update-bloks', 'install-or-update-bloks') registry = anyblok.start(application, configuration_groups=configuration_groups, loadwithoutmigration=True, **kwargs) installed_bloks = registry.System.Blok.list_by_state('installed') required_install_bloks = [] required_update_bloks = [] for blok in (Configuration.get('install_or_update_bloks') or []): if blok in installed_bloks: required_update_bloks.append(blok) else: required_install_bloks.append(blok) if Configuration.get('install_all_bloks'): install_bloks = registry.System.Blok.list_by_state('uninstalled') else: install_bloks = Configuration.get('install_bloks') or [] install_bloks = list(set(install_bloks + required_install_bloks)) if Configuration.get('update_all_bloks'): update_bloks = registry.System.Blok.list_by_state('installed') else: update_bloks = Configuration.get('update_bloks') or [] update_bloks = list(set(update_bloks + required_update_bloks)) uninstall_bloks = Configuration.get('uninstall_bloks') if registry: registry.update_blok_list() # case, new blok added registry.upgrade(install=install_bloks, update=update_bloks, uninstall=uninstall_bloks) registry.commit() registry.close()
def anyblok_nose(): """Run nose unit test after giving it the registry """ warnings.simplefilter('default') warnings.warn( "This script is deprecated and will be removed soon. " "The Nose test machinery has been removed from the framework in order " "to be replaced with Pytest. " "If you need to run your tests with nose, install the Nose package.", DeprecationWarning, stacklevel=2) try: from nose import main except ImportError: logger.error('"Nosetest" is not installed, try: pip install nose') anyblok_registry = anyblok.start('nose', useseparator=True, unittest=True) if anyblok_registry: installed_bloks = anyblok_registry.System.Blok.list_by_state( "installed") selected_bloks = return_list( Configuration.get('selected_bloks')) or installed_bloks unwanted_bloks = return_list( Configuration.get('unwanted_bloks')) or [] unwanted_bloks.extend(['anyblok-core', 'anyblok-test', 'model_authz']) defaultTest = [] for blok in installed_bloks: if blok not in selected_bloks or blok in unwanted_bloks: continue startpath = BlokManager.getPath(blok) for root, dirs, _ in walk(startpath): if 'tests' in dirs: defaultTest.append(join(root, 'tests')) anyblok_registry.close() # free the registry to force create it again sys.exit(main(defaultTest=defaultTest))
def add(): """Add more data to the example DB. This was merely useful to prepare incrementally the example. """ registry = anyblok.start('basic', configuration_groups=[], loadwithoutmigration=True, isolation_level='SERIALIZABLE') if registry is None: logging.critical("Couldn't initialize registry") sys.exit(1) init = Initializer(registry) try: init.add_data() except: import pdb pdb.post_mortem(sys.exc_info()[2]) registry.rollback() raise registry.commit() registry.close()
def anyblok2doc(): """Return auto documentation for the registry """ anyblok_registry = anyblok.start('autodoc') if anyblok_registry: anyblok_registry.commit() doc = anyblok_registry.Documentation() doc.auto_doc() if Configuration.get('doc_format') == 'RST': with open(Configuration.get('doc_output'), 'w') as fp: doc.toRST(fp) elif Configuration.get('doc_format') == 'UML': format_ = Configuration.get('schema_format') name_ = Configuration.get('schema_output') dot = ModelSchema(name_, format=format_) doc.toUML(dot) dot.save() elif Configuration.get('doc_format') == 'SQL': format_ = Configuration.get('schema_format') name_ = Configuration.get('schema_output') dot = SQLSchema(name_, format=format_) doc.toSQL(dot) dot.save()
def exampleblok(): # Initialise the application, with a name and a version number # select the groupe of options to display # return a registry if the database are selected registry = anyblok.start( 'Example Blok', configuration_groups=['message', 'logging']) message_before = Configuration.get('message_before') message_after = Configuration.get('message_after') if not registry: logger.warning("No database database selected") exit(0) if message_before: logger.info(message_before) for address in registry.Address.query().all(): for room in address.rooms: for worker in room.workers: logger.info(worker) if message_after: logger.info(message_after)
def anyblok2doc(): """Return auto documentation for the registry """ registry = anyblok.start('autodoc') if registry: registry.commit() doc = registry.Documentation() doc.auto_doc() if Configuration.get('doc_format') == 'RST': with open(Configuration.get('doc_output'), 'w') as fp: doc.toRST(fp) elif Configuration.get('doc_format') == 'UML': format_ = Configuration.get('schema_format') name_ = Configuration.get('schema_output') dot = ModelSchema(name_, format=format_) doc.toUML(dot) dot.save() elif Configuration.get('doc_format') == 'SQL': format_ = Configuration.get('schema_format') name_ = Configuration.get('schema_output') dot = SQLSchema(name_, format=format_) doc.toSQL(dot) dot.save()
def interpreter(application, configuration_groups, **kwargs): """Execute a script or open an interpreter :param application: name of the application :param configuration_groups: list configuration groupe to load :param \**kwargs: ArgumentParser named arguments """ format_configuration(configuration_groups, 'interpreter') registry = anyblok.start(application, configuration_groups=configuration_groups, **kwargs) if registry: registry.commit() python_script = Configuration.get('python_script') if python_script: with open(python_script, "r") as fh: exec(fh.read(), None, locals()) else: try: from IPython import embed embed() except ImportError: import code code.interact(local=locals())
def anyblok_bus(): # noqa """Run consumer workers process to consume queue """ registry = start('bus', loadwithoutmigration=True) if not registry: exit(1) unexisting_queues = registry.Bus.get_unexisting_queues() if unexisting_queues: profile_name = Configuration.get('bus_profile') profile = registry.Bus.Profile.query().filter_by( name=profile_name).one_or_none() logger.critical("Some queues (%s) are required by consumers on %r", ', '.join(unexisting_queues), profile) exit(1) all_consumers = registry.Bus.get_consumers() registry.close() # close the registry to recreate it in each process worker_pipes = [] worker_processes = [] for processes, consumers in all_consumers: logger.debug('Consume %r, with %r processes', consumers, processes) for worker_id in range(processes): read_fd, write_fd = os.pipe() pid = os.fork() if pid != 0: os.close(write_fd) worker_pipes.append(os.fdopen(read_fd)) worker_processes.append(pid) continue os.close(read_fd) return bus_worker_process(write_fd, consumers) def sighandler(signum, frame): nonlocal worker_processes signum = { signal.SIGINT: signal.SIGTERM, signal.SIGTERM: signal.SIGTERM, signal.SIGHUP: signal.SIGHUP, }[signum] logger.info("Sending %r to worker processes...", signum.name) for pid in worker_processes: try: os.kill(pid, signum) except OSError: logger.warning("Failed to send %r to pid %d.", signum.name, pid) retcode = 0 signal.signal(signal.SIGINT, sighandler) signal.signal(signal.SIGTERM, sighandler) signal.signal(signal.SIGHUP, sighandler) for pid in worker_processes: pid, rc = os.waitpid(pid, 0) retcode = max(retcode, rc >> 8) running = False # noqa for pipe in worker_pipes: pipe.close() return retcode
def export_i18n(): registry = anyblok.start( 'export-i18n', loadwithoutmigration=True) blok_name = Configuration.get('blok-name') registry.FuretUI.export_i18n(blok_name)