def continuous(wtype,
arguments,
isolation_level=DEFAULT_ISOLATION,
cleanup=False):
"""Start a continuous worker.
:param bool cleanup: if ``True`` remove all existing records of
the same worker type. They are considered stale
from previous runs.
"""
registry = anyblok.start('basic',
configuration_groups=[],
loadwithoutmigration=True,
isolation_level=DEFAULT_ISOLATION)
if registry is None:
logging.critical("continuous worker(type=%s): couldn't init registry",
wtype)
sys.exit(1)
Worker = getattr(registry.Wms.Worker, wtype)
if cleanup:
logger.info("Cleaning up any stale %s record due to previous runs",
Worker.__registry_name__)
Worker.query().delete()
registry.commit()
process = Worker.insert(pid=os.getpid())
registry.commit()
while not process.should_proceed():
logger.info("Regular workers not yet running. Waiting a bit")
time.sleep(0.1)
registry.rollback()
process.run()
def registry2doc(application, configuration_groups, **kwargs):
"""Return auto documentation for the registry
:param application: name of the application
:param configuration_groups: list configuration groupe to load
:param \**kwargs: ArgumentParser named arguments
"""
format_configuration(configuration_groups, 'doc', 'schema')
registry = anyblok.start(application,
configuration_groups=configuration_groups,
**kwargs)
if registry:
registry.commit()
doc = registry.Documentation()
doc.auto_doc()
if Configuration.get('doc_format') == 'RST':
with open(Configuration.get('doc_output'), 'w') as fp:
doc.toRST(fp)
elif Configuration.get('doc_format') == 'UML':
format_ = Configuration.get('schema_format')
name_ = Configuration.get('schema_output')
dot = ModelSchema(name_, format=format_)
doc.toUML(dot)
dot.save()
elif Configuration.get('doc_format') == 'SQL':
format_ = Configuration.get('schema_format')
name_ = Configuration.get('schema_output')
dot = SQLSchema(name_, format=format_)
doc.toSQL(dot)
dot.save()
def translate_en():
"""To switch the examples to the english version.
Once this has run, add_data() can't work anymore.
This is obviously suboptimal: in a real application, one would
probably refactor stuff, but we don't want to change the commands
that were run during the presentations, which makes more constraints.
"""
registry = anyblok.start('basic', configuration_groups=[],
loadwithoutmigration=True,
isolation_level='SERIALIZABLE')
if registry is None:
logging.critical("Couldn't initialize registry")
sys.exit(1)
init = Initializer(registry)
try:
init.translate_en()
except:
import pdb
pdb.post_mortem(sys.exc_info()[2])
registry.rollback()
raise
registry.commit()
registry.close()
def anyblok_nose():
"""Run nose unit test for the registry
"""
registry = anyblok.start('nose', useseparator=True, unittest=True)
defaultTest = []
if registry:
installed_bloks = registry.System.Blok.list_by_state("installed")
selected_bloks = return_list(
Configuration.get('selected_bloks')) or installed_bloks
unwanted_bloks = return_list(
Configuration.get('unwanted_bloks')) or []
defaultTest = []
for blok in installed_bloks:
if blok not in selected_bloks or blok in unwanted_bloks:
continue
startpath = BlokManager.getPath(blok)
for root, dirs, _ in walk(startpath):
if 'tests' in dirs:
defaultTest.append(join(root, 'tests'))
registry.close() # free the registry to force create it again
sys.exit(main(defaultTest=defaultTest))
def anyblok_nose():
"""Run nose unit test for the registry
"""
warnings.simplefilter('default')
registry = anyblok.start('nose', useseparator=True, unittest=True)
defaultTest = []
if registry:
installed_bloks = registry.System.Blok.list_by_state("installed")
selected_bloks = return_list(
Configuration.get('selected_bloks')) or installed_bloks
unwanted_bloks = return_list(
Configuration.get('unwanted_bloks')) or []
defaultTest = []
for blok in installed_bloks:
if blok not in selected_bloks or blok in unwanted_bloks:
continue
startpath = BlokManager.getPath(blok)
for root, dirs, _ in walk(startpath):
if 'tests' in dirs:
defaultTest.append(join(root, 'tests'))
registry.close() # free the registry to force create it again
sys.exit(main(defaultTest=defaultTest))
def regular_worker(arguments):
registry = anyblok.start('basic',
configuration_groups=[],
loadwithoutmigration=True,
isolation_level=DEFAULT_ISOLATION)
if registry is None:
logging.critical("regular_worker: couldn't init registry")
sys.exit(1)
Worker = registry.Wms.Worker.Regular
previous_run_timeslice = Worker.query(func.max(
Worker.done_timeslice)).first()[0]
if previous_run_timeslice is None:
previous_run_timeslice = 0
timeslices = arguments.timeslices
process = Worker.insert(
pid=os.getpid(),
active=True,
done_timeslice=previous_run_timeslice,
max_timeslice=previous_run_timeslice + timeslices,
)
for i in range(1, 1 + timeslices):
registry.commit()
try:
process.run_timeslice()
except KeyboardInterrupt:
process.stop()
break
process.wait_others(i)
registry.commit()
def run_exit(application, configuration_groups, **kwargs):
"""Run nose unit test for the registry
:param application: name of the application
:param configuration_groups: list configuration groupe to load
:param \**kwargs: ArgumentParser named arguments
"""
format_configuration(configuration_groups, 'unittest')
registry = anyblok.start(application,
configuration_groups=configuration_groups,
useseparator=True, unittest=True, **kwargs)
defaultTest = []
if registry:
installed_bloks = registry.System.Blok.list_by_state("installed")
selected_bloks = Configuration.get('selected_bloks')
if not selected_bloks:
selected_bloks = installed_bloks
unwanted_bloks = Configuration.get('unwanted_bloks') or []
defaultTest = [path
for blok in installed_bloks
if blok in selected_bloks and blok not in unwanted_bloks
for path in [join(BlokManager.getPath(blok), 'tests')]
if exists(path)]
registry.close() # free the registry to force create it again
sys.exit(main(defaultTest=defaultTest))
def anyblok_updatedb():
"""Update an existing database"""
registry = anyblok.start('updatedb', loadwithoutmigration=True)
installed_bloks = registry.System.Blok.list_by_state('installed')
toupdate_bloks = registry.System.Blok.list_by_state('toupdate')
required_install_bloks = []
required_update_bloks = []
for blok in (Configuration.get('install_or_update_bloks') or []):
if blok in installed_bloks:
required_update_bloks.append(blok)
elif blok not in toupdate_bloks:
required_install_bloks.append(blok)
if Configuration.get('install_all_bloks'):
install_bloks = registry.System.Blok.list_by_state('uninstalled')
else:
install_bloks = Configuration.get('install_bloks') or []
install_bloks = list(set(install_bloks + required_install_bloks))
if Configuration.get('update_all_bloks'):
update_bloks = registry.System.Blok.list_by_state('installed')
else:
update_bloks = Configuration.get('update_bloks') or []
update_bloks = list(set(update_bloks + required_update_bloks))
uninstall_bloks = Configuration.get('uninstall_bloks')
if registry:
registry.update_blok_list() # case, new blok added
registry.upgrade(install=install_bloks, update=update_bloks,
uninstall=uninstall_bloks)
registry.commit()
registry.close()
def anyblok_updatedb():
"""Update an existing database"""
anyblok_registry = anyblok.start('updatedb', loadwithoutmigration=True)
installed_bloks = anyblok_registry.System.Blok.list_by_state('installed')
toupdate_bloks = anyblok_registry.System.Blok.list_by_state('toupdate')
required_install_bloks = []
required_update_bloks = []
for blok in (Configuration.get('install_or_update_bloks') or []):
if blok in installed_bloks:
required_update_bloks.append(blok)
elif blok not in toupdate_bloks:
required_install_bloks.append(blok)
if Configuration.get('install_all_bloks'):
install_bloks = anyblok_registry.System.Blok.list_by_state(
'uninstalled')
else:
install_bloks = Configuration.get('install_bloks') or []
install_bloks = list(set(install_bloks + required_install_bloks))
if Configuration.get('update_all_bloks'):
update_bloks = anyblok_registry.System.Blok.list_by_state('installed')
else:
update_bloks = Configuration.get('update_bloks') or []
update_bloks = list(set(update_bloks + required_update_bloks))
uninstall_bloks = Configuration.get('uninstall_bloks')
if anyblok_registry:
anyblok_registry.update_blok_list() # case, new blok added
anyblok_registry.upgrade(install=install_bloks, update=update_bloks,
uninstall=uninstall_bloks)
anyblok_registry.commit()
anyblok_registry.close()
def furetui_user():
"""Update an existing database"""
registry = anyblok.start(
'furetui-user', loadwithoutmigration=True)
login = Configuration.get('furetui_user_login')
password = Configuration.get('furetui_user_password')
roles = Configuration.get('furetui_user_roles') or []
registry.FuretUI.user_management(login, password, roles)
def test_start_function(self):
BlokManager.unload()
db_name = Configuration.get('db_name') or 'test_anyblok'
db_driver_name = Configuration.get('db_driver_name') or 'postgresql'
testargs = ['default', '--db-name', db_name, '--db-driver-name',
db_driver_name]
with patch.object(sys, 'argv', testargs):
registry = start('default')
assert registry is not None
def test_start_function(self):
BlokManager.unload()
db_name = Configuration.get('db_name') or 'test_anyblok'
db_driver_name = Configuration.get('db_driver_name') or 'postgresql'
testargs = ['default', '--db-name', db_name, '--db-driver-name',
db_driver_name]
with patch.object(sys, 'argv', testargs):
registry = start('default')
self.assertIsNotNone(registry)
def cron_worker(application, configuration_groups, **kwargs):
"""Execute a cron worker
:param application: name of the application
:param configuration_groups: list configuration groupe to load
:param \**kwargs: ArgumentParser named arguments
"""
registry = anyblok.start(application,
configuration_groups=configuration_groups,
**kwargs)
if registry:
registry.commit()
registry.System.Cron.run()
def populate_stations():
"""This is the script called by using anyblok_populate_stations -c app.dev.cfg
This script can be used to perform a complete population of
Model.Station table using data/stations.csv file.
This is mainly intented for development purposes, in case of update of
CSV parsing methods, in order to be able to run a full population with
having to update completely database.
[NOTE] This can be ran on production server to update Model.Station
table."""
registry = anyblok.start("populate_stations", argparse_groups=["logging"])
file_path = Configuration.get("stations_data")
stations_populate(registry=registry, path=file_path)
def populate_reduction_cards():
"""This is the script called by using anyblok_populate_reduction_cards -c
app.dev.cfg
This script can be used to perform a complete population of
Model.ReductionCard
This is mainly intented for development purposes, in case of update of
model Model.ReductionCard, in order to be able to run a full population
with having to update completely database.
[NOTE] This can be ran on production server to update Model.ReductionCard
table."""
registry = anyblok.start("populate_reduction_cards",
argparse_groups=["logging"])
reduction_cards_populate(registry=registry)
def update_labels_status():
"""Execute a script or open an interpreter
"""
registry = anyblok.start('update_labels_status')
if registry:
Shipment = registry.Delivery.Shipment
query = Shipment.query()
query = query.filter(Shipment.status.in_(status))
for ship in query.all():
try:
ship.get_label_status()
except Exception:
logger.exception('failed to get label')
registry.commit()
registry.close()
def anyblok_interpreter():
"""Execute a script or open an interpreter
"""
registry = anyblok.start('interpreter')
if registry:
registry.commit()
python_script = Configuration.get('python_script')
if python_script:
with open(python_script, "r") as fh:
exec(fh.read(), None, locals())
else:
try:
from IPython import embed
embed()
except ImportError:
import code
code.interact(local=locals())
def anyblok_interpreter():
"""Execute a script or open an interpreter
"""
registry = anyblok.start('interpreter')
if registry:
registry.commit()
python_script = Configuration.get('python_script')
if python_script:
with open(python_script, "r") as fh:
exec(fh.read(), None, locals())
else:
try:
from IPython import embed
embed()
except ImportError:
import code
code.interact(local=locals())
def updatedb(application, configuration_groups, **kwargs):
""" Update an existing database
:param application: name of the application
:param configuration_groups: list configuration groupe to load
:param \**kwargs: ArgumentParser named arguments
"""
format_configuration(configuration_groups, 'install-bloks',
'uninstall-bloks', 'update-bloks',
'install-or-update-bloks')
registry = anyblok.start(application,
configuration_groups=configuration_groups,
loadwithoutmigration=True, **kwargs)
installed_bloks = registry.System.Blok.list_by_state('installed')
required_install_bloks = []
required_update_bloks = []
for blok in (Configuration.get('install_or_update_bloks') or []):
if blok in installed_bloks:
required_update_bloks.append(blok)
else:
required_install_bloks.append(blok)
if Configuration.get('install_all_bloks'):
install_bloks = registry.System.Blok.list_by_state('uninstalled')
else:
install_bloks = Configuration.get('install_bloks') or []
install_bloks = list(set(install_bloks + required_install_bloks))
if Configuration.get('update_all_bloks'):
update_bloks = registry.System.Blok.list_by_state('installed')
else:
update_bloks = Configuration.get('update_bloks') or []
update_bloks = list(set(update_bloks + required_update_bloks))
uninstall_bloks = Configuration.get('uninstall_bloks')
if registry:
registry.update_blok_list() # case, new blok added
registry.upgrade(install=install_bloks, update=update_bloks,
uninstall=uninstall_bloks)
registry.commit()
registry.close()
def anyblok_nose():
"""Run nose unit test after giving it the registry
"""
warnings.simplefilter('default')
warnings.warn(
"This script is deprecated and will be removed soon. "
"The Nose test machinery has been removed from the framework in order "
"to be replaced with Pytest. "
"If you need to run your tests with nose, install the Nose package.",
DeprecationWarning, stacklevel=2)
try:
from nose import main
except ImportError:
logger.error('"Nosetest" is not installed, try: pip install nose')
anyblok_registry = anyblok.start('nose', useseparator=True, unittest=True)
if anyblok_registry:
installed_bloks = anyblok_registry.System.Blok.list_by_state(
"installed")
selected_bloks = return_list(
Configuration.get('selected_bloks')) or installed_bloks
unwanted_bloks = return_list(
Configuration.get('unwanted_bloks')) or []
unwanted_bloks.extend(['anyblok-core', 'anyblok-test', 'model_authz'])
defaultTest = []
for blok in installed_bloks:
if blok not in selected_bloks or blok in unwanted_bloks:
continue
startpath = BlokManager.getPath(blok)
for root, dirs, _ in walk(startpath):
if 'tests' in dirs:
defaultTest.append(join(root, 'tests'))
anyblok_registry.close() # free the registry to force create it again
sys.exit(main(defaultTest=defaultTest))
def add():
"""Add more data to the example DB.
This was merely useful to prepare incrementally the example.
"""
registry = anyblok.start('basic', configuration_groups=[],
loadwithoutmigration=True,
isolation_level='SERIALIZABLE')
if registry is None:
logging.critical("Couldn't initialize registry")
sys.exit(1)
init = Initializer(registry)
try:
init.add_data()
except:
import pdb
pdb.post_mortem(sys.exc_info()[2])
registry.rollback()
raise
registry.commit()
registry.close()
def anyblok2doc():
"""Return auto documentation for the registry
"""
anyblok_registry = anyblok.start('autodoc')
if anyblok_registry:
anyblok_registry.commit()
doc = anyblok_registry.Documentation()
doc.auto_doc()
if Configuration.get('doc_format') == 'RST':
with open(Configuration.get('doc_output'), 'w') as fp:
doc.toRST(fp)
elif Configuration.get('doc_format') == 'UML':
format_ = Configuration.get('schema_format')
name_ = Configuration.get('schema_output')
dot = ModelSchema(name_, format=format_)
doc.toUML(dot)
dot.save()
elif Configuration.get('doc_format') == 'SQL':
format_ = Configuration.get('schema_format')
name_ = Configuration.get('schema_output')
dot = SQLSchema(name_, format=format_)
doc.toSQL(dot)
dot.save()
def exampleblok():
# Initialise the application, with a name and a version number
# select the groupe of options to display
# return a registry if the database are selected
registry = anyblok.start(
'Example Blok', configuration_groups=['message', 'logging'])
message_before = Configuration.get('message_before')
message_after = Configuration.get('message_after')
if not registry:
logger.warning("No database database selected")
exit(0)
if message_before:
logger.info(message_before)
for address in registry.Address.query().all():
for room in address.rooms:
for worker in room.workers:
logger.info(worker)
if message_after:
logger.info(message_after)
def anyblok2doc():
"""Return auto documentation for the registry
"""
registry = anyblok.start('autodoc')
if registry:
registry.commit()
doc = registry.Documentation()
doc.auto_doc()
if Configuration.get('doc_format') == 'RST':
with open(Configuration.get('doc_output'), 'w') as fp:
doc.toRST(fp)
elif Configuration.get('doc_format') == 'UML':
format_ = Configuration.get('schema_format')
name_ = Configuration.get('schema_output')
dot = ModelSchema(name_, format=format_)
doc.toUML(dot)
dot.save()
elif Configuration.get('doc_format') == 'SQL':
format_ = Configuration.get('schema_format')
name_ = Configuration.get('schema_output')
dot = SQLSchema(name_, format=format_)
doc.toSQL(dot)
dot.save()
def interpreter(application, configuration_groups, **kwargs):
"""Execute a script or open an interpreter
:param application: name of the application
:param configuration_groups: list configuration groupe to load
:param \**kwargs: ArgumentParser named arguments
"""
format_configuration(configuration_groups, 'interpreter')
registry = anyblok.start(application,
configuration_groups=configuration_groups,
**kwargs)
if registry:
registry.commit()
python_script = Configuration.get('python_script')
if python_script:
with open(python_script, "r") as fh:
exec(fh.read(), None, locals())
else:
try:
from IPython import embed
embed()
except ImportError:
import code
code.interact(local=locals())
def anyblok_bus(): # noqa
"""Run consumer workers process to consume queue
"""
registry = start('bus', loadwithoutmigration=True)
if not registry:
exit(1)
unexisting_queues = registry.Bus.get_unexisting_queues()
if unexisting_queues:
profile_name = Configuration.get('bus_profile')
profile = registry.Bus.Profile.query().filter_by(
name=profile_name).one_or_none()
logger.critical("Some queues (%s) are required by consumers on %r",
', '.join(unexisting_queues), profile)
exit(1)
all_consumers = registry.Bus.get_consumers()
registry.close() # close the registry to recreate it in each process
worker_pipes = []
worker_processes = []
for processes, consumers in all_consumers:
logger.debug('Consume %r, with %r processes', consumers, processes)
for worker_id in range(processes):
read_fd, write_fd = os.pipe()
pid = os.fork()
if pid != 0:
os.close(write_fd)
worker_pipes.append(os.fdopen(read_fd))
worker_processes.append(pid)
continue
os.close(read_fd)
return bus_worker_process(write_fd, consumers)
def sighandler(signum, frame):
nonlocal worker_processes
signum = {
signal.SIGINT: signal.SIGTERM,
signal.SIGTERM: signal.SIGTERM,
signal.SIGHUP: signal.SIGHUP,
}[signum]
logger.info("Sending %r to worker processes...", signum.name)
for pid in worker_processes:
try:
os.kill(pid, signum)
except OSError:
logger.warning("Failed to send %r to pid %d.", signum.name,
pid)
retcode = 0
signal.signal(signal.SIGINT, sighandler)
signal.signal(signal.SIGTERM, sighandler)
signal.signal(signal.SIGHUP, sighandler)
for pid in worker_processes:
pid, rc = os.waitpid(pid, 0)
retcode = max(retcode, rc >> 8)
running = False # noqa
for pipe in worker_pipes:
pipe.close()
return retcode
def export_i18n():
registry = anyblok.start(
'export-i18n', loadwithoutmigration=True)
blok_name = Configuration.get('blok-name')
registry.FuretUI.export_i18n(blok_name)
