def reset_modules_state(db_name):
"""
Resets modules flagged as "to x" to their original state
"""
# Warning, this function was introduced in response to commit 763d714
# which locks cron jobs for dbs which have modules marked as 'to %'.
# The goal of this function is to be called ONLY when module
# installation/upgrade/uninstallation fails, which is the only known case
# for which modules can stay marked as 'to %' for an indefinite amount
# of time
db = flectra.sql_db.db_connect(db_name)
with db.cursor() as cr:
cr.execute(
"UPDATE ir_module_module SET state='installed' WHERE state IN ('to remove', 'to upgrade')"
)
cr.execute(
"UPDATE ir_module_module SET state='uninstalled' WHERE state='to install'"
)
_logger.warning("Transient module states were reset")
def load_modules(db, force_demo=False, status=None, update_module=False):
initialize_sys_path()
force = []
if force_demo:
force.append('demo')
cr = db.cursor()
try:
if not flectra.modules.db.is_initialized(cr):
_logger.info("init db")
flectra.modules.db.initialize(cr)
update_module = True # process auto-installed modules
tools.config["init"]["all"] = 1
tools.config['update']['all'] = 1
if not tools.config['without_demo']:
tools.config["demo"]['all'] = 1
# This is a brand new registry, just created in
# flectra.modules.registry.Registry.new().
registry = flectra.registry(cr.dbname)
if 'base' in tools.config['update'] or 'all' in tools.config['update']:
cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
graph = flectra.modules.graph.Graph()
graph.add_module(cr, 'base', force)
if not graph:
_logger.critical('module base cannot be loaded! (hint: verify addons-path)')
raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)')
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = registry._assertion_report
loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report)
load_lang = tools.config.pop('load_language')
if load_lang or update_module:
# some base models are used below, so make sure they are set up
registry.setup_models(cr)
if load_lang:
for lang in load_lang.split(','):
tools.load_language(cr, lang)
# STEP 2: Mark other modules to be loaded/updated
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
Module = env['ir.module.module']
_logger.info('updating modules list')
Module.update_list()
_check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update']))
module_names = [k for k, v in tools.config['init'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)])
if modules:
modules.button_install()
module_names = [k for k, v in tools.config['update'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'installed'), ('name', 'in', module_names)])
if modules:
modules.button_upgrade()
cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
Module.invalidate_cache(['state'])
# STEP 3: Load marked modules (skipping base which was done in STEP 1)
# IMPORTANT: this is done in two parts, first loading all installed or
# partially installed modules (i.e. installed/to upgrade), to
# offer a consistent system to the second part: installing
# newly selected modules.
# We include the modules 'to remove' in the first step, because
# they are part of the "currently installed" modules. They will
# be dropped in STEP 6 later, before restarting the loading
# process.
# IMPORTANT 2: We have to loop here until all relevant modules have been
# processed, because in some rare cases the dependencies have
# changed, and modules that depend on an uninstalled module
# will not be processed on the first pass.
# It's especially useful for migrations.
previously_processed = -1
while previously_processed < len(processed_modules):
previously_processed = len(processed_modules)
processed_modules += load_marked_modules(cr, graph,
['installed', 'to upgrade', 'to remove'],
force, status, report, loaded_modules, update_module)
if update_module:
processed_modules += load_marked_modules(cr, graph,
['to install'], force, status, report,
loaded_modules, update_module)
registry.loaded = True
registry.setup_models(cr)
# STEP 3.5: execute migration end-scripts
migrations = flectra.modules.migration.MigrationManager(cr, graph)
for package in graph:
migrations.migrate_module(package, 'end')
# STEP 4: Finish and cleanup installations
if processed_modules:
env = api.Environment(cr, SUPERUSER_ID, {})
cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
for (model, name) in cr.fetchall():
if model in registry and not registry[model]._abstract and not registry[model]._transient:
_logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,,1,0,0,0',
model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
# Temporary warning while we remove access rights on osv_memory objects, as they have
# been replaced by owner-only access rights
cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
for (model, name) in cr.fetchall():
if model in registry and registry[model]._transient:
_logger.warning('The transient model %s (%s) should not have explicit access rules!', model, name)
cr.execute("SELECT model from ir_model")
for (model,) in cr.fetchall():
if model in registry:
env[model]._check_removed_columns(log=True)
elif _logger.isEnabledFor(logging.INFO): # more an info that a warning...
_logger.warning("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
# Cleanup orphan records
env['ir.model.data']._process_end(processed_modules)
for kind in ('init', 'demo', 'update'):
tools.config[kind] = {}
cr.commit()
# STEP 5: Uninstall modules to remove
if update_module:
# Remove records referenced from ir_model_data for modules to be
# removed (and removed the references from ir_model_data).
cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',))
modules_to_remove = dict(cr.fetchall())
if modules_to_remove:
env = api.Environment(cr, SUPERUSER_ID, {})
pkgs = reversed([p for p in graph if p.name in modules_to_remove])
for pkg in pkgs:
uninstall_hook = pkg.info.get('uninstall_hook')
if uninstall_hook:
py_module = sys.modules['flectra.addons.%s' % (pkg.name,)]
getattr(py_module, uninstall_hook)(cr, registry)
Module = env['ir.module.module']
Module.browse(modules_to_remove.values()).module_uninstall()
# Recursive reload, should only happen once, because there should be no
# modules to remove next time
cr.commit()
_logger.info('Reloading registry once more after uninstalling modules')
api.Environment.reset()
return flectra.modules.registry.Registry.new(cr.dbname, force_demo, status, update_module)
# STEP 6: verify custom views on every model
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
View = env['ir.ui.view']
for model in registry:
try:
View._validate_custom_views(model)
except Exception as e:
_logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e))
if report.failures:
_logger.error('At least one test failed when loading the modules.')
else:
_logger.info('Modules loaded.')
# STEP 8: call _register_hook on every model
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
model._register_hook()
# STEP 9: save installed/updated modules for post-install tests
registry.updated_modules += processed_modules
cr.commit()
finally:
cr.close()
def load_modules(db, force_demo=False, status=None, update_module=False):
initialize_sys_path()
force = []
if force_demo:
force.append('demo')
models_to_check = set()
with db.cursor() as cr:
# prevent endless wait for locks on schema changes (during online
# installs) if a concurrent transaction has accessed the table;
# connection settings are automatically reset when the connection is
# borrowed from the pool
cr.execute("SET SESSION lock_timeout = '15s'")
if not flectra.modules.db.is_initialized(cr):
if not update_module:
_logger.error("Database %s not initialized, you can force it with `-i base`", cr.dbname)
return
_logger.info("init db")
flectra.modules.db.initialize(cr)
update_module = True # process auto-installed modules
tools.config["init"]["all"] = 1
if not tools.config['without_demo']:
tools.config["demo"]['all'] = 1
# This is a brand new registry, just created in
# flectra.modules.registry.Registry.new().
registry = flectra.registry(cr.dbname)
if 'base' in tools.config['update'] or 'all' in tools.config['update']:
cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
graph = flectra.modules.graph.Graph()
graph.add_module(cr, 'base', force)
if not graph:
_logger.critical('module base cannot be loaded! (hint: verify addons-path)')
raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)')
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = registry._assertion_report
loaded_modules, processed_modules = load_module_graph(
cr, graph, status, perform_checks=update_module,
report=report, models_to_check=models_to_check)
load_lang = tools.config.pop('load_language')
if load_lang or update_module:
# some base models are used below, so make sure they are set up
registry.setup_models(cr)
if load_lang:
for lang in load_lang.split(','):
tools.load_language(cr, lang)
# STEP 2: Mark other modules to be loaded/updated
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
Module = env['ir.module.module']
_logger.info('updating modules list')
Module.update_list()
_check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update']))
module_names = [k for k, v in tools.config['init'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)])
if modules:
modules.button_install()
module_names = [k for k, v in tools.config['update'].items() if v]
if module_names:
modules = Module.search([('state', 'in', ('installed', 'to upgrade')), ('name', 'in', module_names)])
if modules:
modules.button_upgrade()
cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
Module.invalidate_cache(['state'])
Module.flush()
# STEP 3: Load marked modules (skipping base which was done in STEP 1)
# IMPORTANT: this is done in two parts, first loading all installed or
# partially installed modules (i.e. installed/to upgrade), to
# offer a consistent system to the second part: installing
# newly selected modules.
# We include the modules 'to remove' in the first step, because
# they are part of the "currently installed" modules. They will
# be dropped in STEP 6 later, before restarting the loading
# process.
# IMPORTANT 2: We have to loop here until all relevant modules have been
# processed, because in some rare cases the dependencies have
# changed, and modules that depend on an uninstalled module
# will not be processed on the first pass.
# It's especially useful for migrations.
previously_processed = -1
while previously_processed < len(processed_modules):
previously_processed = len(processed_modules)
processed_modules += load_marked_modules(cr, graph,
['installed', 'to upgrade', 'to remove'],
force, status, report, loaded_modules, update_module, models_to_check)
if update_module:
processed_modules += load_marked_modules(cr, graph,
['to install'], force, status, report,
loaded_modules, update_module, models_to_check)
# check that all installed modules have been loaded by the registry after a migration/upgrade
cr.execute("SELECT name from ir_module_module WHERE state = 'installed' and name != 'studio_customization'")
module_list = [name for (name,) in cr.fetchall() if name not in graph]
if module_list:
_logger.error("Some modules are not loaded, some dependencies or manifest may be missing: %s", sorted(module_list))
registry.loaded = True
registry.setup_models(cr)
# STEP 3.5: execute migration end-scripts
migrations = flectra.modules.migration.MigrationManager(cr, graph)
for package in graph:
migrations.migrate_module(package, 'end')
# check that new module dependencies have been properly installed after a migration/upgrade
cr.execute("SELECT name from ir_module_module WHERE state IN ('to install', 'to upgrade')")
module_list = [name for (name,) in cr.fetchall()]
if module_list:
_logger.error("Some modules have inconsistent states, some dependencies may be missing: %s", sorted(module_list))
# STEP 3.6: apply remaining constraints in case of an upgrade
registry.finalize_constraints()
# STEP 4: Finish and cleanup installations
if processed_modules:
env = api.Environment(cr, SUPERUSER_ID, {})
cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
for (model, name) in cr.fetchall():
if model in registry and not registry[model]._abstract:
_logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0',
model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
cr.execute("SELECT model from ir_model")
for (model,) in cr.fetchall():
if model in registry:
env[model]._check_removed_columns(log=True)
elif _logger.isEnabledFor(logging.INFO): # more an info that a warning...
_logger.runbot("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
# Cleanup orphan records
env['ir.model.data']._process_end(processed_modules)
env['base'].flush()
for kind in ('init', 'demo', 'update'):
tools.config[kind] = {}
# STEP 5: Uninstall modules to remove
if update_module:
# Remove records referenced from ir_model_data for modules to be
# removed (and removed the references from ir_model_data).
cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',))
modules_to_remove = dict(cr.fetchall())
if modules_to_remove:
env = api.Environment(cr, SUPERUSER_ID, {})
pkgs = reversed([p for p in graph if p.name in modules_to_remove])
for pkg in pkgs:
uninstall_hook = pkg.info.get('uninstall_hook')
if uninstall_hook:
py_module = sys.modules['flectra.addons.%s' % (pkg.name,)]
getattr(py_module, uninstall_hook)(cr, registry)
Module = env['ir.module.module']
Module.browse(modules_to_remove.values()).module_uninstall()
# Recursive reload, should only happen once, because there should be no
# modules to remove next time
cr.commit()
_logger.info('Reloading registry once more after uninstalling modules')
api.Environment.reset()
registry = flectra.modules.registry.Registry.new(
cr.dbname, force_demo, status, update_module
)
registry.check_tables_exist(cr)
cr.commit()
return registry
# STEP 5.5: Verify extended fields on every model
# This will fix the schema of all models in a situation such as:
# - module A is loaded and defines model M;
# - module B is installed/upgraded and extends model M;
# - module C is loaded and extends model M;
# - module B and C depend on A but not on each other;
# The changes introduced by module C are not taken into account by the upgrade of B.
if models_to_check:
registry.init_models(cr, list(models_to_check), {'models_to_check': True})
# STEP 6: verify custom views on every model
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
env['res.groups']._update_user_groups_view()
View = env['ir.ui.view']
for model in registry:
try:
View._validate_custom_views(model)
except Exception as e:
_logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e))
if report.wasSuccessful():
_logger.info('Modules loaded.')
else:
_logger.error('At least one test failed when loading the modules.')
# STEP 8: call _register_hook on every model
# This is done *exactly once* when the registry is being loaded. See the
# management of those hooks in `Registry.setup_models`: all the calls to
# setup_models() done here do not mess up with hooks, as registry.ready
# is False.
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
model._register_hook()
env['base'].flush()
# STEP 9: save installed/updated modules for post-install tests
registry.updated_modules += processed_modules