def load_modules(db, force_demo=False, status=None, update_module=False):
initialize_sys_path()
force = []
if force_demo:
force.append('demo')
models_to_check = set()
cr = db.cursor()
try:
if not odoo.modules.db.is_initialized(cr):
_logger.info("init db")
odoo.modules.db.initialize(cr)
update_module = True # process auto-installed modules
tools.config["init"]["all"] = 1
tools.config['update']['all'] = 1
if not tools.config['without_demo']:
tools.config["demo"]['all'] = 1
# This is a brand new registry, just created in
# odoo.modules.registry.Registry.new().
registry = odoo.registry(cr.dbname)
env = api.Environment(cr, SUPERUSER_ID, {})
if 'base' in tools.config['update'] or 'all' in tools.config['update']:
cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
graph = odoo.modules.graph.Graph()
graph.add_module(cr, 'base', force)
if not graph:
_logger.critical('module base cannot be loaded! (hint: verify addons-path)')
raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)')
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = registry._assertion_report
loaded_modules, processed_modules = load_module_graph(
cr, graph, status, perform_checks=update_module,
report=report, models_to_check=models_to_check)
load_lang = tools.config.pop('load_language')
if load_lang or update_module:
# some base models are used below, so make sure they are set up
registry.setup_models(cr, partial=True)
if load_lang:
for lang in load_lang.split(','):
tools.load_language(cr, lang)
# STEP 2: Mark other modules to be loaded/updated
if update_module:
Module = env['ir.module.module']
if ('base' in tools.config['init']) or ('base' in tools.config['update']):
_logger.info('updating modules list')
Module.update_list()
_check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
module_names = [k for k, v in tools.config['init'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)])
if modules:
modules.button_install()
module_names = [k for k, v in tools.config['update'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'installed'), ('name', 'in', module_names)])
if modules:
modules.button_upgrade()
cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
Module.invalidate_cache(['state'])
# STEP 3: Load marked modules (skipping base which was done in STEP 1)
# IMPORTANT: this is done in two parts, first loading all installed or
# partially installed modules (i.e. installed/to upgrade), to
# offer a consistent system to the second part: installing
# newly selected modules.
# We include the modules 'to remove' in the first step, because
# they are part of the "currently installed" modules. They will
# be dropped in STEP 6 later, before restarting the loading
# process.
# IMPORTANT 2: We have to loop here until all relevant modules have been
# processed, because in some rare cases the dependencies have
# changed, and modules that depend on an uninstalled module
# will not be processed on the first pass.
# It's especially useful for migrations.
previously_processed = -1
while previously_processed < len(processed_modules):
previously_processed = len(processed_modules)
processed_modules += load_marked_modules(cr, graph,
['installed', 'to upgrade', 'to remove'],
force, status, report, loaded_modules, update_module, models_to_check)
if update_module:
processed_modules += load_marked_modules(cr, graph,
['to install'], force, status, report,
loaded_modules, update_module, models_to_check)
registry.setup_models(cr)
# STEP 3.5: execute migration end-scripts
migrations = odoo.modules.migration.MigrationManager(cr, graph)
for package in graph:
migrations.migrate_module(package, 'end')
# STEP 4: Finish and cleanup installations
if processed_modules:
cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
for (model, name) in cr.fetchall():
if model in registry and not registry[model]._abstract and not registry[model]._transient:
_logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0',
model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
# Temporary warning while we remove access rights on osv_memory objects, as they have
# been replaced by owner-only access rights
cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
for (model, name) in cr.fetchall():
if model in registry and registry[model]._transient:
_logger.warning('The transient model %s (%s) should not have explicit access rules!', model, name)
cr.execute("SELECT model from ir_model")
for (model,) in cr.fetchall():
if model in registry:
env[model]._check_removed_columns(log=True)
elif _logger.isEnabledFor(logging.INFO): # more an info that a warning...
_logger.warning("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
# Cleanup orphan records
env['ir.model.data']._process_end(processed_modules)
for kind in ('init', 'demo', 'update'):
tools.config[kind] = {}
cr.commit()
# STEP 5: Uninstall modules to remove
if update_module:
# Remove records referenced from ir_model_data for modules to be
# removed (and removed the references from ir_model_data).
cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',))
modules_to_remove = dict(cr.fetchall())
if modules_to_remove:
pkgs = reversed([p for p in graph if p.name in modules_to_remove])
for pkg in pkgs:
uninstall_hook = pkg.info.get('uninstall_hook')
if uninstall_hook:
py_module = sys.modules['odoo.addons.%s' % (pkg.name,)]
getattr(py_module, uninstall_hook)(cr, registry)
Module = env['ir.module.module']
Module.browse(modules_to_remove.values()).module_uninstall()
# Recursive reload, should only happen once, because there should be no
# modules to remove next time
cr.commit()
_logger.info('Reloading registry once more after uninstalling modules')
api.Environment.reset()
return odoo.modules.registry.Registry.new(cr.dbname, force_demo, status, update_module)
# STEP 5.5: Verify extended fields on every model
# This will fix the schema of all models in a situation such as:
# - module A is loaded and defines model M;
# - module B is installed/upgraded and extends model M;
# - module C is loaded and extends model M;
# - module B and C depend on A but not on each other;
# The changes introduced by module C are not taken into account by the upgrade of B.
if models_to_check:
registry.init_models(cr, list(models_to_check), {'models_to_check': True})
# STEP 6: verify custom views on every model
if update_module:
View = env['ir.ui.view']
for model in registry:
try:
View._validate_custom_views(model)
except Exception as e:
_logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e))
if report.failures:
_logger.error('At least one test failed when loading the modules.')
else:
_logger.info('Modules loaded.')
# STEP 8: call _register_hook on every model
for model in env.values():
model._register_hook()
# STEP 9: Run the post-install tests
cr.commit()
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
if odoo.tools.config['test_enable']:
if update_module:
cr.execute("SELECT name FROM ir_module_module WHERE state='installed' and name = ANY(%s)", (processed_modules,))
else:
cr.execute("SELECT name FROM ir_module_module WHERE state='installed'")
for module_name in cr.fetchall():
report.record_result(odoo.modules.module.run_unit_tests(module_name[0], cr.dbname, position=runs_post_install))
_logger.log(25, "All post-tested in %.2fs, %s queries", time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
finally:
cr.close()
def load_module_graph(cr, graph, status=None, perform_checks=True,
skip_modules=None, report=None, models_to_check=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(module_name, idref, mode):
cr.commit()
try:
_load_data(cr, module_name, idref, mode, 'test')
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', module_name)
return False
finally:
if tools.config.options['test_commit']:
cr.commit()
else:
cr.rollback()
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
def _get_files_of_kind(kind):
if kind == 'demo':
kind = ['demo_xml', 'demo']
elif kind == 'data':
kind = ['init_xml', 'update_xml', 'data']
if isinstance(kind, str):
kind = [kind]
files = []
for k in kind:
for f in package.data[k]:
files.append(f)
if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')):
# init_xml, update_xml and demo_xml are deprecated except
# for the case of init_xml with yaml, csv and sql files as
# we can't specify noupdate for those file.
correct_key = 'demo' if k.count('demo') else 'data'
_logger.warning(
"module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
package.name, k, correct_key, f
)
return files
def _load_data(cr, module_name, idref, mode, kind):
"""
kind: data, demo, test, init_xml, update_xml, demo_xml.
noupdate is False, unless it is demo data or it is csv data in
init mode.
"""
try:
if kind in ('demo', 'test'):
threading.currentThread().testing = True
for filename in _get_files_of_kind(kind):
_logger.info("loading %s/%s", module_name, filename)
noupdate = False
if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')):
noupdate = True
tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)
finally:
if kind in ('demo', 'test'):
threading.currentThread().testing = False
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
registry.clear_manual_fields()
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if skip_modules and module_name in skip_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index, module_count)
needs_update = (
hasattr(package, "init")
or hasattr(package, "update")
or package.state in ("to install", "to upgrade")
)
if needs_update:
if package.name != 'base':
registry.setup_models(cr, partial=True)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name,)]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr, partial=True)
registry.init_models(cr, model_names, {'module': package.name})
cr.commit()
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module.check()
if package.state=='to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
_load_data(cr, module_name, idref, mode, kind='data')
has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed')
if has_demo:
_load_data(cr, module_name, idref, mode, kind='demo')
cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
module.invalidate_cache(['demo'])
migrations.migrate_module(package, 'post')
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite).update_translations()
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
# validate all the views at a whole
env['ir.ui.view']._validate_module_views(module_name)
if has_demo:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(module_name, idref, mode))
# Python tests
env['ir.http']._clear_routing_map() # force routing map to be rebuilt
report.record_result(odoo.modules.module.run_unit_tests(module_name, cr.dbname))
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
registry._init_modules.add(package.name)
cr.commit()
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
registry.clear_manual_fields()
cr.commit()
return loaded_modules, processed_modules
def load_modules(db, force_demo=False, status=None, update_module=False):
initialize_sys_path()
force = []
if force_demo:
force.append('demo')
cr = db.cursor()
try:
if not odoo.modules.db.is_initialized(cr):
_logger.info("init db")
odoo.modules.db.initialize(cr)
update_module = True # process auto-installed modules
tools.config["init"]["all"] = 1
tools.config['update']['all'] = 1
if not tools.config['without_demo']:
tools.config["demo"]['all'] = 1
# This is a brand new registry, just created in
# odoo.modules.registry.Registry.new().
registry = odoo.registry(cr.dbname)
if 'base' in tools.config['update'] or 'all' in tools.config['update']:
cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
graph = odoo.modules.graph.Graph()
graph.add_module(cr, 'base', force)
if not graph:
_logger.critical('module base cannot be loaded! (hint: verify addons-path)')
raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)')
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = registry._assertion_report
loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report)
load_lang = tools.config.pop('load_language')
if load_lang or update_module:
# some base models are used below, so make sure they are set up
registry.setup_models(cr)
if load_lang:
for lang in load_lang.split(','):
tools.load_language(cr, lang)
# STEP 2: Mark other modules to be loaded/updated
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
Module = env['ir.module.module']
if ('base' in tools.config['init']) or ('base' in tools.config['update']):
_logger.info('updating modules list')
Module.update_list()
_check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update']))
module_names = [k for k, v in tools.config['init'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)])
if modules:
modules.button_install()
module_names = [k for k, v in tools.config['update'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'installed'), ('name', 'in', module_names)])
if modules:
modules.button_upgrade()
cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
Module.invalidate_cache(['state'])
# STEP 3: Load marked modules (skipping base which was done in STEP 1)
# IMPORTANT: this is done in two parts, first loading all installed or
# partially installed modules (i.e. installed/to upgrade), to
# offer a consistent system to the second part: installing
# newly selected modules.
# We include the modules 'to remove' in the first step, because
# they are part of the "currently installed" modules. They will
# be dropped in STEP 6 later, before restarting the loading
# process.
# IMPORTANT 2: We have to loop here until all relevant modules have been
# processed, because in some rare cases the dependencies have
# changed, and modules that depend on an uninstalled module
# will not be processed on the first pass.
# It's especially useful for migrations.
previously_processed = -1
while previously_processed < len(processed_modules):
previously_processed = len(processed_modules)
processed_modules += load_marked_modules(cr, graph,
['installed', 'to upgrade', 'to remove'],
force, status, report, loaded_modules, update_module)
if update_module:
processed_modules += load_marked_modules(cr, graph,
['to install'], force, status, report,
loaded_modules, update_module)
registry.loaded = True
registry.setup_models(cr)
# STEP 3.5: execute migration end-scripts
migrations = odoo.modules.migration.MigrationManager(cr, graph)
for package in graph:
migrations.migrate_module(package, 'end')
# STEP 4: Finish and cleanup installations
if processed_modules:
env = api.Environment(cr, SUPERUSER_ID, {})
cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
for (model, name) in cr.fetchall():
if model in registry and not registry[model]._abstract and not registry[model]._transient:
_logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,,1,0,0,0',
model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
# Temporary warning while we remove access rights on osv_memory objects, as they have
# been replaced by owner-only access rights
cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
for (model, name) in cr.fetchall():
if model in registry and registry[model]._transient:
_logger.warning('The transient model %s (%s) should not have explicit access rules!', model, name)
cr.execute("SELECT model from ir_model")
for (model,) in cr.fetchall():
if model in registry:
env[model]._check_removed_columns(log=True)
elif _logger.isEnabledFor(logging.INFO): # more an info that a warning...
_logger.warning("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
# Cleanup orphan records
env['ir.model.data']._process_end(processed_modules)
for kind in ('init', 'demo', 'update'):
tools.config[kind] = {}
cr.commit()
# STEP 5: Uninstall modules to remove
if update_module:
# Remove records referenced from ir_model_data for modules to be
# removed (and removed the references from ir_model_data).
cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',))
modules_to_remove = dict(cr.fetchall())
if modules_to_remove:
env = api.Environment(cr, SUPERUSER_ID, {})
pkgs = reversed([p for p in graph if p.name in modules_to_remove])
for pkg in pkgs:
uninstall_hook = pkg.info.get('uninstall_hook')
if uninstall_hook:
py_module = sys.modules['odoo.addons.%s' % (pkg.name,)]
getattr(py_module, uninstall_hook)(cr, registry)
Module = env['ir.module.module']
Module.browse(modules_to_remove.values()).module_uninstall()
# Recursive reload, should only happen once, because there should be no
# modules to remove next time
cr.commit()
_logger.info('Reloading registry once more after uninstalling modules')
api.Environment.reset()
return odoo.modules.registry.Registry.new(cr.dbname, force_demo, status, update_module)
# STEP 6: verify custom views on every model
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
View = env['ir.ui.view']
for model in registry:
try:
View._validate_custom_views(model)
except Exception as e:
_logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e))
if report.failures:
_logger.error('At least one test failed when loading the modules.')
else:
_logger.info('Modules loaded.')
# STEP 8: call _register_hook on every model
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
model._register_hook()
# STEP 9: save installed/updated modules for post-install tests
registry.updated_modules += processed_modules
cr.commit()
finally:
cr.close()
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(module_name, idref, mode):
cr.commit()
try:
_load_data(cr, module_name, idref, mode, 'test')
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', module_name)
return False
finally:
if tools.config.options['test_commit']:
cr.commit()
else:
cr.rollback()
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
def _get_files_of_kind(kind):
if kind == 'demo':
kind = ['demo_xml', 'demo']
elif kind == 'data':
kind = ['init_xml', 'update_xml', 'data']
if isinstance(kind, str):
kind = [kind]
files = []
for k in kind:
for f in package.data[k]:
files.append(f)
if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')):
# init_xml, update_xml and demo_xml are deprecated except
# for the case of init_xml with yaml, csv and sql files as
# we can't specify noupdate for those file.
correct_key = 'demo' if k.count('demo') else 'data'
_logger.warning(
"module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
package.name, k, correct_key, f
)
return files
def _load_data(cr, module_name, idref, mode, kind):
"""
kind: data, demo, test, init_xml, update_xml, demo_xml.
noupdate is False, unless it is demo data or it is csv data in
init mode.
"""
try:
if kind in ('demo', 'test'):
threading.currentThread().testing = True
for filename in _get_files_of_kind(kind):
_logger.info("loading %s/%s", module_name, filename)
noupdate = False
if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')):
noupdate = True
tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)
finally:
if kind in ('demo', 'test'):
threading.currentThread().testing = False
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
registry.clear_caches()
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if skip_modules and module_name in skip_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index, module_count)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name,)]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
registry.setup_models(cr)
registry.init_models(cr, model_names, {'module': package.name})
cr.commit()
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module.check()
if package.state=='to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
_load_data(cr, module_name, idref, mode, kind='data')
has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed')
if has_demo:
_load_data(cr, module_name, idref, mode, kind='demo')
cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
module.invalidate_cache(['demo'])
migrations.migrate_module(package, 'post')
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite).update_translations()
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
# validate all the views at a whole
env['ir.ui.view']._validate_module_views(module_name)
if has_demo:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(module_name, idref, mode))
# Python tests
env['ir.http']._clear_routing_map() # force routing map to be rebuilt
report.record_result(odoo.modules.module.run_unit_tests(module_name, cr.dbname))
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
registry._init_modules.add(package.name)
cr.commit()
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
registry.clear_caches()
cr.commit()
return loaded_modules, processed_modules
def load_modules(db, force_demo=False, status=None, update_module=False):
initialize_sys_path()
force = []
if force_demo:
force.append('demo')
upg_registry = {}
models_to_check = set()
cr = db.cursor()
try:
if not odoo.modules.db.is_initialized(cr):
if not update_module:
_logger.error(
"Database %s not initialized, you can force it with `-i base`",
cr.dbname)
return
_logger.info("init db")
odoo.modules.db.initialize(cr)
update_module = True # process auto-installed modules
tools.config["init"]["all"] = 1
tools.config['update']['all'] = 1
if not tools.config['without_demo']:
tools.config["demo"]['all'] = 1
# This is a brand new registry, just created in
# odoo.modules.registry.Registry.new().
registry = odoo.registry(cr.dbname)
if 'base' in tools.config['update'] or 'all' in tools.config['update']:
cr.execute(
"update ir_module_module set state=%s where name=%s and state=%s",
('to upgrade', 'base', 'installed'))
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
graph = odoo.modules.graph.Graph()
graph.add_module(cr, 'base', force)
if not graph:
_logger.critical(
'module base cannot be loaded! (hint: verify addons-path)')
raise ImportError(
'Module `base` cannot be loaded! (hint: verify addons-path)')
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = registry._assertion_report
loaded_modules, processed_modules = load_module_graph(
cr,
graph,
status,
perform_checks=update_module,
report=report,
models_to_check=models_to_check,
upg_registry=upg_registry)
load_lang = tools.config.pop('load_language')
if load_lang or update_module:
# some base models are used below, so make sure they are set up
registry.setup_models(cr)
if load_lang:
for lang in load_lang.split(','):
tools.load_language(cr, lang)
# STEP 2: Mark other modules to be loaded/updated
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
Module = env['ir.module.module']
_logger.info('updating modules list')
Module.update_list()
_check_module_names(
cr,
itertools.chain(tools.config['init'], tools.config['update']))
module_names = [k for k, v in tools.config['init'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'uninstalled'),
('name', 'in', module_names)])
if modules:
modules.button_install()
module_names = [k for k, v in tools.config['update'].items() if v]
if module_names:
# OpenUpgrade: in standard Odoo, '--update all' just means:
# '--update base + upward (installed) dependencies. This breaks
# the chain when new glue modules are encountered.
# E.g. purchase in 8.0 depends on stock_account and report,
# both of which are new. They may be installed, but purchase as
# an upward dependency is not selected for upgrade.
# Therefore, explicitely select all installed modules for
# upgrading in OpenUpgrade in that case.
domain = [('state', '=', 'installed')]
if 'all' not in module_names:
domain.append(('name', 'in', module_names))
modules = Module.search(domain)
if modules:
modules.button_upgrade()
cr.execute("update ir_module_module set state=%s where name=%s",
('installed', 'base'))
Module.invalidate_cache(['state'])
# STEP 3: Load marked modules (skipping base which was done in STEP 1)
# IMPORTANT: this is done in two parts, first loading all installed or
# partially installed modules (i.e. installed/to upgrade), to
# offer a consistent system to the second part: installing
# newly selected modules.
# We include the modules 'to remove' in the first step, because
# they are part of the "currently installed" modules. They will
# be dropped in STEP 6 later, before restarting the loading
# process.
# IMPORTANT 2: We have to loop here until all relevant modules have been
# processed, because in some rare cases the dependencies have
# changed, and modules that depend on an uninstalled module
# will not be processed on the first pass.
# It's especially useful for migrations.
previously_processed = -1
while previously_processed < len(processed_modules):
previously_processed = len(processed_modules)
processed_modules += load_marked_modules(
cr, graph, ['installed', 'to upgrade', 'to remove'], force,
status, report, loaded_modules, update_module, models_to_check,
upg_registry)
if update_module:
processed_modules += load_marked_modules(
cr, graph, ['to install'], force, status, report,
loaded_modules, update_module, models_to_check,
upg_registry)
registry.loaded = True
registry.setup_models(cr)
# STEP 3.5: execute migration end-scripts
migrations = odoo.modules.migration.MigrationManager(cr, graph)
for package in graph:
migrations.migrate_module(package, 'end')
# STEP 3.6: warn about missing NOT NULL constraints
for (table, column), err_msg in registry._notnull_errors.items():
msg = "Table %r: column %r: unable to set constraint NOT NULL\n%s"
_logger.warning(msg, table, column, err_msg)
registry._notnull_errors.clear()
# STEP 4: Finish and cleanup installations
if processed_modules:
env = api.Environment(cr, SUPERUSER_ID, {})
cr.execute(
"""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)"""
)
for (model, name) in cr.fetchall():
if model in registry and not registry[
model]._abstract and not registry[model]._transient:
_logger.warning(
'The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0',
model, model.replace('.',
'_'), model.replace('.', '_'),
model.replace('.', '_'))
# Temporary warning while we remove access rights on osv_memory objects, as they have
# been replaced by owner-only access rights
cr.execute(
"""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id"""
)
for (model, name) in cr.fetchall():
if model in registry and registry[model]._transient:
_logger.warning(
'The transient model %s (%s) should not have explicit access rules!',
model, name)
cr.execute("SELECT model from ir_model")
for (model, ) in cr.fetchall():
if model in registry:
env[model]._check_removed_columns(log=True)
elif _logger.isEnabledFor(
logging.INFO): # more an info that a warning...
_logger.warning(
"Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)",
model)
# Cleanup orphan records
env['ir.model.data']._process_end(processed_modules)
for kind in ('init', 'demo', 'update'):
tools.config[kind] = {}
cr.commit()
# STEP 5: Uninstall modules to remove
if update_module:
# Remove records referenced from ir_model_data for modules to be
# removed (and removed the references from ir_model_data).
cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s",
('to remove', ))
modules_to_remove = dict(cr.fetchall())
if modules_to_remove:
env = api.Environment(cr, SUPERUSER_ID, {})
pkgs = reversed(
[p for p in graph if p.name in modules_to_remove])
for pkg in pkgs:
uninstall_hook = pkg.info.get('uninstall_hook')
if uninstall_hook:
py_module = sys.modules['odoo.addons.%s' %
(pkg.name, )]
getattr(py_module, uninstall_hook)(cr, registry)
Module = env['ir.module.module']
Module.browse(modules_to_remove.values()).module_uninstall()
# Recursive reload, should only happen once, because there should be no
# modules to remove next time
cr.commit()
_logger.info(
'Reloading registry once more after uninstalling modules')
api.Environment.reset()
registry = odoo.modules.registry.Registry.new(
cr.dbname, force_demo, status, update_module)
registry.check_tables_exist(cr)
cr.commit()
return registry
# STEP 5.5: Verify extended fields on every model
# This will fix the schema of all models in a situation such as:
# - module A is loaded and defines model M;
# - module B is installed/upgraded and extends model M;
# - module C is loaded and extends model M;
# - module B and C depend on A but not on each other;
# The changes introduced by module C are not taken into account by the upgrade of B.
if models_to_check:
registry.init_models(cr, list(models_to_check),
{'models_to_check': True})
# STEP 6: verify custom views on every model
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
View = env['ir.ui.view']
for model in registry:
try:
View._validate_custom_views(model)
except Exception as e:
_logger.warning('invalid custom view(s) for model %s: %s',
model, tools.ustr(e))
if report.failures:
_logger.error('At least one test failed when loading the modules.')
else:
_logger.info('Modules loaded.')
# STEP 8: call _register_hook on every model
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
model._register_hook()
# STEP 9: save installed/updated modules for post-install tests
registry.updated_modules += processed_modules
cr.commit()
# OpenUpgrade: run deferred tests
tests_dir = os.path.join(
os.path.dirname(os.path.abspath(openupgrade_loading.__file__)),
'tests_deferred')
if os.environ.get('OPENUPGRADE_TESTS') and os.path.exists(tests_dir):
import unittest
threading.currentThread().testing = True
tests = unittest.defaultTestLoader.discover(
tests_dir, top_level_dir=tests_dir)
report.record_result(
unittest.TextTestRunner(
verbosity=2,
stream=odoo.modules.module.TestStream('deferred'),
).run(tests).wasSuccessful())
threading.currentThread().testing = False
# OpenUpgrade edit end
finally:
cr.close()
def load_module_graph(cr,
graph,
status=None,
perform_checks=True,
skip_modules=None,
report=None,
models_to_check=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(idref, mode):
cr.execute("SAVEPOINT load_test_data_file")
try:
load_data(cr, idref, mode, 'test', package, report)
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', package.name)
return False
finally:
cr.execute("ROLLBACK TO SAVEPOINT load_test_data_file")
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if skip_modules and module_name in skip_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index,
module_count)
needs_update = (hasattr(package, "init") or hasattr(package, "update")
or package.state in ("to install", "to upgrade"))
if needs_update:
if package.name != 'base':
registry.setup_models(cr)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name, )]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr)
registry.init_models(cr, model_names, {'module': package.name})
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module._check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
load_data(cr,
idref,
mode,
kind='data',
package=package,
report=report)
demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode,
report)
cr.execute('update ir_module_module set demo=%s where id=%s',
(demo_loaded, module_id))
module.invalidate_cache(['demo'])
migrations.migrate_module(package, 'post')
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite)._update_translations()
if package.name is not None:
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
if mode == 'update':
# validate the views that have not been checked yet
env['ir.ui.view']._validate_module_views(module_name)
# need to commit any modification the module's installation or
# update made to the schema or data so the tests can run
# (separately in their own transaction)
cr.commit()
if demo_loaded:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(idref, mode))
# Python tests
env['ir.http']._clear_routing_map(
) # force routing map to be rebuilt
report.record_result(
odoo.modules.module.run_unit_tests(
module_name, cr.dbname))
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
if package.name is not None:
registry._init_modules.add(package.name)
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph),
time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
return loaded_modules, processed_modules
def load_modules(registry, force_demo=False, status=None, update_module=False):
""" Load the modules for a registry object that has just been created. This
function is part of Registry.new() and should not be used anywhere else.
"""
initialize_sys_path()
force = []
if force_demo:
force.append('demo')
models_to_check = set()
with registry.cursor() as cr:
# prevent endless wait for locks on schema changes (during online
# installs) if a concurrent transaction has accessed the table;
# connection settings are automatically reset when the connection is
# borrowed from the pool
cr.execute("SET SESSION lock_timeout = '15s'")
if not odoo.modules.db.is_initialized(cr):
if not update_module:
_logger.error(
"Database %s not initialized, you can force it with `-i base`",
cr.dbname)
return
_logger.info("init db")
odoo.modules.db.initialize(cr)
update_module = True # process auto-installed modules
tools.config["init"]["all"] = 1
if not tools.config['without_demo']:
tools.config["demo"]['all'] = 1
if 'base' in tools.config['update'] or 'all' in tools.config['update']:
cr.execute(
"update ir_module_module set state=%s where name=%s and state=%s",
('to upgrade', 'base', 'installed'))
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
graph = odoo.modules.graph.Graph()
graph.add_module(cr, 'base', force)
if not graph:
_logger.critical(
'module base cannot be loaded! (hint: verify addons-path)')
raise ImportError(
'Module `base` cannot be loaded! (hint: verify addons-path)')
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = registry._assertion_report
loaded_modules, processed_modules = load_module_graph(
cr,
graph,
status,
perform_checks=update_module,
report=report,
models_to_check=models_to_check)
load_lang = tools.config.pop('load_language')
if load_lang or update_module:
# some base models are used below, so make sure they are set up
registry.setup_models(cr)
if load_lang:
for lang in load_lang.split(','):
tools.load_language(cr, lang)
# STEP 2: Mark other modules to be loaded/updated
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
Module = env['ir.module.module']
_logger.info('updating modules list')
Module.update_list()
_check_module_names(
cr,
itertools.chain(tools.config['init'], tools.config['update']))
module_names = [k for k, v in tools.config['init'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'uninstalled'),
('name', 'in', module_names)])
if modules:
modules.button_install()
module_names = [k for k, v in tools.config['update'].items() if v]
if module_names:
modules = Module.search([('state', 'in', ('installed',
'to upgrade')),
('name', 'in', module_names)])
if modules:
modules.button_upgrade()
cr.execute("update ir_module_module set state=%s where name=%s",
('installed', 'base'))
Module.invalidate_cache(['state'])
Module.flush()
# STEP 3: Load marked modules (skipping base which was done in STEP 1)
# IMPORTANT: this is done in two parts, first loading all installed or
# partially installed modules (i.e. installed/to upgrade), to
# offer a consistent system to the second part: installing
# newly selected modules.
# We include the modules 'to remove' in the first step, because
# they are part of the "currently installed" modules. They will
# be dropped in STEP 6 later, before restarting the loading
# process.
# IMPORTANT 2: We have to loop here until all relevant modules have been
# processed, because in some rare cases the dependencies have
# changed, and modules that depend on an uninstalled module
# will not be processed on the first pass.
# It's especially useful for migrations.
previously_processed = -1
while previously_processed < len(processed_modules):
previously_processed = len(processed_modules)
processed_modules += load_marked_modules(
cr, graph, ['installed', 'to upgrade', 'to remove'], force,
status, report, loaded_modules, update_module, models_to_check)
if update_module:
processed_modules += load_marked_modules(
cr, graph, ['to install'], force, status, report,
loaded_modules, update_module, models_to_check)
# check that all installed modules have been loaded by the registry after a migration/upgrade
cr.execute(
"SELECT name from ir_module_module WHERE state = 'installed' and name != 'studio_customization'"
)
module_list = [name for (name, ) in cr.fetchall() if name not in graph]
if module_list:
_logger.error(
"Some modules are not loaded, some dependencies or manifest may be missing: %s",
sorted(module_list))
registry.loaded = True
registry.setup_models(cr)
# STEP 3.5: execute migration end-scripts
migrations = odoo.modules.migration.MigrationManager(cr, graph)
for package in graph:
migrations.migrate_module(package, 'end')
# check that new module dependencies have been properly installed after a migration/upgrade
cr.execute(
"SELECT name from ir_module_module WHERE state IN ('to install', 'to upgrade')"
)
module_list = [name for (name, ) in cr.fetchall()]
if module_list:
_logger.error(
"Some modules have inconsistent states, some dependencies may be missing: %s",
sorted(module_list))
# STEP 3.6: apply remaining constraints in case of an upgrade
registry.finalize_constraints()
# STEP 4: Finish and cleanup installations
if processed_modules:
env = api.Environment(cr, SUPERUSER_ID, {})
cr.execute("SELECT model from ir_model")
for (model, ) in cr.fetchall():
if model in registry:
env[model]._check_removed_columns(log=True)
elif _logger.isEnabledFor(
logging.INFO): # more an info that a warning...
_logger.runbot(
"Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)",
model)
# Cleanup orphan records
env['ir.model.data']._process_end(processed_modules)
env['base'].flush()
for kind in ('init', 'demo', 'update'):
tools.config[kind] = {}
# STEP 5: Uninstall modules to remove
if update_module:
# Remove records referenced from ir_model_data for modules to be
# removed (and removed the references from ir_model_data).
cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s",
('to remove', ))
modules_to_remove = dict(cr.fetchall())
if modules_to_remove:
env = api.Environment(cr, SUPERUSER_ID, {})
pkgs = reversed(
[p for p in graph if p.name in modules_to_remove])
for pkg in pkgs:
uninstall_hook = pkg.info.get('uninstall_hook')
if uninstall_hook:
py_module = sys.modules['odoo.addons.%s' %
(pkg.name, )]
getattr(py_module, uninstall_hook)(cr, registry)
Module = env['ir.module.module']
Module.browse(modules_to_remove.values()).module_uninstall()
# Recursive reload, should only happen once, because there should be no
# modules to remove next time
cr.commit()
_logger.info(
'Reloading registry once more after uninstalling modules')
registry = odoo.modules.registry.Registry.new(
cr.dbname, force_demo, status, update_module)
cr.reset()
registry.check_tables_exist(cr)
cr.commit()
return registry
# STEP 5.5: Verify extended fields on every model
# This will fix the schema of all models in a situation such as:
# - module A is loaded and defines model M;
# - module B is installed/upgraded and extends model M;
# - module C is loaded and extends model M;
# - module B and C depend on A but not on each other;
# The changes introduced by module C are not taken into account by the upgrade of B.
if models_to_check:
registry.init_models(cr, list(models_to_check),
{'models_to_check': True})
# STEP 6: verify custom views on every model
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
env['res.groups']._update_user_groups_view()
View = env['ir.ui.view']
for model in registry:
try:
View._validate_custom_views(model)
except Exception as e:
_logger.warning('invalid custom view(s) for model %s: %s',
model, tools.ustr(e))
if report.wasSuccessful():
_logger.info('Modules loaded.')
else:
_logger.error('At least one test failed when loading the modules.')
# STEP 8: call _register_hook on every model
# This is done *exactly once* when the registry is being loaded. See the
# management of those hooks in `Registry.setup_models`: all the calls to
# setup_models() done here do not mess up with hooks, as registry.ready
# is False.
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
model._register_hook()
env['base'].flush()
# STEP 9: save installed/updated modules for post-install tests
registry.updated_modules += processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True,
skip_modules=None, report=None, models_to_check=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(idref, mode):
cr.execute("SAVEPOINT load_test_data_file")
try:
load_data(cr, idref, mode, 'test', package, report)
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', package.name)
return False
finally:
cr.execute("ROLLBACK TO SAVEPOINT load_test_data_file")
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
registry.clear_caches()
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if skip_modules and module_name in skip_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index, module_count)
needs_update = (
hasattr(package, "init")
or hasattr(package, "update")
or package.state in ("to install", "to upgrade")
)
if needs_update:
if package.name != 'base':
registry.setup_models(cr)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name,)]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr)
registry.init_models(cr, model_names, {'module': package.name})
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module._check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
load_data(cr, idref, mode, kind='data', package=package, report=report)
demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode, report)
cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id))
module.invalidate_cache(['demo'])
migrations.migrate_module(package, 'post')
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite)._update_translations()
if package.name is not None:
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
# validate all the views at a whole
env['ir.ui.view']._validate_module_views(module_name)
# need to commit any modification the module's installation or
# update made to the schema or data so the tests can run
# (separately in their own transaction)
cr.commit()
if demo_loaded:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(idref, mode))
# Python tests
env['ir.http']._clear_routing_map() # force routing map to be rebuilt
report.record_result(odoo.modules.module.run_unit_tests(module_name, cr.dbname))
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
if package.name is not None:
registry._init_modules.add(package.name)
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
registry.clear_caches()
return loaded_modules, processed_modules
def load_module_graph(cr,
graph,
status=None,
perform_checks=True,
skip_modules=None,
report=None,
models_to_check=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
# register, instantiate and initialize models for each modules
t0 = time.time()
loading_extra_query_count = odoo.sql_db.sql_counter
loading_cursor_query_count = cr.sql_log_count
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if skip_modules and module_name in skip_modules:
continue
module_t0 = time.time()
module_cursor_query_count = cr.sql_log_count
module_extra_query_count = odoo.sql_db.sql_counter
needs_update = (hasattr(package, "init") or hasattr(package, "update")
or package.state in ("to install", "to upgrade"))
module_log_level = logging.DEBUG
if needs_update:
module_log_level = logging.INFO
_logger.log(module_log_level, 'Loading module %s (%d/%d)', module_name,
index, module_count)
new_install = package.state == 'to install'
if needs_update:
if not new_install:
if package.name != 'base':
registry.setup_models(cr)
migrations.migrate_module(package, 'pre')
if package.name != 'base':
env = api.Environment(cr, SUPERUSER_ID, {})
env['base'].flush()
load_openerp_module(package.name)
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name, )]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr)
registry.init_models(cr, model_names, {'module': package.name},
new_install)
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module._check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
load_data(cr, idref, mode, kind='data', package=package)
demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode)
cr.execute('update ir_module_module set demo=%s where id=%s',
(demo_loaded, module_id))
module.invalidate_cache(['demo'])
migrations.migrate_module(package, 'post')
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module._update_translations(overwrite=overwrite)
if package.name is not None:
registry._init_modules.add(package.name)
if needs_update:
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
if mode == 'update':
# validate the views that have not been checked yet
env['ir.ui.view']._validate_module_views(module_name)
# need to commit any modification the module's installation or
# update made to the schema or data so the tests can run
# (separately in their own transaction)
cr.commit()
concrete_models = [
model for model in model_names if not registry[model]._abstract
]
if concrete_models:
cr.execute(
"""
SELECT model FROM ir_model
WHERE id NOT IN (SELECT DISTINCT model_id FROM ir_model_access) AND model IN %s
""", [tuple(concrete_models)])
models = [model for [model] in cr.fetchall()]
if models:
lines = [
f"The models {models} have no access rules in module {module_name}, consider adding some, like:",
"id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink"
]
for model in models:
xmlid = model.replace('.', '_')
lines.append(
f"{module_name}.access_{xmlid},access_{xmlid},{module_name}.model_{xmlid},base.group_user,1,0,0,0"
)
_logger.warning('\n'.join(lines))
updating = tools.config.options['init'] or tools.config.options[
'update']
test_time = test_queries = 0
test_results = None
if tools.config.options['test_enable'] and (needs_update
or not updating):
env = api.Environment(cr, SUPERUSER_ID, {})
loader = odoo.tests.loader
suite = loader.make_suite([module_name], 'at_install')
if suite.countTestCases():
if not needs_update:
registry.setup_models(cr)
# Python tests
env['ir.http']._clear_routing_map(
) # force routing map to be rebuilt
tests_t0, tests_q0 = time.time(), odoo.sql_db.sql_counter
test_results = loader.run_suite(suite, module_name)
report.update(test_results)
test_time = time.time() - tests_t0
test_queries = odoo.sql_db.sql_counter - tests_q0
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
if needs_update:
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
module.flush()
extra_queries = odoo.sql_db.sql_counter - module_extra_query_count - test_queries
extras = []
if test_queries:
extras.append(f'+{test_queries} test')
if extra_queries:
extras.append(f'+{extra_queries} other')
_logger.log(module_log_level,
"Module %s loaded in %.2fs%s, %s queries%s", module_name,
time.time() - module_t0,
f' (incl. {test_time:.2f}s test)' if test_time else '',
cr.sql_log_count - module_cursor_query_count,
f' ({", ".join(extras)})' if extras else '')
if test_results and not test_results.wasSuccessful():
_logger.error("Module %s: %d failures, %d errors of %d tests",
module_name, len(test_results.failures),
len(test_results.errors), test_results.testsRun)
_logger.runbot("%s modules loaded in %.2fs, %s queries (+%s extra)",
len(graph),
time.time() - t0,
cr.sql_log_count - loading_cursor_query_count,
odoo.sql_db.sql_counter - loading_extra_query_count
) # extra queries: testes, notify, any other closed cursor
return loaded_modules, processed_modules
def load_module_graph(cr,
graph,
status=None,
perform_checks=True,
skip_modules=None,
report=None,
models_to_check=None,
upg_registry=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(idref, mode):
cr.execute("SAVEPOINT load_test_data_file")
try:
load_data(cr, idref, mode, 'test', package, report)
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', package.name)
return False
finally:
cr.execute("ROLLBACK TO SAVEPOINT load_test_data_file")
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
if skip_modules is None:
skip_modules = []
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
# suppress commits to have the upgrade of one module in just one transaction
cr.commit_org = cr.commit
cr.commit = lambda *args: None
cr.rollback_org = cr.rollback
cr.rollback = lambda *args: None
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if module_name in skip_modules or module_name in loaded_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index,
module_count)
needs_update = (hasattr(package, "init") or hasattr(package, "update")
or package.state in ("to install", "to upgrade"))
if needs_update:
if package.name != 'base':
registry.setup_models(cr)
migrations.migrate_module(package, 'pre')
if package.name != 'base':
env = api.Environment(cr, SUPERUSER_ID, {})
env['base'].flush()
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name, )]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr)
# OpenUpgrade: rebuild the local registry based on the loaded models
local_registry = {}
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
if not model._auto:
continue
openupgrade_loading.log_model(model, local_registry)
openupgrade_loading.compare_registries(cr, package.name,
upg_registry,
local_registry)
# OpenUpgrade end
registry.init_models(cr, model_names, {'module': package.name},
new_install)
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module._check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
load_data(cr,
idref,
mode,
kind='data',
package=package,
report=report)
demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode,
report)
cr.execute('update ir_module_module set demo=%s where id=%s',
(demo_loaded, module_id))
module.invalidate_cache(['demo'])
# OpenUpgrade: add 'try' block for logging exceptions
# as errors in post scripts seem to be dropped
try:
migrations.migrate_module(package, 'post')
except Exception as exc:
_logger.error(
'Error executing post migration script for module %s: %s',
package, exc)
raise
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite)._update_translations()
if package.name is not None:
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
if mode == 'update':
# validate the views that have not been checked yet
env['ir.ui.view']._validate_module_views(module_name)
# need to commit any modification the module's installation or
# update made to the schema or data so the tests can run
# (separately in their own transaction)
# OpenUpgrade: commit after processing every module as well, for
# easier debugging and continuing an interrupted migration
cr.commit_org()
if tools.config.options['test_enable']:
report.record_result(load_test(idref, mode))
# Python tests
env['ir.http']._clear_routing_map(
) # force routing map to be rebuilt
report.record_result(
odoo.modules.module.run_unit_tests(module_name))
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
# OpenUpgrade: run tests
if os.environ.get(
'OPENUPGRADE_TESTS') and package.name is not None:
# Load tests in <module>/migrations/tests and enable standard tags if necessary
prefix = '.migrations'
registry.openupgrade_test_prefixes[package.name] = prefix
test_tags = tools.config['test_tags']
if not test_tags:
tools.config['test_tags'] = '+standard'
report.record_result(
odoo.modules.module.run_unit_tests(
module_name, openupgrade_prefix=prefix))
tools.config['test_tags'] = test_tags
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
# OpenUpgrade: commit module_n state and version immediatly
# to avoid invalid database state if module_n+1 raises an
# exception
cr.commit_org()
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
module.flush()
if package.name is not None:
registry._init_modules.add(package.name)
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph),
time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
# Openupgrade: restore commit method
cr.commit = cr.commit_org
cr.commit()
return loaded_modules, processed_modules
def load_modules(db, force_demo=False, status=None, update_module=False):
initialize_sys_path()
force = []
if force_demo:
force.append("demo")
cr = db.cursor()
try:
if not odoo.modules.db.is_initialized(cr):
_logger.info("init db")
odoo.modules.db.initialize(cr)
update_module = True # process auto-installed modules
tools.config["init"]["all"] = 1
tools.config["update"]["all"] = 1
if not tools.config["without_demo"]:
tools.config["demo"]["all"] = 1
# This is a brand new registry, just created in
# odoo.modules.registry.Registry.new().
registry = odoo.registry(cr.dbname)
env = api.Environment(cr, SUPERUSER_ID, {})
if "base" in tools.config["update"] or "all" in tools.config["update"]:
cr.execute(
"update ir_module_module set state=%s where name=%s and state=%s", ("to upgrade", "base", "installed")
)
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
graph = odoo.modules.graph.Graph()
graph.add_module(cr, "base", force)
if not graph:
_logger.critical("module base cannot be loaded! (hint: verify addons-path)")
raise ImportError("Module `base` cannot be loaded! (hint: verify addons-path)")
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = registry._assertion_report
loaded_modules, processed_modules = load_module_graph(
cr, graph, status, perform_checks=update_module, report=report
)
load_lang = tools.config.pop("load_language")
if load_lang or update_module:
# some base models are used below, so make sure they are set up
registry.setup_models(cr, partial=True)
if load_lang:
for lang in load_lang.split(","):
tools.load_language(cr, lang)
# STEP 2: Mark other modules to be loaded/updated
if update_module:
Module = env["ir.module.module"]
if ("base" in tools.config["init"]) or ("base" in tools.config["update"]):
_logger.info("updating modules list")
Module.update_list()
_check_module_names(cr, itertools.chain(tools.config["init"].keys(), tools.config["update"].keys()))
module_names = [k for k, v in tools.config["init"].items() if v]
if module_names:
modules = Module.search([("state", "=", "uninstalled"), ("name", "in", module_names)])
if modules:
modules.button_install()
module_names = [k for k, v in tools.config["update"].items() if v]
if module_names:
modules = Module.search([("state", "=", "installed"), ("name", "in", module_names)])
if modules:
modules.button_upgrade()
cr.execute("update ir_module_module set state=%s where name=%s", ("installed", "base"))
Module.invalidate_cache(["state"])
# STEP 3: Load marked modules (skipping base which was done in STEP 1)
# IMPORTANT: this is done in two parts, first loading all installed or
# partially installed modules (i.e. installed/to upgrade), to
# offer a consistent system to the second part: installing
# newly selected modules.
# We include the modules 'to remove' in the first step, because
# they are part of the "currently installed" modules. They will
# be dropped in STEP 6 later, before restarting the loading
# process.
# IMPORTANT 2: We have to loop here until all relevant modules have been
# processed, because in some rare cases the dependencies have
# changed, and modules that depend on an uninstalled module
# will not be processed on the first pass.
# It's especially useful for migrations.
previously_processed = -1
while previously_processed < len(processed_modules):
previously_processed = len(processed_modules)
processed_modules += load_marked_modules(
cr,
graph,
["installed", "to upgrade", "to remove"],
force,
status,
report,
loaded_modules,
update_module,
)
if update_module:
processed_modules += load_marked_modules(
cr, graph, ["to install"], force, status, report, loaded_modules, update_module
)
registry.setup_models(cr)
# STEP 4: Finish and cleanup installations
if processed_modules:
cr.execute(
"""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)"""
)
for (model, name) in cr.fetchall():
if model in registry and not registry[model]._abstract and not registry[model]._transient:
_logger.warning(
"The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,,1,0,0,0",
model,
model.replace(".", "_"),
model.replace(".", "_"),
model.replace(".", "_"),
)
# Temporary warning while we remove access rights on osv_memory objects, as they have
# been replaced by owner-only access rights
cr.execute(
"""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id"""
)
for (model, name) in cr.fetchall():
if model in registry and registry[model]._transient:
_logger.warning("The transient model %s (%s) should not have explicit access rules!", model, name)
cr.execute("SELECT model from ir_model")
for (model,) in cr.fetchall():
if model in registry:
env[model]._check_removed_columns(log=True)
else:
_logger.warning(
"Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)",
model,
)
# Cleanup orphan records
env["ir.model.data"]._process_end(processed_modules)
for kind in ("init", "demo", "update"):
tools.config[kind] = {}
cr.commit()
# STEP 5: Uninstall modules to remove
if update_module:
# Remove records referenced from ir_model_data for modules to be
# removed (and removed the references from ir_model_data).
cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ("to remove",))
modules_to_remove = dict(cr.fetchall())
if modules_to_remove:
pkgs = reversed([p for p in graph if p.name in modules_to_remove])
for pkg in pkgs:
uninstall_hook = pkg.info.get("uninstall_hook")
if uninstall_hook:
py_module = sys.modules["odoo.addons.%s" % (pkg.name,)]
getattr(py_module, uninstall_hook)(cr, registry)
Module = env["ir.module.module"]
Module.browse(modules_to_remove.values()).module_uninstall()
# Recursive reload, should only happen once, because there should be no
# modules to remove next time
cr.commit()
_logger.info("Reloading registry once more after uninstalling modules")
api.Environment.reset()
return odoo.modules.registry.Registry.new(cr.dbname, force_demo, status, update_module)
# STEP 6: verify custom views on every model
if update_module:
View = env["ir.ui.view"]
custom_view_test = True
for model in registry:
if not View._validate_custom_views(model):
custom_view_test = False
_logger.error("invalid custom view(s) for model %s", model)
report.record_result(custom_view_test)
if report.failures:
_logger.error("At least one test failed when loading the modules.")
else:
_logger.info("Modules loaded.")
# STEP 8: call _register_hook on every model
for model in env.values():
model._register_hook()
# STEP 9: Run the post-install tests
cr.commit()
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
if odoo.tools.config["test_enable"]:
if update_module:
cr.execute(
"SELECT name FROM ir_module_module WHERE state='installed' and name = ANY(%s)", (processed_modules,)
)
else:
cr.execute("SELECT name FROM ir_module_module WHERE state='installed'")
for module_name in cr.fetchall():
report.record_result(
odoo.modules.module.run_unit_tests(module_name[0], cr.dbname, position=runs_post_install)
)
_logger.log(25, "All post-tested in %.2fs, %s queries", time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
finally:
cr.close()
def load_module_graph(cr,
graph,
status=None,
perform_checks=True,
skip_modules=None,
report=None,
upg_registry=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(module_name, idref, mode):
cr.commit()
try:
_load_data(cr, module_name, idref, mode, 'test')
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', module_name)
return False
finally:
if tools.config.options['test_commit']:
cr.commit()
else:
cr.rollback()
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
def _get_files_of_kind(kind):
if kind == 'demo':
kind = ['demo_xml', 'demo']
elif kind == 'data':
kind = ['init_xml', 'update_xml', 'data']
if isinstance(kind, str):
kind = [kind]
files = []
for k in kind:
for f in package.data[k]:
files.append(f)
if k.endswith('_xml') and not (k == 'init_xml'
and not f.endswith('.xml')):
# init_xml, update_xml and demo_xml are deprecated except
# for the case of init_xml with yaml, csv and sql files as
# we can't specify noupdate for those file.
correct_key = 'demo' if k.count('demo') else 'data'
_logger.warning(
"module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
package.name, k, correct_key, f)
return files
def _load_data(cr, module_name, idref, mode, kind):
"""
kind: data, demo, test, init_xml, update_xml, demo_xml.
noupdate is False, unless it is demo data or it is csv data in
init mode.
"""
try:
if kind in ('demo', 'test'):
threading.currentThread().testing = True
for filename in _get_files_of_kind(kind):
_logger.info("loading %s/%s", module_name, filename)
noupdate = False
if kind in ('demo',
'demo_xml') or (filename.endswith('.csv')
and kind in ('init', 'init_xml')):
noupdate = True
tools.convert_file(cr, module_name, filename, idref, mode,
noupdate, kind, report)
finally:
if kind in ('demo', 'test'):
threading.currentThread().testing = False
if status is None:
status = {}
if skip_modules is None:
skip_modules = []
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
registry.clear_manual_fields()
# suppress commits to have the upgrade of one module in just one transaction
cr.commit_org = cr.commit
cr.commit = lambda *args: None
cr.rollback_org = cr.rollback
cr.rollback = lambda *args: None
fields.set_migration_cursor(cr)
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if module_name in skip_modules or module_name in loaded_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index,
module_count)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name, )]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if hasattr(package, 'init') or hasattr(
package,
'update') or package.state in ('to install', 'to upgrade'):
registry.setup_models(cr, partial=True)
# OpenUpgrade: rebuild the local registry based on the loaded models
local_registry = {}
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
if not model._auto:
continue
openupgrade_loading.log_model(model, local_registry)
openupgrade_loading.compare_registries(cr, package.name,
upg_registry,
local_registry)
registry.init_models(cr, model_names, {'module': package.name})
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if hasattr(package, 'init') or hasattr(
package,
'update') or package.state in ('to install', 'to upgrade'):
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module.check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
_load_data(cr, module_name, idref, mode, kind='data')
has_demo = hasattr(package,
'demo') or (package.dbdemo
and package.state != 'installed')
if has_demo:
_load_data(cr, module_name, idref, mode, kind='demo')
cr.execute('update ir_module_module set demo=%s where id=%s',
(True, module_id))
module.invalidate_cache(['demo'])
# OpenUpgrade: add 'try' block for logging exceptions
# as errors in post scripts seem to be dropped
try:
migrations.migrate_module(package, 'post')
except Exception as exc:
_logger.error(
'Error executing post migration script for module %s: %s',
package, exc)
raise
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite).update_translations()
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
# validate all the views at a whole
env['ir.ui.view']._validate_module_views(module_name)
if has_demo:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(module_name, idref, mode))
# Python tests
env['ir.http']._clear_routing_map(
) # force routing map to be rebuilt
report.record_result(
odoo.modules.module.run_unit_tests(
module_name, cr.dbname))
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
registry._init_modules.add(package.name)
cr.commit_org()
# OpenUpgrade edit start:
# if there's a tests directory, run those if tests are enabled
tests_dir = os.path.join(
odoo.modules.module.get_module_path(package.name),
'migrations',
adapt_version(package.data['version']),
'tests',
)
# check for an environment variable because we don't want to mess
# with odoo's config.py, but we also don't want to run existing
# tests
if os.environ.get('OPENUPGRADE_TESTS') and os.path.exists(tests_dir):
import unittest
threading.currentThread().testing = True
tests = unittest.defaultTestLoader.discover(
tests_dir, top_level_dir=tests_dir)
report.record_result(
unittest.TextTestRunner(
verbosity=2,
stream=odoo.modules.module.TestStream(package.name),
).run(tests).wasSuccessful())
threading.currentThread().testing = False
# OpenUpgrade edit end
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph),
time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
registry.clear_manual_fields()
cr.commit = cr.commit_org
cr.commit()
fields.set_migration_cursor()
return loaded_modules, processed_modules
def load_modules(db, force_demo=False, status=None, update_module=False):
initialize_sys_path()
force = []
if force_demo:
force.append('demo')
models_to_check = set()
with db.cursor() as cr:
if not odoo.modules.db.is_initialized(cr):
if not update_module:
_logger.error("Database %s not initialized, you can force it with `-i base`", cr.dbname)
return
_logger.info("init db")
odoo.modules.db.initialize(cr)
update_module = True # process auto-installed modules
tools.config["init"]["all"] = 1
if not tools.config['without_demo']:
tools.config["demo"]['all'] = 1
# This is a brand new registry, just created in
# odoo.modules.registry.Registry.new().
registry = odoo.registry(cr.dbname)
if 'base' in tools.config['update'] or 'all' in tools.config['update']:
cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
graph = odoo.modules.graph.Graph()
graph.add_module(cr, 'base', force)
if not graph:
_logger.critical('module base cannot be loaded! (hint: verify addons-path)')
raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)')
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = registry._assertion_report
loaded_modules, processed_modules = load_module_graph(
cr, graph, status, perform_checks=update_module,
report=report, models_to_check=models_to_check)
load_lang = tools.config.pop('load_language')
if load_lang or update_module:
# some base models are used below, so make sure they are set up
registry.setup_models(cr)
if load_lang:
for lang in load_lang.split(','):
tools.load_language(cr, lang)
# STEP 2: Mark other modules to be loaded/updated
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
Module = env['ir.module.module']
_logger.info('updating modules list')
Module.update_list()
_check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update']))
module_names = [k for k, v in tools.config['init'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)])
if modules:
modules.button_install()
module_names = [k for k, v in tools.config['update'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'installed'), ('name', 'in', module_names)])
if modules:
modules.button_upgrade()
cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
Module.invalidate_cache(['state'])
Module.flush()
# STEP 3: Load marked modules (skipping base which was done in STEP 1)
# IMPORTANT: this is done in two parts, first loading all installed or
# partially installed modules (i.e. installed/to upgrade), to
# offer a consistent system to the second part: installing
# newly selected modules.
# We include the modules 'to remove' in the first step, because
# they are part of the "currently installed" modules. They will
# be dropped in STEP 6 later, before restarting the loading
# process.
# IMPORTANT 2: We have to loop here until all relevant modules have been
# processed, because in some rare cases the dependencies have
# changed, and modules that depend on an uninstalled module
# will not be processed on the first pass.
# It's especially useful for migrations.
previously_processed = -1
while previously_processed < len(processed_modules):
previously_processed = len(processed_modules)
processed_modules += load_marked_modules(cr, graph,
['installed', 'to upgrade', 'to remove'],
force, status, report, loaded_modules, update_module, models_to_check)
if update_module:
processed_modules += load_marked_modules(cr, graph,
['to install'], force, status, report,
loaded_modules, update_module, models_to_check)
# check that new module dependencies have been properly installed after a migration/upgrade
cr.execute("SELECT name from ir_module_module WHERE state IN ('to install', 'to upgrade')")
module_list = [name for (name,) in cr.fetchall()]
if module_list:
_logger.error("Some modules have inconsistent states, some dependencies may be missing: %s", sorted(module_list))
# check that all installed modules have been loaded by the registry after a migration/upgrade
cr.execute("SELECT name from ir_module_module WHERE state = 'installed' and name != 'studio_customization'")
module_list = [name for (name,) in cr.fetchall() if name not in graph]
if module_list:
_logger.error("Some modules are not loaded, some dependencies or manifest may be missing: %s", sorted(module_list))
registry.loaded = True
registry.setup_models(cr)
# STEP 3.5: execute migration end-scripts
migrations = odoo.modules.migration.MigrationManager(cr, graph)
for package in graph:
migrations.migrate_module(package, 'end')
# STEP 3.6: apply remaining constraints in case of an upgrade
registry.finalize_constraints()
# STEP 4: Finish and cleanup installations
if processed_modules:
env = api.Environment(cr, SUPERUSER_ID, {})
cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
for (model, name) in cr.fetchall():
if model in registry and not registry[model]._abstract:
_logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0',
model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
cr.execute("SELECT model from ir_model")
for (model,) in cr.fetchall():
if model in registry:
env[model]._check_removed_columns(log=True)
elif _logger.isEnabledFor(logging.INFO): # more an info that a warning...