def run_tests(package, report):
if package == '_deferred':
name = 'deferred'
tests_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'tests_deferred')
else:
name = package.name
tests_dir = os.path.join(
get_module_path(name),
'migrations',
adapt_version(package.data['version']),
'tests',
)
# check for an environment variable because we don't want to mess
# with odoo's config.py, but we also don't want to run existing
# tests
if os.environ.get('OPENUPGRADE_TESTS') and os.path.exists(tests_dir):
import unittest
threading.currentThread().testing = True
tests = unittest.defaultTestLoader.discover(tests_dir,
top_level_dir=tests_dir)
report.record_result(
unittest.TextTestRunner(
verbosity=2,
stream=TestStream(name)).run(tests).wasSuccessful())
threading.currentThread().testing = False
def load_information_from_description_file(module, mod_path):
"""
:param module: The name of the module (sale, purchase, ...)
:param mod_path: Physical path of module, if not providedThe name of the module (sale, purchase, ...)
"""
manifest_file = module_manifest(mod_path)
if manifest_file:
# default values for descriptor
info = {
'application': False,
'author': 'Odoo S.A.',
'auto_install': False,
'category': 'Uncategorized',
'depends': [],
'description': '',
'installable': True,
'license': 'LGPL-3',
'post_load': None,
'version': '1.0',
'web': False,
'sequence': 100,
'summary': '',
'website': '',
}
info.update(
pycompat.izip(
'depends data demo test init_xml update_xml demo_xml'.split(),
iter(list, None)))
f = file_open(manifest_file, mode='rb')
try:
info.update(ast.literal_eval(pycompat.to_native(f.read())))
finally:
f.close()
if not info.get('description'):
readme_path = [
opj(mod_path, x) for x in README
if os.path.isfile(opj(mod_path, x))
]
if readme_path:
readme_text = file_open(readme_path[0]).read()
info['description'] = readme_text
if 'active' in info:
# 'active' has been renamed 'auto_install'
info['auto_install'] = info['active']
info['version'] = adapt_version(info['version'])
return info
_logger.debug('module %s: no manifest file found %s', module,
MANIFEST_NAMES)
return {}
def load_module_graph(cr, graph, status=None, perform_checks=True,
skip_modules=None, report=None, models_to_check=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(module_name, idref, mode):
cr.commit()
try:
_load_data(cr, module_name, idref, mode, 'test')
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', module_name)
return False
finally:
if tools.config.options['test_commit']:
cr.commit()
else:
cr.rollback()
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
def _get_files_of_kind(kind):
if kind == 'demo':
kind = ['demo_xml', 'demo']
elif kind == 'data':
kind = ['init_xml', 'update_xml', 'data']
if isinstance(kind, str):
kind = [kind]
files = []
for k in kind:
for f in package.data[k]:
files.append(f)
if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')):
# init_xml, update_xml and demo_xml are deprecated except
# for the case of init_xml with yaml, csv and sql files as
# we can't specify noupdate for those file.
correct_key = 'demo' if k.count('demo') else 'data'
_logger.warning(
"module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
package.name, k, correct_key, f
)
return files
def _load_data(cr, module_name, idref, mode, kind):
"""
kind: data, demo, test, init_xml, update_xml, demo_xml.
noupdate is False, unless it is demo data or it is csv data in
init mode.
"""
try:
if kind in ('demo', 'test'):
threading.currentThread().testing = True
for filename in _get_files_of_kind(kind):
_logger.info("loading %s/%s", module_name, filename)
noupdate = False
if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')):
noupdate = True
tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)
finally:
if kind in ('demo', 'test'):
threading.currentThread().testing = False
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
registry.clear_manual_fields()
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if skip_modules and module_name in skip_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index, module_count)
needs_update = (
hasattr(package, "init")
or hasattr(package, "update")
or package.state in ("to install", "to upgrade")
)
if needs_update:
if package.name != 'base':
registry.setup_models(cr, partial=True)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name,)]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr, partial=True)
registry.init_models(cr, model_names, {'module': package.name})
cr.commit()
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module.check()
if package.state=='to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
_load_data(cr, module_name, idref, mode, kind='data')
has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed')
if has_demo:
_load_data(cr, module_name, idref, mode, kind='demo')
cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
module.invalidate_cache(['demo'])
migrations.migrate_module(package, 'post')
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite).update_translations()
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
# validate all the views at a whole
env['ir.ui.view']._validate_module_views(module_name)
if has_demo:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(module_name, idref, mode))
# Python tests
env['ir.http']._clear_routing_map() # force routing map to be rebuilt
report.record_result(odoo.modules.module.run_unit_tests(module_name, cr.dbname))
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
registry._init_modules.add(package.name)
cr.commit()
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
registry.clear_manual_fields()
cr.commit()
return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(module_name, idref, mode):
cr.commit()
try:
_load_data(cr, module_name, idref, mode, 'test')
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', module_name)
return False
finally:
if tools.config.options['test_commit']:
cr.commit()
else:
cr.rollback()
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
def _get_files_of_kind(kind):
if kind == 'demo':
kind = ['demo_xml', 'demo']
elif kind == 'data':
kind = ['init_xml', 'update_xml', 'data']
if isinstance(kind, str):
kind = [kind]
files = []
for k in kind:
for f in package.data[k]:
files.append(f)
if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')):
# init_xml, update_xml and demo_xml are deprecated except
# for the case of init_xml with yaml, csv and sql files as
# we can't specify noupdate for those file.
correct_key = 'demo' if k.count('demo') else 'data'
_logger.warning(
"module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
package.name, k, correct_key, f
)
return files
def _load_data(cr, module_name, idref, mode, kind):
"""
kind: data, demo, test, init_xml, update_xml, demo_xml.
noupdate is False, unless it is demo data or it is csv data in
init mode.
"""
try:
if kind in ('demo', 'test'):
threading.currentThread().testing = True
for filename in _get_files_of_kind(kind):
_logger.info("loading %s/%s", module_name, filename)
noupdate = False
if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')):
noupdate = True
tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)
finally:
if kind in ('demo', 'test'):
threading.currentThread().testing = False
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
registry.clear_caches()
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if skip_modules and module_name in skip_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index, module_count)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name,)]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
registry.setup_models(cr)
registry.init_models(cr, model_names, {'module': package.name})
cr.commit()
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module.check()
if package.state=='to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
_load_data(cr, module_name, idref, mode, kind='data')
has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed')
if has_demo:
_load_data(cr, module_name, idref, mode, kind='demo')
cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
module.invalidate_cache(['demo'])
migrations.migrate_module(package, 'post')
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite).update_translations()
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
# validate all the views at a whole
env['ir.ui.view']._validate_module_views(module_name)
if has_demo:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(module_name, idref, mode))
# Python tests
env['ir.http']._clear_routing_map() # force routing map to be rebuilt
report.record_result(odoo.modules.module.run_unit_tests(module_name, cr.dbname))
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
registry._init_modules.add(package.name)
cr.commit()
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
registry.clear_caches()
cr.commit()
return loaded_modules, processed_modules
def load_module_graph(cr,
graph,
status=None,
perform_checks=True,
skip_modules=None,
report=None,
models_to_check=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(idref, mode):
cr.execute("SAVEPOINT load_test_data_file")
try:
load_data(cr, idref, mode, 'test', package, report)
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', package.name)
return False
finally:
cr.execute("ROLLBACK TO SAVEPOINT load_test_data_file")
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if skip_modules and module_name in skip_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index,
module_count)
needs_update = (hasattr(package, "init") or hasattr(package, "update")
or package.state in ("to install", "to upgrade"))
if needs_update:
if package.name != 'base':
registry.setup_models(cr)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name, )]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr)
registry.init_models(cr, model_names, {'module': package.name})
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module._check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
load_data(cr,
idref,
mode,
kind='data',
package=package,
report=report)
demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode,
report)
cr.execute('update ir_module_module set demo=%s where id=%s',
(demo_loaded, module_id))
module.invalidate_cache(['demo'])
migrations.migrate_module(package, 'post')
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite)._update_translations()
if package.name is not None:
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
if mode == 'update':
# validate the views that have not been checked yet
env['ir.ui.view']._validate_module_views(module_name)
# need to commit any modification the module's installation or
# update made to the schema or data so the tests can run
# (separately in their own transaction)
cr.commit()
if demo_loaded:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(idref, mode))
# Python tests
env['ir.http']._clear_routing_map(
) # force routing map to be rebuilt
report.record_result(
odoo.modules.module.run_unit_tests(
module_name, cr.dbname))
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
if package.name is not None:
registry._init_modules.add(package.name)
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph),
time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True,
skip_modules=None, report=None, models_to_check=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(idref, mode):
cr.execute("SAVEPOINT load_test_data_file")
try:
load_data(cr, idref, mode, 'test', package, report)
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', package.name)
return False
finally:
cr.execute("ROLLBACK TO SAVEPOINT load_test_data_file")
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
registry.clear_caches()
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if skip_modules and module_name in skip_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index, module_count)
needs_update = (
hasattr(package, "init")
or hasattr(package, "update")
or package.state in ("to install", "to upgrade")
)
if needs_update:
if package.name != 'base':
registry.setup_models(cr)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name,)]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr)
registry.init_models(cr, model_names, {'module': package.name})
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module._check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
load_data(cr, idref, mode, kind='data', package=package, report=report)
demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode, report)
cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id))
module.invalidate_cache(['demo'])
migrations.migrate_module(package, 'post')
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite)._update_translations()
if package.name is not None:
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
# validate all the views at a whole
env['ir.ui.view']._validate_module_views(module_name)
# need to commit any modification the module's installation or
# update made to the schema or data so the tests can run
# (separately in their own transaction)
cr.commit()
if demo_loaded:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(idref, mode))
# Python tests
env['ir.http']._clear_routing_map() # force routing map to be rebuilt
report.record_result(odoo.modules.module.run_unit_tests(module_name, cr.dbname))
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
if package.name is not None:
registry._init_modules.add(package.name)
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
registry.clear_caches()
return loaded_modules, processed_modules
def load_module_graph(cr,
graph,
status=None,
perform_checks=True,
skip_modules=None,
report=None,
models_to_check=None,
upg_registry=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(idref, mode):
cr.execute("SAVEPOINT load_test_data_file")
try:
load_data(cr, idref, mode, 'test', package, report)
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', package.name)
return False
finally:
cr.execute("ROLLBACK TO SAVEPOINT load_test_data_file")
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
if skip_modules is None:
skip_modules = []
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
# suppress commits to have the upgrade of one module in just one transaction
cr.commit_org = cr.commit
cr.commit = lambda *args: None
cr.rollback_org = cr.rollback
cr.rollback = lambda *args: None
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if module_name in skip_modules or module_name in loaded_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index,
module_count)
needs_update = (hasattr(package, "init") or hasattr(package, "update")
or package.state in ("to install", "to upgrade"))
if needs_update:
if package.name != 'base':
registry.setup_models(cr)
migrations.migrate_module(package, 'pre')
if package.name != 'base':
env = api.Environment(cr, SUPERUSER_ID, {})
env['base'].flush()
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name, )]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr)
# OpenUpgrade: rebuild the local registry based on the loaded models
local_registry = {}
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
if not model._auto:
continue
openupgrade_loading.log_model(model, local_registry)
openupgrade_loading.compare_registries(cr, package.name,
upg_registry,
local_registry)
# OpenUpgrade end
registry.init_models(cr, model_names, {'module': package.name},
new_install)
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module._check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
load_data(cr,
idref,
mode,
kind='data',
package=package,
report=report)
demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode,
report)
cr.execute('update ir_module_module set demo=%s where id=%s',
(demo_loaded, module_id))
module.invalidate_cache(['demo'])
# OpenUpgrade: add 'try' block for logging exceptions
# as errors in post scripts seem to be dropped
try:
migrations.migrate_module(package, 'post')
except Exception as exc:
_logger.error(
'Error executing post migration script for module %s: %s',
package, exc)
raise
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite)._update_translations()
if package.name is not None:
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
if mode == 'update':
# validate the views that have not been checked yet
env['ir.ui.view']._validate_module_views(module_name)
# need to commit any modification the module's installation or
# update made to the schema or data so the tests can run
# (separately in their own transaction)
# OpenUpgrade: commit after processing every module as well, for
# easier debugging and continuing an interrupted migration
cr.commit_org()
if tools.config.options['test_enable']:
report.record_result(load_test(idref, mode))
# Python tests
env['ir.http']._clear_routing_map(
) # force routing map to be rebuilt
report.record_result(
odoo.modules.module.run_unit_tests(module_name))
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
# OpenUpgrade: run tests
if os.environ.get(
'OPENUPGRADE_TESTS') and package.name is not None:
# Load tests in <module>/migrations/tests and enable standard tags if necessary
prefix = '.migrations'
registry.openupgrade_test_prefixes[package.name] = prefix
test_tags = tools.config['test_tags']
if not test_tags:
tools.config['test_tags'] = '+standard'
report.record_result(
odoo.modules.module.run_unit_tests(
module_name, openupgrade_prefix=prefix))
tools.config['test_tags'] = test_tags
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
# OpenUpgrade: commit module_n state and version immediatly
# to avoid invalid database state if module_n+1 raises an
# exception
cr.commit_org()
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
module.flush()
if package.name is not None:
registry._init_modules.add(package.name)
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph),
time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
# Openupgrade: restore commit method
cr.commit = cr.commit_org
cr.commit()
return loaded_modules, processed_modules
def load_module_graph(cr,
graph,
status=None,
perform_checks=True,
skip_modules=None,
report=None,
upg_registry=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
def load_test(module_name, idref, mode):
cr.commit()
try:
_load_data(cr, module_name, idref, mode, 'test')
return True
except Exception:
_test_logger.exception(
'module %s: an exception occurred in a test', module_name)
return False
finally:
if tools.config.options['test_commit']:
cr.commit()
else:
cr.rollback()
# avoid keeping stale xml_id, etc. in cache
odoo.registry(cr.dbname).clear_caches()
def _get_files_of_kind(kind):
if kind == 'demo':
kind = ['demo_xml', 'demo']
elif kind == 'data':
kind = ['init_xml', 'update_xml', 'data']
if isinstance(kind, str):
kind = [kind]
files = []
for k in kind:
for f in package.data[k]:
files.append(f)
if k.endswith('_xml') and not (k == 'init_xml'
and not f.endswith('.xml')):
# init_xml, update_xml and demo_xml are deprecated except
# for the case of init_xml with yaml, csv and sql files as
# we can't specify noupdate for those file.
correct_key = 'demo' if k.count('demo') else 'data'
_logger.warning(
"module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
package.name, k, correct_key, f)
return files
def _load_data(cr, module_name, idref, mode, kind):
"""
kind: data, demo, test, init_xml, update_xml, demo_xml.
noupdate is False, unless it is demo data or it is csv data in
init mode.
"""
try:
if kind in ('demo', 'test'):
threading.currentThread().testing = True
for filename in _get_files_of_kind(kind):
_logger.info("loading %s/%s", module_name, filename)
noupdate = False
if kind in ('demo',
'demo_xml') or (filename.endswith('.csv')
and kind in ('init', 'init_xml')):
noupdate = True
tools.convert_file(cr, module_name, filename, idref, mode,
noupdate, kind, report)
finally:
if kind in ('demo', 'test'):
threading.currentThread().testing = False
if status is None:
status = {}
if skip_modules is None:
skip_modules = []
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
registry.clear_manual_fields()
# suppress commits to have the upgrade of one module in just one transaction
cr.commit_org = cr.commit
cr.commit = lambda *args: None
cr.rollback_org = cr.rollback
cr.rollback = lambda *args: None
fields.set_migration_cursor(cr)
# register, instantiate and initialize models for each modules
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
if module_name in skip_modules or module_name in loaded_modules:
continue
_logger.debug('loading module %s (%d/%d)', module_name, index,
module_count)
migrations.migrate_module(package, 'pre')
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name, )]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
loaded_modules.append(package.name)
if hasattr(package, 'init') or hasattr(
package,
'update') or package.state in ('to install', 'to upgrade'):
registry.setup_models(cr, partial=True)
# OpenUpgrade: rebuild the local registry based on the loaded models
local_registry = {}
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
if not model._auto:
continue
openupgrade_loading.log_model(model, local_registry)
openupgrade_loading.compare_registries(cr, package.name,
upg_registry,
local_registry)
registry.init_models(cr, model_names, {'module': package.name})
idref = {}
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
if hasattr(package, 'init') or hasattr(
package,
'update') or package.state in ('to install', 'to upgrade'):
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module.check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
_load_data(cr, module_name, idref, mode, kind='data')
has_demo = hasattr(package,
'demo') or (package.dbdemo
and package.state != 'installed')
if has_demo:
_load_data(cr, module_name, idref, mode, kind='demo')
cr.execute('update ir_module_module set demo=%s where id=%s',
(True, module_id))
module.invalidate_cache(['demo'])
# OpenUpgrade: add 'try' block for logging exceptions
# as errors in post scripts seem to be dropped
try:
migrations.migrate_module(package, 'post')
except Exception as exc:
_logger.error(
'Error executing post migration script for module %s: %s',
package, exc)
raise
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite).update_translations()
registry._init_modules.add(package.name)
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
# validate all the views at a whole
env['ir.ui.view']._validate_module_views(module_name)
if has_demo:
# launch tests only in demo mode, allowing tests to use demo data.
if tools.config.options['test_enable']:
# Yamel test
report.record_result(load_test(module_name, idref, mode))
# Python tests
env['ir.http']._clear_routing_map(
) # force routing map to be rebuilt
report.record_result(
odoo.modules.module.run_unit_tests(
module_name, cr.dbname))
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
registry._init_modules.add(package.name)
cr.commit_org()
# OpenUpgrade edit start:
# if there's a tests directory, run those if tests are enabled
tests_dir = os.path.join(
odoo.modules.module.get_module_path(package.name),
'migrations',
adapt_version(package.data['version']),
'tests',
)
# check for an environment variable because we don't want to mess
# with odoo's config.py, but we also don't want to run existing
# tests
if os.environ.get('OPENUPGRADE_TESTS') and os.path.exists(tests_dir):
import unittest
threading.currentThread().testing = True
tests = unittest.defaultTestLoader.discover(
tests_dir, top_level_dir=tests_dir)
report.record_result(
unittest.TextTestRunner(
verbosity=2,
stream=odoo.modules.module.TestStream(package.name),
).run(tests).wasSuccessful())
threading.currentThread().testing = False
# OpenUpgrade edit end
_logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph),
time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
registry.clear_manual_fields()
cr.commit = cr.commit_org
cr.commit()
fields.set_migration_cursor()
return loaded_modules, processed_modules
