def run_tests(package, report): if package == '_deferred': name = 'deferred' tests_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests_deferred') else: name = package.name tests_dir = os.path.join( get_module_path(name), 'migrations', adapt_version(package.data['version']), 'tests', ) # check for an environment variable because we don't want to mess # with odoo's config.py, but we also don't want to run existing # tests if os.environ.get('OPENUPGRADE_TESTS') and os.path.exists(tests_dir): import unittest threading.currentThread().testing = True tests = unittest.defaultTestLoader.discover(tests_dir, top_level_dir=tests_dir) report.record_result( unittest.TextTestRunner( verbosity=2, stream=TestStream(name)).run(tests).wasSuccessful()) threading.currentThread().testing = False
def load_information_from_description_file(module, mod_path): """ :param module: The name of the module (sale, purchase, ...) :param mod_path: Physical path of module, if not providedThe name of the module (sale, purchase, ...) """ manifest_file = module_manifest(mod_path) if manifest_file: # default values for descriptor info = { 'application': False, 'author': 'Odoo S.A.', 'auto_install': False, 'category': 'Uncategorized', 'depends': [], 'description': '', 'installable': True, 'license': 'LGPL-3', 'post_load': None, 'version': '1.0', 'web': False, 'sequence': 100, 'summary': '', 'website': '', } info.update( pycompat.izip( 'depends data demo test init_xml update_xml demo_xml'.split(), iter(list, None))) f = file_open(manifest_file, mode='rb') try: info.update(ast.literal_eval(pycompat.to_native(f.read()))) finally: f.close() if not info.get('description'): readme_path = [ opj(mod_path, x) for x in README if os.path.isfile(opj(mod_path, x)) ] if readme_path: readme_text = file_open(readme_path[0]).read() info['description'] = readme_text if 'active' in info: # 'active' has been renamed 'auto_install' info['auto_install'] = info['active'] info['version'] = adapt_version(info['version']) return info _logger.debug('module %s: no manifest file found %s', module, MANIFEST_NAMES) return {}
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None, models_to_check=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(module_name, idref, mode): cr.commit() try: _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', module_name) return False finally: if tools.config.options['test_commit']: cr.commit() else: cr.rollback() # avoid keeping stale xml_id, etc. in cache odoo.registry(cr.dbname).clear_caches() def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package.data[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): # init_xml, update_xml and demo_xml are deprecated except # for the case of init_xml with yaml, csv and sql files as # we can't specify noupdate for those file. correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", package.name, k, correct_key, f ) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ try: if kind in ('demo', 'test'): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("loading %s/%s", module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report) finally: if kind in ('demo', 'test'): threading.currentThread().testing = False if models_to_check is None: models_to_check = set() processed_modules = [] loaded_modules = [] registry = odoo.registry(cr.dbname) migrations = odoo.modules.migration.MigrationManager(cr, graph) module_count = len(graph) _logger.info('loading %d modules...', module_count) registry.clear_manual_fields() # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = odoo.sql_db.sql_counter models_updated = set() for index, package in enumerate(graph, 1): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue _logger.debug('loading module %s (%d/%d)', module_name, index, module_count) needs_update = ( hasattr(package, "init") or hasattr(package, "update") or package.state in ("to install", "to upgrade") ) if needs_update: if package.name != 'base': registry.setup_models(cr, partial=True) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) new_install = package.state == 'to install' if new_install: py_module = sys.modules['odoo.addons.%s' % (module_name,)] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) model_names = registry.load(cr, package) loaded_modules.append(package.name) if needs_update: models_updated |= set(model_names) models_to_check -= set(model_names) registry.setup_models(cr, partial=True) registry.init_models(cr, model_names, {'module': package.name}) cr.commit() elif package.state != 'to remove': # The current module has simply been loaded. The models extended by this module # and for which we updated the schema, must have their schema checked again. # This is because the extension may have changed the model, # e.g. adding required=True to an existing field, but the schema has not been # updated by this module because it's not marked as 'to upgrade/to install'. models_to_check |= set(model_names) & models_updated idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if needs_update: env = api.Environment(cr, SUPERUSER_ID, {}) # Can't put this line out of the loop: ir.module.module will be # registered by init_models() above. module = env['ir.module.module'].browse(module_id) if perform_checks: module.check() if package.state=='to upgrade': # upgrading the module information module.write(module.get_values_from_terp(package.data)) _load_data(cr, module_name, idref, mode, kind='data') has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed') if has_demo: _load_data(cr, module_name, idref, mode, kind='demo') cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) module.invalidate_cache(['demo']) migrations.migrate_module(package, 'post') # Update translations for all installed languages overwrite = odoo.tools.config["overwrite_existing_translations"] module.with_context(overwrite=overwrite).update_translations() registry._init_modules.add(package.name) if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) # validate all the views at a whole env['ir.ui.view']._validate_module_views(module_name) if has_demo: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options['test_enable']: # Yamel test report.record_result(load_test(module_name, idref, mode)) # Python tests env['ir.http']._clear_routing_map() # force routing map to be rebuilt report.record_result(odoo.modules.module.run_unit_tests(module_name, cr.dbname)) processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies module.write({'state': 'installed', 'latest_version': ver}) package.load_state = package.state package.load_version = package.installed_version package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) registry._init_modules.add(package.name) cr.commit() _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql) registry.clear_manual_fields() cr.commit() return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(module_name, idref, mode): cr.commit() try: _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', module_name) return False finally: if tools.config.options['test_commit']: cr.commit() else: cr.rollback() # avoid keeping stale xml_id, etc. in cache odoo.registry(cr.dbname).clear_caches() def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package.data[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): # init_xml, update_xml and demo_xml are deprecated except # for the case of init_xml with yaml, csv and sql files as # we can't specify noupdate for those file. correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", package.name, k, correct_key, f ) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ try: if kind in ('demo', 'test'): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("loading %s/%s", module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report) finally: if kind in ('demo', 'test'): threading.currentThread().testing = False processed_modules = [] loaded_modules = [] registry = odoo.registry(cr.dbname) migrations = odoo.modules.migration.MigrationManager(cr, graph) module_count = len(graph) _logger.info('loading %d modules...', module_count) registry.clear_caches() # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = odoo.sql_db.sql_counter for index, package in enumerate(graph, 1): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue _logger.debug('loading module %s (%d/%d)', module_name, index, module_count) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) new_install = package.state == 'to install' if new_install: py_module = sys.modules['odoo.addons.%s' % (module_name,)] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) model_names = registry.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): registry.setup_models(cr) registry.init_models(cr, model_names, {'module': package.name}) cr.commit() idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): env = api.Environment(cr, SUPERUSER_ID, {}) # Can't put this line out of the loop: ir.module.module will be # registered by init_models() above. module = env['ir.module.module'].browse(module_id) if perform_checks: module.check() if package.state=='to upgrade': # upgrading the module information module.write(module.get_values_from_terp(package.data)) _load_data(cr, module_name, idref, mode, kind='data') has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed') if has_demo: _load_data(cr, module_name, idref, mode, kind='demo') cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) module.invalidate_cache(['demo']) migrations.migrate_module(package, 'post') # Update translations for all installed languages overwrite = odoo.tools.config["overwrite_existing_translations"] module.with_context(overwrite=overwrite).update_translations() registry._init_modules.add(package.name) if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) # validate all the views at a whole env['ir.ui.view']._validate_module_views(module_name) if has_demo: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options['test_enable']: # Yamel test report.record_result(load_test(module_name, idref, mode)) # Python tests env['ir.http']._clear_routing_map() # force routing map to be rebuilt report.record_result(odoo.modules.module.run_unit_tests(module_name, cr.dbname)) # tests may have reset the environment env = api.Environment(cr, SUPERUSER_ID, {}) module = env['ir.module.module'].browse(module_id) processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies module.write({'state': 'installed', 'latest_version': ver}) package.load_state = package.state package.load_version = package.installed_version package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) registry._init_modules.add(package.name) cr.commit() _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql) registry.clear_caches() cr.commit() return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None, models_to_check=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(idref, mode): cr.execute("SAVEPOINT load_test_data_file") try: load_data(cr, idref, mode, 'test', package, report) return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', package.name) return False finally: cr.execute("ROLLBACK TO SAVEPOINT load_test_data_file") # avoid keeping stale xml_id, etc. in cache odoo.registry(cr.dbname).clear_caches() if models_to_check is None: models_to_check = set() processed_modules = [] loaded_modules = [] registry = odoo.registry(cr.dbname) migrations = odoo.modules.migration.MigrationManager(cr, graph) module_count = len(graph) _logger.info('loading %d modules...', module_count) # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = odoo.sql_db.sql_counter models_updated = set() for index, package in enumerate(graph, 1): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue _logger.debug('loading module %s (%d/%d)', module_name, index, module_count) needs_update = (hasattr(package, "init") or hasattr(package, "update") or package.state in ("to install", "to upgrade")) if needs_update: if package.name != 'base': registry.setup_models(cr) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) new_install = package.state == 'to install' if new_install: py_module = sys.modules['odoo.addons.%s' % (module_name, )] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) model_names = registry.load(cr, package) loaded_modules.append(package.name) if needs_update: models_updated |= set(model_names) models_to_check -= set(model_names) registry.setup_models(cr) registry.init_models(cr, model_names, {'module': package.name}) elif package.state != 'to remove': # The current module has simply been loaded. The models extended by this module # and for which we updated the schema, must have their schema checked again. # This is because the extension may have changed the model, # e.g. adding required=True to an existing field, but the schema has not been # updated by this module because it's not marked as 'to upgrade/to install'. models_to_check |= set(model_names) & models_updated idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if needs_update: env = api.Environment(cr, SUPERUSER_ID, {}) # Can't put this line out of the loop: ir.module.module will be # registered by init_models() above. module = env['ir.module.module'].browse(module_id) if perform_checks: module._check() if package.state == 'to upgrade': # upgrading the module information module.write(module.get_values_from_terp(package.data)) load_data(cr, idref, mode, kind='data', package=package, report=report) demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode, report) cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id)) module.invalidate_cache(['demo']) migrations.migrate_module(package, 'post') # Update translations for all installed languages overwrite = odoo.tools.config["overwrite_existing_translations"] module.with_context(overwrite=overwrite)._update_translations() if package.name is not None: registry._init_modules.add(package.name) if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) if mode == 'update': # validate the views that have not been checked yet env['ir.ui.view']._validate_module_views(module_name) # need to commit any modification the module's installation or # update made to the schema or data so the tests can run # (separately in their own transaction) cr.commit() if demo_loaded: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options['test_enable']: # Yamel test report.record_result(load_test(idref, mode)) # Python tests env['ir.http']._clear_routing_map( ) # force routing map to be rebuilt report.record_result( odoo.modules.module.run_unit_tests( module_name, cr.dbname)) # tests may have reset the environment env = api.Environment(cr, SUPERUSER_ID, {}) module = env['ir.module.module'].browse(module_id) processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies module.write({'state': 'installed', 'latest_version': ver}) package.load_state = package.state package.load_version = package.installed_version package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) if package.name is not None: registry._init_modules.add(package.name) _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql) return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None, models_to_check=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(idref, mode): cr.execute("SAVEPOINT load_test_data_file") try: load_data(cr, idref, mode, 'test', package, report) return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', package.name) return False finally: cr.execute("ROLLBACK TO SAVEPOINT load_test_data_file") # avoid keeping stale xml_id, etc. in cache odoo.registry(cr.dbname).clear_caches() if models_to_check is None: models_to_check = set() processed_modules = [] loaded_modules = [] registry = odoo.registry(cr.dbname) migrations = odoo.modules.migration.MigrationManager(cr, graph) module_count = len(graph) _logger.info('loading %d modules...', module_count) registry.clear_caches() # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = odoo.sql_db.sql_counter models_updated = set() for index, package in enumerate(graph, 1): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue _logger.debug('loading module %s (%d/%d)', module_name, index, module_count) needs_update = ( hasattr(package, "init") or hasattr(package, "update") or package.state in ("to install", "to upgrade") ) if needs_update: if package.name != 'base': registry.setup_models(cr) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) new_install = package.state == 'to install' if new_install: py_module = sys.modules['odoo.addons.%s' % (module_name,)] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) model_names = registry.load(cr, package) loaded_modules.append(package.name) if needs_update: models_updated |= set(model_names) models_to_check -= set(model_names) registry.setup_models(cr) registry.init_models(cr, model_names, {'module': package.name}) elif package.state != 'to remove': # The current module has simply been loaded. The models extended by this module # and for which we updated the schema, must have their schema checked again. # This is because the extension may have changed the model, # e.g. adding required=True to an existing field, but the schema has not been # updated by this module because it's not marked as 'to upgrade/to install'. models_to_check |= set(model_names) & models_updated idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if needs_update: env = api.Environment(cr, SUPERUSER_ID, {}) # Can't put this line out of the loop: ir.module.module will be # registered by init_models() above. module = env['ir.module.module'].browse(module_id) if perform_checks: module._check() if package.state == 'to upgrade': # upgrading the module information module.write(module.get_values_from_terp(package.data)) load_data(cr, idref, mode, kind='data', package=package, report=report) demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode, report) cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id)) module.invalidate_cache(['demo']) migrations.migrate_module(package, 'post') # Update translations for all installed languages overwrite = odoo.tools.config["overwrite_existing_translations"] module.with_context(overwrite=overwrite)._update_translations() if package.name is not None: registry._init_modules.add(package.name) if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) # validate all the views at a whole env['ir.ui.view']._validate_module_views(module_name) # need to commit any modification the module's installation or # update made to the schema or data so the tests can run # (separately in their own transaction) cr.commit() if demo_loaded: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options['test_enable']: # Yamel test report.record_result(load_test(idref, mode)) # Python tests env['ir.http']._clear_routing_map() # force routing map to be rebuilt report.record_result(odoo.modules.module.run_unit_tests(module_name, cr.dbname)) # tests may have reset the environment env = api.Environment(cr, SUPERUSER_ID, {}) module = env['ir.module.module'].browse(module_id) processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies module.write({'state': 'installed', 'latest_version': ver}) package.load_state = package.state package.load_version = package.installed_version package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) if package.name is not None: registry._init_modules.add(package.name) _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql) registry.clear_caches() return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None, models_to_check=None, upg_registry=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(idref, mode): cr.execute("SAVEPOINT load_test_data_file") try: load_data(cr, idref, mode, 'test', package, report) return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', package.name) return False finally: cr.execute("ROLLBACK TO SAVEPOINT load_test_data_file") # avoid keeping stale xml_id, etc. in cache odoo.registry(cr.dbname).clear_caches() if skip_modules is None: skip_modules = [] if models_to_check is None: models_to_check = set() processed_modules = [] loaded_modules = [] registry = odoo.registry(cr.dbname) migrations = odoo.modules.migration.MigrationManager(cr, graph) module_count = len(graph) _logger.info('loading %d modules...', module_count) # suppress commits to have the upgrade of one module in just one transaction cr.commit_org = cr.commit cr.commit = lambda *args: None cr.rollback_org = cr.rollback cr.rollback = lambda *args: None # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = odoo.sql_db.sql_counter models_updated = set() for index, package in enumerate(graph, 1): module_name = package.name module_id = package.id if module_name in skip_modules or module_name in loaded_modules: continue _logger.debug('loading module %s (%d/%d)', module_name, index, module_count) needs_update = (hasattr(package, "init") or hasattr(package, "update") or package.state in ("to install", "to upgrade")) if needs_update: if package.name != 'base': registry.setup_models(cr) migrations.migrate_module(package, 'pre') if package.name != 'base': env = api.Environment(cr, SUPERUSER_ID, {}) env['base'].flush() load_openerp_module(package.name) new_install = package.state == 'to install' if new_install: py_module = sys.modules['odoo.addons.%s' % (module_name, )] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) model_names = registry.load(cr, package) mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' loaded_modules.append(package.name) if needs_update: models_updated |= set(model_names) models_to_check -= set(model_names) registry.setup_models(cr) # OpenUpgrade: rebuild the local registry based on the loaded models local_registry = {} env = api.Environment(cr, SUPERUSER_ID, {}) for model in env.values(): if not model._auto: continue openupgrade_loading.log_model(model, local_registry) openupgrade_loading.compare_registries(cr, package.name, upg_registry, local_registry) # OpenUpgrade end registry.init_models(cr, model_names, {'module': package.name}, new_install) elif package.state != 'to remove': # The current module has simply been loaded. The models extended by this module # and for which we updated the schema, must have their schema checked again. # This is because the extension may have changed the model, # e.g. adding required=True to an existing field, but the schema has not been # updated by this module because it's not marked as 'to upgrade/to install'. models_to_check |= set(model_names) & models_updated idref = {} if needs_update: env = api.Environment(cr, SUPERUSER_ID, {}) # Can't put this line out of the loop: ir.module.module will be # registered by init_models() above. module = env['ir.module.module'].browse(module_id) if perform_checks: module._check() if package.state == 'to upgrade': # upgrading the module information module.write(module.get_values_from_terp(package.data)) load_data(cr, idref, mode, kind='data', package=package, report=report) demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode, report) cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id)) module.invalidate_cache(['demo']) # OpenUpgrade: add 'try' block for logging exceptions # as errors in post scripts seem to be dropped try: migrations.migrate_module(package, 'post') except Exception as exc: _logger.error( 'Error executing post migration script for module %s: %s', package, exc) raise # Update translations for all installed languages overwrite = odoo.tools.config["overwrite_existing_translations"] module.with_context(overwrite=overwrite)._update_translations() if package.name is not None: registry._init_modules.add(package.name) if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) if mode == 'update': # validate the views that have not been checked yet env['ir.ui.view']._validate_module_views(module_name) # need to commit any modification the module's installation or # update made to the schema or data so the tests can run # (separately in their own transaction) # OpenUpgrade: commit after processing every module as well, for # easier debugging and continuing an interrupted migration cr.commit_org() if tools.config.options['test_enable']: report.record_result(load_test(idref, mode)) # Python tests env['ir.http']._clear_routing_map( ) # force routing map to be rebuilt report.record_result( odoo.modules.module.run_unit_tests(module_name)) # tests may have reset the environment env = api.Environment(cr, SUPERUSER_ID, {}) module = env['ir.module.module'].browse(module_id) # OpenUpgrade: run tests if os.environ.get( 'OPENUPGRADE_TESTS') and package.name is not None: # Load tests in <module>/migrations/tests and enable standard tags if necessary prefix = '.migrations' registry.openupgrade_test_prefixes[package.name] = prefix test_tags = tools.config['test_tags'] if not test_tags: tools.config['test_tags'] = '+standard' report.record_result( odoo.modules.module.run_unit_tests( module_name, openupgrade_prefix=prefix)) tools.config['test_tags'] = test_tags processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies module.write({'state': 'installed', 'latest_version': ver}) # OpenUpgrade: commit module_n state and version immediatly # to avoid invalid database state if module_n+1 raises an # exception cr.commit_org() package.load_state = package.state package.load_version = package.installed_version package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) module.flush() if package.name is not None: registry._init_modules.add(package.name) _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql) # Openupgrade: restore commit method cr.commit = cr.commit_org cr.commit() return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None, upg_registry=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(module_name, idref, mode): cr.commit() try: _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', module_name) return False finally: if tools.config.options['test_commit']: cr.commit() else: cr.rollback() # avoid keeping stale xml_id, etc. in cache odoo.registry(cr.dbname).clear_caches() def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package.data[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): # init_xml, update_xml and demo_xml are deprecated except # for the case of init_xml with yaml, csv and sql files as # we can't specify noupdate for those file. correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", package.name, k, correct_key, f) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ try: if kind in ('demo', 'test'): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("loading %s/%s", module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report) finally: if kind in ('demo', 'test'): threading.currentThread().testing = False if status is None: status = {} if skip_modules is None: skip_modules = [] processed_modules = [] loaded_modules = [] registry = odoo.registry(cr.dbname) migrations = odoo.modules.migration.MigrationManager(cr, graph) module_count = len(graph) _logger.info('loading %d modules...', module_count) registry.clear_manual_fields() # suppress commits to have the upgrade of one module in just one transaction cr.commit_org = cr.commit cr.commit = lambda *args: None cr.rollback_org = cr.rollback cr.rollback = lambda *args: None fields.set_migration_cursor(cr) # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = odoo.sql_db.sql_counter for index, package in enumerate(graph, 1): module_name = package.name module_id = package.id if module_name in skip_modules or module_name in loaded_modules: continue _logger.debug('loading module %s (%d/%d)', module_name, index, module_count) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) new_install = package.state == 'to install' if new_install: py_module = sys.modules['odoo.addons.%s' % (module_name, )] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) model_names = registry.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr( package, 'update') or package.state in ('to install', 'to upgrade'): registry.setup_models(cr, partial=True) # OpenUpgrade: rebuild the local registry based on the loaded models local_registry = {} env = api.Environment(cr, SUPERUSER_ID, {}) for model in env.values(): if not model._auto: continue openupgrade_loading.log_model(model, local_registry) openupgrade_loading.compare_registries(cr, package.name, upg_registry, local_registry) registry.init_models(cr, model_names, {'module': package.name}) idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if hasattr(package, 'init') or hasattr( package, 'update') or package.state in ('to install', 'to upgrade'): env = api.Environment(cr, SUPERUSER_ID, {}) # Can't put this line out of the loop: ir.module.module will be # registered by init_models() above. module = env['ir.module.module'].browse(module_id) if perform_checks: module.check() if package.state == 'to upgrade': # upgrading the module information module.write(module.get_values_from_terp(package.data)) _load_data(cr, module_name, idref, mode, kind='data') has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed') if has_demo: _load_data(cr, module_name, idref, mode, kind='demo') cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) module.invalidate_cache(['demo']) # OpenUpgrade: add 'try' block for logging exceptions # as errors in post scripts seem to be dropped try: migrations.migrate_module(package, 'post') except Exception as exc: _logger.error( 'Error executing post migration script for module %s: %s', package, exc) raise # Update translations for all installed languages overwrite = odoo.tools.config["overwrite_existing_translations"] module.with_context(overwrite=overwrite).update_translations() registry._init_modules.add(package.name) if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) # validate all the views at a whole env['ir.ui.view']._validate_module_views(module_name) if has_demo: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options['test_enable']: # Yamel test report.record_result(load_test(module_name, idref, mode)) # Python tests env['ir.http']._clear_routing_map( ) # force routing map to be rebuilt report.record_result( odoo.modules.module.run_unit_tests( module_name, cr.dbname)) processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies module.write({'state': 'installed', 'latest_version': ver}) package.load_state = package.state package.load_version = package.installed_version package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) registry._init_modules.add(package.name) cr.commit_org() # OpenUpgrade edit start: # if there's a tests directory, run those if tests are enabled tests_dir = os.path.join( odoo.modules.module.get_module_path(package.name), 'migrations', adapt_version(package.data['version']), 'tests', ) # check for an environment variable because we don't want to mess # with odoo's config.py, but we also don't want to run existing # tests if os.environ.get('OPENUPGRADE_TESTS') and os.path.exists(tests_dir): import unittest threading.currentThread().testing = True tests = unittest.defaultTestLoader.discover( tests_dir, top_level_dir=tests_dir) report.record_result( unittest.TextTestRunner( verbosity=2, stream=odoo.modules.module.TestStream(package.name), ).run(tests).wasSuccessful()) threading.currentThread().testing = False # OpenUpgrade edit end _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, odoo.sql_db.sql_counter - t0_sql) registry.clear_manual_fields() cr.commit = cr.commit_org cr.commit() fields.set_migration_cursor() return loaded_modules, processed_modules