def _compile(self): def filter_pyc(parents, name, contents): return name.endswith(u'.pyc') try: # STEP 1: Backup of all `pyc` files within the module information self.log(1, 'backing_up', self.name) files = self._get_resource_paths(filter_pyc) for pyc_file in files: self._backup(pyc_file) self.log(1, 'backing_done', self.name) # STEP 2: Compile the module try: self.log(1, 'compiling', self.name) module_path = get_module_path(self.name) compileall.compile_dir(module_path, True) for pyc_file in files: _logger.warning('Reload %s' % pyc_file) reload(pyc_file) self.log(1, 'compilation_done', self.name) except Exception as ex: # STEP 3: Restore old files if compilation fails self.log(1, 'compilation_fail', self.name) for pyc_file in files: self._backup(pyc_file, restore=True) self.log(1, 'restoring_done', self.name) raise ex # STEP 4: Update modules m_data_domain = [ ('module', '=', self.name), ('model', '=', 'ir.model.data') ] m_data_obj = self.env['ir.model.data'] m_data_set = m_data_obj.search(m_data_domain) ids = [record.res_id for record in m_data_set] model_obj = self.env['ir.model'] model_set = model_obj.browse(ids) for model in model_set: model = self.env['dev.code.tester'] init_module_models(self.env.cr, 'development_tools', model) except Exception as ex: _logger.error('compile %s' % ex)
def _compile(self): def filter_pyc(parents, name, contents): return name.endswith(u'.pyc') try: # STEP 1: Backup of all `pyc` files within the module information self.log(1, 'backing_up', self.name) files = self._get_resource_paths(filter_pyc) for pyc_file in files: self._backup(pyc_file) self.log(1, 'backing_done', self.name) # STEP 2: Compile the module try: self.log(1, 'compiling', self.name) module_path = get_module_path(self.name) compileall.compile_dir(module_path, True) for pyc_file in files: _logger.warning('Reload %s' % pyc_file) reload(pyc_file) self.log(1, 'compilation_done', self.name) except Exception as ex: # STEP 3: Restore old files if compilation fails self.log(1, 'compilation_fail', self.name) for pyc_file in files: self._backup(pyc_file, restore=True) self.log(1, 'restoring_done', self.name) raise ex # STEP 4: Update modules m_data_domain = [('module', '=', self.name), ('model', '=', 'ir.model.data')] m_data_obj = self.env['ir.model.data'] m_data_set = m_data_obj.search(m_data_domain) ids = [record.res_id for record in m_data_set] model_obj = self.env['ir.model'] model_set = model_obj.browse(ids) for model in model_set: model = self.env['dev.code.tester'] init_module_models(self.env.cr, 'development_tools', model) except Exception as ex: _logger.error('compile %s' % ex)
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: status dictionary for keeping track of progress :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(module_name, idref, mode): cr.commit() try: _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', module_name) return False finally: if tools.config.options['test_commit']: cr.commit() else: cr.rollback() # avoid keeping stale xml_id, etc. in cache openerp.modules.registry.RegistryManager.clear_caches(cr.dbname) def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package.data[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): # init_xml, update_xml and demo_xml are deprecated except # for the case of init_xml with yaml, csv and sql files as # we can't specify noupdate for those file. correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", package.name, k, correct_key, f ) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ try: if kind in ('demo', 'test'): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("loading %s/%s", module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report) finally: if kind in ('demo', 'test'): threading.currentThread().testing = False if status is None: status = {} processed_modules = [] loaded_modules = [] registry = openerp.registry(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) _logger.info('loading %d modules...', len(graph)) # Query manual fields for all models at once and save them on the registry # so the initialization code for each model does not have to do it # one model at a time. registry.fields_by_model = {} cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',)) for field in cr.dictfetchall(): registry.fields_by_model.setdefault(field['model'], []).append(field) # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = openerp.sql_db.sql_counter for index, package in enumerate(graph): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue migrations.migrate_module(package, 'pre') load_openerp_module(package.name) new_install = package.installed_version is None if new_install: py_module = sys.modules['openerp.addons.%s' % (module_name,)] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) models = registry.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) status['progress'] = float(index) / len(graph) # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = registry['ir.module.module'] if perform_checks: modobj.check(cr, SUPERUSER_ID, [module_id]) idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): if package.state=='to upgrade': # upgrading the module information modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data)) _load_data(cr, module_name, idref, mode, kind='data') has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed') if has_demo: status['progress'] = (index + 0.75) / len(graph) _load_data(cr, module_name, idref, mode, kind='demo') cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) migrations.migrate_module(package, 'post') if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) registry._init_modules.add(package.name) # validate all the views at a whole registry['ir.ui.view']._validate_module_views(cr, SUPERUSER_ID, module_name) if has_demo: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options['test_enable']: # Yamel test report.record_result(load_test(module_name, idref, mode)) # Python tests ir_http = registry['ir.http'] if hasattr(ir_http, '_routing_map'): # Force routing map to be rebuilt between each module test suite del(ir_http._routing_map) report.record_result(openerp.modules.module.run_unit_tests(module_name, cr.dbname)) processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver}) # Update translations for all installed languages modobj.update_translations(cr, SUPERUSER_ID, [module_id], None, {'overwrite': openerp.tools.config["overwrite_existing_translations"]}) package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) registry._init_modules.add(package.name) cr.commit() _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql) # The query won't be valid for models created later (i.e. custom model # created after the registry has been loaded), so empty its result. registry.fields_by_model = None cr.commit() return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(module_name, idref, mode): cr.commit() try: _load_data(cr, module_name, idref, mode, "test") return True except Exception: _test_logger.exception("module %s: an exception occurred in a test", module_name) return False finally: if tools.config.options["test_commit"]: cr.commit() else: cr.rollback() # avoid keeping stale xml_id, etc. in cache openerp.modules.registry.RegistryManager.clear_caches(cr.dbname) def _get_files_of_kind(kind): if kind == "demo": kind = ["demo_xml", "demo"] elif kind == "data": kind = ["init_xml", "update_xml", "data"] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package.data[k]: files.append(f) if k.endswith("_xml") and not (k == "init_xml" and not f.endswith(".xml")): # init_xml, update_xml and demo_xml are deprecated except # for the case of init_xml with yaml, csv and sql files as # we can't specify noupdate for those file. correct_key = "demo" if k.count("demo") else "data" _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", package.name, k, correct_key, f, ) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ try: if kind in ("demo", "test"): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("loading %s/%s", module_name, filename) noupdate = False if kind in ("demo", "demo_xml") or (filename.endswith(".csv") and kind in ("init", "init_xml")): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report) finally: if kind in ("demo", "test"): threading.currentThread().testing = False processed_modules = [] loaded_modules = [] registry = openerp.registry(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) _logger.info("loading %d modules...", len(graph)) registry.clear_manual_fields() # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = openerp.sql_db.sql_counter for index, package in enumerate(graph): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue migrations.migrate_module(package, "pre") load_openerp_module(package.name) new_install = package.state == "to install" if new_install: py_module = sys.modules["openerp.addons.%s" % (module_name,)] pre_init = package.info.get("pre_init_hook") if pre_init: getattr(py_module, pre_init)(cr) models = registry.load(cr, package) loaded_modules.append(package.name) if hasattr(package, "init") or hasattr(package, "update") or package.state in ("to install", "to upgrade"): registry.setup_models(cr, partial=True) init_module_models(cr, package.name, models) idref = {} mode = "update" if hasattr(package, "init") or package.state == "to install": mode = "init" if hasattr(package, "init") or hasattr(package, "update") or package.state in ("to install", "to upgrade"): # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = registry["ir.module.module"] if perform_checks: modobj.check(cr, SUPERUSER_ID, [module_id]) if package.state == "to upgrade": # upgrading the module information modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data)) _load_data(cr, module_name, idref, mode, kind="data") has_demo = hasattr(package, "demo") or (package.dbdemo and package.state != "installed") if has_demo: _load_data(cr, module_name, idref, mode, kind="demo") cr.execute("update ir_module_module set demo=%s where id=%s", (True, module_id)) modobj.invalidate_cache(cr, SUPERUSER_ID, ["demo"], [module_id]) migrations.migrate_module(package, "post") # Update translations for all installed languages modobj.update_translations( cr, SUPERUSER_ID, [module_id], None, {"overwrite": openerp.tools.config["overwrite_existing_translations"]}, ) registry._init_modules.add(package.name) if new_install: post_init = package.info.get("post_init_hook") if post_init: getattr(py_module, post_init)(cr, registry) # validate all the views at a whole registry["ir.ui.view"]._validate_module_views(cr, SUPERUSER_ID, module_name) if has_demo: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options["test_enable"]: # Yamel test report.record_result(load_test(module_name, idref, mode)) # Python tests ir_http = registry["ir.http"] # Force routing map to be rebuilt between each module test suite vars(ir_http).pop("routing_map", None) report.record_result(openerp.modules.module.run_unit_tests(module_name, cr.dbname)) processed_modules.append(package.name) ver = adapt_version(package.data["version"]) # Set new modules and dependencies modobj.write(cr, SUPERUSER_ID, [module_id], {"state": "installed", "latest_version": ver}) package.state = "installed" for kind in ("init", "demo", "update"): if hasattr(package, kind): delattr(package, kind) registry._init_modules.add(package.name) cr.commit() _logger.log( 25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql ) registry.clear_manual_fields() cr.commit() return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: status dictionary for keeping track of progress :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def process_sql_file(cr, fp): queries = fp.read().split(';') for query in queries: new_query = ' '.join(query.split()) if new_query: cr.execute(new_query) load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml') load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml') load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml') load_data = lambda *args: _load_data(cr, *args, kind='data') load_demo = lambda *args: _load_data(cr, *args, kind='demo') def load_test(module_name, idref, mode): cr.commit() try: threading.currentThread().testing = True _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _logger.exception( 'module %s: an exception occurred in a test', module_name) return False finally: threading.currentThread().testing = False if tools.config.options['test_commit']: cr.commit() else: cr.rollback() def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ for filename in package.data[kind]: if kind == 'test': _logger.log(logging.TEST, "module %s: loading %s", module_name, filename) else: _logger.info("module %s: loading %s", module_name, filename) _, ext = os.path.splitext(filename) pathname = os.path.join(module_name, filename) fp = tools.file_open(pathname) noupdate = False if kind in ('demo', 'demo_xml'): noupdate = True try: ext = ext.lower() if ext == '.csv': if kind in ('init', 'init_xml'): noupdate = True tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate) elif ext == '.sql': process_sql_file(cr, fp) elif ext == '.yml': tools.convert_yaml_import(cr, module_name, fp, kind, idref, mode, noupdate, report) elif ext == '.xml': tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report) elif ext == '.js': pass # .js files are valid but ignored here. else: _logger.warning("Can't load unknown file type %s.", filename) finally: fp.close() if status is None: status = {} processed_modules = [] loaded_modules = [] pool = pooler.get_pool(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) _logger.debug('loading %d packages...', len(graph)) # Query manual fields for all models at once and save them on the registry # so the initialization code for each model does not have to do it # one model at a time. pool.fields_by_model = {} cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',)) for field in cr.dictfetchall(): pool.fields_by_model.setdefault(field['model'], []).append(field) # register, instantiate and initialize models for each modules for index, package in enumerate(graph): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue _logger.info('module %s: loading objects', package.name) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) models = pool.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) pool._init_modules.add(package.name) status['progress'] = float(index) / len(graph) # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = pool.get('ir.module.module') if perform_checks: modobj.check(cr, SUPERUSER_ID, [module_id]) idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): if package.state=='to upgrade': # upgrading the module information modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data)) load_init_xml(module_name, idref, mode) load_update_xml(module_name, idref, mode) load_data(module_name, idref, mode) if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): status['progress'] = (index + 0.75) / len(graph) load_demo_xml(module_name, idref, mode) load_demo(module_name, idref, mode) cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) # launch tests only in demo mode, as most tests will depend # on demo data. Other tests can be added into the regular # 'data' section, but should probably not alter the data, # as there is no rollback. if tools.config.options['test_enable']: report.record_result(load_test(module_name, idref, mode)) # Run the `fast_suite` and `checks` tests given by the module. if module_name == 'base': # Also run the core tests after the database is created. report.record_result(openerp.modules.module.run_unit_tests('openerp')) report.record_result(openerp.modules.module.run_unit_tests(module_name)) processed_modules.append(package.name) migrations.migrate_module(package, 'post') ver = adapt_version(package.data['version']) # Set new modules and dependencies modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver}) # Update translations for all installed languages modobj.update_translations(cr, SUPERUSER_ID, [module_id], None) package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) cr.commit() # The query won't be valid for models created later (i.e. custom model # created after the registry has been loaded), so empty its result. pool.fields_by_model = None cr.commit() return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: status dictionary for keeping track of progress :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def process_sql_file(cr, fp): queries = fp.read().split(';') for query in queries: new_query = ' '.join(query.split()) if new_query: cr.execute(new_query) load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml') load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml') load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml') load_data = lambda *args: _load_data(cr, *args, kind='data') load_demo = lambda *args: _load_data(cr, *args, kind='demo') def load_test(module_name, idref, mode): cr.commit() try: threading.currentThread().testing = True _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _logger.exception('module %s: an exception occurred in a test', module_name) return False finally: threading.currentThread().testing = False if tools.config.options['test_commit']: cr.commit() else: cr.rollback() def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ for filename in package.data[kind]: _logger.info("module %s: loading %s", module_name, filename) _, ext = os.path.splitext(filename) pathname = os.path.join(module_name, filename) fp = tools.file_open(pathname) noupdate = False if kind in ('demo', 'demo_xml'): noupdate = True try: ext = ext.lower() if ext == '.csv': if kind in ('init', 'init_xml'): noupdate = True tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate) elif ext == '.sql': process_sql_file(cr, fp) elif ext == '.yml': tools.convert_yaml_import(cr, module_name, fp, kind, idref, mode, noupdate, report) elif ext == '.xml': tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report) elif ext == '.js': pass # .js files are valid but ignored here. else: _logger.warning("Can't load unknown file type %s.", filename) finally: fp.close() if status is None: status = {} processed_modules = [] loaded_modules = [] pool = pooler.get_pool(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) _logger.info('loading %d modules...', len(graph)) # Query manual fields for all models at once and save them on the registry # so the initialization code for each model does not have to do it # one model at a time. pool.fields_by_model = {} cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual', )) for field in cr.dictfetchall(): pool.fields_by_model.setdefault(field['model'], []).append(field) # register, instantiate and initialize models for each modules for index, package in enumerate(graph): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue _logger.debug('module %s: loading objects', package.name) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) models = pool.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr( package, 'update') or package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) pool._init_modules.add(package.name) status['progress'] = float(index) / len(graph) # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = pool.get('ir.module.module') if perform_checks: modobj.check(cr, SUPERUSER_ID, [module_id]) idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if hasattr(package, 'init') or hasattr( package, 'update') or package.state in ('to install', 'to upgrade'): if package.state == 'to upgrade': # upgrading the module information modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data)) load_init_xml(module_name, idref, mode) load_update_xml(module_name, idref, mode) load_data(module_name, idref, mode) if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): status['progress'] = (index + 0.75) / len(graph) load_demo_xml(module_name, idref, mode) load_demo(module_name, idref, mode) cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) # launch tests only in demo mode, as most tests will depend # on demo data. Other tests can be added into the regular # 'data' section, but should probably not alter the data, # as there is no rollback. if tools.config.options['test_enable']: report.record_result(load_test(module_name, idref, mode)) # Run the `fast_suite` and `checks` tests given by the module. if module_name == 'base': # Also run the core tests after the database is created. report.record_result( openerp.modules.module.run_unit_tests('openerp')) report.record_result( openerp.modules.module.run_unit_tests(module_name)) processed_modules.append(package.name) migrations.migrate_module(package, 'post') ver = adapt_version(package.data['version']) # Set new modules and dependencies modobj.write(cr, SUPERUSER_ID, [module_id], { 'state': 'installed', 'latest_version': ver }) # Update translations for all installed languages modobj.update_translations(cr, SUPERUSER_ID, [module_id], None) package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) cr.commit() # The query won't be valid for models created later (i.e. custom model # created after the registry has been loaded), so empty its result. pool.fields_by_model = None cr.commit() return loaded_modules, processed_modules
for index, package in enumerate(graph): pkg_data = False module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue _logger.info('module %s: loading objects', package.name) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) models = pool.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) if git: update_git_information(cr, package.name) status['progress'] = float(index) / len(graph) # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = pool.get('ir.module.module') if perform_checks: modobj.check(cr, 1, [module_id]) idref = {} mode = 'update'
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None, upg_registry=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(module_name, idref, mode): cr.commit() try: _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', module_name) return False finally: if tools.config.options['test_commit']: cr.commit() else: cr.rollback() # avoid keeping stale xml_id, etc. in cache openerp.modules.registry.RegistryManager.clear_caches(cr.dbname) def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package.data[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): # init_xml, update_xml and demo_xml are deprecated except # for the case of init_xml with yaml, csv and sql files as # we can't specify noupdate for those file. correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", package.name, k, correct_key, f ) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ try: if kind in ('demo', 'test'): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("loading %s/%s", module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report) finally: if kind in ('demo', 'test'): threading.currentThread().testing = False if status is None: status = {} if skip_modules is None: skip_modules = [] processed_modules = [] loaded_modules = [] registry = openerp.registry(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) _logger.info('loading %d modules...', len(graph)) registry.clear_manual_fields() # suppress commits to have the upgrade of one module in just one transaction cr.commit_org = cr.commit cr.commit = lambda *args: None cr.rollback_org = cr.rollback cr.rollback = lambda *args: None openerp.osv.fields.set_migration_cursor(cr) # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = openerp.sql_db.sql_counter for index, package in enumerate(graph): module_name = package.name module_id = package.id if module_name in skip_modules or module_name in loaded_modules: continue migrations.migrate_module(package, 'pre') load_openerp_module(package.name) new_install = package.state == 'to install' if new_install: py_module = sys.modules['openerp.addons.%s' % (module_name,)] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) models = registry.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): registry.setup_models(cr, partial=True) # OpenUpgrade: add this module's models to the registry local_registry = {} for model in models: if not model._auto: continue openupgrade_loading.log_model(model, local_registry) openupgrade_loading.compare_registries( cr, package.name, upg_registry, local_registry) init_module_models(cr, package.name, models) idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = registry['ir.module.module'] if perform_checks: modobj.check(cr, SUPERUSER_ID, [module_id]) if package.state=='to upgrade': # upgrading the module information modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data)) _load_data(cr, module_name, idref, mode, kind='data') has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed') if has_demo: _load_data(cr, module_name, idref, mode, kind='demo') cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) modobj.invalidate_cache(cr, SUPERUSER_ID, ['demo'], [module_id]) # OpenUpgrade: add 'try' block for logging exceptions # as errors in post scripts seem to be dropped try: migrations.migrate_module(package, 'post') except Exception as exc: _logger.error('Error executing post migration script for module %s: %s', package, exc) raise # Update translations for all installed languages modobj.update_translations(cr, SUPERUSER_ID, [module_id], None, {'overwrite': openerp.tools.config["overwrite_existing_translations"]}) registry._init_modules.add(package.name) if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) # validate all the views at a whole registry['ir.ui.view']._validate_module_views(cr, SUPERUSER_ID, module_name) if has_demo: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options['test_enable']: # Yamel test report.record_result(load_test(module_name, idref, mode)) # Python tests ir_http = registry['ir.http'] if hasattr(ir_http, '_routing_map'): # Force routing map to be rebuilt between each module test suite del(ir_http._routing_map) report.record_result(openerp.modules.module.run_unit_tests(module_name, cr.dbname)) processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver}) package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) registry._init_modules.add(package.name) cr.commit_org() # OpenUpgrade edit start: # if there's a tests directory, run those if tests are enabled tests_dir = os.path.join( openerp.modules.module.get_module_path(package.name), 'migrations', adapt_version(package.data['version']), 'tests', ) # check for an environment variable because we don't want to mess # with odoo's config.py, but we also don't want to run existing # tests if os.environ.get('OPENUPGRADE_TESTS') and os.path.exists( tests_dir ): import unittest threading.currentThread().testing = True tests = unittest.defaultTestLoader.discover(tests_dir, top_level_dir=tests_dir) report.record_result( unittest.TextTestRunner( verbosity=2, stream=openerp.modules.module.TestStream(package.name), ).run(tests) .wasSuccessful() ) threading.currentThread().testing = False # OpenUpgrade edit end _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql) registry.clear_manual_fields() cr.commit = cr.commit_org cr.commit() openerp.osv.fields.set_migration_cursor() return loaded_modules, processed_modules
def load_modules(db, force_demo=False, status=None, update_module=False): initialize_sys_path() force = [] if force_demo: force.append('demo') models_to_check = set() cr = db.cursor() try: if not openerp.modules.db.is_initialized(cr): if not update_module: _logger.error( "Database %s not initialized, you can force it with `-i base`", cr.dbname) return _logger.info("init db") openerp.modules.db.initialize(cr) update_module = True # process auto-installed modules tools.config["init"]["all"] = 1 tools.config['update']['all'] = 1 if not tools.config['without_demo']: tools.config["demo"]['all'] = 1 # This is a brand new registry, just created in # openerp.modules.registry.RegistryManager.new(). registry = openerp.registry(cr.dbname) if 'base' in tools.config['update'] or 'all' in tools.config['update']: cr.execute( "update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) graph = openerp.modules.graph.Graph() graph.add_module(cr, 'base', force) if not graph: _logger.critical( 'module base cannot be loaded! (hint: verify addons-path)') raise ImportError( 'Module `base` cannot be loaded! (hint: verify addons-path)') # processed_modules: for cleanup step after install # loaded_modules: to avoid double loading report = registry._assertion_report loaded_modules, processed_modules = load_module_graph( cr, graph, status, perform_checks=update_module, report=report, models_to_check=models_to_check) load_lang = tools.config.pop('load_language') if load_lang or update_module: # some base models are used below, so make sure they are set up registry.setup_models(cr, partial=True) if load_lang: for lang in load_lang.split(','): tools.load_language(cr, lang) # STEP 2: Mark other modules to be loaded/updated if update_module: modobj = registry['ir.module.module'] _logger.info('updating modules list') modobj.update_list(cr, SUPERUSER_ID) _check_module_names( cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys())) mods = [k for k in tools.config['init'] if tools.config['init'][k]] if mods: ids = modobj.search( cr, SUPERUSER_ID, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)]) if ids: modobj.button_install(cr, SUPERUSER_ID, ids) mods = [ k for k in tools.config['update'] if tools.config['update'][k] ] if mods: ids = modobj.search( cr, SUPERUSER_ID, ['&', ('state', '=', 'installed'), ('name', 'in', mods)]) if ids: modobj.button_upgrade(cr, SUPERUSER_ID, ids) cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) modobj.invalidate_cache(cr, SUPERUSER_ID, ['state']) # STEP 3: Load marked modules (skipping base which was done in STEP 1) # IMPORTANT: this is done in two parts, first loading all installed or # partially installed modules (i.e. installed/to upgrade), to # offer a consistent system to the second part: installing # newly selected modules. # We include the modules 'to remove' in the first step, because # they are part of the "currently installed" modules. They will # be dropped in STEP 6 later, before restarting the loading # process. # IMPORTANT 2: We have to loop here until all relevant modules have been # processed, because in some rare cases the dependencies have # changed, and modules that depend on an uninstalled module # will not be processed on the first pass. # It's especially useful for migrations. previously_processed = -1 while previously_processed < len(processed_modules): previously_processed = len(processed_modules) processed_modules += load_marked_modules( cr, graph, ['installed', 'to upgrade', 'to remove'], force, status, report, loaded_modules, update_module, models_to_check) if update_module: processed_modules += load_marked_modules( cr, graph, ['to install'], force, status, report, loaded_modules, update_module, models_to_check) registry.setup_models(cr) # STEP 3.5: execute migration end-scripts migrations = openerp.modules.migration.MigrationManager(cr, graph) for package in graph: migrations.migrate_module(package, 'end') # STEP 4: Finish and cleanup installations if processed_modules: cr.execute( """select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""" ) for (model, name) in cr.fetchall(): if model in registry and not registry[model].is_transient( ) and not isinstance(registry[model], openerp.osv.orm.AbstractModel): _logger.warning( 'The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0', model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_')) # Temporary warning while we remove access rights on osv_memory objects, as they have # been replaced by owner-only access rights cr.execute( """select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""" ) for (model, name) in cr.fetchall(): if model in registry and registry[model].is_transient(): _logger.warning( 'The transient model %s (%s) should not have explicit access rules!', model, name) cr.execute("SELECT model from ir_model") for (model, ) in cr.fetchall(): if model in registry: registry[model]._check_removed_columns(cr, log=True) elif _logger.isEnabledFor( logging.INFO): # more an info that a warning... _logger.warning( "Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model) # Cleanup orphan records registry['ir.model.data']._process_end(cr, SUPERUSER_ID, processed_modules) for kind in ('init', 'demo', 'update'): tools.config[kind] = {} cr.commit() # STEP 5: Uninstall modules to remove if update_module: # Remove records referenced from ir_model_data for modules to be # removed (and removed the references from ir_model_data). cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove', )) modules_to_remove = dict(cr.fetchall()) if modules_to_remove: pkgs = reversed( [p for p in graph if p.name in modules_to_remove]) for pkg in pkgs: uninstall_hook = pkg.info.get('uninstall_hook') if uninstall_hook: py_module = sys.modules['openerp.addons.%s' % (pkg.name, )] getattr(py_module, uninstall_hook)(cr, registry) registry['ir.module.module'].module_uninstall( cr, SUPERUSER_ID, modules_to_remove.values()) # Recursive reload, should only happen once, because there should be no # modules to remove next time cr.commit() _logger.info( 'Reloading registry once more after uninstalling modules') openerp.api.Environment.reset() return openerp.modules.registry.RegistryManager.new( cr.dbname, force_demo, status, update_module) # STEP 5.5: Verify extended fields on every model # This will fix the schema of all models in a situation such as: # - module A is loaded and defines model M; # - module B is installed/upgraded and extends model M; # - module C is loaded and extends model M; # - module B and C depend on A but not on each other; # The changes introduced by module C are not taken into account by the upgrade of B. if models_to_check: module_objs_to_check = [ registry.models[m] for m in models_to_check ] init_module_models(cr, 'verification in progress', module_objs_to_check) # STEP 6: verify custom views on every model if update_module: Views = registry['ir.ui.view'] for model in registry.models.keys(): try: Views._validate_custom_views(cr, SUPERUSER_ID, model) except Exception as e: _logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e)) if report.failures: _logger.error('At least one test failed when loading the modules.') else: _logger.info('Modules loaded.') # STEP 8: call _register_hook on every model for model in registry.models.values(): model._register_hook(cr) # STEP 9: Run the post-install tests cr.commit() t0 = time.time() t0_sql = openerp.sql_db.sql_counter if openerp.tools.config['test_enable']: if update_module: cr.execute( "SELECT name FROM ir_module_module WHERE state='installed' and name = ANY(%s)", (processed_modules, )) else: cr.execute( "SELECT name FROM ir_module_module WHERE state='installed'" ) for module_name in cr.fetchall(): report.record_result( openerp.modules.module.run_unit_tests( module_name[0], cr.dbname, position=runs_post_install)) _logger.log(25, "All post-tested in %.2fs, %s queries", time.time() - t0, openerp.sql_db.sql_counter - t0_sql) finally: cr.close()
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None, models_to_check=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(module_name, idref, mode): cr.commit() try: _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', module_name) return False finally: if tools.config.options['test_commit']: cr.commit() else: cr.rollback() # avoid keeping stale xml_id, etc. in cache openerp.modules.registry.RegistryManager.clear_caches( cr.dbname) def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package.data[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): # init_xml, update_xml and demo_xml are deprecated except # for the case of init_xml with yaml, csv and sql files as # we can't specify noupdate for those file. correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", package.name, k, correct_key, f) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ try: if kind in ('demo', 'test'): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("loading %s/%s", module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report) finally: if kind in ('demo', 'test'): threading.currentThread().testing = False if models_to_check is None: models_to_check = set() processed_modules = [] loaded_modules = [] registry = openerp.registry(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) _logger.info('loading %d modules...', len(graph)) registry.clear_manual_fields() # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = openerp.sql_db.sql_counter models_updated = set() for index, package in enumerate(graph): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue needs_update = (hasattr(package, "init") or hasattr(package, "update") or package.state in ("to install", "to upgrade")) if needs_update: if package.name != 'base': registry.setup_models(cr, partial=True) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) new_install = package.state == 'to install' if new_install: py_module = sys.modules['openerp.addons.%s' % (module_name, )] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) models = registry.load(cr, package) model_names = [m._name for m in models] loaded_modules.append(package.name) if needs_update: models_updated |= set(model_names) models_to_check -= set(model_names) registry.setup_models(cr, partial=True) init_module_models(cr, package.name, models) elif package.state != 'to remove': # The current module has simply been loaded. The models extended by this module # and for which we updated the schema, must have their schema checked again. # This is because the extension may have changed the model, # e.g. adding required=True to an existing field, but the schema has not been # updated by this module because it's not marked as 'to upgrade/to install'. models_to_check |= set(model_names) & models_updated idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if needs_update: # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = registry['ir.module.module'] if perform_checks: modobj.check(cr, SUPERUSER_ID, [module_id]) if package.state == 'to upgrade': # upgrading the module information modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data)) _load_data(cr, module_name, idref, mode, kind='data') has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed') if has_demo: _load_data(cr, module_name, idref, mode, kind='demo') cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) modobj.invalidate_cache(cr, SUPERUSER_ID, ['demo'], [module_id]) migrations.migrate_module(package, 'post') # Update translations for all installed languages modobj.update_translations( cr, SUPERUSER_ID, [module_id], None, { 'overwrite': openerp.tools.config["overwrite_existing_translations"] }) registry._init_modules.add(package.name) if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) # validate all the views at a whole registry['ir.ui.view']._validate_module_views( cr, SUPERUSER_ID, module_name) if has_demo: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options['test_enable']: # Yamel test report.record_result(load_test(module_name, idref, mode)) # Python tests ir_http = registry['ir.http'] if hasattr(ir_http, '_routing_map'): # Force routing map to be rebuilt between each module test suite del (ir_http._routing_map) report.record_result( openerp.modules.module.run_unit_tests( module_name, cr.dbname)) processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies modobj.write(cr, SUPERUSER_ID, [module_id], { 'state': 'installed', 'latest_version': ver }) package.load_state = package.state package.load_version = package.installed_version package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) registry._init_modules.add(package.name) cr.commit() _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql) registry.clear_manual_fields() cr.commit() return loaded_modules, processed_modules
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: status dictionary for keeping track of progress :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(module_name, idref, mode): cr.commit() try: threading.currentThread().testing = True _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', module_name) return False finally: threading.currentThread().testing = False if tools.config.options['test_commit']: cr.commit() else: cr.rollback() def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package.data[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): # init_xml, update_xml and demo_xml are deprecated except # for the case of init_xml with yaml, csv and sql files as # we can't specify noupdate for those file. correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", package.name, k, correct_key, f ) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ for filename in _get_files_of_kind(kind): _logger.info("module %s: loading %s", module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report) if status is None: status = {} processed_modules = [] loaded_modules = [] registry = openerp.registry(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) _logger.info('loading %d modules...', len(graph)) # Query manual fields for all models at once and save them on the registry # so the initialization code for each model does not have to do it # one model at a time. registry.fields_by_model = {} cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',)) for field in cr.dictfetchall(): registry.fields_by_model.setdefault(field['model'], []).append(field) # register, instantiate and initialize models for each modules for index, package in enumerate(graph): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue _logger.debug('module %s: loading objects', package.name) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) models = registry.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) registry._init_modules.add(package.name) status['progress'] = float(index) / len(graph) # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = registry['ir.module.module'] if perform_checks: modobj.check(cr, SUPERUSER_ID, [module_id]) idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): if package.state=='to upgrade': # upgrading the module information modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data)) _load_data(cr, module_name, idref, mode, kind='data') if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): status['progress'] = (index + 0.75) / len(graph) _load_data(cr, module_name, idref, mode, kind='demo') cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) # launch tests only in demo mode, as most tests will depend # on demo data. Other tests can be added into the regular # 'data' section, but should probably not alter the data, # as there is no rollback. if tools.config.options['test_enable']: report.record_result(load_test(module_name, idref, mode)) # Run the `fast_suite` and `checks` tests given by the module. if module_name == 'base': # Also run the core tests after the database is created. report.record_result(openerp.modules.module.run_unit_tests('openerp')) report.record_result(openerp.modules.module.run_unit_tests(module_name)) processed_modules.append(package.name) migrations.migrate_module(package, 'post') ver = adapt_version(package.data['version']) # Set new modules and dependencies modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver}) # Update translations for all installed languages modobj.update_translations(cr, SUPERUSER_ID, [module_id], None) package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) cr.commit() # The query won't be valid for models created later (i.e. custom model # created after the registry has been loaded), so empty its result. registry.fields_by_model = None cr.commit() return loaded_modules, processed_modules
def button_update_self(self, cr, uid, ids, context=None): registry = openerp.registry(cr.dbname) def process_sql_file(cr, fp): queries = fp.read().split(';') for query in queries: new_query = ' '.join(query.split()) if new_query: cr.execute(new_query) def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", module['name'], k, correct_key, f) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ try: if kind in ('demo', 'test'): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("loading %s/%s", module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, registry._assertion_report) finally: if kind in ('demo', 'test'): threading.currentThread().testing = False mode = 'update' for module in self.read(cr, uid, ids, [ 'name', ], context=context): idref = {} models = [] for cls in openerp.models.MetaModel.module_to_models.get( module['name'], []): model_name = cls._name if model_name == None: if cls._inherit != None: if isinstance(cls._inherit, (set, list, tuple)): model_name = cls._inherit[0] else: model_name = cls._inherit else: continue if model_name != None and model_name not in models: models.append(model_name) models = [self.pool.get(m) for m in models] init_module_models(cr, module['name'], models) registry.setup_models(cr, partial=True) package = openerp.modules.module.load_information_from_description_file( module['name']) modobj = self.pool.get('ir.module.module') modobj.check(cr, SUPERUSER_ID, [module['id']]) _load_data(cr, module['name'], idref, mode, kind='data') registry['ir.ui.view']._validate_module_views( cr, SUPERUSER_ID, module['name']) modobj.update_translations(cr, SUPERUSER_ID, [module['id']], None, {'overwrite': True}) cr.commit() return True
module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue _logger.info('module %s: loading objects', package.name) migrations.migrate_module(package, 'pre') load_openerp_module(package.name) models = pool.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr( package, 'update') or package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) status['progress'] = float(index) / len(graph) # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = pool.get('ir.module.module') if perform_checks: modobj.check(cr, 1, [module_id]) idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init'
def button_update_self(self, cr, uid, ids, context=None): registry = openerp.registry(cr.dbname) def process_sql_file(cr, fp): queries = fp.read().split(';') for query in queries: new_query = ' '.join(query.split()) if new_query: cr.execute(new_query) def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", module['name'], k, correct_key, f ) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ try: if kind in ('demo', 'test'): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("loading %s/%s", module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, registry._assertion_report) finally: if kind in ('demo', 'test'): threading.currentThread().testing = False mode = 'update' for module in self.read(cr, uid, ids, ['name',], context=context): idref = {} models = [] for cls in openerp.models.MetaModel.module_to_models.get(module['name'], []): model_name = cls._name if model_name == None: if cls._inherit != None: if isinstance(cls._inherit, (set, list, tuple)): model_name = cls._inherit[0] else: model_name = cls._inherit else: continue if model_name != None and model_name not in models: models.append(model_name) models = [self.pool.get(m) for m in models] init_module_models(cr, module['name'], models) registry.setup_models(cr, partial=True) package = openerp.modules.module.load_information_from_description_file(module['name']) modobj = self.pool.get('ir.module.module') modobj.check(cr, SUPERUSER_ID, [module['id']]) _load_data(cr, module['name'], idref, mode, kind='data') registry['ir.ui.view']._validate_module_views(cr, SUPERUSER_ID, module['name']) modobj.update_translations(cr, SUPERUSER_ID, [module['id']], None, {'overwrite': True}) cr.commit() return True
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases) :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ def load_test(module_name, idref, mode): cr.commit() try: _load_data(cr, module_name, idref, mode, 'test') return True except Exception: _test_logger.exception( 'module %s: an exception occurred in a test', module_name) return False finally: if tools.config.options['test_commit']: cr.commit() else: cr.rollback() # avoid keeping stale xml_id, etc. in cache openerp.modules.registry.RegistryManager.clear_caches( cr.dbname) def _get_files_of_kind(kind): if kind == 'demo': kind = ['demo_xml', 'demo'] elif kind == 'data': kind = ['init_xml', 'update_xml', 'data'] if isinstance(kind, str): kind = [kind] files = [] for k in kind: for f in package.data[k]: files.append(f) if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')): # init_xml, update_xml and demo_xml are deprecated except # for the case of init_xml with yaml, csv and sql files as # we can't specify noupdate for those file. correct_key = 'demo' if k.count('demo') else 'data' _logger.warning( "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.", package.name, k, correct_key, f) return files def _load_data(cr, module_name, idref, mode, kind): """ kind: data, demo, test, init_xml, update_xml, demo_xml. noupdate is False, unless it is demo data or it is csv data in init mode. """ module_files = [] module_modified = True if mode == 'update' and kind == 'data': cr.execute("select id from ir_module_module where name=%s", (module_name, )) module_id = cr.fetchone()[0] for filename in _get_files_of_kind(kind): cr.execute( "select id,checksum from ir_module_module_file" " where module_id=%s and filename=%s", (module_id, filename)) checksum = cr.fetchone() module_file_id = checksum and checksum[0] or False checksum = checksum and checksum[1] or '' sha256 = hashlib.sha256() sha256.update( misc.file_open(os.path.join(module_name, filename)).read()) module_files += [{ 'checksum': checksum, 'checksum_new': sha256.hexdigest(), 'module_file_id': module_file_id, 'module_id': module_id, 'filename': filename }] if len(module_files) == len([ 1 for f in module_files if f['checksum'] == f['checksum_new'] ]): module_modified = bool( tools.config.options.get('force_update', False)) try: if kind in ('demo', 'test'): threading.currentThread().testing = True for filename in _get_files_of_kind(kind): _logger.info("%s %s/%s", module_modified and 'loading' or 'no changes', module_name, filename) noupdate = False if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')): noupdate = True if module_modified: tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report) finally: if kind in ('demo', 'test'): threading.currentThread().testing = False if module_modified and mode == 'update' and kind == 'data': for module_file in [ f for f in module_files if f['checksum'] != f['checksum_new'] ]: if module_file['module_file_id']: cr.execute( "update ir_module_module_file set checksum=%s, write_date=now() where id=%s", (module_file['checksum_new'], module_file['module_file_id'])) else: cr.execute( "insert into ir_module_module_file(filename,module_id,checksum,create_date,create_uid,write_date,write_uid)" " values(%s,%s,%s,now(),1,now(),1)", (module_file['filename'], module_file['module_id'], module_file['checksum_new'])) return module_modified processed_modules = [] loaded_modules = [] registry = openerp.registry(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) _logger.info('loading %d modules...', len(graph)) registry.clear_manual_fields() # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = openerp.sql_db.sql_counter for index, package in enumerate(graph): module_name = package.name module_id = package.id if skip_modules and module_name in skip_modules: continue migrations.migrate_module(package, 'pre') load_openerp_module(package.name) new_install = package.state == 'to install' if new_install: py_module = sys.modules['openerp.addons.%s' % (module_name, )] pre_init = package.info.get('pre_init_hook') if pre_init: getattr(py_module, pre_init)(cr) models = registry.load(cr, package) loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr( package, 'update') or package.state in ('to install', 'to upgrade'): registry.setup_models(cr, partial=True) init_module_models(cr, package.name, models) idref = {} mode = 'update' if hasattr(package, 'init') or package.state == 'to install': mode = 'init' if hasattr(package, 'init') or hasattr( package, 'update') or package.state in ('to install', 'to upgrade'): # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. modobj = registry['ir.module.module'] if perform_checks: modobj.check(cr, SUPERUSER_ID, [module_id]) if package.state == 'to upgrade': # upgrading the module information modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data)) module_processed = _load_data(cr, module_name, idref, mode, kind='data') has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed') if has_demo: _load_data(cr, module_name, idref, mode, kind='demo') cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) modobj.invalidate_cache(cr, SUPERUSER_ID, ['demo'], [module_id]) migrations.migrate_module(package, 'post') if new_install: post_init = package.info.get('post_init_hook') if post_init: getattr(py_module, post_init)(cr, registry) registry._init_modules.add(package.name) # validate all the views at a whole registry['ir.ui.view']._validate_module_views( cr, SUPERUSER_ID, module_name) if has_demo: # launch tests only in demo mode, allowing tests to use demo data. if tools.config.options['test_enable']: # Yamel test report.record_result(load_test(module_name, idref, mode)) # Python tests ir_http = registry['ir.http'] if hasattr(ir_http, '_routing_map'): # Force routing map to be rebuilt between each module test suite del (ir_http._routing_map) report.record_result( openerp.modules.module.run_unit_tests( module_name, cr.dbname)) if module_processed: processed_modules.append(package.name) ver = adapt_version(package.data['version']) # Set new modules and dependencies modobj.write(cr, SUPERUSER_ID, [module_id], { 'state': 'installed', 'latest_version': ver }) # Update translations for all installed languages modobj.update_translations( cr, SUPERUSER_ID, [module_id], None, { 'overwrite': openerp.tools.config["overwrite_existing_translations"] }) package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): delattr(package, kind) registry._init_modules.add(package.name) cr.commit() _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql) registry.clear_manual_fields() cr.commit() return loaded_modules, processed_modules