Exemple #1
0
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
    """Migrates+Updates or Installs all module nodes from ``graph``
       :param graph: graph of module nodes to load
       :param status: deprecated parameter, unused, left to avoid changing signature in 8.0
       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
                              for same cases)
       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
       :return: list of modules that were installed or updated
    """

    def load_test(module_name, idref, mode):
        cr.commit()
        try:
            _load_data(cr, module_name, idref, mode, "test")
            return True
        except Exception:
            _test_logger.exception("module %s: an exception occurred in a test", module_name)
            return False
        finally:
            if tools.config.options["test_commit"]:
                cr.commit()
            else:
                cr.rollback()
                # avoid keeping stale xml_id, etc. in cache
                openerp.modules.registry.RegistryManager.clear_caches(cr.dbname)

    def _get_files_of_kind(kind):
        if kind == "demo":
            kind = ["demo_xml", "demo"]
        elif kind == "data":
            kind = ["init_xml", "update_xml", "data"]
        if isinstance(kind, str):
            kind = [kind]
        files = []
        for k in kind:
            for f in package.data[k]:
                files.append(f)
                if k.endswith("_xml") and not (k == "init_xml" and not f.endswith(".xml")):
                    # init_xml, update_xml and demo_xml are deprecated except
                    # for the case of init_xml with yaml, csv and sql files as
                    # we can't specify noupdate for those file.
                    correct_key = "demo" if k.count("demo") else "data"
                    _logger.warning(
                        "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
                        package.name,
                        k,
                        correct_key,
                        f,
                    )
        return files

    def _load_data(cr, module_name, idref, mode, kind):
        """

        kind: data, demo, test, init_xml, update_xml, demo_xml.

        noupdate is False, unless it is demo data or it is csv data in
        init mode.

        """
        try:
            if kind in ("demo", "test"):
                threading.currentThread().testing = True
            for filename in _get_files_of_kind(kind):
                _logger.info("loading %s/%s", module_name, filename)
                noupdate = False
                if kind in ("demo", "demo_xml") or (filename.endswith(".csv") and kind in ("init", "init_xml")):
                    noupdate = True
                tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)
        finally:
            if kind in ("demo", "test"):
                threading.currentThread().testing = False

    processed_modules = []
    loaded_modules = []
    registry = openerp.registry(cr.dbname)
    migrations = openerp.modules.migration.MigrationManager(cr, graph)
    _logger.info("loading %d modules...", len(graph))

    registry.clear_manual_fields()

    # register, instantiate and initialize models for each modules
    t0 = time.time()
    t0_sql = openerp.sql_db.sql_counter

    for index, package in enumerate(graph):
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        migrations.migrate_module(package, "pre")
        load_openerp_module(package.name)

        new_install = package.state == "to install"
        if new_install:
            py_module = sys.modules["openerp.addons.%s" % (module_name,)]
            pre_init = package.info.get("pre_init_hook")
            if pre_init:
                getattr(py_module, pre_init)(cr)

        models = registry.load(cr, package)

        loaded_modules.append(package.name)
        if hasattr(package, "init") or hasattr(package, "update") or package.state in ("to install", "to upgrade"):
            registry.setup_models(cr, partial=True)
            init_module_models(cr, package.name, models)

        idref = {}

        mode = "update"
        if hasattr(package, "init") or package.state == "to install":
            mode = "init"

        if hasattr(package, "init") or hasattr(package, "update") or package.state in ("to install", "to upgrade"):
            # Can't put this line out of the loop: ir.module.module will be
            # registered by init_module_models() above.
            modobj = registry["ir.module.module"]

            if perform_checks:
                modobj.check(cr, SUPERUSER_ID, [module_id])

            if package.state == "to upgrade":
                # upgrading the module information
                modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data))
            _load_data(cr, module_name, idref, mode, kind="data")
            has_demo = hasattr(package, "demo") or (package.dbdemo and package.state != "installed")
            if has_demo:
                _load_data(cr, module_name, idref, mode, kind="demo")
                cr.execute("update ir_module_module set demo=%s where id=%s", (True, module_id))
                modobj.invalidate_cache(cr, SUPERUSER_ID, ["demo"], [module_id])

            migrations.migrate_module(package, "post")

            # Update translations for all installed languages
            modobj.update_translations(
                cr,
                SUPERUSER_ID,
                [module_id],
                None,
                {"overwrite": openerp.tools.config["overwrite_existing_translations"]},
            )

            registry._init_modules.add(package.name)

            if new_install:
                post_init = package.info.get("post_init_hook")
                if post_init:
                    getattr(py_module, post_init)(cr, registry)

            # validate all the views at a whole
            registry["ir.ui.view"]._validate_module_views(cr, SUPERUSER_ID, module_name)

            if has_demo:
                # launch tests only in demo mode, allowing tests to use demo data.
                if tools.config.options["test_enable"]:
                    # Yamel test
                    report.record_result(load_test(module_name, idref, mode))
                    # Python tests
                    ir_http = registry["ir.http"]
                    # Force routing map to be rebuilt between each module test suite
                    vars(ir_http).pop("routing_map", None)

                    report.record_result(openerp.modules.module.run_unit_tests(module_name, cr.dbname))

            processed_modules.append(package.name)

            ver = adapt_version(package.data["version"])
            # Set new modules and dependencies
            modobj.write(cr, SUPERUSER_ID, [module_id], {"state": "installed", "latest_version": ver})

            package.state = "installed"
            for kind in ("init", "demo", "update"):
                if hasattr(package, kind):
                    delattr(package, kind)

        registry._init_modules.add(package.name)
        cr.commit()

    _logger.log(
        25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql
    )

    registry.clear_manual_fields()

    cr.commit()

    return loaded_modules, processed_modules
Exemple #2
0
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
    """Migrates+Updates or Installs all module nodes from ``graph``
       :param graph: graph of module nodes to load
       :param status: status dictionary for keeping track of progress
       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
                              for same cases)
       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
       :return: list of modules that were installed or updated
    """
    def load_test(module_name, idref, mode):
        cr.commit()
        try:
            _load_data(cr, module_name, idref, mode, 'test')
            return True
        except Exception:
            _test_logger.exception(
                'module %s: an exception occurred in a test', module_name)
            return False
        finally:
            if tools.config.options['test_commit']:
                cr.commit()
            else:
                cr.rollback()
                # avoid keeping stale xml_id, etc. in cache 
                openerp.modules.registry.RegistryManager.clear_caches(cr.dbname)


    def _get_files_of_kind(kind):
        if kind == 'demo':
            kind = ['demo_xml', 'demo']
        elif kind == 'data':
            kind = ['init_xml', 'update_xml', 'data']
        if isinstance(kind, str):
            kind = [kind]
        files = []
        for k in kind:
            for f in package.data[k]:
                files.append(f)
                if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')):
                    # init_xml, update_xml and demo_xml are deprecated except
                    # for the case of init_xml with yaml, csv and sql files as
                    # we can't specify noupdate for those file.
                    correct_key = 'demo' if k.count('demo') else 'data'
                    _logger.warning(
                        "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
                        package.name, k, correct_key, f
                    )
        return files

    def _load_data(cr, module_name, idref, mode, kind):
        """

        kind: data, demo, test, init_xml, update_xml, demo_xml.

        noupdate is False, unless it is demo data or it is csv data in
        init mode.

        """
        try:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = True
            for filename in _get_files_of_kind(kind):
                _logger.info("loading %s/%s", module_name, filename)
                noupdate = False
                if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')):
                    noupdate = True
                tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)
        finally:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = False

    if status is None:
        status = {}

    processed_modules = []
    loaded_modules = []
    registry = openerp.registry(cr.dbname)
    migrations = openerp.modules.migration.MigrationManager(cr, graph)
    _logger.info('loading %d modules...', len(graph))

    # Query manual fields for all models at once and save them on the registry
    # so the initialization code for each model does not have to do it
    # one model at a time.
    registry.fields_by_model = {}
    cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',))
    for field in cr.dictfetchall():
        registry.fields_by_model.setdefault(field['model'], []).append(field)

    # register, instantiate and initialize models for each modules
    t0 = time.time()
    t0_sql = openerp.sql_db.sql_counter

    for index, package in enumerate(graph):
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        new_install = package.installed_version is None
        if new_install:
            py_module = sys.modules['openerp.addons.%s' % (module_name,)]
            pre_init = package.info.get('pre_init_hook')
            if pre_init:
                getattr(py_module, pre_init)(cr)

        models = registry.load(cr, package)

        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            init_module_models(cr, package.name, models)
        status['progress'] = float(index) / len(graph)

        # Can't put this line out of the loop: ir.module.module will be
        # registered by init_module_models() above.
        modobj = registry['ir.module.module']

        if perform_checks:
            modobj.check(cr, SUPERUSER_ID, [module_id])

        idref = {}

        mode = 'update'
        if hasattr(package, 'init') or package.state == 'to install':
            mode = 'init'

        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            if package.state=='to upgrade':
                # upgrading the module information
                modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data))
            _load_data(cr, module_name, idref, mode, kind='data')
            has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed')
            if has_demo:
                status['progress'] = (index + 0.75) / len(graph)
                _load_data(cr, module_name, idref, mode, kind='demo')
                cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))

            migrations.migrate_module(package, 'post')

            if new_install:
                post_init = package.info.get('post_init_hook')
                if post_init:
                    getattr(py_module, post_init)(cr, registry)

            registry._init_modules.add(package.name)
            # validate all the views at a whole
            registry['ir.ui.view']._validate_module_views(cr, SUPERUSER_ID, module_name)

            if has_demo:
                # launch tests only in demo mode, allowing tests to use demo data.
                if tools.config.options['test_enable']:
                    # Yamel test
                    report.record_result(load_test(module_name, idref, mode))
                    # Python tests
                    ir_http = registry['ir.http']
                    if hasattr(ir_http, '_routing_map'):
                        # Force routing map to be rebuilt between each module test suite
                        del(ir_http._routing_map)
                    report.record_result(openerp.modules.module.run_unit_tests(module_name, cr.dbname))

            processed_modules.append(package.name)

            ver = adapt_version(package.data['version'])
            # Set new modules and dependencies
            modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver})
            # Update translations for all installed languages
            modobj.update_translations(cr, SUPERUSER_ID, [module_id], None, {'overwrite': openerp.tools.config["overwrite_existing_translations"]})

            package.state = 'installed'
            for kind in ('init', 'demo', 'update'):
                if hasattr(package, kind):
                    delattr(package, kind)

        registry._init_modules.add(package.name)
        cr.commit()

    _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql)

    # The query won't be valid for models created later (i.e. custom model
    # created after the registry has been loaded), so empty its result.
    registry.fields_by_model = None

    cr.commit()

    return loaded_modules, processed_modules
Exemple #3
0
def load_module_graph(cr,
                      graph,
                      status=None,
                      perform_checks=True,
                      skip_modules=None,
                      report=None):
    """Migrates+Updates or Installs all module nodes from ``graph``
       :param graph: graph of module nodes to load
       :param status: status dictionary for keeping track of progress
       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
                              for same cases)
       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
       :return: list of modules that were installed or updated
    """
    def process_sql_file(cr, fp):
        queries = fp.read().split(';')
        for query in queries:
            new_query = ' '.join(query.split())
            if new_query:
                cr.execute(new_query)

    load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml')
    load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml')
    load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml')
    load_data = lambda *args: _load_data(cr, *args, kind='data')
    load_demo = lambda *args: _load_data(cr, *args, kind='demo')

    def load_test(module_name, idref, mode):
        cr.commit()
        try:
            threading.currentThread().testing = True
            _load_data(cr, module_name, idref, mode, 'test')
            return True
        except Exception:
            _logger.exception('module %s: an exception occurred in a test',
                              module_name)
            return False
        finally:
            threading.currentThread().testing = False
            if tools.config.options['test_commit']:
                cr.commit()
            else:
                cr.rollback()

    def _load_data(cr, module_name, idref, mode, kind):
        """

        kind: data, demo, test, init_xml, update_xml, demo_xml.

        noupdate is False, unless it is demo data or it is csv data in
        init mode.

        """
        for filename in package.data[kind]:
            _logger.info("module %s: loading %s", module_name, filename)
            _, ext = os.path.splitext(filename)
            pathname = os.path.join(module_name, filename)
            fp = tools.file_open(pathname)
            noupdate = False
            if kind in ('demo', 'demo_xml'):
                noupdate = True
            try:
                ext = ext.lower()
                if ext == '.csv':
                    if kind in ('init', 'init_xml'):
                        noupdate = True
                    tools.convert_csv_import(cr, module_name, pathname,
                                             fp.read(), idref, mode, noupdate)
                elif ext == '.sql':
                    process_sql_file(cr, fp)
                elif ext == '.yml':
                    tools.convert_yaml_import(cr, module_name, fp, kind, idref,
                                              mode, noupdate, report)
                elif ext == '.xml':
                    tools.convert_xml_import(cr, module_name, fp, idref, mode,
                                             noupdate, report)
                elif ext == '.js':
                    pass  # .js files are valid but ignored here.
                else:
                    _logger.warning("Can't load unknown file type %s.",
                                    filename)
            finally:
                fp.close()

    if status is None:
        status = {}

    processed_modules = []
    loaded_modules = []
    pool = pooler.get_pool(cr.dbname)
    migrations = openerp.modules.migration.MigrationManager(cr, graph)
    _logger.info('loading %d modules...', len(graph))

    # Query manual fields for all models at once and save them on the registry
    # so the initialization code for each model does not have to do it
    # one model at a time.
    pool.fields_by_model = {}
    cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual', ))
    for field in cr.dictfetchall():
        pool.fields_by_model.setdefault(field['model'], []).append(field)

    # register, instantiate and initialize models for each modules
    for index, package in enumerate(graph):
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        _logger.debug('module %s: loading objects', package.name)
        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        models = pool.load(cr, package)

        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(
                package,
                'update') or package.state in ('to install', 'to upgrade'):
            init_module_models(cr, package.name, models)
        pool._init_modules.add(package.name)
        status['progress'] = float(index) / len(graph)

        # Can't put this line out of the loop: ir.module.module will be
        # registered by init_module_models() above.
        modobj = pool.get('ir.module.module')

        if perform_checks:
            modobj.check(cr, SUPERUSER_ID, [module_id])

        idref = {}

        mode = 'update'
        if hasattr(package, 'init') or package.state == 'to install':
            mode = 'init'

        if hasattr(package, 'init') or hasattr(
                package,
                'update') or package.state in ('to install', 'to upgrade'):
            if package.state == 'to upgrade':
                # upgrading the module information
                modobj.write(cr, SUPERUSER_ID, [module_id],
                             modobj.get_values_from_terp(package.data))
            load_init_xml(module_name, idref, mode)
            load_update_xml(module_name, idref, mode)
            load_data(module_name, idref, mode)
            if hasattr(package, 'demo') or (package.dbdemo
                                            and package.state != 'installed'):
                status['progress'] = (index + 0.75) / len(graph)
                load_demo_xml(module_name, idref, mode)
                load_demo(module_name, idref, mode)
                cr.execute('update ir_module_module set demo=%s where id=%s',
                           (True, module_id))

                # launch tests only in demo mode, as most tests will depend
                # on demo data. Other tests can be added into the regular
                # 'data' section, but should probably not alter the data,
                # as there is no rollback.
                if tools.config.options['test_enable']:
                    report.record_result(load_test(module_name, idref, mode))

                    # Run the `fast_suite` and `checks` tests given by the module.
                    if module_name == 'base':
                        # Also run the core tests after the database is created.
                        report.record_result(
                            openerp.modules.module.run_unit_tests('openerp'))
                    report.record_result(
                        openerp.modules.module.run_unit_tests(module_name))

            processed_modules.append(package.name)

            migrations.migrate_module(package, 'post')

            ver = adapt_version(package.data['version'])
            # Set new modules and dependencies
            modobj.write(cr, SUPERUSER_ID, [module_id], {
                'state': 'installed',
                'latest_version': ver
            })
            # Update translations for all installed languages
            modobj.update_translations(cr, SUPERUSER_ID, [module_id], None)

            package.state = 'installed'
            for kind in ('init', 'demo', 'update'):
                if hasattr(package, kind):
                    delattr(package, kind)

        cr.commit()

    # The query won't be valid for models created later (i.e. custom model
    # created after the registry has been loaded), so empty its result.
    pool.fields_by_model = None

    cr.commit()

    return loaded_modules, processed_modules
Exemple #4
0
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
    """Migrates+Updates or Installs all module nodes from ``graph``
       :param graph: graph of module nodes to load
       :param status: status dictionary for keeping track of progress
       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
                              for same cases)
       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
       :return: list of modules that were installed or updated
    """
    def process_sql_file(cr, fp):
        queries = fp.read().split(';')
        for query in queries:
            new_query = ' '.join(query.split())
            if new_query:
                cr.execute(new_query)

    load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml')
    load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml')
    load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml')
    load_data = lambda *args: _load_data(cr, *args, kind='data')
    load_demo = lambda *args: _load_data(cr, *args, kind='demo')

    def load_test(module_name, idref, mode):
        cr.commit()
        try:
            threading.currentThread().testing = True
            _load_data(cr, module_name, idref, mode, 'test')
            return True
        except Exception:
            _logger.exception(
                'module %s: an exception occurred in a test', module_name)
            return False
        finally:
            threading.currentThread().testing = False
            if tools.config.options['test_commit']:
                cr.commit()
            else:
                cr.rollback()

    def _load_data(cr, module_name, idref, mode, kind):
        """

        kind: data, demo, test, init_xml, update_xml, demo_xml.

        noupdate is False, unless it is demo data or it is csv data in
        init mode.

        """
        for filename in package.data[kind]:
            if kind == 'test':
                _logger.log(logging.TEST, "module %s: loading %s", module_name, filename)
            else:
                _logger.info("module %s: loading %s", module_name, filename)
            _, ext = os.path.splitext(filename)
            pathname = os.path.join(module_name, filename)
            fp = tools.file_open(pathname)
            noupdate = False
            if kind in ('demo', 'demo_xml'):
                noupdate = True
            try:
                ext = ext.lower()
                if ext == '.csv':
                    if kind in ('init', 'init_xml'):
                        noupdate = True
                    tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate)
                elif ext == '.sql':
                    process_sql_file(cr, fp)
                elif ext == '.yml':
                    tools.convert_yaml_import(cr, module_name, fp, kind, idref, mode, noupdate, report)
                elif ext == '.xml':
                    tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report)
                elif ext == '.js':
                    pass # .js files are valid but ignored here.
                else:
                    _logger.warning("Can't load unknown file type %s.", filename)
            finally:
                fp.close()

    if status is None:
        status = {}

    processed_modules = []
    loaded_modules = []
    pool = pooler.get_pool(cr.dbname)
    migrations = openerp.modules.migration.MigrationManager(cr, graph)
    _logger.debug('loading %d packages...', len(graph))

    # Query manual fields for all models at once and save them on the registry
    # so the initialization code for each model does not have to do it
    # one model at a time.
    pool.fields_by_model = {}
    cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',))
    for field in cr.dictfetchall():
        pool.fields_by_model.setdefault(field['model'], []).append(field)

    # register, instantiate and initialize models for each modules
    for index, package in enumerate(graph):
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        _logger.info('module %s: loading objects', package.name)
        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        models = pool.load(cr, package)

        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            init_module_models(cr, package.name, models)
        pool._init_modules.add(package.name)
        status['progress'] = float(index) / len(graph)

        # Can't put this line out of the loop: ir.module.module will be
        # registered by init_module_models() above.
        modobj = pool.get('ir.module.module')

        if perform_checks:
            modobj.check(cr, SUPERUSER_ID, [module_id])

        idref = {}

        mode = 'update'
        if hasattr(package, 'init') or package.state == 'to install':
            mode = 'init'

        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            if package.state=='to upgrade':
                # upgrading the module information
                modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data))
            load_init_xml(module_name, idref, mode)
            load_update_xml(module_name, idref, mode)
            load_data(module_name, idref, mode)
            if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'):
                status['progress'] = (index + 0.75) / len(graph)
                load_demo_xml(module_name, idref, mode)
                load_demo(module_name, idref, mode)
                cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))

                # launch tests only in demo mode, as most tests will depend
                # on demo data. Other tests can be added into the regular
                # 'data' section, but should probably not alter the data,
                # as there is no rollback.
                if tools.config.options['test_enable']:
                    report.record_result(load_test(module_name, idref, mode))

                    # Run the `fast_suite` and `checks` tests given by the module.
                    if module_name == 'base':
                        # Also run the core tests after the database is created.
                        report.record_result(openerp.modules.module.run_unit_tests('openerp'))
                    report.record_result(openerp.modules.module.run_unit_tests(module_name))

            processed_modules.append(package.name)

            migrations.migrate_module(package, 'post')

            ver = adapt_version(package.data['version'])
            # Set new modules and dependencies
            modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver})
            # Update translations for all installed languages
            modobj.update_translations(cr, SUPERUSER_ID, [module_id], None)

            package.state = 'installed'
            for kind in ('init', 'demo', 'update'):
                if hasattr(package, kind):
                    delattr(package, kind)

        cr.commit()

    # The query won't be valid for models created later (i.e. custom model
    # created after the registry has been loaded), so empty its result.
    pool.fields_by_model = None
    
    cr.commit()

    return loaded_modules, processed_modules
Exemple #5
0
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None, upg_registry=None):
    """Migrates+Updates or Installs all module nodes from ``graph``
       :param graph: graph of module nodes to load
       :param status: deprecated parameter, unused, left to avoid changing signature in 8.0
       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
                              for same cases)
       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
       :return: list of modules that were installed or updated
    """
    def load_test(module_name, idref, mode):
        cr.commit()
        try:
            _load_data(cr, module_name, idref, mode, 'test')
            return True
        except Exception:
            _test_logger.exception(
                'module %s: an exception occurred in a test', module_name)
            return False
        finally:
            if tools.config.options['test_commit']:
                cr.commit()
            else:
                cr.rollback()
                # avoid keeping stale xml_id, etc. in cache 
                openerp.modules.registry.RegistryManager.clear_caches(cr.dbname)


    def _get_files_of_kind(kind):
        if kind == 'demo':
            kind = ['demo_xml', 'demo']
        elif kind == 'data':
            kind = ['init_xml', 'update_xml', 'data']
        if isinstance(kind, str):
            kind = [kind]
        files = []
        for k in kind:
            for f in package.data[k]:
                files.append(f)
                if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')):
                    # init_xml, update_xml and demo_xml are deprecated except
                    # for the case of init_xml with yaml, csv and sql files as
                    # we can't specify noupdate for those file.
                    correct_key = 'demo' if k.count('demo') else 'data'
                    _logger.warning(
                        "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
                        package.name, k, correct_key, f
                    )
        return files

    def _load_data(cr, module_name, idref, mode, kind):
        """

        kind: data, demo, test, init_xml, update_xml, demo_xml.

        noupdate is False, unless it is demo data or it is csv data in
        init mode.

        """
        try:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = True
            for filename in _get_files_of_kind(kind):
                _logger.info("loading %s/%s", module_name, filename)
                noupdate = False
                if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')):
                    noupdate = True
                tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)
        finally:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = False

    if status is None:
        status = {}

    if skip_modules is None:
        skip_modules = []

    processed_modules = []
    loaded_modules = []
    registry = openerp.registry(cr.dbname)
    migrations = openerp.modules.migration.MigrationManager(cr, graph)
    _logger.info('loading %d modules...', len(graph))

    registry.clear_manual_fields()

    # suppress commits to have the upgrade of one module in just one transaction
    cr.commit_org = cr.commit
    cr.commit = lambda *args: None
    cr.rollback_org = cr.rollback
    cr.rollback = lambda *args: None
    openerp.osv.fields.set_migration_cursor(cr)

    # register, instantiate and initialize models for each modules
    t0 = time.time()
    t0_sql = openerp.sql_db.sql_counter

    for index, package in enumerate(graph):
        module_name = package.name
        module_id = package.id

        if module_name in skip_modules or module_name in loaded_modules:
            continue

        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        new_install = package.state == 'to install'
        if new_install:
            py_module = sys.modules['openerp.addons.%s' % (module_name,)]
            pre_init = package.info.get('pre_init_hook')
            if pre_init:
                getattr(py_module, pre_init)(cr)

        models = registry.load(cr, package)

        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            registry.setup_models(cr, partial=True)
            # OpenUpgrade: add this module's models to the registry
            local_registry = {}
            for model in models:
                if not model._auto:
                    continue
                openupgrade_loading.log_model(model, local_registry)
            openupgrade_loading.compare_registries(
                cr, package.name, upg_registry, local_registry)
            init_module_models(cr, package.name, models)

        idref = {}

        mode = 'update'
        if hasattr(package, 'init') or package.state == 'to install':
            mode = 'init'

        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            # Can't put this line out of the loop: ir.module.module will be
            # registered by init_module_models() above.
            modobj = registry['ir.module.module']

            if perform_checks:
                modobj.check(cr, SUPERUSER_ID, [module_id])

            if package.state=='to upgrade':
                # upgrading the module information
                modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data))
            _load_data(cr, module_name, idref, mode, kind='data')
            has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed')
            if has_demo:
                _load_data(cr, module_name, idref, mode, kind='demo')
                cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
                modobj.invalidate_cache(cr, SUPERUSER_ID, ['demo'], [module_id])

            # OpenUpgrade: add 'try' block for logging exceptions
            # as errors in post scripts seem to be dropped
            try:
                migrations.migrate_module(package, 'post')
            except Exception as exc:
                _logger.error('Error executing post migration script for module %s: %s',
                              package, exc)
                raise

            # Update translations for all installed languages
            modobj.update_translations(cr, SUPERUSER_ID, [module_id], None, {'overwrite': openerp.tools.config["overwrite_existing_translations"]})

            registry._init_modules.add(package.name)

            if new_install:
                post_init = package.info.get('post_init_hook')
                if post_init:
                    getattr(py_module, post_init)(cr, registry)

            # validate all the views at a whole
            registry['ir.ui.view']._validate_module_views(cr, SUPERUSER_ID, module_name)

            if has_demo:
                # launch tests only in demo mode, allowing tests to use demo data.
                if tools.config.options['test_enable']:
                    # Yamel test
                    report.record_result(load_test(module_name, idref, mode))
                    # Python tests
                    ir_http = registry['ir.http']
                    if hasattr(ir_http, '_routing_map'):
                        # Force routing map to be rebuilt between each module test suite
                        del(ir_http._routing_map)
                    report.record_result(openerp.modules.module.run_unit_tests(module_name, cr.dbname))

            processed_modules.append(package.name)

            ver = adapt_version(package.data['version'])
            # Set new modules and dependencies
            modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver})

            package.state = 'installed'
            for kind in ('init', 'demo', 'update'):
                if hasattr(package, kind):
                    delattr(package, kind)

        registry._init_modules.add(package.name)
        cr.commit_org()

        # OpenUpgrade edit start:
        # if there's a tests directory, run those if tests are enabled
        tests_dir = os.path.join(
            openerp.modules.module.get_module_path(package.name),
            'migrations',
            adapt_version(package.data['version']),
            'tests',
        )
        # check for an environment variable because we don't want to mess
        # with odoo's config.py, but we also don't want to run existing
        # tests
        if os.environ.get('OPENUPGRADE_TESTS') and os.path.exists(
            tests_dir
        ):
            import unittest
            threading.currentThread().testing = True
            tests = unittest.defaultTestLoader.discover(tests_dir, top_level_dir=tests_dir)
            report.record_result(
                unittest.TextTestRunner(
                    verbosity=2,
                    stream=openerp.modules.module.TestStream(package.name),
                ).run(tests)
                .wasSuccessful()
            )
            threading.currentThread().testing = False
        # OpenUpgrade edit end

    _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql)

    registry.clear_manual_fields()

    cr.commit = cr.commit_org
    cr.commit()
    openerp.osv.fields.set_migration_cursor()

    return loaded_modules, processed_modules
    dt_before_load = cr.fetchone()[0]

    # register, instantiate and initialize models for each modules
    pkgs_data = []
    inits_data = {}
    for index, package in enumerate(graph):
        pkg_data = False
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        _logger.info('module %s: loading objects', package.name)
        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        models = pool.load(cr, package)
        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            init_module_models(cr, package.name, models)
            if git:
                update_git_information(cr, package.name)

        status['progress'] = float(index) / len(graph)

        # Can't put this line out of the loop: ir.module.module will be
        # registered by init_module_models() above.
        modobj = pool.get('ir.module.module')

        if perform_checks:
Exemple #7
0
def load_module_graph(cr,
                      graph,
                      status=None,
                      perform_checks=True,
                      skip_modules=None,
                      report=None):
    """Migrates+Updates or Installs all module nodes from ``graph``
       :param graph: graph of module nodes to load
       :param status: deprecated parameter, unused, left to avoid changing signature in 8.0
       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
                              for same cases)
       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
       :return: list of modules that were installed or updated
    """
    def load_test(module_name, idref, mode):
        cr.commit()
        try:
            _load_data(cr, module_name, idref, mode, 'test')
            return True
        except Exception:
            _test_logger.exception(
                'module %s: an exception occurred in a test', module_name)
            return False
        finally:
            if tools.config.options['test_commit']:
                cr.commit()
            else:
                cr.rollback()
                # avoid keeping stale xml_id, etc. in cache
                openerp.modules.registry.RegistryManager.clear_caches(
                    cr.dbname)

    def _get_files_of_kind(kind):
        if kind == 'demo':
            kind = ['demo_xml', 'demo']
        elif kind == 'data':
            kind = ['init_xml', 'update_xml', 'data']
        if isinstance(kind, str):
            kind = [kind]
        files = []
        for k in kind:
            for f in package.data[k]:
                files.append(f)
                if k.endswith('_xml') and not (k == 'init_xml'
                                               and not f.endswith('.xml')):
                    # init_xml, update_xml and demo_xml are deprecated except
                    # for the case of init_xml with yaml, csv and sql files as
                    # we can't specify noupdate for those file.
                    correct_key = 'demo' if k.count('demo') else 'data'
                    _logger.warning(
                        "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
                        package.name, k, correct_key, f)
        return files

    def _load_data(cr, module_name, idref, mode, kind):
        """

        kind: data, demo, test, init_xml, update_xml, demo_xml.

        noupdate is False, unless it is demo data or it is csv data in
        init mode.

        """
        try:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = True
            for filename in _get_files_of_kind(kind):
                _logger.info("loading %s/%s", module_name, filename)
                noupdate = False
                if kind in ('demo',
                            'demo_xml') or (filename.endswith('.csv')
                                            and kind in ('init', 'init_xml')):
                    noupdate = True
                tools.convert_file(cr, module_name, filename, idref, mode,
                                   noupdate, kind, report)
        finally:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = False

    processed_modules = []
    loaded_modules = []
    registry = openerp.registry(cr.dbname)
    migrations = openerp.modules.migration.MigrationManager(cr, graph)
    module_count = len(graph)
    _logger.info('loading %d modules...', module_count)

    registry.clear_manual_fields()

    # register, instantiate and initialize models for each modules
    t0 = time.time()
    t0_sql = openerp.sql_db.sql_counter

    for index, package in enumerate(graph, 1):
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        _logger.debug('loading module %s (%d/%d)', module_name, index,
                      module_count)
        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        new_install = package.state == 'to install'
        if new_install:
            py_module = sys.modules['openerp.addons.%s' % (module_name, )]
            pre_init = package.info.get('pre_init_hook')
            if pre_init:
                getattr(py_module, pre_init)(cr)

        model_names = registry.load(cr, package)

        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(
                package,
                'update') or package.state in ('to install', 'to upgrade'):
            registry.setup_models(cr, partial=True)
            registry.init_models(cr, model_names, {'module': package.name})

        idref = {}

        mode = 'update'
        if hasattr(package, 'init') or package.state == 'to install':
            mode = 'init'

        if hasattr(package, 'init') or hasattr(
                package,
                'update') or package.state in ('to install', 'to upgrade'):
            env = api.Environment(cr, SUPERUSER_ID, {})
            # Can't put this line out of the loop: ir.module.module will be
            # registered by init_models() above.
            module = env['ir.module.module'].browse(module_id)

            if perform_checks:
                module.check()

            if package.state == 'to upgrade':
                # upgrading the module information
                module.write(module.get_values_from_terp(package.data))
            _load_data(cr, module_name, idref, mode, kind='data')
            has_demo = hasattr(package,
                               'demo') or (package.dbdemo
                                           and package.state != 'installed')
            if has_demo:
                _load_data(cr, module_name, idref, mode, kind='demo')
                cr.execute('update ir_module_module set demo=%s where id=%s',
                           (True, module_id))
                module.invalidate_cache(['demo'])

            migrations.migrate_module(package, 'post')

            # Update translations for all installed languages
            overwrite = openerp.tools.config["overwrite_existing_translations"]
            module.with_context(overwrite=overwrite).update_translations()

            registry._init_modules.add(package.name)

            if new_install:
                post_init = package.info.get('post_init_hook')
                if post_init:
                    getattr(py_module, post_init)(cr, registry)

            # validate all the views at a whole
            env['ir.ui.view']._validate_module_views(module_name)

            if has_demo:
                # launch tests only in demo mode, allowing tests to use demo data.
                if tools.config.options['test_enable']:
                    # Yamel test
                    report.record_result(load_test(module_name, idref, mode))
                    # Python tests
                    env['ir.http']._clear_routing_map(
                    )  # force routing map to be rebuilt
                    report.record_result(
                        openerp.modules.module.run_unit_tests(
                            module_name, cr.dbname))

            processed_modules.append(package.name)

            ver = adapt_version(package.data['version'])
            # Set new modules and dependencies
            module.write({'state': 'installed', 'latest_version': ver})

            package.state = 'installed'
            for kind in ('init', 'demo', 'update'):
                if hasattr(package, kind):
                    delattr(package, kind)

        registry._init_modules.add(package.name)
        cr.commit()

    _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph),
                time.time() - t0, openerp.sql_db.sql_counter - t0_sql)

    registry.clear_manual_fields()

    cr.commit()

    return loaded_modules, processed_modules
Exemple #8
0
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
    """Migrates+Updates or Installs all module nodes from ``graph``
       :param graph: graph of module nodes to load
       :param status: status dictionary for keeping track of progress
       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
                              for same cases)
       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
       :return: list of modules that were installed or updated
    """
    def load_test(module_name, idref, mode):
        cr.commit()
        try:
            threading.currentThread().testing = True
            _load_data(cr, module_name, idref, mode, 'test')
            return True
        except Exception:
            _test_logger.exception(
                'module %s: an exception occurred in a test', module_name)
            return False
        finally:
            threading.currentThread().testing = False
            if tools.config.options['test_commit']:
                cr.commit()
            else:
                cr.rollback()

    def _get_files_of_kind(kind):
        if kind == 'demo':
            kind = ['demo_xml', 'demo']
        elif kind == 'data':
            kind = ['init_xml', 'update_xml', 'data']
        if isinstance(kind, str):
            kind = [kind]
        files = []
        for k in kind:
            for f in package.data[k]:
                files.append(f)
                if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')):
                    # init_xml, update_xml and demo_xml are deprecated except
                    # for the case of init_xml with yaml, csv and sql files as
                    # we can't specify noupdate for those file.
                    correct_key = 'demo' if k.count('demo') else 'data'
                    _logger.warning(
                        "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
                        package.name, k, correct_key, f
                    )
        return files

    def _load_data(cr, module_name, idref, mode, kind):
        """

        kind: data, demo, test, init_xml, update_xml, demo_xml.

        noupdate is False, unless it is demo data or it is csv data in
        init mode.

        """
        for filename in _get_files_of_kind(kind):
            _logger.info("module %s: loading %s", module_name, filename)
            noupdate = False
            if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')):
                noupdate = True
            tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)

    if status is None:
        status = {}

    processed_modules = []
    loaded_modules = []
    registry = openerp.registry(cr.dbname)
    migrations = openerp.modules.migration.MigrationManager(cr, graph)
    _logger.info('loading %d modules...', len(graph))

    # Query manual fields for all models at once and save them on the registry
    # so the initialization code for each model does not have to do it
    # one model at a time.
    registry.fields_by_model = {}
    cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',))
    for field in cr.dictfetchall():
        registry.fields_by_model.setdefault(field['model'], []).append(field)

    # register, instantiate and initialize models for each modules
    for index, package in enumerate(graph):
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        _logger.debug('module %s: loading objects', package.name)
        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        models = registry.load(cr, package)

        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            init_module_models(cr, package.name, models)
        registry._init_modules.add(package.name)
        status['progress'] = float(index) / len(graph)

        # Can't put this line out of the loop: ir.module.module will be
        # registered by init_module_models() above.
        modobj = registry['ir.module.module']

        if perform_checks:
            modobj.check(cr, SUPERUSER_ID, [module_id])

        idref = {}

        mode = 'update'
        if hasattr(package, 'init') or package.state == 'to install':
            mode = 'init'

        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            if package.state=='to upgrade':
                # upgrading the module information
                modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data))
            _load_data(cr, module_name, idref, mode, kind='data')
            if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'):
                status['progress'] = (index + 0.75) / len(graph)
                _load_data(cr, module_name, idref, mode, kind='demo')
                cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))

                # launch tests only in demo mode, as most tests will depend
                # on demo data. Other tests can be added into the regular
                # 'data' section, but should probably not alter the data,
                # as there is no rollback.
                if tools.config.options['test_enable']:
                    report.record_result(load_test(module_name, idref, mode))

                    # Run the `fast_suite` and `checks` tests given by the module.
                    if module_name == 'base':
                        # Also run the core tests after the database is created.
                        report.record_result(openerp.modules.module.run_unit_tests('openerp'))
                    report.record_result(openerp.modules.module.run_unit_tests(module_name))

            processed_modules.append(package.name)

            migrations.migrate_module(package, 'post')

            ver = adapt_version(package.data['version'])
            # Set new modules and dependencies
            modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver})
            # Update translations for all installed languages
            modobj.update_translations(cr, SUPERUSER_ID, [module_id], None)

            package.state = 'installed'
            for kind in ('init', 'demo', 'update'):
                if hasattr(package, kind):
                    delattr(package, kind)

        cr.commit()

    # The query won't be valid for models created later (i.e. custom model
    # created after the registry has been loaded), so empty its result.
    registry.fields_by_model = None
    
    cr.commit()

    return loaded_modules, processed_modules
Exemple #9
0
def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
    """Migrates+Updates or Installs all module nodes from ``graph``
       :param graph: graph of module nodes to load
       :param status: deprecated parameter, unused, left to avoid changing signature in 8.0
       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
                              for same cases)
       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
       :return: list of modules that were installed or updated
    """
    def load_test(module_name, idref, mode):
        cr.commit()
        try:
            _load_data(cr, module_name, idref, mode, 'test')
            return True
        except Exception:
            _test_logger.exception(
                'module %s: an exception occurred in a test', module_name)
            return False
        finally:
            if tools.config.options['test_commit']:
                cr.commit()
            else:
                cr.rollback()
                # avoid keeping stale xml_id, etc. in cache
                openerp.modules.registry.RegistryManager.clear_caches(cr.dbname)


    def _get_files_of_kind(kind):
        if kind == 'demo':
            kind = ['demo_xml', 'demo']
        elif kind == 'data':
            kind = ['init_xml', 'update_xml', 'data']
        if isinstance(kind, str):
            kind = [kind]
        files = []
        for k in kind:
            for f in package.data[k]:
                files.append(f)
                if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')):
                    # init_xml, update_xml and demo_xml are deprecated except
                    # for the case of init_xml with yaml, csv and sql files as
                    # we can't specify noupdate for those file.
                    correct_key = 'demo' if k.count('demo') else 'data'
                    _logger.warning(
                        "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
                        package.name, k, correct_key, f
                    )
        return files

    def _load_data(cr, module_name, idref, mode, kind):
        """

        kind: data, demo, test, init_xml, update_xml, demo_xml.

        noupdate is False, unless it is demo data or it is csv data in
        init mode.

        """
        try:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = True
            for filename in _get_files_of_kind(kind):
                _logger.info("loading %s/%s", module_name, filename)
                noupdate = False
                if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')):
                    noupdate = True
                tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)
        finally:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = False

    processed_modules = []
    loaded_modules = []
    registry = openerp.registry(cr.dbname)
    migrations = openerp.modules.migration.MigrationManager(cr, graph)
    module_count = len(graph)
    _logger.info('loading %d modules...', module_count)

    registry.clear_manual_fields()

    # register, instantiate and initialize models for each modules
    t0 = time.time()
    t0_sql = openerp.sql_db.sql_counter

    for index, package in enumerate(graph, 1):
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        _logger.debug('loading module %s (%d/%d)', module_name, index, module_count)
        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        new_install = package.state == 'to install'
        if new_install:
            py_module = sys.modules['openerp.addons.%s' % (module_name,)]
            pre_init = package.info.get('pre_init_hook')
            if pre_init:
                getattr(py_module, pre_init)(cr)

        model_names = registry.load(cr, package)

        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            registry.setup_models(cr, partial=True)
            registry.init_models(cr, model_names, {'module': package.name})

        idref = {}

        mode = 'update'
        if hasattr(package, 'init') or package.state == 'to install':
            mode = 'init'

        if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
            env = api.Environment(cr, SUPERUSER_ID, {})
            # Can't put this line out of the loop: ir.module.module will be
            # registered by init_models() above.
            module = env['ir.module.module'].browse(module_id)

            if perform_checks:
                module.check()

            if package.state=='to upgrade':
                # upgrading the module information
                module.write(module.get_values_from_terp(package.data))
            _load_data(cr, module_name, idref, mode, kind='data')
            has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed')
            if has_demo:
                _load_data(cr, module_name, idref, mode, kind='demo')
                cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
                module.invalidate_cache(['demo'])

            migrations.migrate_module(package, 'post')

            # Update translations for all installed languages
            overwrite = openerp.tools.config["overwrite_existing_translations"]
            module.with_context(overwrite=overwrite).update_translations()

            registry._init_modules.add(package.name)

            if new_install:
                post_init = package.info.get('post_init_hook')
                if post_init:
                    getattr(py_module, post_init)(cr, registry)

            # validate all the views at a whole
            env['ir.ui.view']._validate_module_views(module_name)

            if has_demo:
                # launch tests only in demo mode, allowing tests to use demo data.
                if tools.config.options['test_enable']:
                    # Yamel test
                    report.record_result(load_test(module_name, idref, mode))
                    # Python tests
                    env['ir.http']._clear_routing_map()     # force routing map to be rebuilt
                    report.record_result(openerp.modules.module.run_unit_tests(module_name, cr.dbname))

            processed_modules.append(package.name)

            ver = adapt_version(package.data['version'])
            # Set new modules and dependencies
            module.write({'state': 'installed', 'latest_version': ver})

            package.state = 'installed'
            for kind in ('init', 'demo', 'update'):
                if hasattr(package, kind):
                    delattr(package, kind)

        registry._init_modules.add(package.name)
        cr.commit()

    _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql)

    registry.clear_manual_fields()

    cr.commit()

    return loaded_modules, processed_modules
Exemple #10
0
    # get db timestamp
    cr.execute("select (now() at time zone 'UTC')::timestamp")
    dt_before_load = cr.fetchone()[0]

    # register, instantiate and initialize models for each modules
    for index, package in enumerate(graph):
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        _logger.info('module %s: loading objects', package.name)
        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        models = pool.load(cr, package)
        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(
                package,
                'update') or package.state in ('to install', 'to upgrade'):
            init_module_models(cr, package.name, models)

        status['progress'] = float(index) / len(graph)

        # Can't put this line out of the loop: ir.module.module will be
        # registered by init_module_models() above.
        modobj = pool.get('ir.module.module')

        if perform_checks:
Exemple #11
0
def load_module_graph(cr,
                      graph,
                      status=None,
                      perform_checks=True,
                      skip_modules=None,
                      report=None):
    """Migrates+Updates or Installs all module nodes from ``graph``
       :param graph: graph of module nodes to load
       :param status: deprecated parameter, unused, left to avoid changing signature in 8.0
       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
                              for same cases)
       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
       :return: list of modules that were installed or updated
    """
    def load_test(module_name, idref, mode):
        cr.commit()
        try:
            _load_data(cr, module_name, idref, mode, 'test')
            return True
        except Exception:
            _test_logger.exception(
                'module %s: an exception occurred in a test', module_name)
            return False
        finally:
            if tools.config.options['test_commit']:
                cr.commit()
            else:
                cr.rollback()
                # avoid keeping stale xml_id, etc. in cache
                openerp.modules.registry.RegistryManager.clear_caches(
                    cr.dbname)

    def _get_files_of_kind(kind):
        if kind == 'demo':
            kind = ['demo_xml', 'demo']
        elif kind == 'data':
            kind = ['init_xml', 'update_xml', 'data']
        if isinstance(kind, str):
            kind = [kind]
        files = []
        for k in kind:
            for f in package.data[k]:
                files.append(f)
                if k.endswith('_xml') and not (k == 'init_xml'
                                               and not f.endswith('.xml')):
                    # init_xml, update_xml and demo_xml are deprecated except
                    # for the case of init_xml with yaml, csv and sql files as
                    # we can't specify noupdate for those file.
                    correct_key = 'demo' if k.count('demo') else 'data'
                    _logger.warning(
                        "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
                        package.name, k, correct_key, f)
        return files

    def _load_data(cr, module_name, idref, mode, kind):
        """

        kind: data, demo, test, init_xml, update_xml, demo_xml.

        noupdate is False, unless it is demo data or it is csv data in
        init mode.

        """

        module_files = []
        module_modified = True
        if mode == 'update' and kind == 'data':
            cr.execute("select id from ir_module_module where name=%s",
                       (module_name, ))
            module_id = cr.fetchone()[0]
            for filename in _get_files_of_kind(kind):
                cr.execute(
                    "select id,checksum from ir_module_module_file"
                    " where module_id=%s and filename=%s",
                    (module_id, filename))
                checksum = cr.fetchone()
                module_file_id = checksum and checksum[0] or False
                checksum = checksum and checksum[1] or ''
                sha256 = hashlib.sha256()
                sha256.update(
                    misc.file_open(os.path.join(module_name, filename)).read())
                module_files += [{
                    'checksum': checksum,
                    'checksum_new': sha256.hexdigest(),
                    'module_file_id': module_file_id,
                    'module_id': module_id,
                    'filename': filename
                }]
            if len(module_files) == len([
                    1 for f in module_files
                    if f['checksum'] == f['checksum_new']
            ]):
                module_modified = bool(
                    tools.config.options.get('force_update', False))

        try:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = True
            for filename in _get_files_of_kind(kind):
                _logger.info("%s %s/%s", module_modified and 'loading'
                             or 'no changes', module_name, filename)
                noupdate = False
                if kind in ('demo',
                            'demo_xml') or (filename.endswith('.csv')
                                            and kind in ('init', 'init_xml')):
                    noupdate = True
                if module_modified:
                    tools.convert_file(cr, module_name, filename, idref, mode,
                                       noupdate, kind, report)
        finally:
            if kind in ('demo', 'test'):
                threading.currentThread().testing = False

        if module_modified and mode == 'update' and kind == 'data':
            for module_file in [
                    f for f in module_files
                    if f['checksum'] != f['checksum_new']
            ]:
                if module_file['module_file_id']:
                    cr.execute(
                        "update ir_module_module_file set checksum=%s, write_date=now() where id=%s",
                        (module_file['checksum_new'],
                         module_file['module_file_id']))
                else:
                    cr.execute(
                        "insert into ir_module_module_file(filename,module_id,checksum,create_date,create_uid,write_date,write_uid)"
                        " values(%s,%s,%s,now(),1,now(),1)",
                        (module_file['filename'], module_file['module_id'],
                         module_file['checksum_new']))
        return module_modified

    processed_modules = []
    loaded_modules = []
    registry = openerp.registry(cr.dbname)
    migrations = openerp.modules.migration.MigrationManager(cr, graph)
    _logger.info('loading %d modules...', len(graph))

    registry.clear_manual_fields()

    # register, instantiate and initialize models for each modules
    t0 = time.time()
    t0_sql = openerp.sql_db.sql_counter

    for index, package in enumerate(graph):
        module_name = package.name
        module_id = package.id

        if skip_modules and module_name in skip_modules:
            continue

        migrations.migrate_module(package, 'pre')
        load_openerp_module(package.name)

        new_install = package.state == 'to install'
        if new_install:
            py_module = sys.modules['openerp.addons.%s' % (module_name, )]
            pre_init = package.info.get('pre_init_hook')
            if pre_init:
                getattr(py_module, pre_init)(cr)

        models = registry.load(cr, package)

        loaded_modules.append(package.name)
        if hasattr(package, 'init') or hasattr(
                package,
                'update') or package.state in ('to install', 'to upgrade'):
            registry.setup_models(cr, partial=True)
            init_module_models(cr, package.name, models)

        idref = {}

        mode = 'update'
        if hasattr(package, 'init') or package.state == 'to install':
            mode = 'init'

        if hasattr(package, 'init') or hasattr(
                package,
                'update') or package.state in ('to install', 'to upgrade'):
            # Can't put this line out of the loop: ir.module.module will be
            # registered by init_module_models() above.
            modobj = registry['ir.module.module']

            if perform_checks:
                modobj.check(cr, SUPERUSER_ID, [module_id])

            if package.state == 'to upgrade':
                # upgrading the module information
                modobj.write(cr, SUPERUSER_ID, [module_id],
                             modobj.get_values_from_terp(package.data))
            module_processed = _load_data(cr,
                                          module_name,
                                          idref,
                                          mode,
                                          kind='data')
            has_demo = hasattr(package,
                               'demo') or (package.dbdemo
                                           and package.state != 'installed')
            if has_demo:
                _load_data(cr, module_name, idref, mode, kind='demo')
                cr.execute('update ir_module_module set demo=%s where id=%s',
                           (True, module_id))
                modobj.invalidate_cache(cr, SUPERUSER_ID, ['demo'],
                                        [module_id])

            migrations.migrate_module(package, 'post')

            if new_install:
                post_init = package.info.get('post_init_hook')
                if post_init:
                    getattr(py_module, post_init)(cr, registry)

            registry._init_modules.add(package.name)
            # validate all the views at a whole
            registry['ir.ui.view']._validate_module_views(
                cr, SUPERUSER_ID, module_name)

            if has_demo:
                # launch tests only in demo mode, allowing tests to use demo data.
                if tools.config.options['test_enable']:
                    # Yamel test
                    report.record_result(load_test(module_name, idref, mode))
                    # Python tests
                    ir_http = registry['ir.http']
                    if hasattr(ir_http, '_routing_map'):
                        # Force routing map to be rebuilt between each module test suite
                        del (ir_http._routing_map)
                    report.record_result(
                        openerp.modules.module.run_unit_tests(
                            module_name, cr.dbname))

            if module_processed:
                processed_modules.append(package.name)

            ver = adapt_version(package.data['version'])
            # Set new modules and dependencies
            modobj.write(cr, SUPERUSER_ID, [module_id], {
                'state': 'installed',
                'latest_version': ver
            })
            # Update translations for all installed languages
            modobj.update_translations(
                cr, SUPERUSER_ID, [module_id], None, {
                    'overwrite':
                    openerp.tools.config["overwrite_existing_translations"]
                })

            package.state = 'installed'
            for kind in ('init', 'demo', 'update'):
                if hasattr(package, kind):
                    delattr(package, kind)

        registry._init_modules.add(package.name)
        cr.commit()

    _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph),
                time.time() - t0, openerp.sql_db.sql_counter - t0_sql)

    registry.clear_manual_fields()

    cr.commit()

    return loaded_modules, processed_modules