Exemple #1
0
def load_os_config(platform_name,
                   os_name,
                   require_enabled=False,
                   feature_overrides={}):
    """Load configuration for os.

    @param platform_name: platform name to load os config for
    @param os_name: name of os to retrieve config for
    @param require_enabled: if true, raise error if 'enabled' not True
    @param feature_overrides: feature flag overrides to merge with features
    @return_value: config dict
    """
    main_conf = c_util.read_conf(RELEASES_CONF)
    default = main_conf['default_release_config']
    image = main_conf['releases'][os_name]
    conf = merge_config(
        merge_config(get(default, 'default'), get(default, platform_name)),
        merge_config(get(image, 'default'), get(image, platform_name)))

    feature_conf = main_conf['features']
    feature_groups = conf.get('feature_groups', [])
    overrides = merge_config(get(conf, 'features'), feature_overrides)
    conf['arch'] = c_util.get_architecture()
    conf['features'] = merge_feature_groups(feature_conf, feature_groups,
                                            overrides)

    if require_enabled and not enabled(conf):
        raise ValueError('OS is not enabled')
    return conf
Exemple #2
0
def load_os_config(platform_name, os_name, require_enabled=False,
                   feature_overrides=None):
    """Load configuration for os.

    @param platform_name: platform name to load os config for
    @param os_name: name of os to retrieve config for
    @param require_enabled: if true, raise error if 'enabled' not True
    @param feature_overrides: feature flag overrides to merge with features
    @return_value: config dict
    """
    if feature_overrides is None:
        feature_overrides = {}
    main_conf = c_util.read_conf(RELEASES_CONF)
    default = main_conf['default_release_config']
    image = main_conf['releases'][os_name]
    conf = merge_config(merge_config(get(default, 'default'),
                                     get(default, platform_name)),
                        merge_config(get(image, 'default'),
                                     get(image, platform_name)))

    feature_conf = main_conf['features']
    feature_groups = conf.get('feature_groups', [])
    overrides = merge_config(get(conf, 'features'), feature_overrides)
    conf['arch'] = c_util.get_architecture()
    conf['features'] = merge_feature_groups(
        feature_conf, feature_groups, overrides)

    if require_enabled and not enabled(conf):
        raise ValueError('OS is not enabled')
    return conf
Exemple #3
0
def get_default_mirrors(arch=None, target=None):
    """returns the default mirrors for the target. These depend on the
       architecture, for more see:
       https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
    if arch is None:
        arch = util.get_architecture(target)
    if arch in PRIMARY_ARCHES:
        return PRIMARY_ARCH_MIRRORS.copy()
    if arch in PORTS_ARCHES:
        return PORTS_MIRRORS.copy()
    raise ValueError("No default mirror known for arch %s" % arch)
Exemple #4
0
def get_default_mirrors(arch=None, target=None):
    """returns the default mirrors for the target. These depend on the
       architecture, for more see:
       https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
    if arch is None:
        arch = util.get_architecture(target)
    if arch in PRIMARY_ARCHES:
        return PRIMARY_ARCH_MIRRORS.copy()
    if arch in PORTS_ARCHES:
        return PORTS_MIRRORS.copy()
    raise ValueError("No default mirror known for arch %s" % arch)
Exemple #5
0
def handle(name, ocfg, cloud, log, _):
    """process the config for apt_config. This can be called from
       curthooks if a global apt config was provided or via the "apt"
       standalone command."""
    # keeping code close to curtin codebase via entry handler
    target = None
    if log is not None:
        global LOG
        LOG = log
    # feed back converted config, but only work on the subset under 'apt'
    ocfg = convert_to_v3_apt_format(ocfg)
    cfg = ocfg.get('apt', {})

    if not isinstance(cfg, dict):
        raise ValueError("Expected dictionary for 'apt' config, found %s",
                         type(cfg))

    LOG.debug("handling apt (module %s) with apt config '%s'", name, cfg)

    release = util.lsb_release(target=target)['codename']
    arch = util.get_architecture(target)
    mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
    LOG.debug("Apt Mirror info: %s", mirrors)

    apply_debconf_selections(cfg, target)

    if util.is_false(cfg.get('preserve_sources_list', False)):
        generate_sources_list(cfg, release, mirrors, cloud)
        rename_apt_lists(mirrors, target)

    try:
        apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
    except (IOError, OSError):
        LOG.exception("Failed to apply proxy or apt config info:")

    # Process 'apt_source -> sources {dict}'
    if 'sources' in cfg:
        params = mirrors
        params['RELEASE'] = release
        params['MIRROR'] = mirrors["MIRROR"]

        matcher = None
        matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
        if matchcfg:
            matcher = re.compile(matchcfg).search

        add_apt_sources(cfg['sources'],
                        cloud,
                        target=target,
                        template_params=params,
                        aa_repo_match=matcher)
Exemple #6
0
def handle(name, ocfg, cloud, log, _):
    """process the config for apt_config. This can be called from
       curthooks if a global apt config was provided or via the "apt"
       standalone command."""
    # keeping code close to curtin codebase via entry handler
    target = None
    if log is not None:
        global LOG
        LOG = log
    # feed back converted config, but only work on the subset under 'apt'
    ocfg = convert_to_v3_apt_format(ocfg)
    cfg = ocfg.get('apt', {})

    if not isinstance(cfg, dict):
        raise ValueError("Expected dictionary for 'apt' config, found %s",
                         type(cfg))

    LOG.debug("handling apt (module %s) with apt config '%s'", name, cfg)

    release = util.lsb_release(target=target)['codename']
    arch = util.get_architecture(target)
    mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
    LOG.debug("Apt Mirror info: %s", mirrors)

    apply_debconf_selections(cfg, target)

    if util.is_false(cfg.get('preserve_sources_list', False)):
        generate_sources_list(cfg, release, mirrors, cloud)
        rename_apt_lists(mirrors, target)

    try:
        apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
    except (IOError, OSError):
        LOG.exception("Failed to apply proxy or apt config info:")

    # Process 'apt_source -> sources {dict}'
    if 'sources' in cfg:
        params = mirrors
        params['RELEASE'] = release
        params['MIRROR'] = mirrors["MIRROR"]

        matcher = None
        matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
        if matchcfg:
            matcher = re.compile(matchcfg).search

        add_apt_sources(cfg['sources'], cloud, target=target,
                        template_params=params, aa_repo_match=matcher)
Exemple #7
0
def apply_apt(cfg, cloud, target):
    # cfg is the 'apt' top level dictionary already in 'v3' format.
    if not cfg:
        # no config was provided.  If apt configuration does not seem
        # necessary on this system, then return.
        if util.system_is_snappy():
            LOG.debug("Nothing to do: No apt config and running on snappy")
            return
        if not (util.which('apt-get') or util.which('apt')):
            LOG.debug("Nothing to do: No apt config and no apt commands")
            return

    LOG.debug("handling apt config: %s", cfg)

    release = util.lsb_release(target=target)['codename']
    arch = util.get_architecture(target)
    mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
    LOG.debug("Apt Mirror info: %s", mirrors)

    if util.is_false(cfg.get('preserve_sources_list', False)):
        generate_sources_list(cfg, release, mirrors, cloud)
        rename_apt_lists(mirrors, target)

    try:
        apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
    except (IOError, OSError):
        LOG.exception("Failed to apply proxy or apt config info:")

    # Process 'apt_source -> sources {dict}'
    if 'sources' in cfg:
        params = mirrors
        params['RELEASE'] = release
        params['MIRROR'] = mirrors["MIRROR"]

        matcher = None
        matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
        if matchcfg:
            matcher = re.compile(matchcfg).search

        add_apt_sources(cfg['sources'],
                        cloud,
                        target=target,
                        template_params=params,
                        aa_repo_match=matcher)
Exemple #8
0
    def get_image(self, img_conf):
        """Get image using specified image configuration.

        @param img_conf: configuration for image
        @return_value: cloud_tests.images instance
        """
        (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)

        filter = filters.get_filters([
            'arch=%s' % c_util.get_architecture(),
            'release=%s' % img_conf['release'], 'ftype=disk1.img'
        ])
        mirror_config = {
            'filters': filter,
            'keep_items': False,
            'max_items': 1,
            'checksumming_reader': True,
            'item_download': True
        }

        def policy(content, path):
            return s_util.read_signed(content, keyring=img_conf['keyring'])

        smirror = mirrors.UrlMirrorReader(url, policy=policy)
        tstore = objectstores.FileStore(img_conf['mirror_dir'])
        tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
                                             objectstore=tstore)
        tmirror.sync(smirror, path)

        search_d = os.path.join(img_conf['mirror_dir'], '**',
                                img_conf['release'], '**', '*.img')

        images = []
        for fname in glob.iglob(search_d, recursive=True):
            images.append(fname)

        if len(images) != 1:
            raise Exception('No unique images found')

        image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0])
        if img_conf.get('override_templates', False):
            image.update_templates(self.config.get('template_overrides', {}),
                                   self.config.get('template_files', {}))
        return image
Exemple #9
0
    def get_image(self, img_conf):
        """Get image using specified image configuration.

        @param img_conf: configuration for image
        @return_value: cloud_tests.images instance
        """
        (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)

        filter = filters.get_filters([
            'arch=%s' % c_util.get_architecture(),
            'release=%s' % img_conf['release'], 'ftype=disk1.img'
        ])
        mirror_config = {
            'filters': filter,
            'keep_items': False,
            'max_items': 1,
            'checksumming_reader': True,
            'item_download': True
        }

        def policy(content, path):
            return s_util.read_signed(content, keyring=img_conf['keyring'])

        smirror = mirrors.UrlMirrorReader(url, policy=policy)
        tstore = objectstores.FileStore(img_conf['mirror_dir'])
        tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
                                             objectstore=tstore)
        tmirror.sync(smirror, path)

        search_d = os.path.join(img_conf['mirror_dir'], '**',
                                img_conf['release'], '**', '*.img')

        images = []
        for fname in glob.iglob(search_d, recursive=True):
            images.append(fname)

        if len(images) < 1:
            raise RuntimeError("No images found under '%s'" % search_d)
        if len(images) > 1:
            raise RuntimeError("Multiple images found in '%s': %s" %
                               (search_d, ' '.join(images)))

        image = NoCloudKVMImage(self, img_conf, images[0])
        return image
Exemple #10
0
    def get_image(self, img_conf):
        """Get image using specified image configuration.

        @param img_conf: configuration for image
        @return_value: cloud_tests.images instance
        """
        (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)

        filter = filters.get_filters(['arch=%s' % c_util.get_architecture(),
                                      'release=%s' % img_conf['release'],
                                      'ftype=disk1.img'])
        mirror_config = {'filters': filter,
                         'keep_items': False,
                         'max_items': 1,
                         'checksumming_reader': True,
                         'item_download': True
                         }

        def policy(content, path):
            return s_util.read_signed(content, keyring=img_conf['keyring'])

        smirror = mirrors.UrlMirrorReader(url, policy=policy)
        tstore = objectstores.FileStore(img_conf['mirror_dir'])
        tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
                                             objectstore=tstore)
        tmirror.sync(smirror, path)

        search_d = os.path.join(img_conf['mirror_dir'], '**',
                                img_conf['release'], '**', '*.img')

        images = []
        for fname in glob.iglob(search_d, recursive=True):
            images.append(fname)

        if len(images) < 1:
            raise RuntimeError("No images found under '%s'" % search_d)
        if len(images) > 1:
            raise RuntimeError(
                "Multiple images found in '%s': %s" % (search_d,
                                                       ' '.join(images)))

        image = NoCloudKVMImage(self, img_conf, images[0])
        return image
Exemple #11
0
def apply_apt(cfg, cloud, target):
    # cfg is the 'apt' top level dictionary already in 'v3' format.
    if not cfg:
        should_config, msg = _should_configure_on_empty_apt()
        if not should_config:
            LOG.debug("Nothing to do: No apt config and %s", msg)
            return

    LOG.debug("handling apt config: %s", cfg)

    release = util.lsb_release(target=target)['codename']
    arch = util.get_architecture(target)
    mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
    LOG.debug("Apt Mirror info: %s", mirrors)

    if util.is_false(cfg.get('preserve_sources_list', False)):
        generate_sources_list(cfg, release, mirrors, cloud)
        rename_apt_lists(mirrors, target)

    try:
        apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
    except (IOError, OSError):
        LOG.exception("Failed to apply proxy or apt config info:")

    # Process 'apt_source -> sources {dict}'
    if 'sources' in cfg:
        params = mirrors
        params['RELEASE'] = release
        params['MIRROR'] = mirrors["MIRROR"]

        matcher = None
        matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
        if matchcfg:
            matcher = re.compile(matchcfg).search

        add_apt_sources(cfg['sources'],
                        cloud,
                        target=target,
                        template_params=params,
                        aa_repo_match=matcher)
Exemple #12
0
def find_apt_mirror_info(cfg, cloud, arch=None):
    """find_apt_mirror_info
       find an apt_mirror given the cfg provided.
       It can check for separate config of primary and security mirrors
       If only primary is given security is assumed to be equal to primary
       If the generic apt_mirror is given that is defining for both
    """

    if arch is None:
        arch = util.get_architecture()
        LOG.debug("got arch for mirror selection: %s", arch)
    pmirror = get_mirror(cfg, "primary", arch, cloud)
    LOG.debug("got primary mirror: %s", pmirror)
    smirror = get_mirror(cfg, "security", arch, cloud)
    LOG.debug("got security mirror: %s", smirror)

    mirror_info = update_mirror_info(pmirror, smirror, arch, cloud)

    # less complex replacements use only MIRROR, derive from primary
    mirror_info["MIRROR"] = mirror_info["PRIMARY"]

    return mirror_info
Exemple #13
0
def find_apt_mirror_info(cfg, cloud, arch=None):
    """find_apt_mirror_info
       find an apt_mirror given the cfg provided.
       It can check for separate config of primary and security mirrors
       If only primary is given security is assumed to be equal to primary
       If the generic apt_mirror is given that is defining for both
    """

    if arch is None:
        arch = util.get_architecture()
        LOG.debug("got arch for mirror selection: %s", arch)
    pmirror = get_mirror(cfg, "primary", arch, cloud)
    LOG.debug("got primary mirror: %s", pmirror)
    smirror = get_mirror(cfg, "security", arch, cloud)
    LOG.debug("got security mirror: %s", smirror)

    mirror_info = update_mirror_info(pmirror, smirror, arch, cloud)

    # less complex replacements use only MIRROR, derive from primary
    mirror_info["MIRROR"] = mirror_info["PRIMARY"]

    return mirror_info
Exemple #14
0
def rename_apt_lists(new_mirrors, target=None):
    """rename_apt_lists - rename apt lists to preserve old cache data"""
    default_mirrors = get_default_mirrors(util.get_architecture(target))

    pre = util.target_path(target, APT_LISTS)
    for (name, omirror) in default_mirrors.items():
        nmirror = new_mirrors.get(name)
        if not nmirror:
            continue

        oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror)
        nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror)
        if oprefix == nprefix:
            continue
        olen = len(oprefix)
        for filename in glob.glob("%s_*" % oprefix):
            newname = "%s%s" % (nprefix, filename[olen:])
            LOG.debug("Renaming apt list %s to %s", filename, newname)
            try:
                os.rename(filename, newname)
            except OSError:
                # since this is a best effort task, warn with but don't fail
                LOG.warning("Failed to rename apt list:", exc_info=True)
Exemple #15
0
def rename_apt_lists(new_mirrors, target=None):
    """rename_apt_lists - rename apt lists to preserve old cache data"""
    default_mirrors = get_default_mirrors(util.get_architecture(target))

    pre = util.target_path(target, APT_LISTS)
    for (name, omirror) in default_mirrors.items():
        nmirror = new_mirrors.get(name)
        if not nmirror:
            continue

        oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror)
        nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror)
        if oprefix == nprefix:
            continue
        olen = len(oprefix)
        for filename in glob.glob("%s_*" % oprefix):
            newname = "%s%s" % (nprefix, filename[olen:])
            LOG.debug("Renaming apt list %s to %s", filename, newname)
            try:
                os.rename(filename, newname)
            except OSError:
                # since this is a best effort task, warn with but don't fail
                LOG.warn("Failed to rename apt list:", exc_info=True)
def apply_apt(cfg, cloud, target):
    # cfg is the 'apt' top level dictionary already in 'v3' format.
    if not cfg:
        should_config, msg = _should_configure_on_empty_apt()
        if not should_config:
            LOG.debug("Nothing to do: No apt config and %s", msg)
            return

    LOG.debug("handling apt config: %s", cfg)

    release = util.lsb_release(target=target)['codename']
    arch = util.get_architecture(target)
    mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
    LOG.debug("Apt Mirror info: %s", mirrors)

    if util.is_false(cfg.get('preserve_sources_list', False)):
        generate_sources_list(cfg, release, mirrors, cloud)
        rename_apt_lists(mirrors, target)

    try:
        apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
    except (IOError, OSError):
        LOG.exception("Failed to apply proxy or apt config info:")

    # Process 'apt_source -> sources {dict}'
    if 'sources' in cfg:
        params = mirrors
        params['RELEASE'] = release
        params['MIRROR'] = mirrors["MIRROR"]

        matcher = None
        matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
        if matchcfg:
            matcher = re.compile(matchcfg).search

        add_apt_sources(cfg['sources'], cloud, target=target,
                        template_params=params, aa_repo_match=matcher)