def load_os_config(platform_name,
os_name,
require_enabled=False,
feature_overrides={}):
"""Load configuration for os.
@param platform_name: platform name to load os config for
@param os_name: name of os to retrieve config for
@param require_enabled: if true, raise error if 'enabled' not True
@param feature_overrides: feature flag overrides to merge with features
@return_value: config dict
"""
main_conf = c_util.read_conf(RELEASES_CONF)
default = main_conf['default_release_config']
image = main_conf['releases'][os_name]
conf = merge_config(
merge_config(get(default, 'default'), get(default, platform_name)),
merge_config(get(image, 'default'), get(image, platform_name)))
feature_conf = main_conf['features']
feature_groups = conf.get('feature_groups', [])
overrides = merge_config(get(conf, 'features'), feature_overrides)
conf['arch'] = c_util.get_architecture()
conf['features'] = merge_feature_groups(feature_conf, feature_groups,
overrides)
if require_enabled and not enabled(conf):
raise ValueError('OS is not enabled')
return conf
def load_os_config(platform_name, os_name, require_enabled=False,
feature_overrides=None):
"""Load configuration for os.
@param platform_name: platform name to load os config for
@param os_name: name of os to retrieve config for
@param require_enabled: if true, raise error if 'enabled' not True
@param feature_overrides: feature flag overrides to merge with features
@return_value: config dict
"""
if feature_overrides is None:
feature_overrides = {}
main_conf = c_util.read_conf(RELEASES_CONF)
default = main_conf['default_release_config']
image = main_conf['releases'][os_name]
conf = merge_config(merge_config(get(default, 'default'),
get(default, platform_name)),
merge_config(get(image, 'default'),
get(image, platform_name)))
feature_conf = main_conf['features']
feature_groups = conf.get('feature_groups', [])
overrides = merge_config(get(conf, 'features'), feature_overrides)
conf['arch'] = c_util.get_architecture()
conf['features'] = merge_feature_groups(
feature_conf, feature_groups, overrides)
if require_enabled and not enabled(conf):
raise ValueError('OS is not enabled')
return conf
def get_default_mirrors(arch=None, target=None):
"""returns the default mirrors for the target. These depend on the
architecture, for more see:
https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
if arch is None:
arch = util.get_architecture(target)
if arch in PRIMARY_ARCHES:
return PRIMARY_ARCH_MIRRORS.copy()
if arch in PORTS_ARCHES:
return PORTS_MIRRORS.copy()
raise ValueError("No default mirror known for arch %s" % arch)
def get_default_mirrors(arch=None, target=None):
"""returns the default mirrors for the target. These depend on the
architecture, for more see:
https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
if arch is None:
arch = util.get_architecture(target)
if arch in PRIMARY_ARCHES:
return PRIMARY_ARCH_MIRRORS.copy()
if arch in PORTS_ARCHES:
return PORTS_MIRRORS.copy()
raise ValueError("No default mirror known for arch %s" % arch)
def handle(name, ocfg, cloud, log, _):
"""process the config for apt_config. This can be called from
curthooks if a global apt config was provided or via the "apt"
standalone command."""
# keeping code close to curtin codebase via entry handler
target = None
if log is not None:
global LOG
LOG = log
# feed back converted config, but only work on the subset under 'apt'
ocfg = convert_to_v3_apt_format(ocfg)
cfg = ocfg.get('apt', {})
if not isinstance(cfg, dict):
raise ValueError("Expected dictionary for 'apt' config, found %s",
type(cfg))
LOG.debug("handling apt (module %s) with apt config '%s'", name, cfg)
release = util.lsb_release(target=target)['codename']
arch = util.get_architecture(target)
mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
LOG.debug("Apt Mirror info: %s", mirrors)
apply_debconf_selections(cfg, target)
if util.is_false(cfg.get('preserve_sources_list', False)):
generate_sources_list(cfg, release, mirrors, cloud)
rename_apt_lists(mirrors, target)
try:
apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
except (IOError, OSError):
LOG.exception("Failed to apply proxy or apt config info:")
# Process 'apt_source -> sources {dict}'
if 'sources' in cfg:
params = mirrors
params['RELEASE'] = release
params['MIRROR'] = mirrors["MIRROR"]
matcher = None
matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
if matchcfg:
matcher = re.compile(matchcfg).search
add_apt_sources(cfg['sources'],
cloud,
target=target,
template_params=params,
aa_repo_match=matcher)
def handle(name, ocfg, cloud, log, _):
"""process the config for apt_config. This can be called from
curthooks if a global apt config was provided or via the "apt"
standalone command."""
# keeping code close to curtin codebase via entry handler
target = None
if log is not None:
global LOG
LOG = log
# feed back converted config, but only work on the subset under 'apt'
ocfg = convert_to_v3_apt_format(ocfg)
cfg = ocfg.get('apt', {})
if not isinstance(cfg, dict):
raise ValueError("Expected dictionary for 'apt' config, found %s",
type(cfg))
LOG.debug("handling apt (module %s) with apt config '%s'", name, cfg)
release = util.lsb_release(target=target)['codename']
arch = util.get_architecture(target)
mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
LOG.debug("Apt Mirror info: %s", mirrors)
apply_debconf_selections(cfg, target)
if util.is_false(cfg.get('preserve_sources_list', False)):
generate_sources_list(cfg, release, mirrors, cloud)
rename_apt_lists(mirrors, target)
try:
apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
except (IOError, OSError):
LOG.exception("Failed to apply proxy or apt config info:")
# Process 'apt_source -> sources {dict}'
if 'sources' in cfg:
params = mirrors
params['RELEASE'] = release
params['MIRROR'] = mirrors["MIRROR"]
matcher = None
matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
if matchcfg:
matcher = re.compile(matchcfg).search
add_apt_sources(cfg['sources'], cloud, target=target,
template_params=params, aa_repo_match=matcher)
def apply_apt(cfg, cloud, target):
# cfg is the 'apt' top level dictionary already in 'v3' format.
if not cfg:
# no config was provided. If apt configuration does not seem
# necessary on this system, then return.
if util.system_is_snappy():
LOG.debug("Nothing to do: No apt config and running on snappy")
return
if not (util.which('apt-get') or util.which('apt')):
LOG.debug("Nothing to do: No apt config and no apt commands")
return
LOG.debug("handling apt config: %s", cfg)
release = util.lsb_release(target=target)['codename']
arch = util.get_architecture(target)
mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
LOG.debug("Apt Mirror info: %s", mirrors)
if util.is_false(cfg.get('preserve_sources_list', False)):
generate_sources_list(cfg, release, mirrors, cloud)
rename_apt_lists(mirrors, target)
try:
apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
except (IOError, OSError):
LOG.exception("Failed to apply proxy or apt config info:")
# Process 'apt_source -> sources {dict}'
if 'sources' in cfg:
params = mirrors
params['RELEASE'] = release
params['MIRROR'] = mirrors["MIRROR"]
matcher = None
matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
if matchcfg:
matcher = re.compile(matchcfg).search
add_apt_sources(cfg['sources'],
cloud,
target=target,
template_params=params,
aa_repo_match=matcher)
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters([
'arch=%s' % c_util.get_architecture(),
'release=%s' % img_conf['release'], 'ftype=disk1.img'
])
mirror_config = {
'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) != 1:
raise Exception('No unique images found')
image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0])
if img_conf.get('override_templates', False):
image.update_templates(self.config.get('template_overrides', {}),
self.config.get('template_files', {}))
return image
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters([
'arch=%s' % c_util.get_architecture(),
'release=%s' % img_conf['release'], 'ftype=disk1.img'
])
mirror_config = {
'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) < 1:
raise RuntimeError("No images found under '%s'" % search_d)
if len(images) > 1:
raise RuntimeError("Multiple images found in '%s': %s" %
(search_d, ' '.join(images)))
image = NoCloudKVMImage(self, img_conf, images[0])
return image
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters(['arch=%s' % c_util.get_architecture(),
'release=%s' % img_conf['release'],
'ftype=disk1.img'])
mirror_config = {'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) < 1:
raise RuntimeError("No images found under '%s'" % search_d)
if len(images) > 1:
raise RuntimeError(
"Multiple images found in '%s': %s" % (search_d,
' '.join(images)))
image = NoCloudKVMImage(self, img_conf, images[0])
return image
def apply_apt(cfg, cloud, target):
# cfg is the 'apt' top level dictionary already in 'v3' format.
if not cfg:
should_config, msg = _should_configure_on_empty_apt()
if not should_config:
LOG.debug("Nothing to do: No apt config and %s", msg)
return
LOG.debug("handling apt config: %s", cfg)
release = util.lsb_release(target=target)['codename']
arch = util.get_architecture(target)
mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
LOG.debug("Apt Mirror info: %s", mirrors)
if util.is_false(cfg.get('preserve_sources_list', False)):
generate_sources_list(cfg, release, mirrors, cloud)
rename_apt_lists(mirrors, target)
try:
apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
except (IOError, OSError):
LOG.exception("Failed to apply proxy or apt config info:")
# Process 'apt_source -> sources {dict}'
if 'sources' in cfg:
params = mirrors
params['RELEASE'] = release
params['MIRROR'] = mirrors["MIRROR"]
matcher = None
matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
if matchcfg:
matcher = re.compile(matchcfg).search
add_apt_sources(cfg['sources'],
cloud,
target=target,
template_params=params,
aa_repo_match=matcher)
def find_apt_mirror_info(cfg, cloud, arch=None):
"""find_apt_mirror_info
find an apt_mirror given the cfg provided.
It can check for separate config of primary and security mirrors
If only primary is given security is assumed to be equal to primary
If the generic apt_mirror is given that is defining for both
"""
if arch is None:
arch = util.get_architecture()
LOG.debug("got arch for mirror selection: %s", arch)
pmirror = get_mirror(cfg, "primary", arch, cloud)
LOG.debug("got primary mirror: %s", pmirror)
smirror = get_mirror(cfg, "security", arch, cloud)
LOG.debug("got security mirror: %s", smirror)
mirror_info = update_mirror_info(pmirror, smirror, arch, cloud)
# less complex replacements use only MIRROR, derive from primary
mirror_info["MIRROR"] = mirror_info["PRIMARY"]
return mirror_info
def find_apt_mirror_info(cfg, cloud, arch=None):
"""find_apt_mirror_info
find an apt_mirror given the cfg provided.
It can check for separate config of primary and security mirrors
If only primary is given security is assumed to be equal to primary
If the generic apt_mirror is given that is defining for both
"""
if arch is None:
arch = util.get_architecture()
LOG.debug("got arch for mirror selection: %s", arch)
pmirror = get_mirror(cfg, "primary", arch, cloud)
LOG.debug("got primary mirror: %s", pmirror)
smirror = get_mirror(cfg, "security", arch, cloud)
LOG.debug("got security mirror: %s", smirror)
mirror_info = update_mirror_info(pmirror, smirror, arch, cloud)
# less complex replacements use only MIRROR, derive from primary
mirror_info["MIRROR"] = mirror_info["PRIMARY"]
return mirror_info
def rename_apt_lists(new_mirrors, target=None):
"""rename_apt_lists - rename apt lists to preserve old cache data"""
default_mirrors = get_default_mirrors(util.get_architecture(target))
pre = util.target_path(target, APT_LISTS)
for (name, omirror) in default_mirrors.items():
nmirror = new_mirrors.get(name)
if not nmirror:
continue
oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror)
nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror)
if oprefix == nprefix:
continue
olen = len(oprefix)
for filename in glob.glob("%s_*" % oprefix):
newname = "%s%s" % (nprefix, filename[olen:])
LOG.debug("Renaming apt list %s to %s", filename, newname)
try:
os.rename(filename, newname)
except OSError:
# since this is a best effort task, warn with but don't fail
LOG.warning("Failed to rename apt list:", exc_info=True)
def rename_apt_lists(new_mirrors, target=None):
"""rename_apt_lists - rename apt lists to preserve old cache data"""
default_mirrors = get_default_mirrors(util.get_architecture(target))
pre = util.target_path(target, APT_LISTS)
for (name, omirror) in default_mirrors.items():
nmirror = new_mirrors.get(name)
if not nmirror:
continue
oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror)
nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror)
if oprefix == nprefix:
continue
olen = len(oprefix)
for filename in glob.glob("%s_*" % oprefix):
newname = "%s%s" % (nprefix, filename[olen:])
LOG.debug("Renaming apt list %s to %s", filename, newname)
try:
os.rename(filename, newname)
except OSError:
# since this is a best effort task, warn with but don't fail
LOG.warn("Failed to rename apt list:", exc_info=True)
def apply_apt(cfg, cloud, target):
# cfg is the 'apt' top level dictionary already in 'v3' format.
if not cfg:
should_config, msg = _should_configure_on_empty_apt()
if not should_config:
LOG.debug("Nothing to do: No apt config and %s", msg)
return
LOG.debug("handling apt config: %s", cfg)
release = util.lsb_release(target=target)['codename']
arch = util.get_architecture(target)
mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
LOG.debug("Apt Mirror info: %s", mirrors)
if util.is_false(cfg.get('preserve_sources_list', False)):
generate_sources_list(cfg, release, mirrors, cloud)
rename_apt_lists(mirrors, target)
try:
apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
except (IOError, OSError):
LOG.exception("Failed to apply proxy or apt config info:")
# Process 'apt_source -> sources {dict}'
if 'sources' in cfg:
params = mirrors
params['RELEASE'] = release
params['MIRROR'] = mirrors["MIRROR"]
matcher = None
matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
if matchcfg:
matcher = re.compile(matchcfg).search
add_apt_sources(cfg['sources'], cloud, target=target,
template_params=params, aa_repo_match=matcher)