def test_orchestrate_override_build_kwarg(tmpdir):
workflow = mock_workflow(tmpdir)
expected_kwargs = {
'git_uri': SOURCE['uri'],
'git_ref': 'master',
'git_branch': 'master',
'user': '******',
'is_auto': False,
'platform': 'x86_64',
'release': '4242',
'arrangement_version': 1
}
mock_osbs(worker_expect=expected_kwargs)
mock_reactor_config(tmpdir)
plugin_args = {
'platforms': ['x86_64'],
'build_kwargs': make_worker_build_kwargs(),
'worker_build_image': 'fedora:latest',
'osbs_client_config': str(tmpdir),
}
override_build_kwarg(workflow, 'release', '4242')
runner = BuildStepPluginsRunner(
workflow.builder.tasker,
workflow,
[{
'name': OrchestrateBuildPlugin.key,
'args': plugin_args,
}]
)
build_result = runner.run()
assert not build_result.is_failed()
def set_worker_params(self, remote_sources):
for remote_source in remote_sources:
build_args = {}
env_vars = self.cachito_session.get_request_env_vars(
remote_source['request_id'])
for env_var, value_info in env_vars.items():
build_arg_value = value_info['value']
kind = value_info['kind']
if kind == 'path':
name = remote_source['name'] or ''
build_arg_value = os.path.join(REMOTE_SOURCE_DIR, name,
value_info['value'])
self.log.debug(
'Setting the Cachito environment variable "%s" to the absolute path "%s"',
env_var,
build_arg_value,
)
build_args[env_var] = build_arg_value
elif kind == 'literal':
self.log.debug(
'Setting the Cachito environment variable "%s" to a literal value "%s"',
env_var,
build_arg_value,
)
build_args[env_var] = build_arg_value
else:
raise RuntimeError(
f'Unknown kind {kind} got from Cachito.')
remote_source['build_args'] = build_args
override_build_kwarg(self.workflow, 'remote_sources', remote_sources)
def _set_worker_arg(self, replacement_pullspecs):
arg = {
str(old): str(new)
for old, new in replacement_pullspecs.items()
}
override_build_kwarg(self.workflow,
"operator_bundle_replacement_pullspecs", arg)
def forward_composes(self):
repos_by_arch = defaultdict(list)
# set overrides by arch if arches are available
for compose_info in self.composes_info:
result_repofile = compose_info['result_repofile']
try:
arches = compose_info['arches']
except KeyError:
repos_by_arch[None].append(result_repofile)
else:
for arch in arches.split():
repos_by_arch[arch].append(result_repofile)
# we should almost never have a None entry, but if we do, we need to merge
# it with all other repos.
try:
noarch_repos = repos_by_arch.pop(None)
except KeyError:
pass
else:
for repos in repos_by_arch.values():
repos.extend(noarch_repos)
for arch, repofiles in repos_by_arch.items():
override_build_kwarg(self.workflow, 'yum_repourls', repofiles,
arch)
# Only set the None override if there are no other repos
if not repos_by_arch:
override_build_kwarg(self.workflow, 'yum_repourls', noarch_repos,
None)
def test_orchestrate_override_build_kwarg(tmpdir):
workflow = mock_workflow(tmpdir)
expected_kwargs = {
'git_uri': SOURCE['uri'],
'git_ref': 'master',
'git_branch': 'master',
'user': '******',
'is_auto': False,
'platform': 'x86_64',
'release': '4242',
'arrangement_version': 1
}
mock_osbs(worker_expect=expected_kwargs)
mock_reactor_config(tmpdir)
plugin_args = {
'platforms': ['x86_64'],
'build_kwargs': make_worker_build_kwargs(),
'worker_build_image': 'fedora:latest',
'osbs_client_config': str(tmpdir),
}
override_build_kwarg(workflow, 'release', '4242')
runner = BuildStepPluginsRunner(workflow.builder.tasker, workflow,
[{
'name': OrchestrateBuildPlugin.key,
'args': plugin_args,
}])
build_result = runner.run()
assert not build_result.is_failed()
def forward_composes(self):
repos_by_arch = defaultdict(list)
# set overrides by arch if arches are available
for compose_info in self.composes_info:
result_repofile = compose_info['result_repofile']
try:
arches = compose_info['arches']
except KeyError:
repos_by_arch[None].append(result_repofile)
else:
for arch in arches.split():
repos_by_arch[arch].append(result_repofile)
# we should almost never have a None entry from composes,
# but we can have yum_repos added, so if we do, we need to merge
# it with all other repos.
if self.repourls:
repos_by_arch[None].extend(self.repourls)
try:
noarch_repos = repos_by_arch.pop(None)
except KeyError:
pass
else:
for repos in repos_by_arch.values():
repos.extend(noarch_repos)
for arch, repofiles in repos_by_arch.items():
override_build_kwarg(self.workflow, 'yum_repourls', repofiles, arch)
# Only set the None override if there are no other repos
if not repos_by_arch:
override_build_kwarg(self.workflow, 'yum_repourls', noarch_repos, None)
def run(self):
self.adjust_for_autorebuild()
if self.allow_inheritance():
self.adjust_for_inherit()
self.workflow.all_yum_repourls = self.repourls
try:
self.read_configs()
except SkipResolveComposesPlugin as abort_exc:
override_build_kwarg(self.workflow, 'yum_repourls', self.repourls, None)
self.log.info('Aborting plugin execution: %s', abort_exc)
return
self.adjust_compose_config()
self.request_compose_if_needed()
try:
self.wait_for_composes()
except WaitComposeToFinishTimeout as e:
self.log.info(str(e))
preserve_composes = set(self.compose_ids).union(self.parent_compose_ids)
cancel_composes = set(self.all_compose_ids) - preserve_composes
if cancel_composes:
self.log.info('Canceling unfinished composes which were created by the build: %s',
cancel_composes)
for compose_id in cancel_composes:
if self.odcs_client.get_compose_status(compose_id) in ['wait', 'generating']:
self.log.info('Canceling the compose %s', compose_id)
self.odcs_client.cancel_compose(compose_id)
else:
self.log.info('The compose %s is not in progress, skip canceling', compose_id)
raise
self.resolve_signing_intent()
self.forward_composes()
return self.make_result()
def _set_worker_arg(self, replacement_pullspecs):
arg = {
str(repl['original']): str(repl['new'])
for repl in replacement_pullspecs if repl['replaced']
}
override_build_kwarg(self.workflow,
"operator_bundle_replacement_pullspecs", arg)
def run(self):
"""
run the plugin
"""
source = self._load_source()
set_flatpak_source_info(self.workflow, source)
# Create the dockerfile
if source.runtime:
profile = 'runtime'
else:
profile = 'default'
module_info = source.compose.base_module
packages = ' '.join(module_info.mmd.profiles[profile].rpms)
df_path = os.path.join(self.workflow.builder.df_dir, DOCKERFILE_FILENAME)
with open(df_path, 'w') as fp:
fp.write(DOCKERFILE_TEMPLATE.format(name=module_info.name,
stream=module_info.stream,
version=module_info.version,
base_image=self.base_image,
packages=packages,
rpm_qf_args=rpm_qf_args()))
self.workflow.builder.set_df_path(df_path)
# Create the cleanup script
cleanupscript = os.path.join(self.workflow.builder.df_dir, "cleanup.sh")
with open(cleanupscript, 'w') as f:
for line in source.flatpak_json.get('cleanup-commands', []):
f.write(line)
f.write("\n")
os.chmod(cleanupscript, 0o0755)
# Add a yum-repository pointing to the compose
repo_name = 'atomic-reactor-module-{name}-{stream}-{version}'.format(
name=module_info.name,
stream=module_info.stream,
version=module_info.version)
repo = {
'name': repo_name,
'baseurl': source.compose.repo_url,
'enabled': 1,
'gpgcheck': 0,
}
path = os.path.join(YUM_REPOS_DIR, repo_name + '.repo')
self.workflow.files[path] = render_yum_repo(repo, escape_dollars=False)
override_build_kwarg(self.workflow, 'module_compose_id', source.compose.compose_id)
def pull_latest_commit_if_configured(self):
if not self.should_use_latest_commit():
return
git_branch = self.build_labels['git-branch']
self.workflow.source.reset('origin/{}'.format(git_branch))
# Import it here to avoid circular import errors.
from atomic_reactor.plugins.build_orchestrate_build import override_build_kwarg
override_build_kwarg(self.workflow, 'git_ref', self.workflow.source.commit_id)
def run(self):
"""
run the plugin
"""
self.log.info("Resolving module compose")
compose_info = self._resolve_compose()
set_compose_info(self.workflow, compose_info)
override_build_kwarg(self.workflow, 'compose_ids', [compose_info.compose_id])
def run(self):
"""
run the plugin
"""
self.log.info("Resolving module compose")
compose_info = self._resolve_compose()
set_compose_info(self.workflow, compose_info)
override_build_kwarg(self.workflow, 'compose_ids', [compose_info.compose_id])
def pull_latest_commit_if_configured(self):
if not self.should_use_latest_commit():
return
git_branch = self.build_labels['git-branch']
self.workflow.source.reset('origin/{}'.format(git_branch))
# Import it here to avoid circular import errors.
from atomic_reactor.plugins.build_orchestrate_build import override_build_kwarg
override_build_kwarg(self.workflow, 'git_ref',
self.workflow.source.commit_id)
def set_worker_params(self, source_request, remote_source_url):
build_args = {
# Turn the environment variables into absolute paths that
# represent where the remote sources are copied to during
# the build process.
env_var: os.path.join(REMOTE_SOURCE_DIR, value)
for env_var, value in source_request.get('environment_variables',
{}).items()
}
override_build_kwarg(self.workflow, 'remote_source_url',
remote_source_url)
override_build_kwarg(self.workflow, 'remote_source_build_args',
build_args)
def set_worker_params(self, source_request, remote_source_url,
remote_source_conf_url, remote_source_icm_url):
build_args = {}
# This matches values such as 'deps/gomod' but not 'true'
rel_path_regex = re.compile(r'^[^/]+/[^/]+(?:/[^/]+)*$')
for env_var, value in source_request.get('environment_variables',
{}).items():
# Turn the environment variables that are relative paths into absolute paths that
# represent where the remote sources are copied to during the build process.
if re.match(rel_path_regex, value):
abs_path = os.path.join(REMOTE_SOURCE_DIR, value)
self.log.debug(
'Setting the Cachito environment variable "%s" to the absolute path "%s"',
env_var,
abs_path,
)
build_args[env_var] = abs_path
else:
build_args[env_var] = value
# Alias for absolute path to cachito.env script added into buildargs
build_args[CACHITO_ENV_ARG_ALIAS] = os.path.join(
REMOTE_SOURCE_DIR, CACHITO_ENV_FILENAME)
override_build_kwarg(self.workflow, 'remote_source_url',
remote_source_url)
override_build_kwarg(self.workflow, 'remote_source_build_args',
build_args)
override_build_kwarg(self.workflow, 'remote_source_configs',
remote_source_conf_url)
override_build_kwarg(self.workflow, 'remote_source_icm_url',
remote_source_icm_url)
def run(self):
try:
self.adjust_for_autorebuild()
if self.allow_inheritance:
self.adjust_for_inherit()
self.workflow.all_yum_repourls = self.repourls
self.read_configs()
self.adjust_compose_config()
self.request_compose_if_needed()
self.wait_for_composes()
self.resolve_signing_intent()
self.forward_composes()
return self.make_result()
except SkipResolveComposesPlugin as abort_exc:
override_build_kwarg(self.workflow, 'yum_repourls', self.repourls, None)
self.log.info('Aborting plugin execution: %s', abort_exc)
def run(self):
try:
self.adjust_for_autorebuild()
if self.allow_inheritance():
self.adjust_for_inherit()
self.workflow.all_yum_repourls = self.repourls
self.read_configs()
self.adjust_compose_config()
self.request_compose_if_needed()
self.wait_for_composes()
self.resolve_signing_intent()
self.forward_composes()
return self.make_result()
except SkipResolveComposesPlugin as abort_exc:
override_build_kwarg(self.workflow, 'yum_repourls', self.repourls,
None)
self.log.info('Aborting plugin execution: %s', abort_exc)
def forward_composes(self):
repos_by_arch = defaultdict(list)
# set overrides by arch if arches are available
for compose_info in self.composes_info:
result_repofile = compose_info['result_repofile']
try:
arches = compose_info['arches']
except KeyError:
repos_by_arch[None].append(result_repofile)
else:
for arch in arches.split():
repos_by_arch[arch].append(result_repofile)
# we should almost never have a None entry from composes,
# but we can have yum_repos added, so if we do, we need to merge
# it with all other repos.
if self.repourls:
repos_by_arch[None].extend(self.repourls)
try:
noarch_repos = repos_by_arch.pop(None)
except KeyError:
pass
else:
for repos in repos_by_arch.values():
repos.extend(noarch_repos)
for arch, repofiles in repos_by_arch.items():
override_build_kwarg(self.workflow, 'yum_repourls', repofiles,
arch)
# Only set the None override if there are no other repos
if not repos_by_arch:
override_build_kwarg(self.workflow, 'yum_repourls', noarch_repos,
None)
# If we don't think the set of packages available from the user-supplied repourls,
# inherited repourls, and composed repositories is complete, set the 'include_koji_repo'
# kwarg so that the so that the 'yum_repourls' kwarg that we just set doesn't
# result in the 'koji' plugin being omitted.
if not self.has_complete_repos:
override_build_kwarg(self.workflow, 'include_koji_repo', True)
# So that plugins like flatpak_update_dockerfile can get information about the composes
override_build_kwarg(self.workflow, 'compose_ids',
self.all_compose_ids)
def set_worker_params(self, source_request, remote_source_url,
remote_source_conf_url, remote_source_icm_url):
build_args = {}
env_vars = self.cachito_session.get_request_env_vars(
source_request['id'])
for env_var, value_info in env_vars.items():
build_arg_value = value_info['value']
kind = value_info['kind']
if kind == 'path':
build_arg_value = os.path.join(REMOTE_SOURCE_DIR,
value_info['value'])
self.log.debug(
'Setting the Cachito environment variable "%s" to the absolute path "%s"',
env_var,
build_arg_value,
)
build_args[env_var] = build_arg_value
elif kind == 'literal':
self.log.debug(
'Setting the Cachito environment variable "%s" to a literal value "%s"',
env_var,
build_arg_value,
)
build_args[env_var] = build_arg_value
else:
raise RuntimeError(f'Unknown kind {kind} got from Cachito.')
# Alias for absolute path to cachito.env script added into buildargs
build_args[CACHITO_ENV_ARG_ALIAS] = os.path.join(
REMOTE_SOURCE_DIR, CACHITO_ENV_FILENAME)
override_build_kwarg(self.workflow, 'remote_source_url',
remote_source_url)
override_build_kwarg(self.workflow, 'remote_source_build_args',
build_args)
override_build_kwarg(self.workflow, 'remote_source_configs',
remote_source_conf_url)
override_build_kwarg(self.workflow, 'remote_source_icm_url',
remote_source_icm_url)
def run(self):
"""
run the plugin
"""
source = self._load_source()
set_flatpak_source_info(self.workflow, source)
builder = FlatpakBuilder(source, None, None)
builder.precheck()
# Create the dockerfile
module_info = source.base_module
# We need to enable all the modules other than the platform pseudo-module
modules_str = ' '.join(builder.get_enable_modules())
install_packages_str = ' '.join(builder.get_install_packages())
df_path = os.path.join(self.workflow.builder.df_dir,
DOCKERFILE_FILENAME)
with open(df_path, 'w') as fp:
fp.write(
DOCKERFILE_TEMPLATE.format(name=module_info.name,
stream=module_info.stream,
version=module_info.version,
base_image=self.base_image,
modules=modules_str,
packages=install_packages_str,
rpm_qf_args=rpm_qf_args()))
self.workflow.builder.set_df_path(df_path)
includepkgs = builder.get_includepkgs()
includepkgs_path = os.path.join(self.workflow.builder.df_dir,
'atomic-reactor-includepkgs')
with open(includepkgs_path, 'w') as f:
f.write('includepkgs = ' + ','.join(includepkgs) + '\n')
# Create the cleanup script
cleanupscript = os.path.join(self.workflow.builder.df_dir,
"cleanup.sh")
with open(cleanupscript, 'w') as f:
f.write(builder.get_cleanup_script())
os.chmod(cleanupscript, 0o0755)
# Add a yum-repository pointing to the compose
repo_name = 'atomic-reactor-module-{name}-{stream}-{version}'.format(
name=module_info.name,
stream=module_info.stream,
version=module_info.version)
compose_info = get_compose_info(self.workflow)
repo = {
'name': repo_name,
'baseurl': compose_info.repo_url,
'enabled': 1,
'gpgcheck': 0,
}
path = YumRepo(os.path.join(YUM_REPOS_DIR, repo_name)).dst_filename
self.workflow.files[path] = render_yum_repo(repo, escape_dollars=False)
override_build_kwarg(self.workflow, 'module_compose_id',
compose_info.compose_id)
def run(self):
"""
run the plugin
"""
source = self._load_source()
set_flatpak_source_info(self.workflow, source)
# Create the dockerfile
if source.runtime:
profile = 'runtime'
else:
profile = 'default'
module_info = source.compose.base_module
packages = ' '.join(module_info.mmd.profiles[profile].rpms)
df_path = os.path.join(self.workflow.builder.df_dir,
DOCKERFILE_FILENAME)
with open(df_path, 'w') as fp:
fp.write(
DOCKERFILE_TEMPLATE.format(name=module_info.name,
stream=module_info.stream,
version=module_info.version,
base_image=self.base_image,
packages=packages,
rpm_qf_args=rpm_qf_args()))
self.workflow.builder.set_df_path(df_path)
# Create the cleanup script
cleanupscript = os.path.join(self.workflow.builder.df_dir,
"cleanup.sh")
with open(cleanupscript, 'w') as f:
for line in source.flatpak_json.get('cleanup-commands', []):
f.write(line)
f.write("\n")
os.chmod(cleanupscript, 0o0755)
# Add a yum-repository pointing to the compose
repo_name = 'atomic-reactor-module-{name}-{stream}-{version}'.format(
name=module_info.name,
stream=module_info.stream,
version=module_info.version)
repo = {
'name': repo_name,
'baseurl': source.compose.repo_url,
'enabled': 1,
'gpgcheck': 0,
}
path = os.path.join(YUM_REPOS_DIR, repo_name + '.repo')
self.workflow.files[path] = render_yum_repo(repo, escape_dollars=False)
override_build_kwarg(self.workflow, 'module_compose_id',
source.compose.compose_id)
def run(self):
"""
run the plugin
"""
source = self._load_source()
set_flatpak_source_info(self.workflow, source)
module_info = source.compose.base_module
# For a runtime, certain information is duplicated between the container.yaml
# and the modulemd, check that it matches
if source.runtime:
flatpak_yaml = source.flatpak_yaml
flatpak_xmd = module_info.mmd.props.xmd['flatpak']
def check(condition, what):
if not condition:
raise RuntimeError(
"Mismatch for {} betweeen module xmd and container.yaml"
.format(what))
check(flatpak_yaml['branch'] == flatpak_xmd['branch'], "'branch'")
check(source.profile in flatpak_xmd['runtimes'], 'profile name')
profile_xmd = flatpak_xmd['runtimes'][source.profile]
check(flatpak_yaml['id'] == profile_xmd['id'], "'id'")
check(
flatpak_yaml.get('runtime',
None) == profile_xmd.get('runtime', None),
"'runtime'")
check(
flatpak_yaml.get('sdk', None) == profile_xmd.get('sdk', None),
"'sdk'")
# Create the dockerfile
install_packages = module_info.mmd.peek_profiles()[
source.profile].props.rpms.get()
install_packages_str = ' '.join(install_packages)
df_path = os.path.join(self.workflow.builder.df_dir,
DOCKERFILE_FILENAME)
with open(df_path, 'w') as fp:
fp.write(
DOCKERFILE_TEMPLATE.format(name=module_info.name,
stream=module_info.stream,
version=module_info.version,
base_image=self.base_image,
packages=install_packages_str,
rpm_qf_args=rpm_qf_args()))
self.workflow.builder.set_df_path(df_path)
# For a runtime, we want to make sure that the set of RPMs that is installed
# into the filesystem is *exactly* the set that is listed in the runtime
# profile. Requiring the full listed set of RPMs to be listed makes it
# easier to catch unintentional changes in the package list that might break
# applications depending on the runtime. It also simplifies the checking we
# do for application flatpaks, since we can simply look at the runtime
# modulemd to find out what packages are present in the runtime.
#
# For an application, we want to make sure that each RPM that is installed
# into the filesystem is *either* an RPM that is part of the 'runtime'
# profile of the base runtime, or from a module that was built with
# flatpak-rpm-macros in the install root and, thus, prefix=/app.
#
# We achieve this by restricting the set of available packages in the dnf
# configuration to just the ones that we want.
#
# The advantage of doing this upfront, rather than just checking after the
# fact is that this makes sure that when a application is being installed,
# we don't get a different package to satisfy a dependency than the one
# in the runtime - e.g. aajohan-comfortaa-fonts to satisfy font(:lang=en)
# because it's alphabetically first.
if not source.runtime:
runtime_module = source.runtime_module
runtime_profile = runtime_module.mmd.peek_profiles()['runtime']
available_packages = sorted(runtime_profile.props.rpms.get())
for m in source.app_modules:
# Strip off the '.rpm' suffix from the filename to get something
# that DNF can parse.
available_packages.extend(x[:-4] for x in m.rpms)
else:
base_module = source.compose.base_module
runtime_profile = base_module.mmd.peek_profiles()['runtime']
available_packages = sorted(runtime_profile.props.rpms.get())
includepkgs_path = os.path.join(self.workflow.builder.df_dir,
'atomic-reactor-includepkgs')
with open(includepkgs_path, 'w') as f:
f.write('includepkgs = ' + ','.join(available_packages) + '\n')
# Create the cleanup script
cleanupscript = os.path.join(self.workflow.builder.df_dir,
"cleanup.sh")
with open(cleanupscript, 'w') as f:
cleanup_commands = source.flatpak_yaml.get('cleanup-commands')
if cleanup_commands is not None:
f.write(cleanup_commands.rstrip())
f.write("\n")
os.chmod(cleanupscript, 0o0755)
# Add a yum-repository pointing to the compose
repo_name = 'atomic-reactor-module-{name}-{stream}-{version}'.format(
name=module_info.name,
stream=module_info.stream,
version=module_info.version)
repo = {
'name': repo_name,
'baseurl': source.compose.repo_url,
'enabled': 1,
'gpgcheck': 0,
}
path = os.path.join(YUM_REPOS_DIR, repo_name + '.repo')
self.workflow.files[path] = render_yum_repo(repo, escape_dollars=False)
override_build_kwarg(self.workflow, 'module_compose_id',
source.compose.compose_id)
def forward_composes(self):
yum_repourls = [compose_info['result_repofile'] for compose_info in self.composes_info]
override_build_kwarg(self.workflow, 'yum_repourls', yum_repourls)
