def _inject_into_repo_files(self):
"""Inject repo files into a relative directory inside the build context"""
host_repos_path = os.path.join(self.workflow.builder.df_dir,
RELATIVE_REPOS_PATH)
self.log.info("creating directory for yum repos: %s", host_repos_path)
os.mkdir(host_repos_path)
for repo_filename, repo_content in self.workflow.files.items():
# Update every repo accordingly in a repofile
# input_buf ---- updated ----> updated_buf
with StringIO(
repo_content) as input_buf, StringIO() as updated_buf:
for line in input_buf:
updated_buf.write(line)
# Apply sslcacert to every repo in a repofile
if line.lstrip().startswith(
'[') and self._builder_ca_bundle:
updated_buf.write(
f'sslcacert=/tmp/{self._ca_bundle_pem}\n')
yum_repo = YumRepo(repourl=repo_filename,
content=updated_buf.getvalue(),
dst_repos_dir=host_repos_path,
add_hash=False)
yum_repo.write_content()
def _inject_into_repo_files(self, build_dir: BuildDir):
"""Inject repo files into a relative directory inside the build context"""
host_repos_path = build_dir.path / RELATIVE_REPOS_PATH
self.log.info("creating directory for yum repos: %s", host_repos_path)
os.mkdir(host_repos_path)
allow_repo_dir_in_dockerignore(build_dir.path)
for repo in self.yum_repos[build_dir.platform]:
# Update every repo accordingly in a repofile
# input_buf ---- updated ----> updated_buf
with StringIO(repo.content.decode()) as input_buf, StringIO(
) as updated_buf:
for line in input_buf:
updated_buf.write(line)
# Apply sslcacert to every repo in a repofile
if line.lstrip().startswith(
'[') and self._builder_ca_bundle:
updated_buf.write(
f'sslcacert=/tmp/{self._ca_bundle_pem}\n')
yum_repo = YumRepo(repourl=repo.dst_filename,
content=updated_buf.getvalue(),
dst_repos_dir=host_repos_path,
add_hash=False)
yum_repo.write_content()
def run(self):
"""
run the plugin
"""
yum_repos = {
k: v
for k, v in self.workflow.files.items()
if k.startswith(YUM_REPOS_DIR)
}
if not yum_repos:
return
# absolute path in containers -> relative path within context
host_repos_path = os.path.join(self.workflow.builder.df_dir,
RELATIVE_REPOS_PATH)
self.log.info("creating directory for yum repos: %s", host_repos_path)
os.mkdir(host_repos_path)
for repo, repo_content in self.workflow.files.items():
yum_repo = YumRepo(repourl=repo,
content=repo_content,
dst_repos_dir=host_repos_path,
add_hash=False)
yum_repo.write_content()
# Find out the USER inherited from the base image
inspect = self.workflow.builder.base_image_inspect
inherited_user = ''
if not self.workflow.builder.base_from_scratch:
inherited_user = inspect.get(INSPECT_CONFIG).get('User', '')
df = df_parser(self.workflow.builder.df_path, workflow=self.workflow)
yum_repos = list(self.workflow.files)
add_yum_repos_to_dockerfile(yum_repos, df, inherited_user,
self.workflow.builder.base_from_scratch)
for repo in yum_repos:
self.log.info("injected yum repo: %s", repo)
def extract_base_url(self, repo_url):
yum_repo = YumRepo(repo_url)
yum_repo.fetch()
if not yum_repo.is_valid():
return []
repo = yum_repo.config
return [repo.get(section, 'baseurl') for section in repo.sections()
if repo.has_option(section, 'baseurl')]
def add_koji_repo(self):
xmlrpc = get_koji_session(self.workflow.conf)
pathinfo = self.workflow.conf.koji_path_info
proxy = self.workflow.conf.yum_proxy
if not self.target:
self.log.info('no target provided, not adding koji repo')
return
target_info = xmlrpc.getBuildTarget(self.target)
if target_info is None:
self.log.error("provided target '%s' doesn't exist", self.target)
raise RuntimeError("Provided target '%s' doesn't exist!" %
self.target)
tag_info = xmlrpc.getTag(target_info['build_tag_name'])
if not tag_info or 'name' not in tag_info:
self.log.warning("No tag info was retrieved")
return
repo_info = xmlrpc.getRepo(tag_info['id'])
if not repo_info or 'id' not in repo_info:
self.log.warning("No repo info was retrieved")
return
# to use urljoin, we would have to append '/', so let's append everything
baseurl = pathinfo.repo(repo_info['id'],
tag_info['name']) + "/$basearch"
self.log.info("baseurl = '%s'", baseurl)
repo = {
'name': 'atomic-reactor-koji-plugin-%s' % self.target,
'baseurl': baseurl,
'enabled': 1,
'gpgcheck': 0,
}
# yum doesn't accept a certificate path in sslcacert - it requires a db with added cert
# dnf ignores that option completely
# we have to fall back to sslverify=0 everytime we get https repo from brew so we'll surely
# be able to pull from it
if baseurl.startswith("https://"):
self.log.info("Ignoring certificates in the repo")
repo['sslverify'] = 0
if proxy:
self.log.info("Setting yum proxy to %s", proxy)
repo['proxy'] = proxy
yum_repo = YumRepo(os.path.join(YUM_REPOS_DIR, self.target))
path = yum_repo.dst_filename
self.log.info("yum repo of koji target: '%s'", path)
yum_repo.content = render_yum_repo(repo, escape_dollars=False)
for platform in self.platforms:
self.yum_repos[platform].append(yum_repo)
def test_write_content(tmpdir):
test_content = 'test_content'
repo = YumRepo(repourl='http://example.com/a/b/c/myrepo.repo',
content=test_content,
dst_repos_dir=str(tmpdir))
repo.write_content()
with open(os.path.join(str(tmpdir), repo.filename)) as f:
assert f.read() == test_content
def run(self):
"""
run the plugin
"""
if (self.workflow.builder.dockerfile_images.base_from_scratch and
not self.workflow.builder.dockerfile_images):
self.log.info("Skipping add yum repo by url: unsupported for FROM-scratch images")
return
if self.repourls and not is_scratch_build(self.workflow):
self.validate_yum_repo_files_url()
for repourl in self.repourls:
yumrepo = YumRepo(repourl)
self.log.info("fetching yum repo from '%s'", yumrepo.repourl)
try:
yumrepo.fetch()
except Exception as e:
msg = "Failed to fetch yum repo {repo}: {exc}".format(
repo=yumrepo.repourl, exc=e)
raise RuntimeError(msg) from e
else:
self.log.info("fetched yum repo from '%s'", yumrepo.repourl)
if self.inject_proxy:
if yumrepo.is_valid():
yumrepo.set_proxy_for_all_repos(self.inject_proxy)
self.workflow.files[yumrepo.dst_filename] = yumrepo.content.decode()
self.log.debug("saving yum repo '%s', length %d", yumrepo.dst_filename,
len(yumrepo.content))
def run(self):
"""
run the plugin
"""
if not self.workflow.data.dockerfile_images:
self.log.info(
"Skipping plugin, from scratch stage(s) can't add repos")
return
if self.include_koji_repo:
self.add_koji_repo()
else:
self.log.info(
"'include_koji_repo parameter is set to '%s', not including koji repo",
self.include_koji_repo)
if self.repourls and not is_scratch_build(self.workflow):
self.validate_yum_repo_files_url()
fetched_yum_repos = {}
for platform in self.platforms:
for repourl in self.repourls.get(platform, []):
if repourl in fetched_yum_repos:
yum_repo = fetched_yum_repos[repourl]
self.yum_repos[platform].append(yum_repo)
continue
yum_repo = YumRepo(repourl)
self.log.info("fetching yum repo from '%s'", yum_repo.repourl)
try:
yum_repo.fetch()
except Exception as e:
msg = "Failed to fetch yum repo {repo}: {exc}".format(
repo=yum_repo.repourl, exc=e)
raise RuntimeError(msg) from e
else:
self.log.info("fetched yum repo from '%s'",
yum_repo.repourl)
if self.inject_proxy:
if yum_repo.is_valid():
yum_repo.set_proxy_for_all_repos(self.inject_proxy)
self.log.debug("saving yum repo '%s', length %d",
yum_repo.dst_filename, len(yum_repo.content))
self.yum_repos[platform].append(yum_repo)
fetched_yum_repos[repourl] = yum_repo
if not self.yum_repos:
return
self._builder_ca_bundle = self.workflow.conf.builder_ca_bundle
if self._builder_ca_bundle:
self._ca_bundle_pem = os.path.basename(self._builder_ca_bundle)
self.workflow.build_dir.for_each_platform(self._inject_into_repo_files)
self.workflow.build_dir.for_each_platform(self._inject_into_dockerfile)
for platform in self.platforms:
for repo in self.yum_repos[platform]:
self.log.info("injected yum repo: %s for '%s' platform",
repo.dst_filename, platform)
def run(self):
"""
run the plugin
"""
if self.workflow.builder.base_from_scratch and not self.workflow.builder.parent_images:
self.log.info(
"from scratch single stage can't add repos from koji target")
return
target_info = self.xmlrpc.getBuildTarget(self.target)
if target_info is None:
self.log.error("provided target '%s' doesn't exist", self.target)
raise RuntimeError("Provided target '%s' doesn't exist!" %
self.target)
tag_info = self.xmlrpc.getTag(target_info['build_tag_name'])
if not tag_info or 'name' not in tag_info:
self.log.warning("No tag info was retrieved")
return
repo_info = self.xmlrpc.getRepo(tag_info['id'])
if not repo_info or 'id' not in repo_info:
self.log.warning("No repo info was retrieved")
return
# to use urljoin, we would have to append '/', so let's append everything
baseurl = self.pathinfo.repo(repo_info['id'],
tag_info['name']) + "/$basearch"
self.log.info("baseurl = '%s'", baseurl)
repo = {
'name': 'atomic-reactor-koji-plugin-%s' % self.target,
'baseurl': baseurl,
'enabled': 1,
'gpgcheck': 0,
}
# yum doesn't accept a certificate path in sslcacert - it requires a db with added cert
# dnf ignores that option completely
# we have to fall back to sslverify=0 everytime we get https repo from brew so we'll surely
# be able to pull from it
if baseurl.startswith("https://"):
self.log.info("Ignoring certificates in the repo")
repo['sslverify'] = 0
if self.proxy:
self.log.info("Setting yum proxy to %s", self.proxy)
repo['proxy'] = self.proxy
path = YumRepo(os.path.join(YUM_REPOS_DIR, self.target)).dst_filename
self.log.info("yum repo of koji target: '%s'", path)
self.workflow.files[path] = render_yum_repo(repo, escape_dollars=False)
def test_inject_repos(configure_ca_bundle, inherited_user, repos,
dockerfile_content, expected_final_dockerfile, tmpdir):
dockerfile = tmpdir.join('Dockerfile')
dockerfile.write_text(dockerfile_content, encoding='utf8')
tasker, workflow = prepare(str(dockerfile), inherited_user)
config = {
'version': 1,
# Ensure the AddYumRepoByUrlPlugin plugin is able to run
'yum_repo_allowed_domains': ['odcs.example.com', 'repos.host'],
}
if configure_ca_bundle:
config['builder_ca_bundle'] = BUILDER_CA_BUNDLE
workflow.plugin_workspace[ReactorConfigPlugin.key] = {
WORKSPACE_CONF_KEY: ReactorConfig(config)
}
# Ensure the ca_bundle PEM file is copied into build context
flexmock(shutil).should_receive('copyfile').with_args(
BUILDER_CA_BUNDLE,
str(tmpdir.join(CA_BUNDLE_PEM)),
)
for repofile_url, repofile_content, _ in repos:
responses.add(responses.GET, repofile_url, body=repofile_content)
PreBuildPluginsRunner(tasker, workflow, [
{
'name': AddYumRepoByUrlPlugin.key,
'args': {
'repourls': [url for url, _, _ in repos]
},
},
{
'name': InjectYumRepoPlugin.key,
'args': {},
},
]).run()
# Ensure Dockerfile is update correctly
hashes = [
sha256sum(repofile_url, abbrev_len=5) for repofile_url, _, _ in repos
]
expected = expected_final_dockerfile.format(*hashes)
assert expected == df_parser(str(dockerfile)).content
# Ensure the repofile is updated correctly as well
for repofile_url, _, expected_final_repofile in repos:
yum_repo = YumRepo(repofile_url)
updated_repos = tmpdir.join(RELATIVE_REPOS_PATH,
yum_repo.filename).read_text('utf-8')
assert expected_final_repofile == updated_repos
def test_inject_repos(configure_ca_bundle, inherited_user, include_koji_repo,
repos, dockerfile_content, expected_final_dockerfile,
workflow, build_dir):
platforms = ['x86_64', 'ppc64le']
yum_repourls = {}
for platform in platforms:
yum_repourls[platform] = [url for url, _, _ in repos]
workflow = prepare(workflow,
build_dir,
inherited_user,
dockerfile_content,
include_koji_repo=include_koji_repo,
platforms=platforms,
yum_repourls=yum_repourls)
workflow.conf.conf['yum_repo_allowed_domains'] = [
'odcs.example.com', 'repos.host'
]
if configure_ca_bundle:
workflow.conf.conf['builder_ca_bundle'] = BUILDER_CA_BUNDLE
# Ensure the ca_bundle PEM file is copied into build context
flexmock(shutil).should_receive('copyfile').with_args(
BUILDER_CA_BUNDLE,
(workflow.build_dir.any_platform.path / CA_BUNDLE_PEM))
flexmock(shutil).should_receive('copyfile').with_args(
BUILDER_CA_BUNDLE,
(workflow.build_dir.path / 'x86_64' / CA_BUNDLE_PEM))
for repofile_url, repofile_content, _ in repos:
responses.add(responses.GET, repofile_url, body=repofile_content)
PreBuildPluginsRunner(workflow, [
{
'name': InjectYumReposPlugin.key,
'args': {
'target': KOJI_TARGET
},
},
]).run()
# Ensure Dockerfile is update correctly
hashes = [
sha256sum(repofile_url, abbrev_len=5) for repofile_url, _, _ in repos
]
expected = expected_final_dockerfile.format(*hashes)
assert expected == workflow.build_dir.any_platform.dockerfile.content
# Ensure the repofile is updated correctly as well
for repofile_url, _, expected_final_repofile in repos:
yum_repo = YumRepo(repofile_url)
repos_path = workflow.build_dir.any_platform.path / RELATIVE_REPOS_PATH / yum_repo.filename
updated_repos = repos_path.read_text('utf-8')
assert expected_final_repofile == updated_repos
def test_invalid_config():
repo = YumRepo('http://example.com/a/b/c/myrepo.repo', 'line noise')
assert not repo.is_valid()
def test_add_repo_to_url_raises(repourl):
repo = YumRepo(repourl)
with pytest.raises(RuntimeError):
assert repo.filename
def test_add_repo_to_url(repourl, add_hash, pattern):
repo = YumRepo(repourl, add_hash=add_hash)
assert repo.repourl == repourl
assert fnmatch(repo.filename, pattern)
def test_invalid_config():
repo = YumRepo('http://example.com/a/b/c/myrepo.repo', 'line noise')
if (sys.version_info < (3, 0)):
assert not repo.is_valid()
else:
assert True
