def run_cekit(image_dir, args=None, descriptor=None):
if args is None:
args = ['build', '--dry-run', 'docker']
if descriptor is None:
descriptor = image_descriptor
with Chdir(image_dir):
with open('image.yaml', 'w') as fd:
yaml.dump(descriptor, fd, default_flow_style=False)
result = CliRunner().invoke(cli, args, catch_exceptions=False)
assert result.exit_code == 0
return result
def test_osbs_builder_with_fetch_artifacts_url_file_creation_naming(
tmpdir, mocker, caplog):
"""
Checks whether the fetch-artifacts-url.yaml file is generated with name specified.
"""
caplog.set_level(logging.DEBUG, logger="cekit")
res = mocker.Mock()
res.getcode.return_value = 200
res.read.side_effect = [b'test', None]
mocker.patch('cekit.descriptor.resource.urlopen', return_value=res)
mocker.patch('cekit.tools.decision', return_value=True)
mocker.patch.object(subprocess, 'check_output')
mocker.patch('cekit.builders.osbs.DistGit.push')
tmpdir.mkdir('osbs').mkdir('repo')
tmpdir.join('osbs').join('repo').join(
'fetch-artifacts-url.yaml').write_text(u'Some content', 'utf8')
with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')):
subprocess.call(["git", "init"])
subprocess.call(["git", "add", "fetch-artifacts-url.yaml"])
subprocess.call(["git", "commit", "-m", "Dummy"])
descriptor = image_descriptor.copy()
descriptor['artifacts'] = [{
'name': 'myfile.jar',
'sha256': '123456',
'url': 'https://foo/bar.jar'
}]
run_osbs(descriptor, str(tmpdir), mocker)
with open(
os.path.join(str(tmpdir), 'target', 'image',
'fetch-artifacts-url.yaml'), 'r') as _file:
fetch_artifacts = yaml.safe_load(_file)
assert len(fetch_artifacts) == 1
assert fetch_artifacts[0] == {
'sha256': '123456',
'target': 'myfile.jar',
'url': 'https://foo/bar.jar'
}
assert "Artifact 'myfile.jar' (as URL) added to fetch-artifacts-url.yaml" in caplog.text
def run_cekit_return_dockerfile(mocker, workdir, argv, getservbyport=getservbyport_works):
"""utility function to invoke cekit and return the generated Dockerfile"""
mocker.patch.object(sys, 'argv', argv)
mocker.patch.object(socket, 'getservbyport', getservbyport)
with Chdir(str(workdir)):
if os.path.exists('target'):
shutil.rmtree('target')
c = Cekit().parse()
c.configure()
try:
c.run()
except SystemExit:
pass
with open("target/image/Dockerfile", "r") as fd:
return fd.read()
def test_no_override_help_template(mocker, workdir, tmpdir):
cleanup(workdir)
config = setup_config(tmpdir, "")
mocker.patch.object(sys, 'argv',
['cekit', '-v', '--config', config, 'generate'])
with Chdir(workdir):
c = Cekit().parse()
c.configure()
try:
c.run()
except SystemExit:
pass
with open("target/image/help.md", "r") as fd:
contents = fd.read()
assert -1 == contents.find(template_teststr)
def run_cekit(cwd, parameters=None, message=None, env=None):
if parameters is None:
parameters = ['build', '--dry-run', 'docker']
if env is None:
env = {}
with Chdir(cwd):
result = CliRunner(env=env).invoke(cli, parameters, catch_exceptions=False)
assert result.exit_code == 0
if message:
assert message in result.output
return result
def repo_info(path):
with Chdir(path):
if subprocess.check_output(["git", "rev-parse", "--is-inside-work-tree"]).strip().decode("utf8") != "true":
raise Exception("Directory {} doesn't seem to be a git repository. "
"Please make sure you specified correct path.".format(path))
name = os.path.basename(subprocess.check_output(
["git", "rev-parse", "--show-toplevel"]).strip().decode("utf8"))
branch = subprocess.check_output(
["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip().decode("utf8")
commit = subprocess.check_output(
["git", "rev-parse", "HEAD"]).strip().decode("utf8")
return name, branch, commit
def test_osbs_builder_with_cachito_enabled(tmpdir, mocker, caplog):
"""
Checks whether the generated Dockerfile has cachito instructions if container.yaml
file has cachito section.
"""
caplog.set_level(logging.DEBUG, logger="cekit")
mocker.patch('cekit.tools.decision', return_value=True)
mocker.patch('cekit.descriptor.resource.urlopen')
mocker.patch('cekit.generator.osbs.get_brew_url',
return_value='http://random.url/path')
mocker.patch.object(subprocess, 'check_output')
mocker.patch('cekit.builders.osbs.DistGit.push')
tmpdir.mkdir('osbs').mkdir('repo')
with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')):
subprocess.call(["git", "init"])
subprocess.call(["touch", "file"])
subprocess.call(["git", "add", "file"])
subprocess.call(["git", "commit", "-m", "Dummy"])
descriptor = image_descriptor.copy()
descriptor["osbs"]["configuration"] = {
'container': {
'remote_source': {
'ref': '123456',
'repo': 'http://foo.bar.com'
},
'compose': {
'pulp_repos': True
}
}
}
run_osbs(descriptor, str(tmpdir), mocker)
dockerfile_path = os.path.join(str(tmpdir), 'target', 'image',
'Dockerfile')
assert os.path.exists(dockerfile_path) is True
with open(dockerfile_path, 'r') as _file:
dockerfile = _file.read()
assert "COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR" in dockerfile
assert re.search("Cachito definition is .*http://foo.bar.com", caplog.text)
def test_osbs_builder_container_yaml_existence(tmpdir, mocker, caplog, flag):
"""
Make sure that the osbs section is properly merged.
The evidence is that the container.yaml file is generated.
https://github.com/cekit/cekit/issues/631
"""
caplog.set_level(logging.DEBUG, logger="cekit")
mocker.patch('cekit.tools.decision', return_value=True)
mocker.patch('cekit.descriptor.resource.urlopen')
mocker.patch('cekit.generator.osbs.get_brew_url',
return_value='http://random.url/path')
mocker.patch.object(subprocess, 'check_output')
mocker.patch('cekit.builders.osbs.DistGit.push')
tmpdir.mkdir('osbs').mkdir('repo')
with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')):
subprocess.call(["git", "init"])
subprocess.call(["touch", "file"])
subprocess.call(["git", "add", "file"])
subprocess.call(["git", "commit", "-m", "Dummy"])
descriptor = image_descriptor.copy()
descriptor["osbs"]["configuration"] = {
'container': {
'compose': {
'pulp_repos': True
}
}
}
run_osbs(descriptor, str(tmpdir), mocker, general_command=flag)
assert os.path.exists(
os.path.join(str(tmpdir), 'osbs', 'repo', 'container.yaml'))
dockerfile_path = os.path.join(str(tmpdir), 'target', 'image',
'Dockerfile')
assert os.path.exists(dockerfile_path) is True
with open(dockerfile_path, 'r') as _file:
dockerfile = _file.read()
assert "COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR" not in dockerfile
def clean(self):
""" Removes old generated scripts, repos and modules directories """
with Chdir(self.output):
git_files = subprocess.check_output(
["git", "ls-files", "."]).strip().decode("utf8").splitlines()
for d in ["repos", "modules"]:
LOGGER.info("Removing old '{}' directory".format(d))
shutil.rmtree(d, ignore_errors=True)
if d in git_files:
subprocess.check_call(["git", "rm", "-rf", d])
if os.path.exists("fetch-artifacts-url.yaml"):
LOGGER.info("Removing old 'fetch-artifacts-url.yaml' file")
subprocess.check_call(
["git", "rm", "-rf", "fetch-artifacts-url.yaml"])
def update_lookaside_cache(self):
LOGGER.info("Updating lookaside cache...")
if not self.artifacts:
return
cmd = [self._fedpkg]
if self.params.user:
cmd += ['--user', self.params.user]
cmd += ["new-sources"] + self.artifacts
LOGGER.debug("Executing '%s'" % cmd)
with Chdir(self.dist_git_dir):
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
LOGGER.error("Cannot run '%s', ouput: '%s'" % (cmd, ex.output))
raise CekitError("Cannot update sources.")
LOGGER.info("Update finished.")
def test_osbs_builder_with_fetch_artifacts_plain_file_creation(
tmpdir, mocker, caplog):
"""
Checks whether the fetch-artifacts-url.yaml file is generated with plain artifact.
"""
caplog.set_level(logging.DEBUG, logger="cekit")
mocker.patch('cekit.tools.decision', return_value=True)
mocker.patch('cekit.descriptor.resource.urlopen')
mocker.patch('cekit.generator.osbs.get_brew_url',
return_value='http://random.url/path')
mocker.patch.object(subprocess, 'check_output')
mocker.patch('cekit.builders.osbs.DistGit.push')
tmpdir.mkdir('osbs').mkdir('repo')
tmpdir.join('osbs').join('repo').join(
'fetch-artifacts-url.yaml').write_text(u'Some content', 'utf8')
with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')):
subprocess.call(["git", "init"])
subprocess.call(["git", "add", "fetch-artifacts-url.yaml"])
subprocess.call(["git", "commit", "-m", "Dummy"])
descriptor = image_descriptor.copy()
descriptor['artifacts'] = [{'name': 'artifact_name', 'md5': '123456'}]
run_osbs(descriptor, str(tmpdir), mocker)
with open(
os.path.join(str(tmpdir), 'target', 'image',
'fetch-artifacts-url.yaml'), 'r') as _file:
fetch_artifacts = yaml.safe_load(_file)
assert len(fetch_artifacts) == 1
assert fetch_artifacts[0] == {
'md5': '123456',
'target': 'artifact_name',
'url': 'http://random.url/path'
}
assert "Artifact 'artifact_name' (as plain) added to fetch-artifacts-url.yaml" in caplog.text
def run(self, image, run_tags, test_names):
"""Run test suite"""
test_path = os.path.join(self.target, 'test')
logger.debug("Running behave in '%s'." % test_path)
args = [test_path,
'--junit',
'--junit-directory', 'results',
'--no-skipped',
'-t', '~ignore',
'-D', 'IMAGE=%s' % image]
if test_names:
for name in test_names:
args.append('--name')
args.append("%s" % name)
else:
for tag in run_tags:
if ':' in tag:
test_tag = tag.split(':')[0]
args.append('-t')
if '/' in tag:
args.append("@%s,@%s" % (test_tag.split('/')[0], test_tag))
else:
args.append(tag)
# Check if we're running runtests on CI or locally
# If we run tests locally - skip all features that
# are marked with the @ci annotation
if getpass.getuser() != "jenkins":
args.append("-t")
args.append("~ci ")
try:
from behave.__main__ import main as behave_main
with Chdir(os.path.join(self.target, 'test')):
if behave_main(args) != 0:
raise CekitError("Test execution failed, please consult output above")
except CekitError:
raise
except:
raise CekitError("An error occurred while executing tests")
def _build_image(overrides=None):
image_dir = tempfile.mkdtemp(prefix="tmp-cekit-test")
with open(os.path.join(image_dir, 'image.yaml'), 'w') as fd:
yaml.dump(image_descriptor, fd, default_flow_style=False)
args = ["-v", "build"]
if overrides:
args += ["--overrides", overrides]
args.append("docker")
with Chdir(image_dir):
result = CliRunner().invoke(cli, args, catch_exceptions=False)
assert result.exit_code == 0
return image_dir
def test_osbs_builder_with_fetch_artifacts_url_file_creation_1(tmpdir, mocker, caplog, flag):
"""
Checks whether the fetch-artifacts-url.yaml file is generated with URL artifact with md5 checksum.
"""
caplog.set_level(logging.DEBUG, logger="cekit")
mocker.patch('cekit.tools.decision', return_value=True)
mocker.patch('cekit.descriptor.resource.urlopen')
mocker.patch.object(subprocess, 'check_output')
mocker.patch('cekit.builders.osbs.DistGit.push')
tmpdir.mkdir('osbs').mkdir('repo')
tmpdir.join('osbs').join('repo').join(
'fetch-artifacts-url.yaml').write_text(u'Some content', 'utf8')
with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')):
subprocess.call(["git", "init"])
subprocess.call(["git", "add", "fetch-artifacts-url.yaml"])
subprocess.call(["git", "commit", "-m", "Dummy"])
descriptor = image_descriptor.copy()
descriptor['artifacts'] = [
{'name': 'artifact_name', 'md5': '123456', 'url': 'https://foo/bar.jar', 'description': 'http://foo.com/123456'}
]
run_osbs(descriptor, str(tmpdir), mocker, general_command=flag)
fau = os.path.join(str(tmpdir), 'target', 'image', 'fetch-artifacts-url.yaml')
with open(fau, 'r') as _file:
fetch_artifacts = yaml.safe_load(_file)
assert len(fetch_artifacts) == 1
assert fetch_artifacts[0] == {'md5': '123456',
'target': 'artifact_name', 'url': 'https://foo/bar.jar'}
if len(flag):
with open(fau) as myfile:
assert "https://foo/bar.jar # http://foo.com/123456" in myfile.read()
assert "Artifact 'artifact_name' (as URL) added to fetch-artifacts-url.yaml" in caplog.text
def test_config_override_help_template(mocker, workdir, tmpdir):
cleanup(workdir)
help_template = os.path.join(workdir, "help.jinja")
with open(help_template, "w") as fd:
fd.write(template_teststr)
config = setup_config(tmpdir,
"[doc]\nhelp_template = {}".format(help_template))
mocker.patch.object(sys, 'argv',
['cekit', '-v', '--config', config, 'generate'])
with Chdir(workdir):
c = Cekit().parse()
c.configure()
try:
c.run()
except SystemExit:
pass
with open("target/image/help.md", "r") as fd:
contents = fd.read()
assert contents.find(template_teststr) >= 0
def generate(image_dir, command, descriptor=None, exit_code=0):
desc = basic_config.copy()
if descriptor:
desc.update(descriptor)
tmp_image_file = os.path.join(image_dir, 'image.yaml')
with open(tmp_image_file, 'w') as outfile:
yaml.dump(desc, outfile, default_flow_style=False)
with Chdir(image_dir):
result = CliRunner().invoke(cli, command, catch_exceptions=False)
assert result.exit_code == exit_code
if exit_code != 0:
return
with open(os.path.join(image_dir, 'target', 'image.yaml'), 'r') as desc:
return yaml.safe_load(desc)
def test_custom_help_template_path(tmpdir, path_type):
help_template = os.path.join(str(tmpdir), "help.jinja")
if path_type == 'relative':
help_template = "help.jinja"
my_image_descriptor = image_descriptor.copy()
my_image_descriptor['help'] = {'template': help_template, 'add': True}
with Chdir(str(tmpdir)):
with open('help.jinja', "w") as fd:
fd.write(template_teststr)
with open('image.yaml', 'w') as fd:
yaml.dump(my_image_descriptor, fd, default_flow_style=False)
CliRunner().invoke(cli, ['-v', 'build', '--dry-run', 'docker'],
catch_exceptions=False)
with open("target/image/help.md", "r") as fd:
contents = fd.read()
assert contents.find(template_teststr) >= 0
def build(self):
cmd = [self._rhpkg]
if self._user:
cmd += ['--user', self._user]
cmd.append("container-build")
if self._target:
cmd += ['--target', self._target]
if self._nowait:
cmd += ['--nowait']
if self._rhpkg_set_url_repos:
cmd += ['--repo-url']
cmd += self._rhpkg_set_url_repos
if not self._release:
cmd.append("--scratch")
with Chdir(self.dist_git_dir):
self.dist_git.add()
self.update_lookaside_cache()
if self.dist_git.stage_modified():
self.dist_git.commit(self._commit_msg)
self.dist_git.push()
else:
logger.info("No changes made to the code, committing skipped")
logger.info("About to execute '%s'." % ' '.join(cmd))
if tools.decision("Do you want to build the image in OSBS?"):
build_type = "release" if self._release else "scratch"
logger.info("Executing %s container build in OSBS..." %
build_type)
subprocess.check_call(cmd)
def run(self):
cmd = [self._koji]
if self.params.user:
cmd += ['--user', self.params.user]
cmd += ['call', '--python', 'buildContainer', '--kwargs']
with Chdir(self.dist_git_dir):
self.dist_git.add()
self.update_lookaside_cache()
if self.dist_git.stage_modified():
self.dist_git.commit(self.params.commit_message)
self.dist_git.push()
else:
LOGGER.info("No changes made to the code, committing skipped")
# Get the url of the repository
url = subprocess.check_output(
["git", "config", "--get",
"remote.origin.url"]).strip().decode("utf8")
# Get the latest commit hash
commit = subprocess.check_output(["git", "rev-parse",
"HEAD"]).strip().decode("utf8")
# Parse the dist-git repository url
url = urlparse(url)
# Construct the url again, with a hash and removed username and password, if any
src = "git://{}{}#{}".format(url.hostname, url.path, commit)
target = "{}-containers-candidate".format(self.dist_git.branch)
if self.params.koji_target:
target = self.params.koji_target
scratch = True
if self.params.release:
scratch = False
kwargs = "{{'src': '{}', 'target': '{}', 'opts': {{'scratch': {}, 'git_branch': '{}', 'yum_repourls': {}}}}}".format(
src, target, scratch, self.dist_git.branch,
self._rhpkg_set_url_repos)
cmd.append(kwargs)
LOGGER.info("About to execute '%s'." % ' '.join(cmd))
if tools.decision("Do you want to build the image in OSBS?"):
build_type = "scratch" if scratch else "release"
LOGGER.info("Executing %s container build in OSBS..." %
build_type)
try:
task_id = subprocess.check_output(cmd).strip().decode(
"utf8")
except subprocess.CalledProcessError as ex:
raise CekitError("Building container image in OSBS failed",
ex)
LOGGER.info(
"Task {0} was submitted, you can watch the progress here: {1}/taskinfo?taskID={0}"
.format(task_id, self._koji_url))
if self.params.nowait:
return
self._wait_for_osbs_task(task_id)
LOGGER.info("Image was built successfully in OSBS!")
def run_cekit_exception(cwd):
with Chdir(cwd):
# run cekit and check it exits with 0
with pytest.raises(SystemExit) as system_exit:
Cekit().parse().run()
assert system_exit.value.code == 1
def run_cekit(cwd):
with Chdir(cwd):
c = Cekit().parse()
c.configure()
return c.generator._params['addhelp']
def run(self):
if self.params.sync_only:
LOGGER.info(
"The --sync-only parameter was specified, build will not be executed, exiting"
)
return
cmd = [self._koji]
if self.params.user:
cmd += ['--user', self.params.user]
cmd += ['call', '--python', 'buildContainer', '--kwargs']
with Chdir(self.dist_git_dir):
# Get the url of the repository
url = subprocess.check_output(
["git", "config", "--get",
"remote.origin.url"]).strip().decode("utf8")
# Get the latest commit hash
commit = subprocess.check_output(["git", "rev-parse",
"HEAD"]).strip().decode("utf8")
# Parse the dist-git repository url
url = urlparse(url)
# Construct the url again, with a hash and removed username and password, if any
src = "git://{}{}#{}".format(url.hostname, url.path, commit)
target = self.generator.image.get('osbs', {}).get('koji_target')
# If target was not specified in the image descriptor
if not target:
# Default to computed target based on branch
target = "{}-containers-candidate".format(self.dist_git.branch)
# Check if it was specified on command line
# TODO: Remove in 3.6
if self.params.koji_target:
target = self.params.koji_target
scratch = True
if self.params.release:
scratch = False
kwargs = "{{'src': '{}', 'target': '{}', 'opts': {{'scratch': {}, 'git_branch': '{}', 'yum_repourls': {}}}}}".format(
src, target, scratch, self.dist_git.branch,
self._rhpkg_set_url_repos)
cmd.append(kwargs)
LOGGER.info("About to execute '{}'.".format(' '.join(cmd)))
if self.params.assume_yes or tools.decision(
"Do you want to build the image in OSBS?"):
build_type = "scratch" if scratch else "release"
LOGGER.info("Executing {} container build in OSBS...".format(
build_type))
try:
task_id = subprocess.check_output(cmd).strip().decode(
"utf8")
except subprocess.CalledProcessError as ex:
raise CekitError("Building container image in OSBS failed",
ex)
LOGGER.info(
"Task {0} was submitted, you can watch the progress here: {1}/taskinfo?taskID={0}"
.format(task_id, self._koji_url))
if self.params.nowait:
return
self._wait_for_osbs_task(task_id)
LOGGER.info("Image was built successfully in OSBS!")