def run_cekit(image_dir, args=None, descriptor=None): if args is None: args = ['build', '--dry-run', 'docker'] if descriptor is None: descriptor = image_descriptor with Chdir(image_dir): with open('image.yaml', 'w') as fd: yaml.dump(descriptor, fd, default_flow_style=False) result = CliRunner().invoke(cli, args, catch_exceptions=False) assert result.exit_code == 0 return result
def test_osbs_builder_with_fetch_artifacts_url_file_creation_naming( tmpdir, mocker, caplog): """ Checks whether the fetch-artifacts-url.yaml file is generated with name specified. """ caplog.set_level(logging.DEBUG, logger="cekit") res = mocker.Mock() res.getcode.return_value = 200 res.read.side_effect = [b'test', None] mocker.patch('cekit.descriptor.resource.urlopen', return_value=res) mocker.patch('cekit.tools.decision', return_value=True) mocker.patch.object(subprocess, 'check_output') mocker.patch('cekit.builders.osbs.DistGit.push') tmpdir.mkdir('osbs').mkdir('repo') tmpdir.join('osbs').join('repo').join( 'fetch-artifacts-url.yaml').write_text(u'Some content', 'utf8') with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')): subprocess.call(["git", "init"]) subprocess.call(["git", "add", "fetch-artifacts-url.yaml"]) subprocess.call(["git", "commit", "-m", "Dummy"]) descriptor = image_descriptor.copy() descriptor['artifacts'] = [{ 'name': 'myfile.jar', 'sha256': '123456', 'url': 'https://foo/bar.jar' }] run_osbs(descriptor, str(tmpdir), mocker) with open( os.path.join(str(tmpdir), 'target', 'image', 'fetch-artifacts-url.yaml'), 'r') as _file: fetch_artifacts = yaml.safe_load(_file) assert len(fetch_artifacts) == 1 assert fetch_artifacts[0] == { 'sha256': '123456', 'target': 'myfile.jar', 'url': 'https://foo/bar.jar' } assert "Artifact 'myfile.jar' (as URL) added to fetch-artifacts-url.yaml" in caplog.text
def run_cekit_return_dockerfile(mocker, workdir, argv, getservbyport=getservbyport_works): """utility function to invoke cekit and return the generated Dockerfile""" mocker.patch.object(sys, 'argv', argv) mocker.patch.object(socket, 'getservbyport', getservbyport) with Chdir(str(workdir)): if os.path.exists('target'): shutil.rmtree('target') c = Cekit().parse() c.configure() try: c.run() except SystemExit: pass with open("target/image/Dockerfile", "r") as fd: return fd.read()
def test_no_override_help_template(mocker, workdir, tmpdir): cleanup(workdir) config = setup_config(tmpdir, "") mocker.patch.object(sys, 'argv', ['cekit', '-v', '--config', config, 'generate']) with Chdir(workdir): c = Cekit().parse() c.configure() try: c.run() except SystemExit: pass with open("target/image/help.md", "r") as fd: contents = fd.read() assert -1 == contents.find(template_teststr)
def run_cekit(cwd, parameters=None, message=None, env=None): if parameters is None: parameters = ['build', '--dry-run', 'docker'] if env is None: env = {} with Chdir(cwd): result = CliRunner(env=env).invoke(cli, parameters, catch_exceptions=False) assert result.exit_code == 0 if message: assert message in result.output return result
def repo_info(path): with Chdir(path): if subprocess.check_output(["git", "rev-parse", "--is-inside-work-tree"]).strip().decode("utf8") != "true": raise Exception("Directory {} doesn't seem to be a git repository. " "Please make sure you specified correct path.".format(path)) name = os.path.basename(subprocess.check_output( ["git", "rev-parse", "--show-toplevel"]).strip().decode("utf8")) branch = subprocess.check_output( ["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip().decode("utf8") commit = subprocess.check_output( ["git", "rev-parse", "HEAD"]).strip().decode("utf8") return name, branch, commit
def test_osbs_builder_with_cachito_enabled(tmpdir, mocker, caplog): """ Checks whether the generated Dockerfile has cachito instructions if container.yaml file has cachito section. """ caplog.set_level(logging.DEBUG, logger="cekit") mocker.patch('cekit.tools.decision', return_value=True) mocker.patch('cekit.descriptor.resource.urlopen') mocker.patch('cekit.generator.osbs.get_brew_url', return_value='http://random.url/path') mocker.patch.object(subprocess, 'check_output') mocker.patch('cekit.builders.osbs.DistGit.push') tmpdir.mkdir('osbs').mkdir('repo') with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')): subprocess.call(["git", "init"]) subprocess.call(["touch", "file"]) subprocess.call(["git", "add", "file"]) subprocess.call(["git", "commit", "-m", "Dummy"]) descriptor = image_descriptor.copy() descriptor["osbs"]["configuration"] = { 'container': { 'remote_source': { 'ref': '123456', 'repo': 'http://foo.bar.com' }, 'compose': { 'pulp_repos': True } } } run_osbs(descriptor, str(tmpdir), mocker) dockerfile_path = os.path.join(str(tmpdir), 'target', 'image', 'Dockerfile') assert os.path.exists(dockerfile_path) is True with open(dockerfile_path, 'r') as _file: dockerfile = _file.read() assert "COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR" in dockerfile assert re.search("Cachito definition is .*http://foo.bar.com", caplog.text)
def test_osbs_builder_container_yaml_existence(tmpdir, mocker, caplog, flag): """ Make sure that the osbs section is properly merged. The evidence is that the container.yaml file is generated. https://github.com/cekit/cekit/issues/631 """ caplog.set_level(logging.DEBUG, logger="cekit") mocker.patch('cekit.tools.decision', return_value=True) mocker.patch('cekit.descriptor.resource.urlopen') mocker.patch('cekit.generator.osbs.get_brew_url', return_value='http://random.url/path') mocker.patch.object(subprocess, 'check_output') mocker.patch('cekit.builders.osbs.DistGit.push') tmpdir.mkdir('osbs').mkdir('repo') with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')): subprocess.call(["git", "init"]) subprocess.call(["touch", "file"]) subprocess.call(["git", "add", "file"]) subprocess.call(["git", "commit", "-m", "Dummy"]) descriptor = image_descriptor.copy() descriptor["osbs"]["configuration"] = { 'container': { 'compose': { 'pulp_repos': True } } } run_osbs(descriptor, str(tmpdir), mocker, general_command=flag) assert os.path.exists( os.path.join(str(tmpdir), 'osbs', 'repo', 'container.yaml')) dockerfile_path = os.path.join(str(tmpdir), 'target', 'image', 'Dockerfile') assert os.path.exists(dockerfile_path) is True with open(dockerfile_path, 'r') as _file: dockerfile = _file.read() assert "COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR" not in dockerfile
def clean(self): """ Removes old generated scripts, repos and modules directories """ with Chdir(self.output): git_files = subprocess.check_output( ["git", "ls-files", "."]).strip().decode("utf8").splitlines() for d in ["repos", "modules"]: LOGGER.info("Removing old '{}' directory".format(d)) shutil.rmtree(d, ignore_errors=True) if d in git_files: subprocess.check_call(["git", "rm", "-rf", d]) if os.path.exists("fetch-artifacts-url.yaml"): LOGGER.info("Removing old 'fetch-artifacts-url.yaml' file") subprocess.check_call( ["git", "rm", "-rf", "fetch-artifacts-url.yaml"])
def update_lookaside_cache(self): LOGGER.info("Updating lookaside cache...") if not self.artifacts: return cmd = [self._fedpkg] if self.params.user: cmd += ['--user', self.params.user] cmd += ["new-sources"] + self.artifacts LOGGER.debug("Executing '%s'" % cmd) with Chdir(self.dist_git_dir): try: subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as ex: LOGGER.error("Cannot run '%s', ouput: '%s'" % (cmd, ex.output)) raise CekitError("Cannot update sources.") LOGGER.info("Update finished.")
def test_osbs_builder_with_fetch_artifacts_plain_file_creation( tmpdir, mocker, caplog): """ Checks whether the fetch-artifacts-url.yaml file is generated with plain artifact. """ caplog.set_level(logging.DEBUG, logger="cekit") mocker.patch('cekit.tools.decision', return_value=True) mocker.patch('cekit.descriptor.resource.urlopen') mocker.patch('cekit.generator.osbs.get_brew_url', return_value='http://random.url/path') mocker.patch.object(subprocess, 'check_output') mocker.patch('cekit.builders.osbs.DistGit.push') tmpdir.mkdir('osbs').mkdir('repo') tmpdir.join('osbs').join('repo').join( 'fetch-artifacts-url.yaml').write_text(u'Some content', 'utf8') with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')): subprocess.call(["git", "init"]) subprocess.call(["git", "add", "fetch-artifacts-url.yaml"]) subprocess.call(["git", "commit", "-m", "Dummy"]) descriptor = image_descriptor.copy() descriptor['artifacts'] = [{'name': 'artifact_name', 'md5': '123456'}] run_osbs(descriptor, str(tmpdir), mocker) with open( os.path.join(str(tmpdir), 'target', 'image', 'fetch-artifacts-url.yaml'), 'r') as _file: fetch_artifacts = yaml.safe_load(_file) assert len(fetch_artifacts) == 1 assert fetch_artifacts[0] == { 'md5': '123456', 'target': 'artifact_name', 'url': 'http://random.url/path' } assert "Artifact 'artifact_name' (as plain) added to fetch-artifacts-url.yaml" in caplog.text
def run(self, image, run_tags, test_names): """Run test suite""" test_path = os.path.join(self.target, 'test') logger.debug("Running behave in '%s'." % test_path) args = [test_path, '--junit', '--junit-directory', 'results', '--no-skipped', '-t', '~ignore', '-D', 'IMAGE=%s' % image] if test_names: for name in test_names: args.append('--name') args.append("%s" % name) else: for tag in run_tags: if ':' in tag: test_tag = tag.split(':')[0] args.append('-t') if '/' in tag: args.append("@%s,@%s" % (test_tag.split('/')[0], test_tag)) else: args.append(tag) # Check if we're running runtests on CI or locally # If we run tests locally - skip all features that # are marked with the @ci annotation if getpass.getuser() != "jenkins": args.append("-t") args.append("~ci ") try: from behave.__main__ import main as behave_main with Chdir(os.path.join(self.target, 'test')): if behave_main(args) != 0: raise CekitError("Test execution failed, please consult output above") except CekitError: raise except: raise CekitError("An error occurred while executing tests")
def _build_image(overrides=None): image_dir = tempfile.mkdtemp(prefix="tmp-cekit-test") with open(os.path.join(image_dir, 'image.yaml'), 'w') as fd: yaml.dump(image_descriptor, fd, default_flow_style=False) args = ["-v", "build"] if overrides: args += ["--overrides", overrides] args.append("docker") with Chdir(image_dir): result = CliRunner().invoke(cli, args, catch_exceptions=False) assert result.exit_code == 0 return image_dir
def test_osbs_builder_with_fetch_artifacts_url_file_creation_1(tmpdir, mocker, caplog, flag): """ Checks whether the fetch-artifacts-url.yaml file is generated with URL artifact with md5 checksum. """ caplog.set_level(logging.DEBUG, logger="cekit") mocker.patch('cekit.tools.decision', return_value=True) mocker.patch('cekit.descriptor.resource.urlopen') mocker.patch.object(subprocess, 'check_output') mocker.patch('cekit.builders.osbs.DistGit.push') tmpdir.mkdir('osbs').mkdir('repo') tmpdir.join('osbs').join('repo').join( 'fetch-artifacts-url.yaml').write_text(u'Some content', 'utf8') with Chdir(os.path.join(str(tmpdir), 'osbs', 'repo')): subprocess.call(["git", "init"]) subprocess.call(["git", "add", "fetch-artifacts-url.yaml"]) subprocess.call(["git", "commit", "-m", "Dummy"]) descriptor = image_descriptor.copy() descriptor['artifacts'] = [ {'name': 'artifact_name', 'md5': '123456', 'url': 'https://foo/bar.jar', 'description': 'http://foo.com/123456'} ] run_osbs(descriptor, str(tmpdir), mocker, general_command=flag) fau = os.path.join(str(tmpdir), 'target', 'image', 'fetch-artifacts-url.yaml') with open(fau, 'r') as _file: fetch_artifacts = yaml.safe_load(_file) assert len(fetch_artifacts) == 1 assert fetch_artifacts[0] == {'md5': '123456', 'target': 'artifact_name', 'url': 'https://foo/bar.jar'} if len(flag): with open(fau) as myfile: assert "https://foo/bar.jar # http://foo.com/123456" in myfile.read() assert "Artifact 'artifact_name' (as URL) added to fetch-artifacts-url.yaml" in caplog.text
def test_config_override_help_template(mocker, workdir, tmpdir): cleanup(workdir) help_template = os.path.join(workdir, "help.jinja") with open(help_template, "w") as fd: fd.write(template_teststr) config = setup_config(tmpdir, "[doc]\nhelp_template = {}".format(help_template)) mocker.patch.object(sys, 'argv', ['cekit', '-v', '--config', config, 'generate']) with Chdir(workdir): c = Cekit().parse() c.configure() try: c.run() except SystemExit: pass with open("target/image/help.md", "r") as fd: contents = fd.read() assert contents.find(template_teststr) >= 0
def generate(image_dir, command, descriptor=None, exit_code=0): desc = basic_config.copy() if descriptor: desc.update(descriptor) tmp_image_file = os.path.join(image_dir, 'image.yaml') with open(tmp_image_file, 'w') as outfile: yaml.dump(desc, outfile, default_flow_style=False) with Chdir(image_dir): result = CliRunner().invoke(cli, command, catch_exceptions=False) assert result.exit_code == exit_code if exit_code != 0: return with open(os.path.join(image_dir, 'target', 'image.yaml'), 'r') as desc: return yaml.safe_load(desc)
def test_custom_help_template_path(tmpdir, path_type): help_template = os.path.join(str(tmpdir), "help.jinja") if path_type == 'relative': help_template = "help.jinja" my_image_descriptor = image_descriptor.copy() my_image_descriptor['help'] = {'template': help_template, 'add': True} with Chdir(str(tmpdir)): with open('help.jinja', "w") as fd: fd.write(template_teststr) with open('image.yaml', 'w') as fd: yaml.dump(my_image_descriptor, fd, default_flow_style=False) CliRunner().invoke(cli, ['-v', 'build', '--dry-run', 'docker'], catch_exceptions=False) with open("target/image/help.md", "r") as fd: contents = fd.read() assert contents.find(template_teststr) >= 0
def build(self): cmd = [self._rhpkg] if self._user: cmd += ['--user', self._user] cmd.append("container-build") if self._target: cmd += ['--target', self._target] if self._nowait: cmd += ['--nowait'] if self._rhpkg_set_url_repos: cmd += ['--repo-url'] cmd += self._rhpkg_set_url_repos if not self._release: cmd.append("--scratch") with Chdir(self.dist_git_dir): self.dist_git.add() self.update_lookaside_cache() if self.dist_git.stage_modified(): self.dist_git.commit(self._commit_msg) self.dist_git.push() else: logger.info("No changes made to the code, committing skipped") logger.info("About to execute '%s'." % ' '.join(cmd)) if tools.decision("Do you want to build the image in OSBS?"): build_type = "release" if self._release else "scratch" logger.info("Executing %s container build in OSBS..." % build_type) subprocess.check_call(cmd)
def run(self): cmd = [self._koji] if self.params.user: cmd += ['--user', self.params.user] cmd += ['call', '--python', 'buildContainer', '--kwargs'] with Chdir(self.dist_git_dir): self.dist_git.add() self.update_lookaside_cache() if self.dist_git.stage_modified(): self.dist_git.commit(self.params.commit_message) self.dist_git.push() else: LOGGER.info("No changes made to the code, committing skipped") # Get the url of the repository url = subprocess.check_output( ["git", "config", "--get", "remote.origin.url"]).strip().decode("utf8") # Get the latest commit hash commit = subprocess.check_output(["git", "rev-parse", "HEAD"]).strip().decode("utf8") # Parse the dist-git repository url url = urlparse(url) # Construct the url again, with a hash and removed username and password, if any src = "git://{}{}#{}".format(url.hostname, url.path, commit) target = "{}-containers-candidate".format(self.dist_git.branch) if self.params.koji_target: target = self.params.koji_target scratch = True if self.params.release: scratch = False kwargs = "{{'src': '{}', 'target': '{}', 'opts': {{'scratch': {}, 'git_branch': '{}', 'yum_repourls': {}}}}}".format( src, target, scratch, self.dist_git.branch, self._rhpkg_set_url_repos) cmd.append(kwargs) LOGGER.info("About to execute '%s'." % ' '.join(cmd)) if tools.decision("Do you want to build the image in OSBS?"): build_type = "scratch" if scratch else "release" LOGGER.info("Executing %s container build in OSBS..." % build_type) try: task_id = subprocess.check_output(cmd).strip().decode( "utf8") except subprocess.CalledProcessError as ex: raise CekitError("Building container image in OSBS failed", ex) LOGGER.info( "Task {0} was submitted, you can watch the progress here: {1}/taskinfo?taskID={0}" .format(task_id, self._koji_url)) if self.params.nowait: return self._wait_for_osbs_task(task_id) LOGGER.info("Image was built successfully in OSBS!")
def run_cekit_exception(cwd): with Chdir(cwd): # run cekit and check it exits with 0 with pytest.raises(SystemExit) as system_exit: Cekit().parse().run() assert system_exit.value.code == 1
def run_cekit(cwd): with Chdir(cwd): c = Cekit().parse() c.configure() return c.generator._params['addhelp']
def run(self): if self.params.sync_only: LOGGER.info( "The --sync-only parameter was specified, build will not be executed, exiting" ) return cmd = [self._koji] if self.params.user: cmd += ['--user', self.params.user] cmd += ['call', '--python', 'buildContainer', '--kwargs'] with Chdir(self.dist_git_dir): # Get the url of the repository url = subprocess.check_output( ["git", "config", "--get", "remote.origin.url"]).strip().decode("utf8") # Get the latest commit hash commit = subprocess.check_output(["git", "rev-parse", "HEAD"]).strip().decode("utf8") # Parse the dist-git repository url url = urlparse(url) # Construct the url again, with a hash and removed username and password, if any src = "git://{}{}#{}".format(url.hostname, url.path, commit) target = self.generator.image.get('osbs', {}).get('koji_target') # If target was not specified in the image descriptor if not target: # Default to computed target based on branch target = "{}-containers-candidate".format(self.dist_git.branch) # Check if it was specified on command line # TODO: Remove in 3.6 if self.params.koji_target: target = self.params.koji_target scratch = True if self.params.release: scratch = False kwargs = "{{'src': '{}', 'target': '{}', 'opts': {{'scratch': {}, 'git_branch': '{}', 'yum_repourls': {}}}}}".format( src, target, scratch, self.dist_git.branch, self._rhpkg_set_url_repos) cmd.append(kwargs) LOGGER.info("About to execute '{}'.".format(' '.join(cmd))) if self.params.assume_yes or tools.decision( "Do you want to build the image in OSBS?"): build_type = "scratch" if scratch else "release" LOGGER.info("Executing {} container build in OSBS...".format( build_type)) try: task_id = subprocess.check_output(cmd).strip().decode( "utf8") except subprocess.CalledProcessError as ex: raise CekitError("Building container image in OSBS failed", ex) LOGGER.info( "Task {0} was submitted, you can watch the progress here: {1}/taskinfo?taskID={0}" .format(task_id, self._koji_url)) if self.params.nowait: return self._wait_for_osbs_task(task_id) LOGGER.info("Image was built successfully in OSBS!")