def test_constructor_cache(self, tmpdir): tmpdir_path = str(tmpdir.realpath()) df1 = DockerfileParser(tmpdir_path) df1.lines = ["From fedora:latest\n", "LABEL a b\n"] df2 = DockerfileParser(tmpdir_path, True) assert df2.cached_content
def create_metadata_dockerfile(self): """Create a minimal Dockerfile on the metadata repository, copying all manifests inside the image and having nearly the same labels as its corresponding operator Dockerfile But some modifications on the labels are needed: - 'com.redhat.component' label should contain the metadata component name, otherwise it conflicts with the operator. - 'com.redhat.delivery.appregistry' should always be "true", regardless of the value coming from the operator Dockerfile - 'release' label should be removed, because we can't build the same NVR multiple times - 'version' label should contain both 'release' info and the target stream """ operator_dockerfile = DockerfileParser('{}/{}/Dockerfile'.format( self.working_dir, self.operator_name)) metadata_dockerfile = DockerfileParser('{}/{}/Dockerfile'.format( self.working_dir, self.metadata_repo)) metadata_dockerfile.content = 'FROM scratch\nCOPY ./manifests /manifests' metadata_dockerfile.labels = operator_dockerfile.labels metadata_dockerfile.labels['com.redhat.component'] = ( operator_dockerfile.labels['com.redhat.component'].replace( self.operator_name, self.metadata_name)) metadata_dockerfile.labels['com.redhat.delivery.appregistry'] = 'true' metadata_dockerfile.labels['name'] = 'openshift/ose-{}'.format( self.metadata_name) # mangle version according to spec metadata_dockerfile.labels['version'] = '{}.{}.{}'.format( operator_dockerfile.labels['version'], operator_dockerfile.labels['release'], self.stream) del (metadata_dockerfile.labels['release'])
def __init__(self, target, **_): super().__init__() logger.debug("Target is a dockerfile.") if isinstance(target, io.IOBase): logger.debug( "Target is a dockerfile loaded from the file-like object.") self.instance = DockerfileParser(fileobj=target) else: self.instance = DockerfileParser(fileobj=open(target))
def test_bump_release_find_current_release(tmpdir, labelval, expected_attr, expected_key): dflines = [ 'FROM fedora\n', 'ENV ENV=1\n', 'LABEL Label={0}\n'.format(labelval) ] df_path = os.path.join(str(tmpdir), 'Dockerfile') DockerfileParser(df_path).lines = dflines plugin = BumpReleasePlugin(None, None, None, None, None) parser = DockerfileParser(df_path) attrs, key = plugin.find_current_release(parser, 'Label') assert attrs == getattr(parser, expected_attr) assert key == expected_key
def _get_target_instance(target, logging_level): """ Get the Container/Image instance for the given name. (Container is the first choice.) or DockerfileParser instance if the target is path or file-like object. :param target: str or instance of Image/Container or file-like object as Dockerfile :return: Target object """ logger.debug("Finding target '{}'.".format(target)) if isinstance(target, (Image, Container)): logger.debug("Target is a conu object.") return target if isinstance(target, io.IOBase): logger.debug( "Target is a dockerfile loaded from the file-like object.") return DockerfileParser(fileobj=target) if os.path.isfile(target): logger.debug("Target is a dockerfile.") return DockerfileParser(fileobj=open(target)) with DockerBackend(logging_level=logging_level) as backend: try: cont = backend.ContainerClass(image=None, container_id=target) logger.debug("Target is a container.") return cont except NotFound: image_name = ImageName.parse(target) logger.debug("Finding image '{}' with tag '{}'.".format( image_name.name, image_name.tag)) if image_name.tag: image = backend.ImageClass( repository=image_name.name, tag=image_name.tag, pull_policy=DockerImagePullPolicy.NEVER) else: image = backend.ImageClass( repository=image_name.name, pull_policy=DockerImagePullPolicy.NEVER) if image.is_present(): logger.debug("Target is an image.") return image logger.error("Target is neither image nor container.") raise ColinException("Target not found.")
def test_remove_whitespace(self, tmpdir): """ Verify keys are parsed correctly even if there is no final newline. """ with open(os.path.join(str(tmpdir), 'Dockerfile'), 'w') as fp: fp.write('FROM scratch') tmpdir_path = str(tmpdir.realpath()) df1 = DockerfileParser(tmpdir_path) df1.labels['foo'] = 'bar ❤' df2 = DockerfileParser(tmpdir_path, True) assert df2.baseimage == 'scratch' assert df2.labels['foo'] == 'bar ❤'
def test_bump_release_indirect_correct(tmpdir, labelval): dflines = ['FROM fedora\n', 'ENV RELEASE=1\n', 'LABEL release={0}\n'.format(labelval)] with DFWithRelease(lines=dflines) as (df_path, commit): dummy_workflow, dummy_args, runner = prepare(tmpdir, df_path, commit) labels_before = DockerfileParser(df_path, env_replace=False).labels runner.run() parser = DockerfileParser(df_path) assert parser.envs['RELEASE'] == '2' parser.env_replace = False assert parser.labels == labels_before
def dfparser(tmpdir, request): """ :param tmpdir: already existing fixture defined in pytest :param request: parameter, cache_content arg to DockerfileParser :return: DockerfileParser instance """ use_fileobj, cache_content = request.param if use_fileobj: file = six.BytesIO() return DockerfileParser(fileobj=file, cache_content=cache_content) else: tmpdir_path = str(tmpdir.realpath()) return DockerfileParser(path=tmpdir_path, cache_content=cache_content)
def test_increment(self, tmpdir, component, version, next_release, include_target): class MockedClientSession(object): def __init__(self): pass def getNextRelease(self, build_info): assert build_info['name'] == list(component.values())[0] assert build_info['version'] == list(version.values())[0] return next_release session = MockedClientSession() flexmock(koji, ClientSession=session) labels = {} labels.update(component) labels.update(version) plugin = self.prepare(tmpdir, labels=labels, include_target=include_target) plugin.run() parser = DockerfileParser(plugin.workflow.builder.df_path) # Both spellings of release labels should always be set assert parser.labels['release'] == next_release assert parser.labels['Release'] == next_release
def test_adddockerfile_todest(tmpdir, docker_tasker): df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = DockerfileParser(str(tmpdir)) df.content = df_content workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = PreBuildPluginsRunner(docker_tasker, workflow, [{ 'name': AddDockerfilePlugin.key, 'args': { 'nvr': 'jboss-eap-6-docker-6.4-77', 'destdir': '/usr/share/doc/' } }]) runner.run() assert AddDockerfilePlugin.key is not None expected_output = """ FROM fedora RUN yum install -y python-django ADD Dockerfile-jboss-eap-6-docker-6.4-77 /usr/share/doc/Dockerfile-jboss-eap-6-docker-6.4-77 CMD blabla""" assert df.content == expected_output
def test_adddockerfile_plugin(tmpdir, docker_tasker): df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = DockerfileParser(str(tmpdir)) df.content = df_content workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = PreBuildPluginsRunner(docker_tasker, workflow, [{ 'name': AddDockerfilePlugin.key, 'args': { 'nvr': 'rhel-server-docker-7.1-20' } }]) runner.run() assert AddDockerfilePlugin.key is not None expected_output = """ FROM fedora RUN yum install -y python-django ADD Dockerfile-rhel-server-docker-7.1-20 /root/buildinfo/Dockerfile-rhel-server-docker-7.1-20 CMD blabla""" assert df.content == expected_output
def test_adddockerfile_nvr_from_labels2(tmpdir, docker_tasker, add_labels, aliases): df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = DockerfileParser(str(tmpdir)) df.content = df_content if MOCK: mock_docker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') flexmock(workflow, base_image_inspect={"Config": {"Labels": {}}}) workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = PreBuildPluginsRunner(docker_tasker, workflow, [{ 'name': AddLabelsPlugin.key, 'args': { 'labels': add_labels, 'auto_labels': [], 'aliases': aliases } }, { 'name': AddDockerfilePlugin.key }]) runner.run() assert AddDockerfilePlugin.key is not None assert "ADD Dockerfile-jboss-eap-6-docker-6.4-77 /root/buildinfo/Dockerfile-jboss-eap-6-docker-6.4-77" in df.content
def build(self): """ build image inside current environment; it's expected this may run within (privileged) docker container :return: image string (e.g. fedora-python:34) """ try: logger.info("building image '%s' inside current environment", self.image) self._ensure_not_built() logger.debug("using dockerfile:\n%s", DockerfileParser(self.df_path).content) logs_gen = self.tasker.build_image_from_path( self.df_dir, self.image, ) logger.debug("build is submitted, waiting for it to finish") command_result = wait_for_command( logs_gen) # wait for build to finish logger.info("build was %ssuccessful!", 'un' if command_result.is_failed() else '') self.is_built = True if not command_result.is_failed(): self.built_image_info = self.get_built_image_info() # self.base_image_id = self.built_image_info['ParentId'] # parent id is not base image! self.image_id = self.built_image_info['Id'] build_result = BuildResult(command_result, self.image_id) return build_result except: logger.exception("build failed") return ExceptionBuildResult()
def update_image(image, source_url_env, build_date_env): dfparser = DockerfileParser(f'../{image}') version = dfparser.labels["version"] try: base_url = os.path.dirname(dfparser.envs[source_url_env]) pkg_url = os.path.basename(dfparser.envs[source_url_env]) except KeyError: print(f'Docker image {image} does not contain any packages to update') return False session = HTMLSession() req = session.get(base_url) if not req.ok: return new_build = req.html.xpath( f"//a[contains(@href, '{pkg_url}')]/../following-sibling::td", first=True, ).text if should_update_build(dfparser.envs[build_date_env], new_build): print(f"Updating {image} {build_date_env} to {new_build}") # update Dockerfile in-place dfparser.envs[build_date_env] = new_build else: print(f"No updates found for {image} {build_date_env}")
def __init__(self, model_id, config_json=None): ErsiliaBase.__init__(self, config_json=config_json) self.model_id = model_id self.dir = os.path.abspath(self._get_bundle_location(model_id)) self.path = os.path.join(self.dir, DOCKERFILE_FILE) self.exists = os.path.exists(self.path) self.parser = DockerfileParser(path=self.path)
def add_env(docker_file, docker_env, docker_val): dfp = DockerfileParser() docker_content = open(docker_file).read() dfp.content = docker_content env_line = "ENV {}={}".format(docker_env, docker_val) # if env already in Dockerfile, use DockerfileParser to modify it # else, add the env_line just after the 'FROM ' line # The way DockerfileParser deals with adding new envs is not optimal # (adds it to end of file), this is why we need another method to add *new* # env if docker_env in dfp.envs: dfp.envs[docker_env] = docker_val with open(docker_file, 'w') as f: f.write(dfp.content) else: index = -1 lines = docker_content.split('\n') for i, line in enumerate(lines): if line.strip().startswith('FROM '): index = i break if index != -1: lines.insert(index + 1, env_line) with open(docker_file, 'w') as f: f.write('\n'.join(lines))
def __init__(self, tasker, workflow, nvr=None, destdir="/root/buildinfo/", use_final_dockerfile=False): """ constructor :param tasker: DockerTasker instance :param workflow: DockerBuildWorkflow instance :param nvr: name-version-release, will be appended to Dockerfile-. If not specified, try to get it from Name, Version, Release labels. :param destdir: directory in the image to put Dockerfile-N-V-R into :param use_final_dockerfile: bool, when set to True, uses final version of processed dockerfile, when set to False, uses Dockerfile from time when this plugin was executed """ # call parent constructor super(AddDockerfilePlugin, self).__init__(tasker, workflow) self.use_final_dockerfile = use_final_dockerfile if nvr is None: labels = DockerfileParser(self.workflow.builder.df_path).labels name = get_preferred_label(labels, 'name') version = get_preferred_label(labels, 'version') release = get_preferred_label(labels, 'release') if name is None or version is None or release is None: raise ValueError("You have to specify either nvr arg or Name/Version/Release labels.") nvr = "{0}-{1}-{2}".format(name, version, release) nvr = nvr.replace("/", "-") self.df_name = '{0}-{1}'.format(DOCKERFILE_FILENAME, nvr) self.df_dir = destdir self.df_path = os.path.join(self.df_dir, self.df_name) # we are not using final dockerfile, so let's copy current snapshot if not self.use_final_dockerfile: local_df_path = os.path.join(self.workflow.builder.df_dir, self.df_name) shutil.copy2(self.workflow.builder.df_path, local_df_path)
def check(dockerfile: Path) -> List[Tuple[str, str, bool]]: """Return True if everything looks fine, otherwise false.""" config = load_config() dfp = DockerfileParser() with open(dockerfile) as fp: content = fp.read() dfp.content = content checks = [] checks.append(( "Use a trusted base image", dfp.baseimage, is_trusted_base_image(config.trusted_images, dfp.baseimage), )) tag = get_tag(dfp.baseimage) tag_str = "-" if tag is None else tag checks.append(("A tag for the base image is set", tag_str, tag is not None)) checks.append(("Executes as non-root", "", executes_as_non_root(dfp))) checks.append( ("COPY added after apt-get update", "", copy_added_after_update(dfp))) checks.append(( "'apt-get update' always has upgrade/install in same command", "", apt_update_has_upgrade_or_install(dfp), )) checks.append(( "Only install dependencies you're really using", "", use_no_install_recommends(dfp), )) checks.append(("apt-caches are cleaned", "", apt_caches_are_cleaned(dfp))) checks.append(("Don't put secrets as ENV variables in the image", "", no_secrets_as_env(dfp))) return checks
def test_add_labels_plugin(tmpdir, labels_conf_base, labels_conf, dont_overwrite, expected_output): df = DockerfileParser(str(tmpdir)) df.content = DF_CONTENT tasker = DockerTasker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') setattr(workflow, 'builder', X) flexmock(workflow, base_image_inspect=labels_conf_base) setattr(workflow.builder, 'df_path', df.dockerfile_path) runner = PreBuildPluginsRunner(tasker, workflow, [{ 'name': AddLabelsPlugin.key, 'args': { 'labels': labels_conf, "dont_overwrite": dont_overwrite } }]) if isinstance(expected_output, RuntimeError): with pytest.raises(RuntimeError): runner.run() else: runner.run() assert AddLabelsPlugin.key is not None assert df.content in expected_output
def __init__(self, source, image, **kwargs): """ """ LastLogger.__init__(self) BuilderStateMachine.__init__(self) print_version_of_tools() self.tasker = DockerTasker() info, version = self.tasker.get_info(), self.tasker.get_version() logger.debug(json.dumps(info, indent=2)) logger.info(json.dumps(version, indent=2)) # arguments for build self.source = source self.base_image_id = None self.image_id = None self.built_image_info = None self.image = ImageName.parse(image) # get info about base image from dockerfile self.df_path, self.df_dir = self.source.get_dockerfile_path() self.set_base_image(DockerfileParser(self.df_path).baseimage) logger.debug("base image specified in dockerfile = '%s'", self.base_image) if not self.base_image.tag: self.base_image.tag = 'latest'
def _get_import_path_override(self, srcdir): """Look inside the Dockerfile for a named label. :param srcdir: path to source code to examine :return: import path override, or None :rtype str/None: """ label = 'io.openshift.source-repo-url' try: df = DockerfileParser(srcdir, cache_content=True) except IOError: log.exception('Unable to read Dockerfile') return None try: repo = df.labels[label] except KeyError: log.debug(f'No {label} label in Dockerfile') return None except: # noqa:E722 log.exception('Failed to process Dockerfile; ignoring') return None # Convert it to an import path by stripping off the scheme. (_, _, import_path) = repo.rpartition('://') if not import_path: return None return import_path
def test_add_labels_plugin(tmpdir, docker_tasker, df_content, labels_conf_base, labels_conf, dont_overwrite, aliases, expected_output, caplog): df = DockerfileParser(str(tmpdir)) df.content = df_content if MOCK: mock_docker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') setattr(workflow, 'builder', X) flexmock(workflow, base_image_inspect=labels_conf_base) setattr(workflow.builder, 'df_path', df.dockerfile_path) runner = PreBuildPluginsRunner(docker_tasker, workflow, [{ 'name': AddLabelsPlugin.key, 'args': { 'labels': labels_conf, 'dont_overwrite': dont_overwrite, 'auto_labels': [], 'aliases': aliases, } }]) runner.run() if isinstance(expected_output, RuntimeError): assert "plugin 'add_labels_in_dockerfile' raised an exception: RuntimeError" in caplog.text( ) else: assert AddLabelsPlugin.key is not None assert df.content in expected_output
def test_parse_dockerfile_again_after_data_is_loaded(context_dir, build_dir, tmpdir): context_dir = ContextDir(Path(tmpdir.join("context_dir"))) wf_data = ImageBuildWorkflowData.load_from_dir(context_dir) # Note that argument source is None, that causes a DummySource is created # and "FROM scratch" is included in the Dockerfile. workflow = DockerBuildWorkflow(context_dir, build_dir, NAMESPACE, PIPELINE_RUN_NAME, wf_data) assert ["scratch"] == workflow.data.dockerfile_images.original_parents # Now, save the workflow data and load it again wf_data.save(context_dir) another_source = DummySource("git", "https://git.host/") dfp = DockerfileParser(another_source.source_path) dfp.content = 'FROM fedora:35\nCMD ["bash", "--version"]' wf_data = ImageBuildWorkflowData.load_from_dir(context_dir) flexmock(DockerBuildWorkflow).should_receive( "_parse_dockerfile_images").never() flexmock(wf_data.dockerfile_images).should_receive( "set_source_registry").never() workflow = DockerBuildWorkflow(context_dir, build_dir, NAMESPACE, PIPELINE_RUN_NAME, wf_data, source=another_source) assert ["scratch"] == workflow.data.dockerfile_images.original_parents, \ "The dockerfile_images should not be changed."
def test_yuminject_multiline_notwrapped(tmpdir): df_content = """\ FROM fedora RUN yum install -y httpd \ uwsgi CMD blabla""" df = DockerfileParser(str(tmpdir)) df.content = df_content tasker, workflow = prepare(df.dockerfile_path) metalink = r'https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch' workflow.files[os.path.join( YUM_REPOS_DIR, DEFAULT_YUM_REPOFILE_NAME)] = render_yum_repo( OrderedDict((('name', 'my-repo'), ('metalink', metalink), ('enabled', "1"), ('gpgcheck', "0")), )) runner = PreBuildPluginsRunner(tasker, workflow, [{ 'name': InjectYumRepoPlugin.key, 'args': { "wrap_commands": False } }]) runner.run() assert InjectYumRepoPlugin.key is not None expected_output = r"""FROM fedora ADD atomic-reactor-repos/* '/etc/yum.repos.d/' RUN yum install -y httpd uwsgi CMD blabla RUN rm -f '/etc/yum.repos.d/atomic-reactor-injected.repo' """ assert df.content == expected_output
def parse(self, dockerfile): # Parse the Dockerfile with open(dockerfile, 'r') as infile: dockercontent = infile.read() self.dfp = DockerfileParser() self.dfp.content = dockercontent
def dockerfile(self) -> DockerfileParser: """Return the parsed Dockerfile. :return: the parsed Dockerfile. :rtype: DockerfileParser """ return DockerfileParser(str(self.dockerfile_path))
def test_assertlabels_plugin(tmpdir, docker_tasker, df_content, req_labels, expected): df = DockerfileParser(str(tmpdir)) df.content = df_content workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = PreBuildPluginsRunner(docker_tasker, workflow, [{ 'name': AssertLabelsPlugin.key, 'args': { 'required_labels': req_labels } }]) assert AssertLabelsPlugin.key is not None if isinstance(expected, PluginFailedException): with pytest.raises(PluginFailedException): runner.run() else: runner.run()
def test_add_labels_plugin_generated(tmpdir, docker_tasker, auto_label, value_re_part): df = DockerfileParser(str(tmpdir)) df.content = DF_CONTENT if MOCK: mock_docker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') setattr(workflow, 'builder', X) flexmock(workflow, source=MockSource()) flexmock(workflow, base_image_inspect=LABELS_CONF_BASE) setattr(workflow.builder, 'df_path', df.dockerfile_path) runner = PreBuildPluginsRunner( docker_tasker, workflow, [{ 'name': AddLabelsPlugin.key, 'args': { 'labels': {}, "dont_overwrite": [], "auto_labels": [auto_label], 'aliases': { 'Build_Host': 'com.redhat.build-host' } } }]) runner.run() assert re.match(value_re_part, df.labels[auto_label])
def run(self): """ run the plugin """ # dict comprehension is syntax error on 2.6 yum_repos = {} for key, value in self.workflow.files.items(): if key.startswith(YUM_REPOS_DIR): yum_repos[key] = value if self.wrap_commands: wrap_yum_commands(yum_repos, self.workflow.builder.df_path) else: if not yum_repos: return # absolute path in containers -> relative path within context repos_host_cont_mapping = {} host_repos_path = os.path.join(self.workflow.builder.df_dir, RELATIVE_REPOS_PATH) self.log.info("creating directory for yum repos: %s", host_repos_path) os.mkdir(host_repos_path) for repo, repo_content in self.workflow.files.items(): repo_basename = os.path.basename(repo) repo_relative_path = os.path.join(RELATIVE_REPOS_PATH, repo_basename) repo_host_path = os.path.join(host_repos_path, repo_basename) self.log.info("writing repo to '%s'", repo_host_path) with open(repo_host_path, "wb") as fp: fp.write(repo_content.encode("utf-8")) repos_host_cont_mapping[repo] = repo_relative_path # Find out the USER inherited from the base image inspect = self.workflow.builder.inspect_base_image() inherited_user = inspect['Config'].get('User', '') df = DockerfileParser(self.workflow.builder.df_path) df.lines = add_yum_repos_to_dockerfile(repos_host_cont_mapping, df, inherited_user)
def __enter__(self): repo = pygit2.init_repository(self.path) repo.remotes.create('origin', '/dev/null') # Set up branch 'master' filename = 'Dockerfile' dockerfile_path = os.path.join(self.path, filename) open(dockerfile_path, mode="w+t").close() index = repo.index index.add(filename) author = pygit2.Signature('Test', '*****@*****.**') committer = pygit2.Signature('Test', '*****@*****.**') oid = repo.create_commit('HEAD', author, committer, '', index.write_tree(), []) master = repo.head # Now set up our branch branch = repo.create_branch(BRANCH, repo.get(oid)) repo.checkout(refname=branch) DockerfileParser(dockerfile_path).lines = self.lines index = repo.index index.add(filename) repo.create_commit(branch.name, author, committer, '', index.write_tree(), [repo.head.peel().hex]) branch.upstream = branch return dockerfile_path, repo.head.peel().hex