def test_build_image_from_path(tmpdir, temp_image_name): if MOCK: mock_docker() tmpdir_path = str(tmpdir.realpath()) clone_git_repo(DOCKERFILE_GIT, tmpdir_path) df = tmpdir.join("Dockerfile") assert df.check() t = DockerTasker() response = t.build_image_from_path(tmpdir_path, temp_image_name, use_cache=True) list(response) assert response is not None assert t.image_exists(temp_image_name) t.remove_image(temp_image_name)
def test_build_image_from_path(tmpdir, temp_image_name): if MOCK: mock_docker() tmpdir_path = str(tmpdir.realpath()) clone_git_repo(DOCKERFILE_GIT, tmpdir_path) df = tmpdir.join("Dockerfile") assert df.check() t = DockerTasker() response = t.build_image_from_path(tmpdir_path, temp_image_name, use_cache=True) list(response) assert response is not None assert t.image_exists(temp_image_name) t.remove_image(temp_image_name)
class InsideBuilder(LastLogger, BuilderStateMachine): """ This is expected to run within container """ def __init__(self, source, image, **kwargs): """ """ LastLogger.__init__(self) BuilderStateMachine.__init__(self) self.tasker = DockerTasker() # arguments for build self.source = source self.base_image_id = None self.image_id = None self.built_image_info = None self.image = ImageName.parse(image) # get info about base image from dockerfile self.df_path, self.df_dir = self.source.get_dockerfile_path() self.base_image = ImageName.parse(DockerfileParser(self.df_path).baseimage) logger.debug("base image specified in dockerfile = '%s'", self.base_image) if not self.base_image.tag: self.base_image.tag = 'latest' def build(self): """ build image inside current environment; it's expected this may run within (privileged) docker container :return: image string (e.g. fedora-python:34) """ logger.info("building image '%s' inside current environment", self.image) self._ensure_not_built() logger.debug("using dockerfile:\n%s", DockerfileParser(self.df_path).content) logs_gen = self.tasker.build_image_from_path( self.df_dir, self.image, ) logger.debug("build is submitted, waiting for it to finish") command_result = wait_for_command(logs_gen) # wait for build to finish logger.info("build was %ssuccesful!", 'un' if command_result.is_failed() else '') self.is_built = True if not command_result.is_failed(): self.built_image_info = self.get_built_image_info() # self.base_image_id = self.built_image_info['ParentId'] # parent id is not base image! self.image_id = self.built_image_info['Id'] build_result = BuildResult(command_result, self.image_id) return build_result def push_built_image(self, registry, insecure=False): """ push built image to provided registry :param registry: str :param insecure: bool, allow connecting to registry over plain http :return: str, image """ logger.info("pushing built image '%s' to registry '%s'", self.image, registry) self._ensure_is_built() if not registry: logger.warning("no registry specified; skipping") return if self.image.registry and self.image.registry != registry: logger.error("registry in image name doesn't match provided target registry, " "image registry = '%s', target = '%s'", self.image.registry, registry) raise RuntimeError( "Registry in image name doesn't match target registry. Image: '%s', Target: '%s'" % (self.image.registry, registry)) target_image = self.image.copy() target_image.registry = registry response = self.tasker.tag_and_push_image(self.image, target_image, insecure=insecure) self.tasker.remove_image(target_image) return response def inspect_base_image(self): """ inspect base image :return: dict """ logger.info("inspecting base image '%s'", self.base_image) inspect_data = self.tasker.inspect_image(self.base_image) return inspect_data def inspect_built_image(self): """ inspect built image :return: dict """ logger.info("inspecting built image '%s'", self.image_id) self._ensure_is_built() inspect_data = self.tasker.inspect_image(self.image_id) # dict with lots of data, see man docker-inspect return inspect_data def get_base_image_info(self): """ query docker about base image :return dict """ logger.info("getting information about base image '%s'", self.base_image) image_info = self.tasker.get_image_info_by_image_name(self.base_image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.base_image) raise RuntimeError("image '%s' not found", self.base_image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.base_image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.base_image)) def get_built_image_info(self): """ query docker about built image :return dict """ logger.info("getting information about built image '%s'", self.image) self._ensure_is_built() image_info = self.tasker.get_image_info_by_image_name(self.image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.image) raise RuntimeError("image '%s' not found" % self.image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.image))
class InsideBuilder(LastLogger, BuilderStateMachine): """ This is expected to run within container """ def __init__(self, source, image, **kwargs): """ """ LastLogger.__init__(self) BuilderStateMachine.__init__(self) print_version_of_tools() self.tasker = DockerTasker() info, version = self.tasker.get_info(), self.tasker.get_version() logger.debug(json.dumps(info, indent=2)) logger.info(json.dumps(version, indent=2)) # arguments for build self.source = source self.base_image_id = None self.image_id = None self.built_image_info = None self.image = ImageName.parse(image) # get info about base image from dockerfile self.df_path, self.df_dir = self.source.get_dockerfile_path() self.set_base_image(df_parser(self.df_path).baseimage) logger.debug("base image specified in dockerfile = '%s'", self.base_image) if not self.base_image.tag: self.base_image.tag = 'latest' def build(self): """ build image inside current environment; it's expected this may run within (privileged) docker container :return: image string (e.g. fedora-python:34) """ try: logger.info("building image '%s' inside current environment", self.image) self._ensure_not_built() logger.debug("using dockerfile:\n%s", df_parser(self.df_path).content) logs_gen = self.tasker.build_image_from_path( self.df_dir, self.image, ) logger.debug("build is submitted, waiting for it to finish") command_result = wait_for_command( logs_gen) # wait for build to finish logger.info( "build %s!", 'failed' if command_result.is_failed() else 'succeeded') self.is_built = True if not command_result.is_failed(): self.built_image_info = self.get_built_image_info() # self.base_image_id = self.built_image_info['ParentId'] # parent id is not base image! self.image_id = self.built_image_info['Id'] build_result = BuildResult(command_result, self.image_id) return build_result except: logger.exception("build failed") return ExceptionBuildResult() def set_base_image(self, base_image): self.base_image = ImageName.parse(base_image) def inspect_base_image(self): """ inspect base image :return: dict """ logger.info("inspecting base image '%s'", self.base_image) inspect_data = self.tasker.inspect_image(self.base_image) return inspect_data def inspect_built_image(self): """ inspect built image :return: dict """ logger.info("inspecting built image '%s'", self.image_id) self._ensure_is_built() inspect_data = self.tasker.inspect_image( self.image_id) # dict with lots of data, see man docker-inspect return inspect_data def get_base_image_info(self): """ query docker about base image :return dict """ logger.info("getting information about base image '%s'", self.base_image) image_info = self.tasker.get_image_info_by_image_name(self.base_image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.base_image) raise RuntimeError("image '%s' not found", self.base_image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.base_image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.base_image)) def get_built_image_info(self): """ query docker about built image :return dict """ logger.info("getting information about built image '%s'", self.image) self._ensure_is_built() image_info = self.tasker.get_image_info_by_image_name(self.image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.image) raise RuntimeError("image '%s' not found" % self.image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.image))
class BuildImageBuilder(object): def __init__(self, reactor_tarball_path=None, reactor_local_path=None, reactor_remote_path=None, use_official_reactor_git=False): self.tasker = DockerTasker() self.reactor_tarball_path = reactor_tarball_path self.reactor_local_path = reactor_local_path self.reactor_remote_path = reactor_remote_path self.use_official_reactor_git = use_official_reactor_git if not self.reactor_tarball_path and \ not self.reactor_local_path and \ not self.reactor_remote_path and \ not self.use_official_reactor_git: logger.error("no atomic_reactor source specified, can't proceed") raise RuntimeError("You have to specify atomic_reactor source: either local gitrepo, " "path to atomic_reactor tarball, or use upstream git repo.") def create_image(self, df_dir_path, image, use_cache=False): """ create image: get atomic-reactor sdist tarball, build image and tag it :param df_path: :param image: :return: """ logger.debug("creating build image: df_dir_path = '%s', image = '%s'", df_dir_path, image) if not os.path.isdir(df_dir_path): raise RuntimeError("Directory '%s' does not exist.", df_dir_path) tmpdir = tempfile.mkdtemp() df_tmpdir = os.path.join(tmpdir, 'df-%s' % uuid.uuid4()) git_tmpdir = os.path.join(tmpdir, 'git-%s' % uuid.uuid4()) os.mkdir(df_tmpdir) logger.debug("tmp dir with dockerfile '%s' created", df_tmpdir) os.mkdir(git_tmpdir) logger.debug("tmp dir with atomic-reactor '%s' created", git_tmpdir) try: for f in glob(os.path.join(df_dir_path, '*')): shutil.copy(f, df_tmpdir) logger.debug("cp '%s' -> '%s'", f, df_tmpdir) logger.debug("df dir: %s", os.listdir(df_tmpdir)) reactor_tarball = self.get_reactor_tarball_path(tmpdir=git_tmpdir) reactor_tb_path = os.path.join(df_tmpdir, DOCKERFILE_REACTOR_TARBALL_NAME) shutil.copy(reactor_tarball, reactor_tb_path) image_name = ImageName.parse(image) logs_gen = self.tasker.build_image_from_path(df_tmpdir, image_name, stream=True, use_cache=use_cache) wait_for_command(logs_gen) finally: shutil.rmtree(tmpdir) def get_reactor_tarball_path(self, tmpdir): """ generate atomic-reactor tarball :return: """ if self.reactor_tarball_path: if not os.path.isfile(self.reactor_tarball_path): logger.error("atomic-reactor sdist tarball does not exist: '%s'", self.reactor_tarball_path) raise RuntimeError("File does not exist: '%s'" % self.reactor_tarball_path) return self.reactor_tarball_path elif self.reactor_local_path: if not os.path.isdir(self.reactor_local_path): logger.error("local atomic-reactor git clone does not exist: '%s'", self.reactor_local_path) raise RuntimeError("Local atomic-reactor git repo does not exist: '%s'" % self.reactor_local_path) local_reactor_git_path = self.reactor_local_path else: if self.use_official_reactor_git: self.reactor_remote_path = REACTOR_GIT_URL g = LazyGit(self.reactor_remote_path, tmpdir=tmpdir) local_reactor_git_path = g.git_path cwd = os.getcwd() os.chdir(local_reactor_git_path) try: logger.debug("executing sdist command in directory '%s'", os.getcwd()) subprocess.check_call(["python", "setup.py", "sdist", "--dist-dir", tmpdir]) finally: os.chdir(cwd) candidates_list = glob(os.path.join(tmpdir, 'atomic-reactor-*.tar.gz')) if len(candidates_list) == 1: return candidates_list[0] else: logger.warning("len(atomic-reactor-*.tar.gz) != 1: '%s'", candidates_list) try: return candidates_list[0] except IndexError: raise RuntimeError("No atomic-reactor tarball built.")
class BuildImageBuilder(object): def __init__(self, reactor_tarball_path=None, reactor_local_path=None, reactor_remote_path=None, use_official_reactor_git=False): self.tasker = DockerTasker() self.reactor_tarball_path = reactor_tarball_path self.reactor_local_path = reactor_local_path self.reactor_remote_path = reactor_remote_path self.use_official_reactor_git = use_official_reactor_git if not self.reactor_tarball_path and \ not self.reactor_local_path and \ not self.reactor_remote_path and \ not self.use_official_reactor_git: logger.error("no atomic_reactor source specified, can't proceed") raise RuntimeError( "You have to specify atomic_reactor source: either local gitrepo, " "path to atomic_reactor tarball, or use upstream git repo.") def create_image(self, df_dir_path, image, use_cache=False): """ create image: get atomic-reactor sdist tarball, build image and tag it :param df_path: :param image: :return: """ logger.debug("creating build image: df_dir_path = '%s', image = '%s'", df_dir_path, image) if not os.path.isdir(df_dir_path): raise RuntimeError("Directory '%s' does not exist.", df_dir_path) tmpdir = tempfile.mkdtemp() df_tmpdir = os.path.join(tmpdir, 'df-%s' % uuid.uuid4()) git_tmpdir = os.path.join(tmpdir, 'git-%s' % uuid.uuid4()) os.mkdir(df_tmpdir) logger.debug("tmp dir with dockerfile '%s' created", df_tmpdir) os.mkdir(git_tmpdir) logger.debug("tmp dir with atomic-reactor '%s' created", git_tmpdir) try: for f in glob(os.path.join(df_dir_path, '*')): shutil.copy(f, df_tmpdir) logger.debug("cp '%s' -> '%s'", f, df_tmpdir) logger.debug("df dir: %s", os.listdir(df_tmpdir)) reactor_tarball = self.get_reactor_tarball_path(tmpdir=git_tmpdir) reactor_tb_path = os.path.join(df_tmpdir, DOCKERFILE_REACTOR_TARBALL_NAME) shutil.copy(reactor_tarball, reactor_tb_path) image_name = ImageName.parse(image) logs_gen = self.tasker.build_image_from_path(df_tmpdir, image_name, stream=True, use_cache=use_cache) wait_for_command(logs_gen) finally: shutil.rmtree(tmpdir) def get_reactor_tarball_path(self, tmpdir): """ generate atomic-reactor tarball :return: """ if self.reactor_tarball_path: if not os.path.isfile(self.reactor_tarball_path): logger.error( "atomic-reactor sdist tarball does not exist: '%s'", self.reactor_tarball_path) raise RuntimeError("File does not exist: '%s'" % self.reactor_tarball_path) return self.reactor_tarball_path elif self.reactor_local_path: if not os.path.isdir(self.reactor_local_path): logger.error( "local atomic-reactor git clone does not exist: '%s'", self.reactor_local_path) raise RuntimeError( "Local atomic-reactor git repo does not exist: '%s'" % self.reactor_local_path) local_reactor_git_path = self.reactor_local_path else: if self.use_official_reactor_git: self.reactor_remote_path = REACTOR_GIT_URL g = LazyGit(self.reactor_remote_path, tmpdir=tmpdir) local_reactor_git_path = g.git_path cwd = os.getcwd() os.chdir(local_reactor_git_path) try: logger.debug("executing sdist command in directory '%s'", os.getcwd()) subprocess.check_call( ["python", "setup.py", "sdist", "--dist-dir", tmpdir]) finally: os.chdir(cwd) candidates_list = glob(os.path.join(tmpdir, 'atomic-reactor-*.tar.gz')) if len(candidates_list) == 1: return candidates_list[0] else: logger.warning("len(atomic-reactor-*.tar.gz) != 1: '%s'", candidates_list) try: return candidates_list[0] except IndexError: raise RuntimeError("No atomic-reactor tarball built.")
class InsideBuilder(LastLogger, BuilderStateMachine): """ This is expected to run within container """ def __init__(self, source, image, **kwargs): """ """ LastLogger.__init__(self) BuilderStateMachine.__init__(self) print_version_of_tools() self.tasker = DockerTasker() info, version = self.tasker.get_info(), self.tasker.get_version() logger.debug(json.dumps(info, indent=2)) logger.info(json.dumps(version, indent=2)) # arguments for build self.source = source self.base_image_id = None self.image_id = None self.built_image_info = None self.image = ImageName.parse(image) # get info about base image from dockerfile self.df_path, self.df_dir = self.source.get_dockerfile_path() self.base_image = ImageName.parse(DockerfileParser(self.df_path).baseimage) logger.debug("base image specified in dockerfile = '%s'", self.base_image) if not self.base_image.tag: self.base_image.tag = 'latest' def build(self): """ build image inside current environment; it's expected this may run within (privileged) docker container :return: image string (e.g. fedora-python:34) """ logger.info("building image '%s' inside current environment", self.image) self._ensure_not_built() logger.debug("using dockerfile:\n%s", DockerfileParser(self.df_path).content) logs_gen = self.tasker.build_image_from_path( self.df_dir, self.image, ) logger.debug("build is submitted, waiting for it to finish") command_result = wait_for_command(logs_gen) # wait for build to finish logger.info("build was %ssuccesful!", 'un' if command_result.is_failed() else '') self.is_built = True if not command_result.is_failed(): self.built_image_info = self.get_built_image_info() # self.base_image_id = self.built_image_info['ParentId'] # parent id is not base image! self.image_id = self.built_image_info['Id'] build_result = BuildResult(command_result, self.image_id) return build_result def push_built_image(self, registry, insecure=False): """ push built image to provided registry :param registry: str :param insecure: bool, allow connecting to registry over plain http :return: str, image """ logger.info("pushing built image '%s' to registry '%s'", self.image, registry) self._ensure_is_built() if not registry: logger.warning("no registry specified; skipping") return if self.image.registry and self.image.registry != registry: logger.error("registry in image name doesn't match provided target registry, " "image registry = '%s', target = '%s'", self.image.registry, registry) raise RuntimeError( "Registry in image name doesn't match target registry. Image: '%s', Target: '%s'" % (self.image.registry, registry)) target_image = self.image.copy() target_image.registry = registry response = self.tasker.tag_and_push_image(self.image, target_image, insecure=insecure) self.tasker.remove_image(target_image) return response def inspect_base_image(self): """ inspect base image :return: dict """ logger.info("inspecting base image '%s'", self.base_image) inspect_data = self.tasker.inspect_image(self.base_image) return inspect_data def inspect_built_image(self): """ inspect built image :return: dict """ logger.info("inspecting built image '%s'", self.image_id) self._ensure_is_built() inspect_data = self.tasker.inspect_image(self.image_id) # dict with lots of data, see man docker-inspect return inspect_data def get_base_image_info(self): """ query docker about base image :return dict """ logger.info("getting information about base image '%s'", self.base_image) image_info = self.tasker.get_image_info_by_image_name(self.base_image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.base_image) raise RuntimeError("image '%s' not found", self.base_image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.base_image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.base_image)) def get_built_image_info(self): """ query docker about built image :return dict """ logger.info("getting information about built image '%s'", self.image) self._ensure_is_built() image_info = self.tasker.get_image_info_by_image_name(self.image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.image) raise RuntimeError("image '%s' not found" % self.image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.image))
class InsideBuilder(LastLogger, BuilderStateMachine): """ This is expected to run within container """ def __init__(self, source, image, **kwargs): """ """ LastLogger.__init__(self) BuilderStateMachine.__init__(self) print_version_of_tools() self.tasker = DockerTasker() info, version = self.tasker.get_info(), self.tasker.get_version() logger.debug(json.dumps(info, indent=2)) logger.info(json.dumps(version, indent=2)) # arguments for build self.source = source self.base_image_id = None self.image_id = None self.built_image_info = None self.image = ImageName.parse(image) # get info about base image from dockerfile self.df_path, self.df_dir = self.source.get_dockerfile_path() self.set_base_image(df_parser(self.df_path).baseimage) logger.debug("base image specified in dockerfile = '%s'", self.base_image) if not self.base_image.tag: self.base_image.tag = 'latest' def build(self): """ build image inside current environment; it's expected this may run within (privileged) docker container :return: image string (e.g. fedora-python:34) """ try: logger.info("building image '%s' inside current environment", self.image) self._ensure_not_built() logger.debug("using dockerfile:\n%s", df_parser(self.df_path).content) logs_gen = self.tasker.build_image_from_path( self.df_dir, self.image, ) logger.debug("build is submitted, waiting for it to finish") command_result = wait_for_command(logs_gen) # wait for build to finish logger.info("build was %ssuccessful!", 'un' if command_result.is_failed() else '') self.is_built = True if not command_result.is_failed(): self.built_image_info = self.get_built_image_info() # self.base_image_id = self.built_image_info['ParentId'] # parent id is not base image! self.image_id = self.built_image_info['Id'] build_result = BuildResult(command_result, self.image_id) return build_result except: logger.exception("build failed") return ExceptionBuildResult() def set_base_image(self, base_image): self.base_image = ImageName.parse(base_image) def inspect_base_image(self): """ inspect base image :return: dict """ logger.info("inspecting base image '%s'", self.base_image) inspect_data = self.tasker.inspect_image(self.base_image) return inspect_data def inspect_built_image(self): """ inspect built image :return: dict """ logger.info("inspecting built image '%s'", self.image_id) self._ensure_is_built() inspect_data = self.tasker.inspect_image(self.image_id) # dict with lots of data, see man docker-inspect return inspect_data def get_base_image_info(self): """ query docker about base image :return dict """ logger.info("getting information about base image '%s'", self.base_image) image_info = self.tasker.get_image_info_by_image_name(self.base_image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.base_image) raise RuntimeError("image '%s' not found", self.base_image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.base_image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.base_image)) def get_built_image_info(self): """ query docker about built image :return dict """ logger.info("getting information about built image '%s'", self.image) self._ensure_is_built() image_info = self.tasker.get_image_info_by_image_name(self.image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.image) raise RuntimeError("image '%s' not found" % self.image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.image))