def test_pull_image(): if MOCK: mock_docker() t = DockerTasker() local_img = input_image_name remote_img = local_img.copy() remote_img.registry = LOCALHOST_REGISTRY t.tag_and_push_image(local_img, remote_img, insecure=True) got_image = t.pull_image(remote_img, insecure=True) assert remote_img.to_str() == got_image assert len(t.last_logs) > 0 t.remove_image(remote_img)
def test_pull_image(): if MOCK: mock_docker() t = DockerTasker() local_img = input_image_name remote_img = local_img.copy() remote_img.registry = LOCALHOST_REGISTRY t.tag_and_push_image(local_img, remote_img, insecure=True) got_image = t.pull_image(remote_img, insecure=True) assert remote_img.to_str() == got_image assert len(t.last_logs) > 0 t.remove_image(remote_img)
def test_tag_and_push(temp_image_name): if MOCK: mock_docker() t = DockerTasker() temp_image_name.registry = LOCALHOST_REGISTRY temp_image_name.tag = "1" output = t.tag_and_push_image(INPUT_IMAGE, temp_image_name, insecure=True) assert output is not None assert t.image_exists(temp_image_name) t.remove_image(temp_image_name)
def test_tag_and_push(temp_image_name): # noqa if MOCK: mock_docker() t = DockerTasker() temp_image_name.registry = LOCALHOST_REGISTRY temp_image_name.tag = "1" output = t.tag_and_push_image(INPUT_IMAGE, temp_image_name, insecure=True) assert output is not None assert t.image_exists(temp_image_name) t.remove_image(temp_image_name)
def test_pull_image(tmpdir, insecure, dockercfg): if MOCK: mock_docker() dockercfg_path = None if dockercfg: dockercfg_path = str(tmpdir.realpath()) file_name = '.dockercfg' dockercfg_secret_path = os.path.join(dockercfg_path, file_name) with open(dockercfg_secret_path, "w+") as dockerconfig: dockerconfig.write(json.dumps(dockercfg)) dockerconfig.flush() t = DockerTasker() local_img = input_image_name remote_img = local_img.copy() remote_img.registry = LOCALHOST_REGISTRY t.tag_and_push_image(local_img, remote_img, insecure=insecure, dockercfg=dockercfg_path) got_image = t.pull_image(remote_img, insecure=insecure, dockercfg_path=dockercfg_path) assert remote_img.to_str() == got_image assert len(t.last_logs) > 0 t.remove_image(remote_img)
class InsideBuilder(LastLogger, BuilderStateMachine): """ This is expected to run within container """ def __init__(self, source, image, **kwargs): """ """ LastLogger.__init__(self) BuilderStateMachine.__init__(self) self.tasker = DockerTasker() # arguments for build self.source = source self.base_image_id = None self.image_id = None self.built_image_info = None self.image = ImageName.parse(image) # get info about base image from dockerfile self.df_path, self.df_dir = self.source.get_dockerfile_path() self.base_image = ImageName.parse(DockerfileParser(self.df_path).baseimage) logger.debug("base image specified in dockerfile = '%s'", self.base_image) if not self.base_image.tag: self.base_image.tag = 'latest' def build(self): """ build image inside current environment; it's expected this may run within (privileged) docker container :return: image string (e.g. fedora-python:34) """ logger.info("building image '%s' inside current environment", self.image) self._ensure_not_built() logger.debug("using dockerfile:\n%s", DockerfileParser(self.df_path).content) logs_gen = self.tasker.build_image_from_path( self.df_dir, self.image, ) logger.debug("build is submitted, waiting for it to finish") command_result = wait_for_command(logs_gen) # wait for build to finish logger.info("build was %ssuccesful!", 'un' if command_result.is_failed() else '') self.is_built = True if not command_result.is_failed(): self.built_image_info = self.get_built_image_info() # self.base_image_id = self.built_image_info['ParentId'] # parent id is not base image! self.image_id = self.built_image_info['Id'] build_result = BuildResult(command_result, self.image_id) return build_result def push_built_image(self, registry, insecure=False): """ push built image to provided registry :param registry: str :param insecure: bool, allow connecting to registry over plain http :return: str, image """ logger.info("pushing built image '%s' to registry '%s'", self.image, registry) self._ensure_is_built() if not registry: logger.warning("no registry specified; skipping") return if self.image.registry and self.image.registry != registry: logger.error("registry in image name doesn't match provided target registry, " "image registry = '%s', target = '%s'", self.image.registry, registry) raise RuntimeError( "Registry in image name doesn't match target registry. Image: '%s', Target: '%s'" % (self.image.registry, registry)) target_image = self.image.copy() target_image.registry = registry response = self.tasker.tag_and_push_image(self.image, target_image, insecure=insecure) self.tasker.remove_image(target_image) return response def inspect_base_image(self): """ inspect base image :return: dict """ logger.info("inspecting base image '%s'", self.base_image) inspect_data = self.tasker.inspect_image(self.base_image) return inspect_data def inspect_built_image(self): """ inspect built image :return: dict """ logger.info("inspecting built image '%s'", self.image_id) self._ensure_is_built() inspect_data = self.tasker.inspect_image(self.image_id) # dict with lots of data, see man docker-inspect return inspect_data def get_base_image_info(self): """ query docker about base image :return dict """ logger.info("getting information about base image '%s'", self.base_image) image_info = self.tasker.get_image_info_by_image_name(self.base_image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.base_image) raise RuntimeError("image '%s' not found", self.base_image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.base_image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.base_image)) def get_built_image_info(self): """ query docker about built image :return dict """ logger.info("getting information about built image '%s'", self.image) self._ensure_is_built() image_info = self.tasker.get_image_info_by_image_name(self.image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.image) raise RuntimeError("image '%s' not found" % self.image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.image))
class InsideBuilder(LastLogger, BuilderStateMachine): """ This is expected to run within container """ def __init__(self, source, image, **kwargs): """ """ LastLogger.__init__(self) BuilderStateMachine.__init__(self) print_version_of_tools() self.tasker = DockerTasker() info, version = self.tasker.get_info(), self.tasker.get_version() logger.debug(json.dumps(info, indent=2)) logger.info(json.dumps(version, indent=2)) # arguments for build self.source = source self.base_image_id = None self.image_id = None self.built_image_info = None self.image = ImageName.parse(image) # get info about base image from dockerfile self.df_path, self.df_dir = self.source.get_dockerfile_path() self.base_image = ImageName.parse(DockerfileParser(self.df_path).baseimage) logger.debug("base image specified in dockerfile = '%s'", self.base_image) if not self.base_image.tag: self.base_image.tag = 'latest' def build(self): """ build image inside current environment; it's expected this may run within (privileged) docker container :return: image string (e.g. fedora-python:34) """ logger.info("building image '%s' inside current environment", self.image) self._ensure_not_built() logger.debug("using dockerfile:\n%s", DockerfileParser(self.df_path).content) logs_gen = self.tasker.build_image_from_path( self.df_dir, self.image, ) logger.debug("build is submitted, waiting for it to finish") command_result = wait_for_command(logs_gen) # wait for build to finish logger.info("build was %ssuccesful!", 'un' if command_result.is_failed() else '') self.is_built = True if not command_result.is_failed(): self.built_image_info = self.get_built_image_info() # self.base_image_id = self.built_image_info['ParentId'] # parent id is not base image! self.image_id = self.built_image_info['Id'] build_result = BuildResult(command_result, self.image_id) return build_result def push_built_image(self, registry, insecure=False): """ push built image to provided registry :param registry: str :param insecure: bool, allow connecting to registry over plain http :return: str, image """ logger.info("pushing built image '%s' to registry '%s'", self.image, registry) self._ensure_is_built() if not registry: logger.warning("no registry specified; skipping") return if self.image.registry and self.image.registry != registry: logger.error("registry in image name doesn't match provided target registry, " "image registry = '%s', target = '%s'", self.image.registry, registry) raise RuntimeError( "Registry in image name doesn't match target registry. Image: '%s', Target: '%s'" % (self.image.registry, registry)) target_image = self.image.copy() target_image.registry = registry response = self.tasker.tag_and_push_image(self.image, target_image, insecure=insecure) self.tasker.remove_image(target_image) return response def inspect_base_image(self): """ inspect base image :return: dict """ logger.info("inspecting base image '%s'", self.base_image) inspect_data = self.tasker.inspect_image(self.base_image) return inspect_data def inspect_built_image(self): """ inspect built image :return: dict """ logger.info("inspecting built image '%s'", self.image_id) self._ensure_is_built() inspect_data = self.tasker.inspect_image(self.image_id) # dict with lots of data, see man docker-inspect return inspect_data def get_base_image_info(self): """ query docker about base image :return dict """ logger.info("getting information about base image '%s'", self.base_image) image_info = self.tasker.get_image_info_by_image_name(self.base_image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.base_image) raise RuntimeError("image '%s' not found", self.base_image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.base_image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.base_image)) def get_built_image_info(self): """ query docker about built image :return dict """ logger.info("getting information about built image '%s'", self.image) self._ensure_is_built() image_info = self.tasker.get_image_info_by_image_name(self.image) items_count = len(image_info) if items_count == 1: return image_info[0] elif items_count <= 0: logger.error("image '%s' not found", self.image) raise RuntimeError("image '%s' not found" % self.image) else: logger.error("multiple (%d) images found for image '%s'", items_count, self.image) raise RuntimeError("multiple (%d) images found for image '%s'" % (items_count, self.image))
class BuildManager(BuilderStateMachine): """ initiates build and waits for it to finish, then it collects data """ def __init__(self, build_image, build_args): BuilderStateMachine.__init__(self) self.build_image = build_image self.build_args = build_args self.image = build_args['image'] self.uri = build_args['source']['uri'] self.temp_dir = None self.build_container_id = None # build image after build self.buildroot_image_id = None self.buildroot_image_name = None self.dt = DockerTasker() def _build(self, build_method): """ build image from provided build_args :return: BuildResults """ logger.info("building image '%s'", self.image) self.ensure_not_built() self.temp_dir = tempfile.mkdtemp() temp_path = os.path.join(self.temp_dir, BUILD_JSON) try: with open(temp_path, 'w') as build_json: json.dump(self.build_args, build_json) self.build_container_id = build_method(self.build_image, self.temp_dir) try: logs_gen = self.dt.logs(self.build_container_id, stream=True) wait_for_command(logs_gen) return_code = self.dt.wait(self.build_container_id) except KeyboardInterrupt: logger.info("killing build container on user's request") self.dt.remove_container(self.build_container_id, force=True) results = BuildResults() results.return_code = 1 return results else: results = self._load_results(self.build_container_id) results.return_code = return_code return results finally: shutil.rmtree(self.temp_dir) def _load_results(self, container_id): """ load results from recent build :return: BuildResults """ if self.temp_dir: dt = DockerTasker() # FIXME: load results only when requested # results_path = os.path.join(self.temp_dir, RESULTS_JSON) # df_path = os.path.join(self.temp_dir, 'Dockerfile') # try: # with open(results_path, 'r') as results_fp: # results = json.load(results_fp, cls=BuildResultsJSONDecoder) # except (IOError, OSError) as ex: # logger.error("Can't open results: '%s'", ex) # for l in self.dt.logs(self.build_container_id, stream=False): # logger.debug(l.strip()) # raise RuntimeError("Can't open results: '%s'" % ex) # results.dockerfile = open(df_path, 'r').read() results = BuildResults() results.build_logs = dt.logs(container_id, stream=False) results.container_id = container_id return results def commit_buildroot(self): """ create image from buildroot :return: """ logger.info("committing buildroot") self.ensure_is_built() commit_message = "docker build of '%s' (%s)" % (self.image, self.uri) self.buildroot_image_name = ImageName( repo="buildroot-%s" % self.image, # save the time when image was built tag=datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) self.buildroot_image_id = self.dt.commit_container(self.build_container_id, commit_message) return self.buildroot_image_id def push_buildroot(self, registry): logger.info("pushing buildroot to registry") self.ensure_is_built() image_name_with_registry = self.buildroot_image_name.copy() image_name_with_registry.registry = registry return self.dt.tag_and_push_image( self.buildroot_image_id, image_name_with_registry)
class BuildManager(BuilderStateMachine): """ initiates build and waits for it to finish, then it collects data """ def __init__(self, build_image, build_args): BuilderStateMachine.__init__(self) self.build_image = build_image self.build_args = build_args self.image = build_args['image'] self.uri = build_args['source']['uri'] self.temp_dir = None # build image after build self.buildroot_image_id = None self.buildroot_image_name = None self.dt = DockerTasker() def _build(self, build_method): """ build image from provided build_args :return: BuildResults """ logger.info("building image '%s'", self.image) self._ensure_not_built() self.temp_dir = tempfile.mkdtemp() temp_path = os.path.join(self.temp_dir, BUILD_JSON) try: with open(temp_path, 'w') as build_json: json.dump(self.build_args, build_json) self.build_container_id = build_method(self.build_image, self.temp_dir) try: logs_gen = self.dt.logs(self.build_container_id, stream=True) wait_for_command(logs_gen) return_code = self.dt.wait(self.build_container_id) except KeyboardInterrupt: logger.info("killing build container on user's request") self.dt.remove_container(self.build_container_id, force=True) results = BuildResults() results.return_code = 1 return results else: results = self._load_results(self.build_container_id) results.return_code = return_code return results finally: shutil.rmtree(self.temp_dir) def _load_results(self, container_id): """ load results from recent build :return: BuildResults """ if self.temp_dir: dt = DockerTasker() # FIXME: load results only when requested # results_path = os.path.join(self.temp_dir, RESULTS_JSON) # df_path = os.path.join(self.temp_dir, 'Dockerfile') # try: # with open(results_path, 'r') as results_fp: # results = json.load(results_fp, cls=BuildResultsJSONDecoder) # except (IOError, OSError) as ex: # logger.error("Can't open results: '%s'", repr(ex)) # for l in self.dt.logs(self.build_container_id, stream=False): # logger.debug(l.strip()) # raise RuntimeError("Can't open results: '%s'" % repr(ex)) # results.dockerfile = open(df_path, 'r').read() results = BuildResults() results.build_logs = dt.logs(container_id, stream=False) results.container_id = container_id return results def commit_buildroot(self): """ create image from buildroot :return: """ logger.info("committing buildroot") self._ensure_is_built() commit_message = "docker build of '%s' (%s)" % (self.image, self.uri) self.buildroot_image_name = ImageName( repo="buildroot-%s" % self.image, # save the time when image was built tag=datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) self.buildroot_image_id = self.dt.commit_container(self.build_container_id, commit_message) return self.buildroot_image_id def push_buildroot(self, registry): logger.info("pushing buildroot to registry") self._ensure_is_built() image_name_with_registry = self.buildroot_image_name.copy() image_name_with_registry.registry = registry return self.dt.tag_and_push_image( self.buildroot_image_id, image_name_with_registry)