def run_task(self): image_info_of_dependencies = DependencyImageInfoCollector( ).get_from_dict_of_inputs(self.input()) image_hash = self._build_context_hasher.generate_image_hash( image_info_of_dependencies) image_info = ImageInfo( source_repository_name=self._source_repository_name, target_repository_name=self._target_repository_name, source_tag=self._source_image_tag, target_tag=self._target_image_tag, hash=image_hash, commit=self.get_commit_id(), build_name=build_config().build_name, depends_on_images=image_info_of_dependencies, image_state=None, image_description=self.image_description) target_image_target = DockerImageTarget( self._target_repository_name, image_info.get_target_complete_tag()) source_image_target = DockerImageTarget( self._source_repository_name, image_info.get_source_complete_tag()) image_state = self.get_image_state(source_image_target, target_image_target, image_info_of_dependencies) image_info.image_state = image_state.name # TODO setter for image_state with self._image_info_target.open("w") as f: f.write(image_info.to_json())
class DockerImageCreatorBaseTask(StoppableTask): logger = logging.getLogger('luigi-interface') image_name = luigi.Parameter() # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown image_info_json = luigi.Parameter( visibility=luigi.parameter.ParameterVisibility.HIDDEN, significant=True) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.client = docker_client_config().get_client() self.image_info = ImageInfo.from_json(self.image_info_json) self.image_target = DockerImageTarget( self.image_info.target_repository_name, self.image_info.get_target_complete_tag()) self.remove_image() def __del__(self): self.client.close() def remove_image(self): if self.image_target.exists(): self.client.images.remove( image=self.image_target.get_complete_name(), force=True) self.logger.warning("Task %s: Removed docker images %s", self.task_id, self.image_target.get_complete_name()) def output(self): return self.image_target
def get_image_state( self, source_image_target: DockerImageTarget, target_image_target: DockerImageTarget, image_info_of_dependencies: Dict[str, ImageInfo]) -> ImageState: if self.is_rebuild_necessary(image_info_of_dependencies): return ImageState.NEEDS_TO_BE_BUILD else: if target_image_target.exists() \ and not build_config().force_pull \ and not build_config().force_load: self.logger.info( f"Task {self.__repr__()}: Checking if target image {target_image_target.get_complete_name()} " f"is locally available, result {target_image_target.exists()}" ) return ImageState.TARGET_LOCALLY_AVAILABLE if source_image_target.exists() \ and not build_config().force_pull \ and not build_config().force_load: self.logger.info( f"Task {self.__repr__()}: Checking if source image {source_image_target.get_complete_name()} " f"is locally available, result {source_image_target.exists()}" ) return ImageState.SOURCE_LOCALLY_AVAILABLE elif self.can_image_be_loaded(source_image_target): return ImageState.CAN_BE_LOADED elif self.is_image_in_registry(source_image_target): return ImageState.REMOTE_AVAILABLE else: return ImageState.NEEDS_TO_BE_BUILD
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.client = docker_client_config().get_client() self.image_info = ImageInfo.from_json(self.image_info_json) self.image_target = DockerImageTarget( self.image_info.target_repository_name, self.image_info.get_target_complete_tag()) self.remove_image()
def is_image_in_registry(self, image_target: DockerImageTarget): try: self.logger.info("Task %s: Try to find image %s in registry", self.__repr__(), image_target.get_complete_name()) exists = DockerRegistryImageChecker().check( image_target.get_complete_name()) if exists: self.logger.info("Task %s: Found image %s in registry", self.__repr__(), image_target.get_complete_name()) return exists except docker.errors.DockerException as e: self.logger.warning( "Task %s: Image %s not in registry, got exception %s", self.__repr__(), image_target.get_complete_name(), e) return False
def is_image_locally_available(self, image_target: DockerImageTarget): exists = image_target.exists() self.logger.info( f"Task {self.__repr__()}: Checking if image {image_target.get_complete_name()} " f"is locally available, result {exists}") return exists