def copy_files( filepath: Path, ctx: dict, out_dir: str, relative_to: Optional[str] = None ): out_dir = Path(out_dir) if relative_to is not None: new_filepath = Path(out_dir) / filepath.relative_to(relative_to) else: new_filepath = Path(out_dir) / filepath if filepath.is_file(): new_filepath.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(filepath, new_filepath) elif filepath.is_dir(): if new_filepath.exists(): shutil.rmtree(new_filepath) shutil.copytree(filepath, new_filepath) return ctx
def _docker_image_create(self, docker_base_image: List[Any]) -> Tuple[str, str]: """ Create docker image: 1. Installing 'build base' if required in alpine images version - https://wiki.alpinelinux.org/wiki/GCC 2. Installing pypi packs - if only pylint required - only pylint installed otherwise all pytest and pylint installed, packages which being install can be found in path demisto_sdk/commands/lint/dev_envs 3. The docker image build done by Dockerfile template located in demisto_sdk/commands/lint/templates/dockerfile.jinja2 Args: docker_base_image(list): docker image to use as base for installing dev deps and python version. Returns: str, str. image name to use and errors string. """ log_prompt = f"{self._pack_name} - Image create" test_image_id = "" # Get requirements file for image requirements = [] if 2 < docker_base_image[1] < 3: requirements = self._req_2 elif docker_base_image[1] > 3: requirements = self._req_3 # Using DockerFile template file_loader = FileSystemLoader(Path(__file__).parent / 'templates') env = Environment(loader=file_loader, lstrip_blocks=True, trim_blocks=True, autoescape=True) template = env.get_template('dockerfile.jinja2') try: dockerfile = template.render( image=docker_base_image[0], pypi_packs=requirements + self._facts["additional_requirements"], pack_type=self._pkg_lint_status["pack_type"], copy_pack=False) except exceptions.TemplateError as e: logger.debug( f"{log_prompt} - Error when build image - {e.message()}") return test_image_id, str(e) # Trying to pull image based on dockerfile hash, will check if something changed errors = "" test_image_name = f'devtest{docker_base_image[0]}-{hashlib.md5(dockerfile.encode("utf-8")).hexdigest()}' test_image = None try: logger.info( f"{log_prompt} - Trying to pull existing image {test_image_name}" ) test_image = self._docker_client.images.pull(test_image_name) except (docker.errors.APIError, docker.errors.ImageNotFound): logger.info( f"{log_prompt} - Unable to find image {test_image_name}") # Creatng new image if existing image isn't found if not test_image: logger.info( f"{log_prompt} - Creating image based on {docker_base_image[0]} - Could take 2-3 minutes at first " f"time") try: with io.BytesIO() as f: f.write(dockerfile.encode('utf-8')) f.seek(0) self._docker_client.images.build(fileobj=f, tag=test_image_name, forcerm=True) if self._docker_hub_login: for trial in range(2): try: self._docker_client.images.push( test_image_name) logger.info( f"{log_prompt} - Image {test_image_name} pushed to repository" ) break except (requests.exceptions.ConnectionError, urllib3.exceptions.ReadTimeoutError): logger.info( f"{log_prompt} - Unable to push image {test_image_name} to repository" ) except (docker.errors.BuildError, docker.errors.APIError, Exception) as e: logger.critical(f"{log_prompt} - Build errors occurred {e}") errors = str(e) else: logger.info( f"{log_prompt} - Found existing image {test_image_name}") for trial in range(2): dockerfile_path = Path(self._pack_abs_dir / ".Dockerfile") try: logger.info( f"{log_prompt} - Copy pack dir to image {test_image_name}") dockerfile = template.render(image=test_image_name, copy_pack=True) with open(file=dockerfile_path, mode="+x") as file: file.write(str(dockerfile)) docker_image_final = self._docker_client.images.build( path=str(dockerfile_path.parent), dockerfile=dockerfile_path.stem, forcerm=True) test_image_name = docker_image_final[0].short_id break except (docker.errors.ImageNotFound, docker.errors.APIError, urllib3.exceptions.ReadTimeoutError, exceptions.TemplateError) as e: logger.info( f"{log_prompt} - errors occurred when copy pack dir {e}") if trial == 2: errors = str(e) if dockerfile_path.exists(): dockerfile_path.unlink() if test_image_id: logger.info( f"{log_prompt} - Image {test_image_id} created successfully") return test_image_name, errors