def copy_files(
filepath: Path, ctx: dict, out_dir: str, relative_to: Optional[str] = None
):
out_dir = Path(out_dir)
if relative_to is not None:
new_filepath = Path(out_dir) / filepath.relative_to(relative_to)
else:
new_filepath = Path(out_dir) / filepath
if filepath.is_file():
new_filepath.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(filepath, new_filepath)
elif filepath.is_dir():
if new_filepath.exists():
shutil.rmtree(new_filepath)
shutil.copytree(filepath, new_filepath)
return ctx
def _docker_image_create(self,
docker_base_image: List[Any]) -> Tuple[str, str]:
""" Create docker image:
1. Installing 'build base' if required in alpine images version - https://wiki.alpinelinux.org/wiki/GCC
2. Installing pypi packs - if only pylint required - only pylint installed otherwise all pytest and pylint
installed, packages which being install can be found in path demisto_sdk/commands/lint/dev_envs
3. The docker image build done by Dockerfile template located in
demisto_sdk/commands/lint/templates/dockerfile.jinja2
Args:
docker_base_image(list): docker image to use as base for installing dev deps and python version.
Returns:
str, str. image name to use and errors string.
"""
log_prompt = f"{self._pack_name} - Image create"
test_image_id = ""
# Get requirements file for image
requirements = []
if 2 < docker_base_image[1] < 3:
requirements = self._req_2
elif docker_base_image[1] > 3:
requirements = self._req_3
# Using DockerFile template
file_loader = FileSystemLoader(Path(__file__).parent / 'templates')
env = Environment(loader=file_loader,
lstrip_blocks=True,
trim_blocks=True,
autoescape=True)
template = env.get_template('dockerfile.jinja2')
try:
dockerfile = template.render(
image=docker_base_image[0],
pypi_packs=requirements +
self._facts["additional_requirements"],
pack_type=self._pkg_lint_status["pack_type"],
copy_pack=False)
except exceptions.TemplateError as e:
logger.debug(
f"{log_prompt} - Error when build image - {e.message()}")
return test_image_id, str(e)
# Trying to pull image based on dockerfile hash, will check if something changed
errors = ""
test_image_name = f'devtest{docker_base_image[0]}-{hashlib.md5(dockerfile.encode("utf-8")).hexdigest()}'
test_image = None
try:
logger.info(
f"{log_prompt} - Trying to pull existing image {test_image_name}"
)
test_image = self._docker_client.images.pull(test_image_name)
except (docker.errors.APIError, docker.errors.ImageNotFound):
logger.info(
f"{log_prompt} - Unable to find image {test_image_name}")
# Creatng new image if existing image isn't found
if not test_image:
logger.info(
f"{log_prompt} - Creating image based on {docker_base_image[0]} - Could take 2-3 minutes at first "
f"time")
try:
with io.BytesIO() as f:
f.write(dockerfile.encode('utf-8'))
f.seek(0)
self._docker_client.images.build(fileobj=f,
tag=test_image_name,
forcerm=True)
if self._docker_hub_login:
for trial in range(2):
try:
self._docker_client.images.push(
test_image_name)
logger.info(
f"{log_prompt} - Image {test_image_name} pushed to repository"
)
break
except (requests.exceptions.ConnectionError,
urllib3.exceptions.ReadTimeoutError):
logger.info(
f"{log_prompt} - Unable to push image {test_image_name} to repository"
)
except (docker.errors.BuildError, docker.errors.APIError,
Exception) as e:
logger.critical(f"{log_prompt} - Build errors occurred {e}")
errors = str(e)
else:
logger.info(
f"{log_prompt} - Found existing image {test_image_name}")
for trial in range(2):
dockerfile_path = Path(self._pack_abs_dir / ".Dockerfile")
try:
logger.info(
f"{log_prompt} - Copy pack dir to image {test_image_name}")
dockerfile = template.render(image=test_image_name,
copy_pack=True)
with open(file=dockerfile_path, mode="+x") as file:
file.write(str(dockerfile))
docker_image_final = self._docker_client.images.build(
path=str(dockerfile_path.parent),
dockerfile=dockerfile_path.stem,
forcerm=True)
test_image_name = docker_image_final[0].short_id
break
except (docker.errors.ImageNotFound, docker.errors.APIError,
urllib3.exceptions.ReadTimeoutError,
exceptions.TemplateError) as e:
logger.info(
f"{log_prompt} - errors occurred when copy pack dir {e}")
if trial == 2:
errors = str(e)
if dockerfile_path.exists():
dockerfile_path.unlink()
if test_image_id:
logger.info(
f"{log_prompt} - Image {test_image_id} created successfully")
return test_image_name, errors
