def checkDockerImageExists(appliance):
"""
Attempts to check a url registryName for the existence of a docker image with a given tag.
:param str appliance: The url of a docker image's registry (with a tag) of the form:
'quay.io/<repo_path>:<tag>' or '<repo_path>:<tag>'.
Examples: 'quay.io/ucsc_cgl/toil:latest', 'ubuntu:latest', or
'broadinstitute/genomes-in-the-cloud:2.0.0'.
:return: Raises an exception if the docker image cannot be found or is invalid. Otherwise, it
will return the appliance string.
:rtype: str
"""
appliance = appliance.lower()
tag = appliance.split(':')[-1]
registryName = appliance[:-(len(':' + tag))] # remove only the tag
if currentCommit in appliance:
return appliance
# docker repo syntax sanity checks
elif '://' in registryName:
raise ImageNotFound(
"Docker images cannot contain a schema (such as '://'): %s"
"" % registryName + ':' + tag)
# docker repo syntax sanity checks
elif registryName[0] == '-':
raise ImageNotFound("Docker images cannot begin with '-': %s"
"" % registryName + ':' + tag)
# check if a valid io host is specified
elif '.io/' in registryName and 'docker.io/' not in registryName:
return requestCheckRegularIo(registryName=registryName, tag=tag)
# otherwise check docker.io
else:
return requestCheckDockerIo(registryName=registryName, tag=tag)
def checkDockerSchema(appliance):
if not appliance:
raise ImageNotFound("No docker image specified.")
elif '://' in appliance:
raise ImageNotFound("Docker images cannot contain a schema (such as '://'): %s"
"" % appliance)
elif len(appliance) > 256:
raise ImageNotFound("Docker image must be less than 256 chars: %s"
"" % appliance)
def test_force_building_image_that_doesnt_already_exists(
self, generate_docker_image_version_patch, build_image_patch):
layer_downloader_mock = Mock()
layer_downloader_mock.download_all.return_value = ["layers1"]
generate_docker_image_version_patch.return_value = "image-version"
docker_client_mock = Mock()
docker_client_mock.images.get.side_effect = ImageNotFound(
"image not found")
lambda_image = LambdaImage(layer_downloader_mock,
False,
True,
docker_client=docker_client_mock)
actual_image_id = lambda_image.build("python3.6", ["layers1"])
self.assertEquals(actual_image_id, "samcli/lambda:image-version")
layer_downloader_mock.download_all.assert_called_once_with(["layers1"],
True)
generate_docker_image_version_patch.assert_called_once_with(
["layers1"], "python3.6")
docker_client_mock.images.get.assert_called_once_with(
"samcli/lambda:image-version")
build_image_patch.assert_called_once_with(
"lambci/lambda:python3.6", "samcli/lambda:image-version",
["layers1"])
def test_ImageObserverException_raised_for_not_exist_image(self):
image_name = "test_image:test_version"
self.docker_client_mock.images.get.side_effect = ImageNotFound("")
with self.assertRaises(ImageObserverException):
self.image_observer.watch(image_name)
self.docker_client_mock.images.get.assert_called_with(image_name)
def test_missing_image(self, docker_run: Mock):
"""Test proper handling of container encountering an error"""
docker_run.side_effect = ImageNotFound("test error")
rpm = ReductionProcessManager(self.message, self.run_name,
self.software)
self.assertRaises(ImageNotFound, rpm.run)
docker_run.assert_called_once()
def test_remove_image_not_found(
cfngin_context: MockCFNginContext,
mock_docker_client: DockerClient,
mocker: MockerFixture,
) -> None:
"""Test remove ImageNotFound."""
args = ImageRemoveArgs(repo="dkr.test.com/image", tags=["latest"])
mocker.patch.object(ImageRemoveArgs, "parse_obj", return_value=args)
mocker.patch.object(DockerHookData, "client", mock_docker_client)
docker_hook_data = DockerHookData()
mocker.patch.object(
DockerHookData, "from_cfngin_context", return_value=docker_hook_data
)
mock_update_context = mocker.patch.object(
DockerHookData, "update_context", return_value=docker_hook_data
)
cfngin_context.hook_data["docker"] = docker_hook_data
docker_hook_data.client.api.remove_image.side_effect = ImageNotFound(
args.repo + ":latest" # type: ignore
)
assert remove(context=cfngin_context, **args.dict()) == docker_hook_data
docker_hook_data.client.api.remove_image.assert_has_calls( # pylint: disable=no-member
[
call(force=False, image="{}:{}".format(args.repo, tag), noprune=False)
for tag in args.tags
]
)
mock_update_context.assert_called_once_with(cfngin_context)
def get(self, name: str) -> MockImage:
assert ":" in name
for image in self.list():
if name in image.tags:
return image
raise ImageNotFound(name)
def _build_image(build_args):
image_tag, dockerfile_path = build_args
log.info(f"Building {image_tag} from {dockerfile_path}")
try:
# this is should be when there are changed in base image.
if FLAGS.overwrite:
docker_client.api.remove_image(image_tag, force=True)
# workaround for not having goto supports in Python.
raise ImageNotFound(f"Building {image_tag}")
if docker_client.api.history(image_tag):
log.info(
f"Image {image_tag} is already built and --overwrite is not specified. Skipping..."
)
self._push_context[
image_tag] = docker_client.images.get(image_tag)
return
except ImageNotFound:
pyenv = get_data(self._tags, image_tag, "envars",
"PYTHON_VERSION")
build_args = {"PYTHON_VERSION": pyenv}
# NOTES: https://warehouse.pypa.io/api-reference/index.html
log.info(
f"Building docker image {image_tag} with {dockerfile_path}"
)
resp = docker_client.api.build(
timeout=FLAGS.timeout,
path=".",
nocache=False,
buildargs=build_args,
dockerfile=dockerfile_path,
tag=image_tag,
rm=True,
)
self.build_logs(resp, image_tag)
def run(self, image, **kwargs):
for container in self.containers:
for tag in container.image.tags:
if tag.startswith(image):
if container.status == 'running':
raise DockerError('Container is already running')
container.status = 'running'
return container
raise ImageNotFound('Image \'{}\' not found'.format(image))
def test_must_raise_if_image_not_found(self):
msg = "some error"
self.mock_docker_client.api.pull.side_effect = ImageNotFound(msg)
with self.assertRaises(DockerImageNotFoundException) as context:
self.manager.pull_image("imagename")
ex = context.exception
self.assertEquals(str(ex), msg)
def test_get_local_image_id(self, image_id):
docker_client = MagicMock(spec=DockerClient)
image = MagicMock(spec=Image)
image.attrs = {'RepoDigests': []}
docker_client.images.pull.side_effect = ImageNotFound(image_id)
docker_client.images.get.return_value = image
fetched_image_sha = get_image_sha(docker_client, image_id)
assert fetched_image_sha == image_id
def pull(self, latest: bool = False) -> Image:
# We could use the Python API to pull, like this:
# return self.client.images.pull(self.image_name, tag=[self.tag, None][latest])
# However, that doesn't include progress bars. So call the `docker` command instead:
name = f"{self.image_name}:{[self.tag, 'latest'][latest]}"
try:
subprocess.check_call(["docker", "pull", name])
for image in self.client.images.list():
if name in image.tags:
return image
except subprocess.CalledProcessError:
pass
raise ImageNotFound(name)
def requestCheckDockerIo(registryName, tag):
"""
Checks docker.io to see if an image exists using the requests library.
URL is based on the docker v2 schema. Requires that an access token be fetched first.
:param str registryName: The url of a docker image's registry. e.g. "ubuntu"
:param str tag: The tag used at that docker image's registry. e.g. "latest"
:return: Return True if match found. Raise otherwise.
"""
pathName = registryName
if pathName.endswith('/'):
pathName = pathName[:-1]
# only official images like 'busybox' or 'ubuntu'
if '/' not in pathName:
pathName = 'library/' + pathName
token_url = 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull'.format(
repo=pathName)
requests_url = 'https://registry-1.docker.io/v2/{repo}/manifests/{tag}'.format(
repo=pathName, tag=tag)
token = requests.get(token_url)
jsonToken = token.json()
bearer = jsonToken["token"]
response = requests.head(
requests_url, headers={'Authorization': 'Bearer {}'.format(bearer)})
if not response.ok:
raise ImageNotFound(
"The docker image that TOIL_APPLIANCE_SELF specifies (%s) is "
"unsupported (or malformed). Please supply a docker image with the "
"format: '<websitehost>.io/<repo_path>:<tag>' or '<repo_path>:<tag>' "
"(for official docker.io images). Examples: "
"'quay.io/ucsc_cgl/toil:latest', 'ubuntu:latest', or "
"'broadinstitute/genomes-in-the-cloud:2.0.0'."
"" % (registryName + ':' + tag))
else:
return registryName + ':' + tag
def requestCheckRegularIo(registryName, tag):
"""
Checks to see if an image exists using the requests library.
URL is based on the docker v2 schema described here:
https://docs.docker.com/registry/spec/manifest-v2-2/
This has the following format:
https://{websitehostname}.io/v2/{repo}/manifests/{tag}
Does not work with the official (docker.io) site, because they require an OAuth token, so a
separate check is done for docker.io images.
:param str registryName: The url of a docker image's registry. e.g. "quay.io/ucsc_cgl/toil"
:param str tag: The tag used at that docker image's registry. e.g. "latest"
:return: Return True if match found. Raise otherwise.
"""
# find initial position of '.io/'
clipPosition = registryName.find('.io/')
# split at '.io/' to get the path
splitHere = clipPosition + len('.io/')
webhostName = registryName[:splitHere]
pathName = registryName[splitHere:]
if pathName.endswith('/'):
pathName = pathName[:-1]
ioURL = 'https://{webhost}v2/{pathName}/manifests/{tag}' \
''.format(pathName=pathName, webhost=webhostName, tag=tag)
response = requests.head(ioURL)
if not response.ok:
raise ImageNotFound(
"The docker image that TOIL_APPLIANCE_SELF specifies (%s) is "
"unsupported (or malformed). Please supply a docker image with the "
"format: '<websitehost>.io/<repo_path>:<tag>' or '<repo_path>:<tag>' "
"(for official docker.io images). Examples: "
"'quay.io/ucsc_cgl/toil:latest', 'ubuntu:latest', or "
"'broadinstitute/genomes-in-the-cloud:2.0.0'."
"" % (registryName + ':' + tag))
else:
return registryName + ':' + tag
def test_not_force_building_image_that_doesnt_already_exists(
self, generate_docker_image_version_patch, build_image_patch):
layer_downloader_mock = Mock()
layer_downloader_mock.download_all.return_value = ["layers1"]
generate_docker_image_version_patch.return_value = "image-version"
docker_client_mock = Mock()
docker_client_mock.images.get.side_effect = ImageNotFound(
"image not found")
stream = io.StringIO()
lambda_image = LambdaImage(layer_downloader_mock,
False,
False,
docker_client=docker_client_mock)
actual_image_id = lambda_image.build("python3.6",
ZIP,
None, ["layers1"],
False,
stream=stream)
self.assertEqual(actual_image_id, "samcli/lambda:image-version")
layer_downloader_mock.download_all.assert_called_once_with(["layers1"],
False)
generate_docker_image_version_patch.assert_called_once_with(
["layers1"], "python3.6")
docker_client_mock.images.get.assert_called_once_with(
"samcli/lambda:image-version")
build_image_patch.assert_called_once_with(
"amazon/aws-sam-cli-emulation-image-python3.6:latest",
"samcli/lambda:image-version",
["layers1"],
False,
stream=stream,
)
def test_must_not_find_image(self):
self.mock_docker_client.images.get.side_effect = ImageNotFound("test")
self.assertFalse(self.manager.has_image(self.image_name))
import sys
docker.api.build.process_dockerfile = lambda dockerfile, path: (
"Dockerfile",
dockerfile,
)
tag = f"jinjamator/jinjamator-runtime:latest"
client = docker.from_env()
try:
client.images.get(tag)
log.info(f"base image {tag} found")
if force_rebuild:
log.info("force_rebuild set -> rebuilding")
raise ImageNotFound("force_rebuild set -> rebuilding")
except ImageNotFound:
log.info("base image not found building")
docker_file = task.run("dockerfile", output_plugin="null")[0]["result"]
log.debug(docker_file)
try:
image, logs = client.images.build(path=".",
dockerfile=docker_file,
tag=tag,
nocache=True)
except BuildError as e:
# for line in logs:
# log.debug(line)
log.error(e)
raise
def raise_ImageNotFound(*args, **kwargs):
raise ImageNotFound(message="")