Ejemplo n.º 1
0
def checkDockerImageExists(appliance):
    """
    Attempts to check a url registryName for the existence of a docker image with a given tag.

    :param str appliance: The url of a docker image's registry (with a tag) of the form:
                          'quay.io/<repo_path>:<tag>' or '<repo_path>:<tag>'.
                          Examples: 'quay.io/ucsc_cgl/toil:latest', 'ubuntu:latest', or
                          'broadinstitute/genomes-in-the-cloud:2.0.0'.
    :return: Raises an exception if the docker image cannot be found or is invalid.  Otherwise, it
             will return the appliance string.
    :rtype: str
    """
    appliance = appliance.lower()
    tag = appliance.split(':')[-1]
    registryName = appliance[:-(len(':' + tag))]  # remove only the tag
    if currentCommit in appliance:
        return appliance
    # docker repo syntax sanity checks
    elif '://' in registryName:
        raise ImageNotFound(
            "Docker images cannot contain a schema (such as '://'): %s"
            "" % registryName + ':' + tag)
    # docker repo syntax sanity checks
    elif registryName[0] == '-':
        raise ImageNotFound("Docker images cannot begin with '-': %s"
                            "" % registryName + ':' + tag)
    # check if a valid io host is specified
    elif '.io/' in registryName and 'docker.io/' not in registryName:
        return requestCheckRegularIo(registryName=registryName, tag=tag)
    # otherwise check docker.io
    else:
        return requestCheckDockerIo(registryName=registryName, tag=tag)
Ejemplo n.º 2
0
def checkDockerSchema(appliance):
    if not appliance:
        raise ImageNotFound("No docker image specified.")
    elif '://' in appliance:
        raise ImageNotFound("Docker images cannot contain a schema (such as '://'): %s"
                            "" % appliance)
    elif len(appliance) > 256:
        raise ImageNotFound("Docker image must be less than 256 chars: %s"
                            "" % appliance)
Ejemplo n.º 3
0
    def test_force_building_image_that_doesnt_already_exists(
            self, generate_docker_image_version_patch, build_image_patch):
        layer_downloader_mock = Mock()
        layer_downloader_mock.download_all.return_value = ["layers1"]

        generate_docker_image_version_patch.return_value = "image-version"

        docker_client_mock = Mock()
        docker_client_mock.images.get.side_effect = ImageNotFound(
            "image not found")

        lambda_image = LambdaImage(layer_downloader_mock,
                                   False,
                                   True,
                                   docker_client=docker_client_mock)
        actual_image_id = lambda_image.build("python3.6", ["layers1"])

        self.assertEquals(actual_image_id, "samcli/lambda:image-version")

        layer_downloader_mock.download_all.assert_called_once_with(["layers1"],
                                                                   True)
        generate_docker_image_version_patch.assert_called_once_with(
            ["layers1"], "python3.6")
        docker_client_mock.images.get.assert_called_once_with(
            "samcli/lambda:image-version")
        build_image_patch.assert_called_once_with(
            "lambci/lambda:python3.6", "samcli/lambda:image-version",
            ["layers1"])
Ejemplo n.º 4
0
    def test_ImageObserverException_raised_for_not_exist_image(self):
        image_name = "test_image:test_version"
        self.docker_client_mock.images.get.side_effect = ImageNotFound("")

        with self.assertRaises(ImageObserverException):
            self.image_observer.watch(image_name)
        self.docker_client_mock.images.get.assert_called_with(image_name)
 def test_missing_image(self, docker_run: Mock):
     """Test proper handling of container encountering an error"""
     docker_run.side_effect = ImageNotFound("test error")
     rpm = ReductionProcessManager(self.message, self.run_name,
                                   self.software)
     self.assertRaises(ImageNotFound, rpm.run)
     docker_run.assert_called_once()
Ejemplo n.º 6
0
def test_remove_image_not_found(
    cfngin_context: MockCFNginContext,
    mock_docker_client: DockerClient,
    mocker: MockerFixture,
) -> None:
    """Test remove ImageNotFound."""
    args = ImageRemoveArgs(repo="dkr.test.com/image", tags=["latest"])
    mocker.patch.object(ImageRemoveArgs, "parse_obj", return_value=args)
    mocker.patch.object(DockerHookData, "client", mock_docker_client)
    docker_hook_data = DockerHookData()
    mocker.patch.object(
        DockerHookData, "from_cfngin_context", return_value=docker_hook_data
    )
    mock_update_context = mocker.patch.object(
        DockerHookData, "update_context", return_value=docker_hook_data
    )
    cfngin_context.hook_data["docker"] = docker_hook_data
    docker_hook_data.client.api.remove_image.side_effect = ImageNotFound(
        args.repo + ":latest"  # type: ignore
    )
    assert remove(context=cfngin_context, **args.dict()) == docker_hook_data
    docker_hook_data.client.api.remove_image.assert_has_calls(  # pylint: disable=no-member
        [
            call(force=False, image="{}:{}".format(args.repo, tag), noprune=False)
            for tag in args.tags
        ]
    )
    mock_update_context.assert_called_once_with(cfngin_context)
Ejemplo n.º 7
0
    def get(self, name: str) -> MockImage:
        assert ":" in name
        for image in self.list():
            if name in image.tags:
                return image

        raise ImageNotFound(name)
Ejemplo n.º 8
0
            def _build_image(build_args):
                image_tag, dockerfile_path = build_args
                log.info(f"Building {image_tag} from {dockerfile_path}")
                try:
                    # this is should be when there are changed in base image.
                    if FLAGS.overwrite:
                        docker_client.api.remove_image(image_tag, force=True)
                        # workaround for not having goto supports in Python.
                        raise ImageNotFound(f"Building {image_tag}")
                    if docker_client.api.history(image_tag):
                        log.info(
                            f"Image {image_tag} is already built and --overwrite is not specified. Skipping..."
                        )
                        self._push_context[
                            image_tag] = docker_client.images.get(image_tag)
                        return
                except ImageNotFound:
                    pyenv = get_data(self._tags, image_tag, "envars",
                                     "PYTHON_VERSION")
                    build_args = {"PYTHON_VERSION": pyenv}
                    # NOTES: https://warehouse.pypa.io/api-reference/index.html

                    log.info(
                        f"Building docker image {image_tag} with {dockerfile_path}"
                    )
                    resp = docker_client.api.build(
                        timeout=FLAGS.timeout,
                        path=".",
                        nocache=False,
                        buildargs=build_args,
                        dockerfile=dockerfile_path,
                        tag=image_tag,
                        rm=True,
                    )
                    self.build_logs(resp, image_tag)
Ejemplo n.º 9
0
 def run(self, image, **kwargs):
     for container in self.containers:
         for tag in container.image.tags:
             if tag.startswith(image):
                 if container.status == 'running':
                     raise DockerError('Container is already running')
                 container.status = 'running'
                 return container
     raise ImageNotFound('Image \'{}\' not found'.format(image))
Ejemplo n.º 10
0
    def test_must_raise_if_image_not_found(self):
        msg = "some error"
        self.mock_docker_client.api.pull.side_effect = ImageNotFound(msg)

        with self.assertRaises(DockerImageNotFoundException) as context:
            self.manager.pull_image("imagename")

        ex = context.exception
        self.assertEquals(str(ex), msg)
Ejemplo n.º 11
0
    def test_get_local_image_id(self, image_id):
        docker_client = MagicMock(spec=DockerClient)

        image = MagicMock(spec=Image)
        image.attrs = {'RepoDigests': []}

        docker_client.images.pull.side_effect = ImageNotFound(image_id)
        docker_client.images.get.return_value = image

        fetched_image_sha = get_image_sha(docker_client, image_id)

        assert fetched_image_sha == image_id
Ejemplo n.º 12
0
 def pull(self, latest: bool = False) -> Image:
     # We could use the Python API to pull, like this:
     #     return self.client.images.pull(self.image_name, tag=[self.tag, None][latest])
     # However, that doesn't include progress bars. So call the `docker` command instead:
     name = f"{self.image_name}:{[self.tag, 'latest'][latest]}"
     try:
         subprocess.check_call(["docker", "pull", name])
         for image in self.client.images.list():
             if name in image.tags:
                 return image
     except subprocess.CalledProcessError:
         pass
     raise ImageNotFound(name)
Ejemplo n.º 13
0
def requestCheckDockerIo(registryName, tag):
    """
    Checks docker.io to see if an image exists using the requests library.

    URL is based on the docker v2 schema.  Requires that an access token be fetched first.

    :param str registryName: The url of a docker image's registry.  e.g. "ubuntu"
    :param str tag: The tag used at that docker image's registry.  e.g. "latest"
    :return: Return True if match found.  Raise otherwise.
    """
    pathName = registryName
    if pathName.endswith('/'):
        pathName = pathName[:-1]

    # only official images like 'busybox' or 'ubuntu'
    if '/' not in pathName:
        pathName = 'library/' + pathName

    token_url = 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull'.format(
        repo=pathName)
    requests_url = 'https://registry-1.docker.io/v2/{repo}/manifests/{tag}'.format(
        repo=pathName, tag=tag)

    token = requests.get(token_url)
    jsonToken = token.json()
    bearer = jsonToken["token"]
    response = requests.head(
        requests_url, headers={'Authorization': 'Bearer {}'.format(bearer)})
    if not response.ok:
        raise ImageNotFound(
            "The docker image that TOIL_APPLIANCE_SELF specifies (%s) is "
            "unsupported (or malformed).  Please supply a docker image with the "
            "format: '<websitehost>.io/<repo_path>:<tag>' or '<repo_path>:<tag>' "
            "(for official docker.io images).  Examples: "
            "'quay.io/ucsc_cgl/toil:latest', 'ubuntu:latest', or "
            "'broadinstitute/genomes-in-the-cloud:2.0.0'."
            "" % (registryName + ':' + tag))
    else:
        return registryName + ':' + tag
Ejemplo n.º 14
0
def requestCheckRegularIo(registryName, tag):
    """
    Checks to see if an image exists using the requests library.

    URL is based on the docker v2 schema described here:
    https://docs.docker.com/registry/spec/manifest-v2-2/

    This has the following format:
    https://{websitehostname}.io/v2/{repo}/manifests/{tag}

    Does not work with the official (docker.io) site, because they require an OAuth token, so a
    separate check is done for docker.io images.

    :param str registryName: The url of a docker image's registry.  e.g. "quay.io/ucsc_cgl/toil"
    :param str tag: The tag used at that docker image's registry.  e.g. "latest"
    :return: Return True if match found.  Raise otherwise.
    """
    # find initial position of '.io/'
    clipPosition = registryName.find('.io/')
    # split at '.io/' to get the path
    splitHere = clipPosition + len('.io/')
    webhostName = registryName[:splitHere]
    pathName = registryName[splitHere:]
    if pathName.endswith('/'):
        pathName = pathName[:-1]
    ioURL = 'https://{webhost}v2/{pathName}/manifests/{tag}' \
              ''.format(pathName=pathName, webhost=webhostName, tag=tag)
    response = requests.head(ioURL)
    if not response.ok:
        raise ImageNotFound(
            "The docker image that TOIL_APPLIANCE_SELF specifies (%s) is "
            "unsupported (or malformed).  Please supply a docker image with the "
            "format: '<websitehost>.io/<repo_path>:<tag>' or '<repo_path>:<tag>' "
            "(for official docker.io images).  Examples: "
            "'quay.io/ucsc_cgl/toil:latest', 'ubuntu:latest', or "
            "'broadinstitute/genomes-in-the-cloud:2.0.0'."
            "" % (registryName + ':' + tag))
    else:
        return registryName + ':' + tag
Ejemplo n.º 15
0
    def test_not_force_building_image_that_doesnt_already_exists(
            self, generate_docker_image_version_patch, build_image_patch):
        layer_downloader_mock = Mock()
        layer_downloader_mock.download_all.return_value = ["layers1"]

        generate_docker_image_version_patch.return_value = "image-version"

        docker_client_mock = Mock()
        docker_client_mock.images.get.side_effect = ImageNotFound(
            "image not found")

        stream = io.StringIO()

        lambda_image = LambdaImage(layer_downloader_mock,
                                   False,
                                   False,
                                   docker_client=docker_client_mock)
        actual_image_id = lambda_image.build("python3.6",
                                             ZIP,
                                             None, ["layers1"],
                                             False,
                                             stream=stream)

        self.assertEqual(actual_image_id, "samcli/lambda:image-version")

        layer_downloader_mock.download_all.assert_called_once_with(["layers1"],
                                                                   False)
        generate_docker_image_version_patch.assert_called_once_with(
            ["layers1"], "python3.6")
        docker_client_mock.images.get.assert_called_once_with(
            "samcli/lambda:image-version")
        build_image_patch.assert_called_once_with(
            "amazon/aws-sam-cli-emulation-image-python3.6:latest",
            "samcli/lambda:image-version",
            ["layers1"],
            False,
            stream=stream,
        )
Ejemplo n.º 16
0
    def test_must_not_find_image(self):

        self.mock_docker_client.images.get.side_effect = ImageNotFound("test")
        self.assertFalse(self.manager.has_image(self.image_name))
import sys

docker.api.build.process_dockerfile = lambda dockerfile, path: (
    "Dockerfile",
    dockerfile,
)

tag = f"jinjamator/jinjamator-runtime:latest"
client = docker.from_env()

try:
    client.images.get(tag)
    log.info(f"base image {tag} found")
    if force_rebuild:
        log.info("force_rebuild set -> rebuilding")
        raise ImageNotFound("force_rebuild set -> rebuilding")
except ImageNotFound:
    log.info("base image not found building")
    docker_file = task.run("dockerfile", output_plugin="null")[0]["result"]
    log.debug(docker_file)
    try:
        image, logs = client.images.build(path=".",
                                          dockerfile=docker_file,
                                          tag=tag,
                                          nocache=True)
    except BuildError as e:
        # for line in logs:
        #    log.debug(line)
        log.error(e)
        raise
Ejemplo n.º 18
0
 def raise_ImageNotFound(*args, **kwargs):
     raise ImageNotFound(message="")