Beispiel #1
0
def extractlayers(dc, args, layers, top_most_layer_id):
    target_path = args.target
    flags = O_WRONLY

    if target_path == _TARGET_STDOUT:
        target_fd = stdout.fileno()
    else:
        flags |= O_CREAT | O_TRUNC

        if not args.force:
            flags |= O_EXCL

        target_fd = logexception(
            _LOGGER, ERROR,
            'unable to open target file "{}": {{e}}'.format(target_path),
            os_open, target_path, flags, 0o666)

    with fdopen(target_fd, 'wb') as target_file:
        if hasattr(target_file, 'seekable'):
            seekable = target_file.seekable()
        else:
            try:
                seekable = not lseek(target_fd, 0, SEEK_CUR) < 0 \
                    and S_ISREG(fstat(target_fd).st_mode)
            except OSError as e:
                if errorcode.get(e.errno) != 'ESPIPE':
                    raise

                seekable = False

        open_args = {'fileobj': target_file}

        if args.compression is None:
            open_args['mode'] = 'w' if seekable else 'w|'
        else:
            if seekable:
                mode = 'w:{}'
                open_args['compresslevel'] = args.compress_level
                _, ext = ospath_splitext(target_path)

                if ext.lower() != '{}{}'.format(ospath_extsep,
                                                args.compression):
                    _LOGGER.warning(
                        'target name "%s" doesn\'t match compression type ("%s")',
                        target_path, args.compression)
            else:
                mode = 'w|{}'
                _LOGGER.warning(
                    'target "%s" is not seekable, ignoring compression level (%d)',
                    target_path, args.compress_level)

            open_args['mode'] = mode.format(args.compression)

        with tarfile_open(**open_args) as tar_file:
            dimgx_extractlayers(dc, layers, tar_file, top_most_layer_id)
Beispiel #2
0
def extractlayers(dc, args, layers, top_most_layer_id):
    target_path = args.target
    flags = O_WRONLY

    if target_path == _TARGET_STDOUT:
        target_fd = stdout.fileno()
    else:
        flags |= O_CREAT | O_TRUNC

        if not args.force:
            flags |= O_EXCL

        target_fd = logexception(_LOGGER, ERROR, 'unable to open target file "{}": {{e}}'.format(target_path), os_open, target_path, flags, 0o666)

    with fdopen(target_fd, 'wb') as target_file:
        if hasattr(target_file, 'seekable'):
            seekable = target_file.seekable()
        else:
            try:
                seekable = not lseek(target_fd, 0, SEEK_CUR) < 0 \
                    and S_ISREG(fstat(target_fd).st_mode)
            except OSError as e:
                if errorcode.get(e.errno) != 'ESPIPE':
                    raise

                seekable = False

        open_args = { 'fileobj': target_file }

        if args.compression is None:
            open_args['mode'] = 'w' if seekable else 'w|'
        else:
            if seekable:
                mode = 'w:{}'
                open_args['compresslevel'] = args.compress_level
                _, ext = ospath_splitext(target_path)

                if ext.lower() != '{}{}'.format(ospath_extsep, args.compression):
                    _LOGGER.warning('target name "%s" doesn\'t match compression type ("%s")', target_path, args.compression)
            else:
                mode = 'w|{}'
                _LOGGER.warning('target "%s" is not seekable, ignoring compression level (%d)', target_path, args.compress_level)

            open_args['mode'] = mode.format(args.compression)

        with tarfile_open(**open_args) as tar_file:
            dimgx_extractlayers(dc, layers, tar_file, top_most_layer_id)
Beispiel #3
0
def inspectlayers(dc, image_spec):
    """
    :param dc: a |docker.Client|_

    :param image_spec: the name or ID of the image to inspect

    :returns: a :class:`dict` containing the descriptions (see below)

    :raises: :class:`docker.errors.APIError` or
        :class:`docker.errors.DockerException` on failure interacting with
        Docker

    Retrieves and normalizes descriptions for the :obj:`image_spec` image
    and each of its ancestors by calling |docker.Client.images|_.

    The returned :class:`dict` is as follows:

    .. code-block:: python

        {
            ':layers': ( image_desc_n, ..., image_desc_0 ),
            image_id_n: n,
            ...
            image_id_0: 0,
            ...
        }

    The :attr:`':layers'` :class:`list` is in desscending order (i.e.,
    from :obj:`image_spec` to the root). The other entries map the layers'
    various IDs, to their respective indexes in the :attr:`':layers'`
    :class:`list`.
    """
    images = logexception(_LOGGER, ERROR, 'unable to retrieve image summaries: {{e}}'.format(), dc.images, all=True)
    images = sorted(( normalizeimage(i) for i in images ), key=imagekey, reverse=True)
    image_spec_len = len(image_spec)
    images_by_id = {}
    children = {}
    layer = None

    for image in images:
        image_id = image[':id']
        image_parent_id = image[':parent_id']
        images_by_id[image_id] = image

        try:
            image[':child_ids'] = children[image_id]
        except KeyError:
            image[':child_ids'] = []

        try:
            children[image_parent_id].append(image_id)
        except KeyError:
            children[image_parent_id] = [ image_id ]

        if image_spec in image[':repo_tags'] \
                or image_spec.lower() == image_id[0:image_spec_len]:
            if layer is not None:
                raise RuntimeError('{} does not resolve to a single image'.format(image_spec))

            layer = image

    if layer is None:
        raise RuntimeError('{} not found among the layers retreieved for that image'.format(image_spec))

    layers = []

    layers_by_id = {
        ':all_images': images_by_id,
        ':layers': layers,
    }

    layer_id = layer[':id']

    if image_spec.lower() not in ( layer_id, layer[':short_id'] ):
        _LOGGER.debug('image "%s" has ID "%s"', image_spec, layer[':short_id'])

    i = 0

    while True:
        layers.append(layer)

        for j in range(1, len(layer_id) + 1):
            layer_id_part = layer_id[0:j]
            layers_by_id[layer_id_part] = None if layer_id_part in layers_by_id else i

        for repo_tag in layer[':repo_tags']:
            layers_by_id[repo_tag] = i

        parent_layer_id = layer[':parent_id']

        if not parent_layer_id:
            _LOGGER.debug('found root layer "%s"', layer[':short_id'])
            break

        layer = images_by_id[parent_layer_id]
        layer_id = layer[':id']
        i += 1

    return layers_by_id
Beispiel #4
0
def extractlayers(dc, layers, tar_file, top_most_layer=0):
    """
    :param dc: a |docker.Client|_

    :param layers: a sequence of inspection objects (likely retrieved with
        :func:`inspectlayers`) corresponding to the layers to extract and
        flatten in order of precedence

    :param tar_file: a :class:`~tarfile.TarFile` open for writing to which
        to write the flattened layer archive

    :param top_most_layer: an image ID or an index into :obj:`layers`
        indicating the most recent layer to retrieve (the default of ``0``
        references the first item in :obj:`layers`; see below)

    :raises docker.errors.APIError: on failure interacting with Docker
        (e.g., failed connection, Docker not running, etc.)

    :raises docker.errors.DockerException: on failure interacting with
        Docker (e.g., bad image ID, etc.)

    :raises UnsafeTarPath: - probably indicative of a bug in Docker

    Retrieves the layers corresponding to the :obj:`layers` parameter and
    extracts them into :obj:`tar_file`. Changes from layers corresponding
    to smaller indexes in :obj:`layers` will overwrite or block those from
    larger ones.

    Callers will need to set the :obj:`top_most_layer` parameter if
    :obj:`layers` is not in descending order. It is always safe to provide
    the same value as the :obj:`image_spec` parameter to
    :func:`inspectlayers`, but this may be ineffecient if that layer does
    not appear in :obj:`layers`.
    """
    if not layers:
        _LOGGER.warning('nothing to extract')

        return

    image_spec = top_most_layer if not isinstance(top_most_layer, int) else layers[top_most_layer][':id']
    tmp_dir = path_realpath(mkdtemp())

    try:
        image = logexception(_LOGGER, ERROR, 'unable to retrieve image layers from "{}": {{e}}'.format(image_spec), dc.get_image, image_spec)

        with tarfile_open(mode='r|*', fileobj=image) as image_tar_file:
            next_info = image_tar_file.next()

            while next_info:
                next_path = path_realpath(path_join(tmp_dir, next_info.name))

                if not next_path.startswith(tmp_dir):
                    exc = UnsafeTarPath('unsafe path: "{}"'.format(next_info.name))
                    logexception(_LOGGER, ERROR, 'unable to retrieve entry from export of "{}": {{e}}'.format(image_spec), exc)

                image_tar_file.extract(next_info, tmp_dir)
                next_info = image_tar_file.next()

        seen = set()
        hides_subtrees = set()

        # Look through each layer's archive (newest to oldest)
        for layer in layers:
            layer_id = layer[':id']
            layer_tar_path = path_join(tmp_dir, layer_id, 'layer.tar')

            with tarfile_open(layer_tar_path) as layer_tar_file:
                next_info = layer_tar_file.next()

                while next_info:
                    next_dirname = posixpath_dirname(next_info.name)
                    next_basename = posixpath_basename(next_info.name)

                    if next_basename.startswith(_WHITEOUT_PFX):
                        removed_path = posixpath_join(next_dirname, next_basename[_WHITEOUT_PFX_LEN:])
                        hides_subtrees.add(( removed_path, 'removal' ))

                        if removed_path in seen:
                            _LOGGER.debug('skipping removal "%s"', removed_path)
                        else:
                            _LOGGER.debug('hiding "%s" as removed', removed_path)
                    elif next_info.name in seen:
                        _LOGGER.debug('skipping "%s" as overwritten', next_info.name)
                    else:
                        next_name_len = len(next_info.name)
                        hidden = None

                        for h, deverbal in hides_subtrees: # https://en.wikipedia.org/wiki/deverbal
                            if len(h) > next_name_len:
                                continue

                            common_pfx = posixpath_commonprefix(( h, next_info.name ))
                            common_pfx_len = len(common_pfx)

                            if next_name_len == common_pfx_len \
                                    or next_info.name[common_pfx_len:].startswith(posixpath_sep):
                                hidden = deverbal, h
                                break

                        if hidden:
                            _LOGGER.debug('skipping "%s" hidden by %s of %s', next_info.name, *hidden)
                        else:
                            mtime = naturaltime(datetime.utcfromtimestamp(next_info.mtime).replace(tzinfo=TZ_UTC))
                            _LOGGER.info('writing "%s" from "%s" to archive (size: %s; mode: %o; mtime: %s)', next_info.name, layer_id, naturalsize(next_info.size), next_info.mode, mtime)

                            if next_info.linkname:
                                # TarFile.extractfile() tries to do
                                # something weird when its parameter
                                # represents a link (see the docs)
                                fileobj = None
                            else:
                                fileobj = layer_tar_file.extractfile(next_info)

                            tar_file.addfile(next_info, fileobj)
                            seen.add(next_info.name)

                            if not next_info.isdir():
                                hides_subtrees.add(( next_info.name, 'presence' ))

                    next_info = layer_tar_file.next()
    finally:
        rmtree(tmp_dir, ignore_errors=True)
Beispiel #5
0
def inspectlayers(dc, image_spec):
    """
    :param dc: a |docker.Client|_

    :param image_spec: the name or ID of the image to inspect

    :returns: a :class:`dict` containing the descriptions (see below)

    :raises: :class:`docker.errors.APIError` or
        :class:`docker.errors.DockerException` on failure interacting with
        Docker

    Retrieves and normalizes descriptions for the :obj:`image_spec` image
    and each of its ancestors by calling |docker.Client.images|_.

    The returned :class:`dict` is as follows:

    .. code-block:: python

        {
            ':layers': ( image_desc_n, ..., image_desc_0 ),
            image_id_n: n,
            ...
            image_id_0: 0,
            ...
        }

    The :attr:`':layers'` :class:`list` is in desscending order (i.e.,
    from :obj:`image_spec` to the root). The other entries map the layers'
    various IDs, to their respective indexes in the :attr:`':layers'`
    :class:`list`.
    """
    images = logexception(_LOGGER,
                          ERROR,
                          'unable to retrieve image summaries: {{e}}'.format(),
                          dc.images,
                          all=True)
    images = sorted((normalizeimage(i) for i in images),
                    key=imagekey,
                    reverse=True)
    image_spec_len = len(image_spec)
    images_by_id = {}
    children = {}
    layer = None

    for image in images:
        image_id = image[':id']
        image_parent_id = image[':parent_id']
        images_by_id[image_id] = image

        try:
            image[':child_ids'] = children[image_id]
        except KeyError:
            image[':child_ids'] = []

        try:
            children[image_parent_id].append(image_id)
        except KeyError:
            children[image_parent_id] = [image_id]

        if image_spec in image[':repo_tags'] \
                or image_spec.lower() == image_id[0:image_spec_len]:
            if layer is not None:
                raise RuntimeError(
                    '{} does not resolve to a single image'.format(image_spec))

            layer = image

    if layer is None:
        raise RuntimeError(
            '{} not found among the layers retreieved for that image'.format(
                image_spec))

    layers = []

    layers_by_id = {
        ':all_images': images_by_id,
        ':layers': layers,
    }

    layer_id = layer[':id']

    if image_spec.lower() not in (layer_id, layer[':short_id']):
        _LOGGER.debug('image "%s" has ID "%s"', image_spec, layer[':short_id'])

    i = 0

    while True:
        layers.append(layer)

        for j in range(1, len(layer_id) + 1):
            layer_id_part = layer_id[0:j]
            layers_by_id[
                layer_id_part] = None if layer_id_part in layers_by_id else i

        for repo_tag in layer[':repo_tags']:
            layers_by_id[repo_tag] = i

        parent_layer_id = layer[':parent_id']

        if not parent_layer_id:
            _LOGGER.debug('found root layer "%s"', layer[':short_id'])
            break

        layer = images_by_id[parent_layer_id]
        layer_id = layer[':id']
        i += 1

    return layers_by_id
Beispiel #6
0
def extractlayers(dc, layers, tar_file, top_most_layer=0):
    """
    :param dc: a |docker.Client|_

    :param layers: a sequence of inspection objects (likely retrieved with
        :func:`inspectlayers`) corresponding to the layers to extract and
        flatten in order of precedence

    :param tar_file: a :class:`~tarfile.TarFile` open for writing to which
        to write the flattened layer archive

    :param top_most_layer: an image ID or an index into :obj:`layers`
        indicating the most recent layer to retrieve (the default of ``0``
        references the first item in :obj:`layers`; see below)

    :raises docker.errors.APIError: on failure interacting with Docker
        (e.g., failed connection, Docker not running, etc.)

    :raises docker.errors.DockerException: on failure interacting with
        Docker (e.g., bad image ID, etc.)

    :raises UnsafeTarPath: - probably indicative of a bug in Docker

    Retrieves the layers corresponding to the :obj:`layers` parameter and
    extracts them into :obj:`tar_file`. Changes from layers corresponding
    to smaller indexes in :obj:`layers` will overwrite or block those from
    larger ones.

    Callers will need to set the :obj:`top_most_layer` parameter if
    :obj:`layers` is not in descending order. It is always safe to provide
    the same value as the :obj:`image_spec` parameter to
    :func:`inspectlayers`, but this may be ineffecient if that layer does
    not appear in :obj:`layers`.
    """
    if not layers:
        _LOGGER.warning('nothing to extract')

        return

    image_spec = top_most_layer if not isinstance(
        top_most_layer, int) else layers[top_most_layer][':id']
    tmp_dir = path_realpath(mkdtemp())

    try:
        image = logexception(
            _LOGGER, ERROR,
            'unable to retrieve image layers from "{}": {{e}}'.format(
                image_spec), dc.get_image, image_spec)

        with tarfile_open(mode='r|*', fileobj=image) as image_tar_file:
            next_info = image_tar_file.next()

            while next_info:
                next_path = path_realpath(path_join(tmp_dir, next_info.name))

                if not next_path.startswith(tmp_dir):
                    exc = UnsafeTarPath('unsafe path: "{}"'.format(
                        next_info.name))
                    logexception(
                        _LOGGER, ERROR,
                        'unable to retrieve entry from export of "{}": {{e}}'.
                        format(image_spec), exc)

                image_tar_file.extract(next_info, tmp_dir)
                next_info = image_tar_file.next()

        seen = set()
        hides_subtrees = set()

        # Look through each layer's archive (newest to oldest)
        for layer in layers:
            layer_id = layer[':id']
            layer_tar_path = path_join(tmp_dir, layer_id, 'layer.tar')

            with tarfile_open(layer_tar_path) as layer_tar_file:
                next_info = layer_tar_file.next()

                while next_info:
                    next_dirname = posixpath_dirname(next_info.name)
                    next_basename = posixpath_basename(next_info.name)

                    if next_basename.startswith(_WHITEOUT_PFX):
                        removed_path = posixpath_join(
                            next_dirname, next_basename[_WHITEOUT_PFX_LEN:])
                        hides_subtrees.add((removed_path, 'removal'))

                        if removed_path in seen:
                            _LOGGER.debug('skipping removal "%s"',
                                          removed_path)
                        else:
                            _LOGGER.debug('hiding "%s" as removed',
                                          removed_path)
                    elif next_info.name in seen:
                        _LOGGER.debug('skipping "%s" as overwritten',
                                      next_info.name)
                    else:
                        next_name_len = len(next_info.name)
                        hidden = None

                        for h, deverbal in hides_subtrees:  # https://en.wikipedia.org/wiki/deverbal
                            if len(h) > next_name_len:
                                continue

                            common_pfx = posixpath_commonprefix(
                                (h, next_info.name))
                            common_pfx_len = len(common_pfx)

                            if next_name_len == common_pfx_len \
                                    or next_info.name[common_pfx_len:].startswith(posixpath_sep):
                                hidden = deverbal, h
                                break

                        if hidden:
                            _LOGGER.debug('skipping "%s" hidden by %s of %s',
                                          next_info.name, *hidden)
                        else:
                            mtime = naturaltime(
                                datetime.utcfromtimestamp(
                                    next_info.mtime).replace(tzinfo=TZ_UTC))
                            _LOGGER.info(
                                'writing "%s" from "%s" to archive (size: %s; mode: %o; mtime: %s)',
                                next_info.name, layer_id,
                                naturalsize(next_info.size), next_info.mode,
                                mtime)

                            if next_info.linkname:
                                # TarFile.extractfile() tries to do
                                # something weird when its parameter
                                # represents a link (see the docs)
                                fileobj = None
                            else:
                                fileobj = layer_tar_file.extractfile(next_info)

                            tar_file.addfile(next_info, fileobj)
                            seen.add(next_info.name)

                            if not next_info.isdir():
                                hides_subtrees.add(
                                    (next_info.name, 'presence'))

                    next_info = layer_tar_file.next()
    finally:
        rmtree(tmp_dir, ignore_errors=True)