Ejemplo n.º 1
0
def printlayerinfo(args, layers, outfile=stdout):
    if args.quiet:
        for l in layers:
            print(l[':short_id'], file=outfile)

        return

    total_size = 0
    fields_fmt = '\t'.join(['{:<23}'] + ['{:<15}'] * 5)
    print(fields_fmt.format('REPO TAG', 'IMAGE ID', 'PARENT ID', 'CREATED',
                            'LAYER SIZE', 'VIRTUAL SIZE'),
          file=outfile)

    total_size = sum((l['Size'] for l in layers))

    for layer in layers:
        try:
            image_tag = layer[':repo_tags'][0]
        except IndexError:
            image_tag = '-'

        image_id = layer[':short_id']
        parent_id = layer[':parent_id'][:12].lower()

        if not parent_id:
            parent_id = '-'

        created = naturaltime(layer[':created_dt'])
        layer_size = naturalsize(layer['Size'])
        virt_size = naturalsize(total_size)
        print(fields_fmt.format(image_tag, image_id, parent_id, created,
                                layer_size, virt_size),
              file=outfile)
        total_size -= layer['Size']
Ejemplo n.º 2
0
def printlayerinfo(args, layers, outfile=stdout):
    if args.quiet:
        for l in layers:
            print(l[':short_id'], file=outfile)

        return

    total_size = 0
    fields_fmt = '\t'.join([ '{:<23}' ] + [ '{:<15}' ] * 5)
    print(fields_fmt.format('REPO TAG', 'IMAGE ID', 'PARENT ID', 'CREATED', 'LAYER SIZE', 'VIRTUAL SIZE'), file=outfile)

    total_size = sum(( l['Size'] for l in layers ))

    for layer in layers:
        try:
            image_tag = layer[':repo_tags'][0]
        except IndexError:
            image_tag = '-'

        image_id = layer[':short_id']
        parent_id = layer[':parent_id'][:12].lower()

        if not parent_id:
            parent_id = '-'

        created = naturaltime(layer[':created_dt'])
        layer_size = naturalsize(layer['Size'])
        virt_size = naturalsize(total_size)
        print(fields_fmt.format(image_tag, image_id, parent_id, created, layer_size, virt_size), file=outfile)
        total_size -= layer['Size']
Ejemplo n.º 3
0
def extractlayers(dc, layers, tar_file, top_most_layer=0):
    """
    :param dc: a |docker.Client|_

    :param layers: a sequence of inspection objects (likely retrieved with
        :func:`inspectlayers`) corresponding to the layers to extract and
        flatten in order of precedence

    :param tar_file: a :class:`~tarfile.TarFile` open for writing to which
        to write the flattened layer archive

    :param top_most_layer: an image ID or an index into :obj:`layers`
        indicating the most recent layer to retrieve (the default of ``0``
        references the first item in :obj:`layers`; see below)

    :raises docker.errors.APIError: on failure interacting with Docker
        (e.g., failed connection, Docker not running, etc.)

    :raises docker.errors.DockerException: on failure interacting with
        Docker (e.g., bad image ID, etc.)

    :raises UnsafeTarPath: - probably indicative of a bug in Docker

    Retrieves the layers corresponding to the :obj:`layers` parameter and
    extracts them into :obj:`tar_file`. Changes from layers corresponding
    to smaller indexes in :obj:`layers` will overwrite or block those from
    larger ones.

    Callers will need to set the :obj:`top_most_layer` parameter if
    :obj:`layers` is not in descending order. It is always safe to provide
    the same value as the :obj:`image_spec` parameter to
    :func:`inspectlayers`, but this may be ineffecient if that layer does
    not appear in :obj:`layers`.
    """
    if not layers:
        _LOGGER.warning('nothing to extract')

        return

    image_spec = top_most_layer if not isinstance(top_most_layer, int) else layers[top_most_layer][':id']
    tmp_dir = path_realpath(mkdtemp())

    try:
        image = logexception(_LOGGER, ERROR, 'unable to retrieve image layers from "{}": {{e}}'.format(image_spec), dc.get_image, image_spec)

        with tarfile_open(mode='r|*', fileobj=image) as image_tar_file:
            next_info = image_tar_file.next()

            while next_info:
                next_path = path_realpath(path_join(tmp_dir, next_info.name))

                if not next_path.startswith(tmp_dir):
                    exc = UnsafeTarPath('unsafe path: "{}"'.format(next_info.name))
                    logexception(_LOGGER, ERROR, 'unable to retrieve entry from export of "{}": {{e}}'.format(image_spec), exc)

                image_tar_file.extract(next_info, tmp_dir)
                next_info = image_tar_file.next()

        seen = set()
        hides_subtrees = set()

        # Look through each layer's archive (newest to oldest)
        for layer in layers:
            layer_id = layer[':id']
            layer_tar_path = path_join(tmp_dir, layer_id, 'layer.tar')

            with tarfile_open(layer_tar_path) as layer_tar_file:
                next_info = layer_tar_file.next()

                while next_info:
                    next_dirname = posixpath_dirname(next_info.name)
                    next_basename = posixpath_basename(next_info.name)

                    if next_basename.startswith(_WHITEOUT_PFX):
                        removed_path = posixpath_join(next_dirname, next_basename[_WHITEOUT_PFX_LEN:])
                        hides_subtrees.add(( removed_path, 'removal' ))

                        if removed_path in seen:
                            _LOGGER.debug('skipping removal "%s"', removed_path)
                        else:
                            _LOGGER.debug('hiding "%s" as removed', removed_path)
                    elif next_info.name in seen:
                        _LOGGER.debug('skipping "%s" as overwritten', next_info.name)
                    else:
                        next_name_len = len(next_info.name)
                        hidden = None

                        for h, deverbal in hides_subtrees: # https://en.wikipedia.org/wiki/deverbal
                            if len(h) > next_name_len:
                                continue

                            common_pfx = posixpath_commonprefix(( h, next_info.name ))
                            common_pfx_len = len(common_pfx)

                            if next_name_len == common_pfx_len \
                                    or next_info.name[common_pfx_len:].startswith(posixpath_sep):
                                hidden = deverbal, h
                                break

                        if hidden:
                            _LOGGER.debug('skipping "%s" hidden by %s of %s', next_info.name, *hidden)
                        else:
                            mtime = naturaltime(datetime.utcfromtimestamp(next_info.mtime).replace(tzinfo=TZ_UTC))
                            _LOGGER.info('writing "%s" from "%s" to archive (size: %s; mode: %o; mtime: %s)', next_info.name, layer_id, naturalsize(next_info.size), next_info.mode, mtime)

                            if next_info.linkname:
                                # TarFile.extractfile() tries to do
                                # something weird when its parameter
                                # represents a link (see the docs)
                                fileobj = None
                            else:
                                fileobj = layer_tar_file.extractfile(next_info)

                            tar_file.addfile(next_info, fileobj)
                            seen.add(next_info.name)

                            if not next_info.isdir():
                                hides_subtrees.add(( next_info.name, 'presence' ))

                    next_info = layer_tar_file.next()
    finally:
        rmtree(tmp_dir, ignore_errors=True)
Ejemplo n.º 4
0
def extractlayers(dc, layers, tar_file, top_most_layer=0):
    """
    :param dc: a |docker.Client|_

    :param layers: a sequence of inspection objects (likely retrieved with
        :func:`inspectlayers`) corresponding to the layers to extract and
        flatten in order of precedence

    :param tar_file: a :class:`~tarfile.TarFile` open for writing to which
        to write the flattened layer archive

    :param top_most_layer: an image ID or an index into :obj:`layers`
        indicating the most recent layer to retrieve (the default of ``0``
        references the first item in :obj:`layers`; see below)

    :raises docker.errors.APIError: on failure interacting with Docker
        (e.g., failed connection, Docker not running, etc.)

    :raises docker.errors.DockerException: on failure interacting with
        Docker (e.g., bad image ID, etc.)

    :raises UnsafeTarPath: - probably indicative of a bug in Docker

    Retrieves the layers corresponding to the :obj:`layers` parameter and
    extracts them into :obj:`tar_file`. Changes from layers corresponding
    to smaller indexes in :obj:`layers` will overwrite or block those from
    larger ones.

    Callers will need to set the :obj:`top_most_layer` parameter if
    :obj:`layers` is not in descending order. It is always safe to provide
    the same value as the :obj:`image_spec` parameter to
    :func:`inspectlayers`, but this may be ineffecient if that layer does
    not appear in :obj:`layers`.
    """
    if not layers:
        _LOGGER.warning('nothing to extract')

        return

    image_spec = top_most_layer if not isinstance(
        top_most_layer, int) else layers[top_most_layer][':id']
    tmp_dir = path_realpath(mkdtemp())

    try:
        image = logexception(
            _LOGGER, ERROR,
            'unable to retrieve image layers from "{}": {{e}}'.format(
                image_spec), dc.get_image, image_spec)

        with tarfile_open(mode='r|*', fileobj=image) as image_tar_file:
            next_info = image_tar_file.next()

            while next_info:
                next_path = path_realpath(path_join(tmp_dir, next_info.name))

                if not next_path.startswith(tmp_dir):
                    exc = UnsafeTarPath('unsafe path: "{}"'.format(
                        next_info.name))
                    logexception(
                        _LOGGER, ERROR,
                        'unable to retrieve entry from export of "{}": {{e}}'.
                        format(image_spec), exc)

                image_tar_file.extract(next_info, tmp_dir)
                next_info = image_tar_file.next()

        seen = set()
        hides_subtrees = set()

        # Look through each layer's archive (newest to oldest)
        for layer in layers:
            layer_id = layer[':id']
            layer_tar_path = path_join(tmp_dir, layer_id, 'layer.tar')

            with tarfile_open(layer_tar_path) as layer_tar_file:
                next_info = layer_tar_file.next()

                while next_info:
                    next_dirname = posixpath_dirname(next_info.name)
                    next_basename = posixpath_basename(next_info.name)

                    if next_basename.startswith(_WHITEOUT_PFX):
                        removed_path = posixpath_join(
                            next_dirname, next_basename[_WHITEOUT_PFX_LEN:])
                        hides_subtrees.add((removed_path, 'removal'))

                        if removed_path in seen:
                            _LOGGER.debug('skipping removal "%s"',
                                          removed_path)
                        else:
                            _LOGGER.debug('hiding "%s" as removed',
                                          removed_path)
                    elif next_info.name in seen:
                        _LOGGER.debug('skipping "%s" as overwritten',
                                      next_info.name)
                    else:
                        next_name_len = len(next_info.name)
                        hidden = None

                        for h, deverbal in hides_subtrees:  # https://en.wikipedia.org/wiki/deverbal
                            if len(h) > next_name_len:
                                continue

                            common_pfx = posixpath_commonprefix(
                                (h, next_info.name))
                            common_pfx_len = len(common_pfx)

                            if next_name_len == common_pfx_len \
                                    or next_info.name[common_pfx_len:].startswith(posixpath_sep):
                                hidden = deverbal, h
                                break

                        if hidden:
                            _LOGGER.debug('skipping "%s" hidden by %s of %s',
                                          next_info.name, *hidden)
                        else:
                            mtime = naturaltime(
                                datetime.utcfromtimestamp(
                                    next_info.mtime).replace(tzinfo=TZ_UTC))
                            _LOGGER.info(
                                'writing "%s" from "%s" to archive (size: %s; mode: %o; mtime: %s)',
                                next_info.name, layer_id,
                                naturalsize(next_info.size), next_info.mode,
                                mtime)

                            if next_info.linkname:
                                # TarFile.extractfile() tries to do
                                # something weird when its parameter
                                # represents a link (see the docs)
                                fileobj = None
                            else:
                                fileobj = layer_tar_file.extractfile(next_info)

                            tar_file.addfile(next_info, fileobj)
                            seen.add(next_info.name)

                            if not next_info.isdir():
                                hides_subtrees.add(
                                    (next_info.name, 'presence'))

                    next_info = layer_tar_file.next()
    finally:
        rmtree(tmp_dir, ignore_errors=True)