def extractlayers(dc, args, layers, top_most_layer_id):
target_path = args.target
flags = O_WRONLY
if target_path == _TARGET_STDOUT:
target_fd = stdout.fileno()
else:
flags |= O_CREAT | O_TRUNC
if not args.force:
flags |= O_EXCL
target_fd = logexception(
_LOGGER, ERROR,
'unable to open target file "{}": {{e}}'.format(target_path),
os_open, target_path, flags, 0o666)
with fdopen(target_fd, 'wb') as target_file:
if hasattr(target_file, 'seekable'):
seekable = target_file.seekable()
else:
try:
seekable = not lseek(target_fd, 0, SEEK_CUR) < 0 \
and S_ISREG(fstat(target_fd).st_mode)
except OSError as e:
if errorcode.get(e.errno) != 'ESPIPE':
raise
seekable = False
open_args = {'fileobj': target_file}
if args.compression is None:
open_args['mode'] = 'w' if seekable else 'w|'
else:
if seekable:
mode = 'w:{}'
open_args['compresslevel'] = args.compress_level
_, ext = ospath_splitext(target_path)
if ext.lower() != '{}{}'.format(ospath_extsep,
args.compression):
_LOGGER.warning(
'target name "%s" doesn\'t match compression type ("%s")',
target_path, args.compression)
else:
mode = 'w|{}'
_LOGGER.warning(
'target "%s" is not seekable, ignoring compression level (%d)',
target_path, args.compress_level)
open_args['mode'] = mode.format(args.compression)
with tarfile_open(**open_args) as tar_file:
dimgx_extractlayers(dc, layers, tar_file, top_most_layer_id)
def extractlayers(dc, args, layers, top_most_layer_id):
target_path = args.target
flags = O_WRONLY
if target_path == _TARGET_STDOUT:
target_fd = stdout.fileno()
else:
flags |= O_CREAT | O_TRUNC
if not args.force:
flags |= O_EXCL
target_fd = logexception(_LOGGER, ERROR, 'unable to open target file "{}": {{e}}'.format(target_path), os_open, target_path, flags, 0o666)
with fdopen(target_fd, 'wb') as target_file:
if hasattr(target_file, 'seekable'):
seekable = target_file.seekable()
else:
try:
seekable = not lseek(target_fd, 0, SEEK_CUR) < 0 \
and S_ISREG(fstat(target_fd).st_mode)
except OSError as e:
if errorcode.get(e.errno) != 'ESPIPE':
raise
seekable = False
open_args = { 'fileobj': target_file }
if args.compression is None:
open_args['mode'] = 'w' if seekable else 'w|'
else:
if seekable:
mode = 'w:{}'
open_args['compresslevel'] = args.compress_level
_, ext = ospath_splitext(target_path)
if ext.lower() != '{}{}'.format(ospath_extsep, args.compression):
_LOGGER.warning('target name "%s" doesn\'t match compression type ("%s")', target_path, args.compression)
else:
mode = 'w|{}'
_LOGGER.warning('target "%s" is not seekable, ignoring compression level (%d)', target_path, args.compress_level)
open_args['mode'] = mode.format(args.compression)
with tarfile_open(**open_args) as tar_file:
dimgx_extractlayers(dc, layers, tar_file, top_most_layer_id)
def inspectlayers(dc, image_spec):
"""
:param dc: a |docker.Client|_
:param image_spec: the name or ID of the image to inspect
:returns: a :class:`dict` containing the descriptions (see below)
:raises: :class:`docker.errors.APIError` or
:class:`docker.errors.DockerException` on failure interacting with
Docker
Retrieves and normalizes descriptions for the :obj:`image_spec` image
and each of its ancestors by calling |docker.Client.images|_.
The returned :class:`dict` is as follows:
.. code-block:: python
{
':layers': ( image_desc_n, ..., image_desc_0 ),
image_id_n: n,
...
image_id_0: 0,
...
}
The :attr:`':layers'` :class:`list` is in desscending order (i.e.,
from :obj:`image_spec` to the root). The other entries map the layers'
various IDs, to their respective indexes in the :attr:`':layers'`
:class:`list`.
"""
images = logexception(_LOGGER, ERROR, 'unable to retrieve image summaries: {{e}}'.format(), dc.images, all=True)
images = sorted(( normalizeimage(i) for i in images ), key=imagekey, reverse=True)
image_spec_len = len(image_spec)
images_by_id = {}
children = {}
layer = None
for image in images:
image_id = image[':id']
image_parent_id = image[':parent_id']
images_by_id[image_id] = image
try:
image[':child_ids'] = children[image_id]
except KeyError:
image[':child_ids'] = []
try:
children[image_parent_id].append(image_id)
except KeyError:
children[image_parent_id] = [ image_id ]
if image_spec in image[':repo_tags'] \
or image_spec.lower() == image_id[0:image_spec_len]:
if layer is not None:
raise RuntimeError('{} does not resolve to a single image'.format(image_spec))
layer = image
if layer is None:
raise RuntimeError('{} not found among the layers retreieved for that image'.format(image_spec))
layers = []
layers_by_id = {
':all_images': images_by_id,
':layers': layers,
}
layer_id = layer[':id']
if image_spec.lower() not in ( layer_id, layer[':short_id'] ):
_LOGGER.debug('image "%s" has ID "%s"', image_spec, layer[':short_id'])
i = 0
while True:
layers.append(layer)
for j in range(1, len(layer_id) + 1):
layer_id_part = layer_id[0:j]
layers_by_id[layer_id_part] = None if layer_id_part in layers_by_id else i
for repo_tag in layer[':repo_tags']:
layers_by_id[repo_tag] = i
parent_layer_id = layer[':parent_id']
if not parent_layer_id:
_LOGGER.debug('found root layer "%s"', layer[':short_id'])
break
layer = images_by_id[parent_layer_id]
layer_id = layer[':id']
i += 1
return layers_by_id
def extractlayers(dc, layers, tar_file, top_most_layer=0):
"""
:param dc: a |docker.Client|_
:param layers: a sequence of inspection objects (likely retrieved with
:func:`inspectlayers`) corresponding to the layers to extract and
flatten in order of precedence
:param tar_file: a :class:`~tarfile.TarFile` open for writing to which
to write the flattened layer archive
:param top_most_layer: an image ID or an index into :obj:`layers`
indicating the most recent layer to retrieve (the default of ``0``
references the first item in :obj:`layers`; see below)
:raises docker.errors.APIError: on failure interacting with Docker
(e.g., failed connection, Docker not running, etc.)
:raises docker.errors.DockerException: on failure interacting with
Docker (e.g., bad image ID, etc.)
:raises UnsafeTarPath: - probably indicative of a bug in Docker
Retrieves the layers corresponding to the :obj:`layers` parameter and
extracts them into :obj:`tar_file`. Changes from layers corresponding
to smaller indexes in :obj:`layers` will overwrite or block those from
larger ones.
Callers will need to set the :obj:`top_most_layer` parameter if
:obj:`layers` is not in descending order. It is always safe to provide
the same value as the :obj:`image_spec` parameter to
:func:`inspectlayers`, but this may be ineffecient if that layer does
not appear in :obj:`layers`.
"""
if not layers:
_LOGGER.warning('nothing to extract')
return
image_spec = top_most_layer if not isinstance(top_most_layer, int) else layers[top_most_layer][':id']
tmp_dir = path_realpath(mkdtemp())
try:
image = logexception(_LOGGER, ERROR, 'unable to retrieve image layers from "{}": {{e}}'.format(image_spec), dc.get_image, image_spec)
with tarfile_open(mode='r|*', fileobj=image) as image_tar_file:
next_info = image_tar_file.next()
while next_info:
next_path = path_realpath(path_join(tmp_dir, next_info.name))
if not next_path.startswith(tmp_dir):
exc = UnsafeTarPath('unsafe path: "{}"'.format(next_info.name))
logexception(_LOGGER, ERROR, 'unable to retrieve entry from export of "{}": {{e}}'.format(image_spec), exc)
image_tar_file.extract(next_info, tmp_dir)
next_info = image_tar_file.next()
seen = set()
hides_subtrees = set()
# Look through each layer's archive (newest to oldest)
for layer in layers:
layer_id = layer[':id']
layer_tar_path = path_join(tmp_dir, layer_id, 'layer.tar')
with tarfile_open(layer_tar_path) as layer_tar_file:
next_info = layer_tar_file.next()
while next_info:
next_dirname = posixpath_dirname(next_info.name)
next_basename = posixpath_basename(next_info.name)
if next_basename.startswith(_WHITEOUT_PFX):
removed_path = posixpath_join(next_dirname, next_basename[_WHITEOUT_PFX_LEN:])
hides_subtrees.add(( removed_path, 'removal' ))
if removed_path in seen:
_LOGGER.debug('skipping removal "%s"', removed_path)
else:
_LOGGER.debug('hiding "%s" as removed', removed_path)
elif next_info.name in seen:
_LOGGER.debug('skipping "%s" as overwritten', next_info.name)
else:
next_name_len = len(next_info.name)
hidden = None
for h, deverbal in hides_subtrees: # https://en.wikipedia.org/wiki/deverbal
if len(h) > next_name_len:
continue
common_pfx = posixpath_commonprefix(( h, next_info.name ))
common_pfx_len = len(common_pfx)
if next_name_len == common_pfx_len \
or next_info.name[common_pfx_len:].startswith(posixpath_sep):
hidden = deverbal, h
break
if hidden:
_LOGGER.debug('skipping "%s" hidden by %s of %s', next_info.name, *hidden)
else:
mtime = naturaltime(datetime.utcfromtimestamp(next_info.mtime).replace(tzinfo=TZ_UTC))
_LOGGER.info('writing "%s" from "%s" to archive (size: %s; mode: %o; mtime: %s)', next_info.name, layer_id, naturalsize(next_info.size), next_info.mode, mtime)
if next_info.linkname:
# TarFile.extractfile() tries to do
# something weird when its parameter
# represents a link (see the docs)
fileobj = None
else:
fileobj = layer_tar_file.extractfile(next_info)
tar_file.addfile(next_info, fileobj)
seen.add(next_info.name)
if not next_info.isdir():
hides_subtrees.add(( next_info.name, 'presence' ))
next_info = layer_tar_file.next()
finally:
rmtree(tmp_dir, ignore_errors=True)
def inspectlayers(dc, image_spec):
"""
:param dc: a |docker.Client|_
:param image_spec: the name or ID of the image to inspect
:returns: a :class:`dict` containing the descriptions (see below)
:raises: :class:`docker.errors.APIError` or
:class:`docker.errors.DockerException` on failure interacting with
Docker
Retrieves and normalizes descriptions for the :obj:`image_spec` image
and each of its ancestors by calling |docker.Client.images|_.
The returned :class:`dict` is as follows:
.. code-block:: python
{
':layers': ( image_desc_n, ..., image_desc_0 ),
image_id_n: n,
...
image_id_0: 0,
...
}
The :attr:`':layers'` :class:`list` is in desscending order (i.e.,
from :obj:`image_spec` to the root). The other entries map the layers'
various IDs, to their respective indexes in the :attr:`':layers'`
:class:`list`.
"""
images = logexception(_LOGGER,
ERROR,
'unable to retrieve image summaries: {{e}}'.format(),
dc.images,
all=True)
images = sorted((normalizeimage(i) for i in images),
key=imagekey,
reverse=True)
image_spec_len = len(image_spec)
images_by_id = {}
children = {}
layer = None
for image in images:
image_id = image[':id']
image_parent_id = image[':parent_id']
images_by_id[image_id] = image
try:
image[':child_ids'] = children[image_id]
except KeyError:
image[':child_ids'] = []
try:
children[image_parent_id].append(image_id)
except KeyError:
children[image_parent_id] = [image_id]
if image_spec in image[':repo_tags'] \
or image_spec.lower() == image_id[0:image_spec_len]:
if layer is not None:
raise RuntimeError(
'{} does not resolve to a single image'.format(image_spec))
layer = image
if layer is None:
raise RuntimeError(
'{} not found among the layers retreieved for that image'.format(
image_spec))
layers = []
layers_by_id = {
':all_images': images_by_id,
':layers': layers,
}
layer_id = layer[':id']
if image_spec.lower() not in (layer_id, layer[':short_id']):
_LOGGER.debug('image "%s" has ID "%s"', image_spec, layer[':short_id'])
i = 0
while True:
layers.append(layer)
for j in range(1, len(layer_id) + 1):
layer_id_part = layer_id[0:j]
layers_by_id[
layer_id_part] = None if layer_id_part in layers_by_id else i
for repo_tag in layer[':repo_tags']:
layers_by_id[repo_tag] = i
parent_layer_id = layer[':parent_id']
if not parent_layer_id:
_LOGGER.debug('found root layer "%s"', layer[':short_id'])
break
layer = images_by_id[parent_layer_id]
layer_id = layer[':id']
i += 1
return layers_by_id
def extractlayers(dc, layers, tar_file, top_most_layer=0):
"""
:param dc: a |docker.Client|_
:param layers: a sequence of inspection objects (likely retrieved with
:func:`inspectlayers`) corresponding to the layers to extract and
flatten in order of precedence
:param tar_file: a :class:`~tarfile.TarFile` open for writing to which
to write the flattened layer archive
:param top_most_layer: an image ID or an index into :obj:`layers`
indicating the most recent layer to retrieve (the default of ``0``
references the first item in :obj:`layers`; see below)
:raises docker.errors.APIError: on failure interacting with Docker
(e.g., failed connection, Docker not running, etc.)
:raises docker.errors.DockerException: on failure interacting with
Docker (e.g., bad image ID, etc.)
:raises UnsafeTarPath: - probably indicative of a bug in Docker
Retrieves the layers corresponding to the :obj:`layers` parameter and
extracts them into :obj:`tar_file`. Changes from layers corresponding
to smaller indexes in :obj:`layers` will overwrite or block those from
larger ones.
Callers will need to set the :obj:`top_most_layer` parameter if
:obj:`layers` is not in descending order. It is always safe to provide
the same value as the :obj:`image_spec` parameter to
:func:`inspectlayers`, but this may be ineffecient if that layer does
not appear in :obj:`layers`.
"""
if not layers:
_LOGGER.warning('nothing to extract')
return
image_spec = top_most_layer if not isinstance(
top_most_layer, int) else layers[top_most_layer][':id']
tmp_dir = path_realpath(mkdtemp())
try:
image = logexception(
_LOGGER, ERROR,
'unable to retrieve image layers from "{}": {{e}}'.format(
image_spec), dc.get_image, image_spec)
with tarfile_open(mode='r|*', fileobj=image) as image_tar_file:
next_info = image_tar_file.next()
while next_info:
next_path = path_realpath(path_join(tmp_dir, next_info.name))
if not next_path.startswith(tmp_dir):
exc = UnsafeTarPath('unsafe path: "{}"'.format(
next_info.name))
logexception(
_LOGGER, ERROR,
'unable to retrieve entry from export of "{}": {{e}}'.
format(image_spec), exc)
image_tar_file.extract(next_info, tmp_dir)
next_info = image_tar_file.next()
seen = set()
hides_subtrees = set()
# Look through each layer's archive (newest to oldest)
for layer in layers:
layer_id = layer[':id']
layer_tar_path = path_join(tmp_dir, layer_id, 'layer.tar')
with tarfile_open(layer_tar_path) as layer_tar_file:
next_info = layer_tar_file.next()
while next_info:
next_dirname = posixpath_dirname(next_info.name)
next_basename = posixpath_basename(next_info.name)
if next_basename.startswith(_WHITEOUT_PFX):
removed_path = posixpath_join(
next_dirname, next_basename[_WHITEOUT_PFX_LEN:])
hides_subtrees.add((removed_path, 'removal'))
if removed_path in seen:
_LOGGER.debug('skipping removal "%s"',
removed_path)
else:
_LOGGER.debug('hiding "%s" as removed',
removed_path)
elif next_info.name in seen:
_LOGGER.debug('skipping "%s" as overwritten',
next_info.name)
else:
next_name_len = len(next_info.name)
hidden = None
for h, deverbal in hides_subtrees: # https://en.wikipedia.org/wiki/deverbal
if len(h) > next_name_len:
continue
common_pfx = posixpath_commonprefix(
(h, next_info.name))
common_pfx_len = len(common_pfx)
if next_name_len == common_pfx_len \
or next_info.name[common_pfx_len:].startswith(posixpath_sep):
hidden = deverbal, h
break
if hidden:
_LOGGER.debug('skipping "%s" hidden by %s of %s',
next_info.name, *hidden)
else:
mtime = naturaltime(
datetime.utcfromtimestamp(
next_info.mtime).replace(tzinfo=TZ_UTC))
_LOGGER.info(
'writing "%s" from "%s" to archive (size: %s; mode: %o; mtime: %s)',
next_info.name, layer_id,
naturalsize(next_info.size), next_info.mode,
mtime)
if next_info.linkname:
# TarFile.extractfile() tries to do
# something weird when its parameter
# represents a link (see the docs)
fileobj = None
else:
fileobj = layer_tar_file.extractfile(next_info)
tar_file.addfile(next_info, fileobj)
seen.add(next_info.name)
if not next_info.isdir():
hides_subtrees.add(
(next_info.name, 'presence'))
next_info = layer_tar_file.next()
finally:
rmtree(tmp_dir, ignore_errors=True)
