def add_base_packages(image_layer, binary, shell): # pylint: disable=too-many-locals
'''Given the image layer, the binary to invoke and shell:
1. get the listing from the base.yml
2. Invoke any commands against the base layer
3. Make a list of packages and add them to the layer'''
origin_layer = 'Layer: ' + image_layer.fs_hash[:10]
if image_layer.created_by:
image_layer.origins.add_notice_to_origins(
origin_layer,
Notice(
formats.layer_created_by.format(
created_by=image_layer.created_by), 'info'))
else:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(formats.no_created_by, 'warning'))
origin_command_lib = formats.invoking_base_commands
# find the binary
listing = command_lib.get_base_listing(binary)
if listing:
# put info notice about what is going to be invoked
snippet_msg = formats.invoke_for_base + '\n' + \
content.print_base_invoke(binary)
image_layer.origins.add_notice_to_origins(origin_layer,
Notice(snippet_msg, 'info'))
shell, _ = command_lib.get_image_shell(listing)
if not shell:
shell = constants.shell
# get all the packages in the base layer
names, n_msg = command_lib.get_pkg_attr_list(shell, listing['names'])
versions, v_msg = command_lib.get_pkg_attr_list(
shell, listing['versions'])
licenses, l_msg = command_lib.get_pkg_attr_list(
shell, listing['licenses'])
src_urls, u_msg = command_lib.get_pkg_attr_list(
shell, listing['src_urls'])
# add a notice to the image if something went wrong
invoke_msg = n_msg + v_msg + l_msg + u_msg
if invoke_msg:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(invoke_msg, 'error'))
if names and len(names) > 1:
for index, name in enumerate(names):
pkg = Package(name)
if len(versions) == len(names):
pkg.version = versions[index]
if len(licenses) == len(names):
pkg.license = licenses[index]
if len(src_urls) == len(names):
pkg.src_url = src_urls[index]
image_layer.add_package(pkg)
# if there is no listing add a notice
else:
image_layer.origins.add_notice_to_origins(
origin_command_lib,
Notice(errors.no_listing_for_base_key.format(listing_key=binary),
'error'))
def print_base_invoke(key):
'''Given the key in the base library, return a string containing
the command_lib/base.yml'''
info = command_lib.get_base_listing(key)
report = ''
for item in command_lib.base_keys:
if item in info.keys():
report = report + print_invoke_list(info, item)
report = report + '\n'
return report
def print_base_invoke(key):
'''Given the key in the base library, return a string containing
the command_lib/base.yml'''
info = command_lib.get_base_listing(key)
report = ''
report = report + print_invoke_list(info, 'names')
report = report + print_invoke_list(info, 'versions')
report = report + print_invoke_list(info, 'licenses')
report = report + print_invoke_list(info, 'src_urls')
report = report + '\n'
return report
def add_base_packages(image_layer, binary, shell, work_dir=None, envs=None):
'''Given the image layer, the binary to invoke and shell:
1. get the listing from the base.yml
2. Invoke any commands against the base layer
3. Make a list of packages and add them to the layer'''
origin_layer = 'Layer {}'.format(image_layer.layer_index)
if image_layer.created_by:
image_layer.origins.add_notice_to_origins(
origin_layer,
Notice(
formats.layer_created_by.format(
created_by=image_layer.created_by), 'info'))
else:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(formats.no_created_by, 'warning'))
origin_command_lib = formats.invoking_base_commands
# find the binary
listing = command_lib.get_base_listing(binary)
if listing:
# put info notice about what is going to be invoked
snippet_msg = (formats.invoke_for_base + '\n' +
content.print_base_invoke(binary))
image_layer.origins.add_notice_to_origins(origin_layer,
Notice(snippet_msg, 'info'))
# get all the packages in the base layer
pkg_dict, invoke_msg, warnings = collate_list_metadata(
shell, listing, work_dir, envs)
if listing.get("pkg_format") == "deb":
pkg_dict["pkg_licenses"] = get_deb_package_licenses(
pkg_dict["copyrights"])
if invoke_msg:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(invoke_msg, 'error'))
if warnings:
image_layer.origins.add_notice_to_origins(
origin_command_lib, Notice(warnings, 'warning'))
if 'names' in pkg_dict and len(pkg_dict['names']) > 1:
pkg_list = convert_to_pkg_dicts(pkg_dict)
for pkg_dict in pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
image_layer.add_package(pkg)
remove_duplicate_layer_files(image_layer)
# if there is no listing add a notice
else:
image_layer.origins.add_notice_to_origins(
origin_command_lib,
Notice(errors.no_listing_for_base_key.format(listing_key=binary),
'error'))
def add_base_packages(image_layer, binary, shell):
'''Given the image layer, the binary to invoke and shell:
1. get the listing from the base.yml
2. Invoke any commands against the base layer
3. Make a list of packages and add them to the layer'''
origin_layer = 'Layer: ' + image_layer.fs_hash[:10]
if image_layer.created_by:
image_layer.origins.add_notice_to_origins(
origin_layer,
Notice(
formats.layer_created_by.format(
created_by=image_layer.created_by), 'info'))
else:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(formats.no_created_by, 'warning'))
origin_command_lib = formats.invoking_base_commands
# find the binary
listing = command_lib.get_base_listing(binary)
if listing:
# put info notice about what is going to be invoked
snippet_msg = formats.invoke_for_base + '\n' + \
content.print_base_invoke(binary)
image_layer.origins.add_notice_to_origins(origin_layer,
Notice(snippet_msg, 'info'))
shell, _ = command_lib.get_image_shell(listing)
if not shell:
shell = constants.shell
# get all the packages in the base layer
pkg_dict, invoke_msg, warnings = collate_list_metadata(shell, listing)
if invoke_msg:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(invoke_msg, 'error'))
if warnings:
image_layer.origins.add_notice_to_origins(
origin_command_lib, Notice(warnings, 'warning'))
if 'names' in pkg_dict and len(pkg_dict['names']) > 1:
pkg_list = convert_to_pkg_dicts(pkg_dict)
for pkg_dict in pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
image_layer.add_package(pkg)
# if there is no listing add a notice
else:
image_layer.origins.add_notice_to_origins(
origin_command_lib,
Notice(errors.no_listing_for_base_key.format(listing_key=binary),
'error'))
def get_shell(image_obj, binary):
# set up a notice origin referring to the base command library listing
origin_command_lib = formats.invoking_base_commands
# find the shell to invoke commands in
shell, _ = command_lib.get_image_shell(
command_lib.get_base_listing(binary))
if not shell:
# add a warning notice for no shell in the command library
logger.warning('No shell listing in command library. '
'Using default shell')
no_shell_message = errors.no_shell_listing.format(
binary=binary, default_shell=constants.shell)
image_obj.layers[0].origins.add_notice_to_origins(
origin_command_lib, Notice(no_shell_message, 'warning'))
# add a hint notice to add the shell to the command library
add_shell_message = errors.no_listing_for_base_key.format(
listing_key='shell')
image_obj.layers[0].origins.add_notice_to_origins(
origin_command_lib, Notice(add_shell_message, 'hint'))
shell = constants.shell
return shell
def analyze_docker_image(image_obj, redo=False, dockerfile=False): # pylint: disable=too-many-locals
'''Given a DockerImage object, for each layer, retrieve the packages, first
looking up in cache and if not there then looking up in the command
library. For looking up in command library first mount the filesystem
and then look up the command library for commands to run in chroot'''
# find the layers that are imported
if dockerfile:
dhelper.set_imported_layers(image_obj)
# add notices for each layer if it is imported
image_setup(image_obj)
shell = ''
# set up empty master list of packages
master_list = []
# find the binary by mounting the base layer
target = rootfs.mount_base_layer(image_obj.layers[0].tar_file)
binary = common.get_base_bin()
# set up a notice origin referring to the base command library listing
origin_command_lib = formats.invoking_base_commands
# set up a notice origin for the first layer
origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10]
# find the shell to invoke commands in
shell, _ = command_lib.get_image_shell(
command_lib.get_base_listing(binary))
if not shell:
# add a warning notice for no shell in the command library
logger.warning('No shell listing in command library. '
'Using default shell')
no_shell_message = errors.no_shell_listing.format(
binary=binary, default_shell=constants.shell)
image_obj.layers[0].origins.add_notice_to_origins(
origin_command_lib, Notice(no_shell_message, 'warning'))
# add a hint notice to add the shell to the command library
add_shell_message = errors.no_listing_for_base_key.format(
listing_key='shell')
image_obj.layers[0].origins.add_notice_to_origins(
origin_command_lib, Notice(add_shell_message, 'hint'))
shell = constants.shell
# only extract packages if there is a known binary and the layer is not
# cached
if binary:
if not common.load_from_cache(image_obj.layers[0], redo):
# get the packages of the first layer
rootfs.prep_rootfs(target)
common.add_base_packages(image_obj.layers[0], binary, shell)
# unmount proc, sys and dev
rootfs.undo_mount()
else:
no_base = errors.unrecognized_base.format(
image_name=image_obj.name, image_tag=image_obj.tag)
logger.warning(no_base)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(no_base, 'warning'))
# no binary means there is no shell so set to default shell
logger.warning('Unknown filesystem. Using default shell')
shell = constants.shell
# unmount the first layer
rootfs.unmount_rootfs()
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
# get packages for subsequent layers
curr_layer = 1
while curr_layer < len(image_obj.layers):
if not common.load_from_cache(image_obj.layers[curr_layer], redo):
# get commands that created the layer
# for docker images this is retrieved from the image history
command_list = dhelper.get_commands_from_history(
image_obj.layers[curr_layer])
if command_list:
# mount diff layers from 0 till the current layer
target = mount_overlay_fs(image_obj, curr_layer)
# mount dev, sys and proc after mounting diff layers
rootfs.prep_rootfs(target)
# for each command look up the snippet library
for command in command_list:
pkg_listing = command_lib.get_package_listing(command.name)
if isinstance(pkg_listing, str):
common.add_base_packages(
image_obj.layers[curr_layer], pkg_listing, shell)
else:
common.add_snippet_packages(
image_obj.layers[curr_layer], command, pkg_listing,
shell)
if command_list:
rootfs.undo_mount()
rootfs.unmount_rootfs()
# update the master list
common.update_master_list(master_list, image_obj.layers[curr_layer])
curr_layer = curr_layer + 1
common.save_to_cache(image_obj)
