def analyze_subsequent_layers(image_obj, shell, master_list, redo):
# get packages for subsequent layers
curr_layer = 1
while curr_layer < len(image_obj.layers):
if not common.load_from_cache(image_obj.layers[curr_layer], redo):
# get commands that created the layer
# for docker images this is retrieved from the image history
command_list = dhelper.get_commands_from_history(
image_obj.layers[curr_layer])
if command_list:
# mount diff layers from 0 till the current layer
target = mount_overlay_fs(image_obj, curr_layer)
# mount dev, sys and proc after mounting diff layers
rootfs.prep_rootfs(target)
# for each command look up the snippet library
for command in command_list:
pkg_listing = command_lib.get_package_listing(command.name)
if isinstance(pkg_listing, str):
common.add_base_packages(image_obj.layers[curr_layer],
pkg_listing, shell)
else:
common.add_snippet_packages(image_obj.layers[curr_layer],
command, pkg_listing, shell)
if command_list:
rootfs.undo_mount()
rootfs.unmount_rootfs()
# update the master list
common.update_master_list(master_list, image_obj.layers[curr_layer])
curr_layer = curr_layer + 1
def analyze_first_layer(image_obj, master_list, redo):
# find the binary of the first layer
binary = common.get_base_bin(image_obj.layers[0])
# see if there is an associated shell
# if there is no binary, this will be set to the default shell
shell = get_shell(image_obj, binary)
# try to load packages from cache
if not common.load_from_cache(image_obj.layers[0], redo):
# set a possible OS
common.get_os_style(image_obj.layers[0], binary)
# set up a notice origin for the first layer
origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10]
# if there is a binary, extract packages
if binary:
try:
target = rootfs.mount_base_layer(image_obj.layers[0].tar_file)
rootfs.prep_rootfs(target)
common.add_base_packages(image_obj.layers[0], binary, shell)
# unmount proc, sys and dev
rootfs.undo_mount()
rootfs.unmount_rootfs()
except KeyboardInterrupt:
logger.critical(errors.keyboard_interrupt)
abort_analysis()
else:
logger.warning(errors.no_package_manager)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_package_manager,
'warning'))
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
return shell
def analyze_first_layer(image_obj, master_list, redo):
# find the binary and shell by mounting the base layer
target = rootfs.mount_base_layer(image_obj.layers[0].tar_file)
binary = common.get_base_bin()
shell = get_shell(image_obj, binary)
# set up a notice origin for the first layer
origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10]
# only extract packages if there is a known binary and the layer is not
# cached
if binary:
if not common.load_from_cache(image_obj.layers[0], redo):
# Determine pacakge/os style from binary in the image layer
common.get_os_style(image_obj.layers[0], binary)
# get the packages of the first layer
rootfs.prep_rootfs(target)
common.add_base_packages(image_obj.layers[0], binary, shell)
# unmount proc, sys and dev
rootfs.undo_mount()
else:
logger.warning(errors.no_package_manager)
# /etc/os-release may still be present even if binary is not
common.get_os_style(image_obj.layers[0], None)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_package_manager, 'warning'))
# no binary means there is no shell so set to default shell
logger.warning('Unknown filesystem. Using default shell')
shell = constants.shell
# unmount the first layer
rootfs.unmount_rootfs()
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
return shell
def analyze_subsequent_layers(
image_obj,
shell,
master_list,
redo,
dfobj=None, # noqa: R0912,R0913
dfile_lock=False):
# get packages for subsequent layers
curr_layer = 1
while curr_layer < len(image_obj.layers): # pylint:disable=too-many-nested-blocks
# if there is no shell, try to see if it exists in the current layer
if not shell:
shell = common.get_shell(image_obj.layers[curr_layer])
if not common.load_from_cache(image_obj.layers[curr_layer], redo):
# get commands that created the layer
# for docker images this is retrieved from the image history
command_list = dhelper.get_commands_from_history(
image_obj.layers[curr_layer])
if command_list:
# mount diff layers from 0 till the current layer
target = mount_overlay_fs(image_obj, curr_layer)
# mount dev, sys and proc after mounting diff layers
rootfs.prep_rootfs(target)
# for each command look up the snippet library
for command in command_list:
pkg_listing = command_lib.get_package_listing(command.name)
if isinstance(pkg_listing, str):
try:
common.add_base_packages(image_obj.layers[curr_layer],
pkg_listing, shell)
except KeyboardInterrupt:
logger.critical(errors.keyboard_interrupt)
abort_analysis()
else:
try:
common.add_snippet_packages(
image_obj.layers[curr_layer], command, pkg_listing,
shell)
except KeyboardInterrupt:
logger.critical(errors.keyboard_interrupt)
abort_analysis()
# pin any installed packages to a locked dockerfile.
if dfile_lock:
# collect list of RUN commands that could install pkgs
run_dict = d_file.get_run_layers(dfobj)
for package in image_obj.layers[curr_layer].packages:
# check that package is in current dfobj RUN line
if d_file.package_in_dockerfile(
run_dict[curr_layer - 1], package.name):
d_file.expand_package(
run_dict[curr_layer - 1], package.name,
package.version,
command_lib.check_pinning_separator(
pkg_listing))
if command_list:
rootfs.undo_mount()
rootfs.unmount_rootfs()
# update the master list
common.update_master_list(master_list, image_obj.layers[curr_layer])
curr_layer = curr_layer + 1
def fresh_analysis(image_obj, curr_layer, shell, driver, envs):
"""This is a subroutine that is run if there is no chached results or if
the user wants to redo the analysis
1 Check if we have a shell, if not then see if we can find a shell
2 Check if we can find any info in the container history (created_by)
3 If this exists then check if we can parse the command to figure out
what package managers were used.
4 Use the prescribed methods for the package managers to retrieve
"""
# if there is no shell, try to see if it exists in the current layer
if not shell:
shell = dcom.get_shell(image_obj.layers[curr_layer])
# get commands that created the layer
# for docker images this is retrieved from the image history
command_list = dcom.get_commands_from_metadata(
image_obj.layers[curr_layer])
if command_list:
# mount diff layers from 0 till the current layer
target = mount_overlay_fs(image_obj, curr_layer, driver)
# mount dev, sys and proc after mounting diff layers
rootfs.prep_rootfs(target)
# for each command look up the snippet library
for command in command_list:
pkg_listing = command_lib.get_package_listing(command.name)
if isinstance(pkg_listing, str):
core.execute_base(image_obj.layers[curr_layer], shell,
pkg_listing, envs)
else:
core.execute_snippets(image_obj.layers[curr_layer], command,
pkg_listing, shell, envs)
rootfs.undo_mount()
rootfs.unmount_rootfs()
def unmount():
"""Go through unmounting the working directory"""
# try to unmount proc, sys and dev
try:
rootfs.undo_mount()
except subprocess.CalledProcessError:
pass
try:
rootfs.unmount_rootfs()
except subprocess.CalledProcessError:
pass
def execute_base_layer(base_layer, binary, shell):
'''Execute retrieving base layer packages'''
try:
target = rootfs.mount_base_layer(base_layer.tar_file)
rootfs.prep_rootfs(target)
common.add_base_packages(base_layer, binary, shell)
except KeyboardInterrupt:
logger.critical(errors.keyboard_interrupt)
abort_analysis()
finally:
# unmount proc, sys and dev
rootfs.undo_mount()
rootfs.unmount_rootfs()
def analyze_first_layer(image_obj, master_list, options):
"""Analyze the first layer of an image. Return the installed shell.
If there is no installed shell, return None
1. Check if the layer is empty. If it is then we can't find a shell
2. See if we can load the layer from cache
3. If we can't load from cache
3.1 See if we can find any information about the rootfs
3.2 If we are able to find any information, use any prescribed methods
to extract package information
4. Process and bundle that information into the image object
5. Return the shell for subsequent layer processing"""
# set up a notice origin for the first layer
origin_first_layer = 'Layer {}'.format(image_obj.layers[0].layer_index)
# check if the layer is empty
if com.is_empty_layer(image_obj.layers[0]):
logger.warning(errors.empty_layer)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.empty_layer, 'warning'))
return None
# create a Prereqs object
prereqs = core.Prereqs()
# find the shell from the first layer
prereqs.shell = dcom.get_shell(image_obj.layers[0])
if not prereqs.shell:
logger.warning(errors.no_shell)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_shell, 'warning'))
# find the binary from the first layer
prereqs.binary = dcom.get_base_bin(image_obj.layers[0])
if not prereqs.binary:
logger.warning(errors.no_package_manager)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_package_manager, 'warning'))
# try to load packages from cache
if not com.load_from_cache(image_obj.layers[0], options.redo):
# set a possible OS
get_os_style(image_obj.layers[0], prereqs.binary)
# if there is a binary, extract packages
if prereqs.shell and prereqs.binary:
# mount the first layer
mount_first_layer(image_obj.layers[0])
# core default execution on the first layer
core.execute_base(image_obj.layers[0], prereqs)
# unmount
rootfs.undo_mount()
rootfs.unmount_rootfs()
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
return prereqs.shell
def recover():
"""Undo all the mounts and clean up directories"""
try:
rootfs.undo_mount()
except subprocess.CalledProcessError:
pass
try:
rootfs.unmount_rootfs()
except subprocess.CalledProcessError:
pass
# we nuke all the directories after mounting
rootfs.clean_up()
working_dir = rootfs.get_working_dir()
if os.path.exists(working_dir):
rootfs.root_command(rootfs.remove, working_dir)
def execute_invoke(image_obj, args):
"""Execution path for checking command library scripts"""
# we set up the image for analysis
run.setup(image_obj)
# we now mount the whole container image
mount_container_image(image_obj, args.driver)
# invoke commands in chroot
invoke_script(args)
# undo the mounts
rootfs.undo_mount()
rootfs.unmount_rootfs()
# cleanup
rootfs.clean_up()
if not args.keep_wd:
prep.clean_image_tars(image_obj)
def fresh_analysis(image_obj, curr_layer, prereqs, options):
"""This is a subroutine that is run if there is no chached results or if
the user wants to redo the analysis
1 Check if we have a shell, if not then see if we can find a shell
2 Check if we can find any info in the container history (created_by)
3 If this exists then check if we can parse the command to figure out
what package managers were used.
4 Use the prescribed methods for the package managers to retrieve
"""
# set up a notice origin for the current layer
origin_curr_layer = 'Layer {}'.format(
image_obj.layers[curr_layer].layer_index)
image_obj.layers[curr_layer].origins.add_notice_to_origins(
origin_curr_layer, Notice(formats.layer_created_by.format(
created_by=image_obj.layers[curr_layer].created_by), 'info'))
# if there is no shell, try to see if it exists in the current layer
if not prereqs.fs_shell:
prereqs.shell = dcom.get_shell(image_obj.layers[curr_layer])
# mount diff layers from 0 till the current layer
target = mount_overlay_fs(image_obj, curr_layer, options.driver)
# set this layer's host path
prereqs.host_path = target
# mount dev, sys and proc after mounting diff layers
rootfs.prep_rootfs(target)
# get commands that created the layer
# for docker images this is retrieved from the image history
command_list = dcom.get_commands_from_metadata(
image_obj.layers[curr_layer])
if command_list:
# for each command look up the snippet library
for command in command_list:
pkg_listing = command_lib.get_package_listing(command.name)
if isinstance(pkg_listing, str):
prereqs.binary = pkg_listing
core.execute_base(
image_obj.layers[curr_layer], prereqs)
else:
prereqs.listing = pkg_listing
core.execute_snippets(
image_obj.layers[curr_layer], command, prereqs)
else:
# fall back to executing what we know
core.execute_base(image_obj.layers[curr_layer], prereqs)
rootfs.undo_mount()
rootfs.unmount_rootfs()
rootfs.prep_rootfs(target)
# invoke commands in chroot
# if we're looking up the snippets library
# we should see 'snippets' in the keys
if 'snippets' in args.keys and 'packages' in args.keys:
# get the package info that corresponds to the package name
# or get the default
last = args.keys.pop()
info_list = look_up_lib(args.keys)
info_dict = command_lib.check_for_unique_package(
info_list, args.package)[last]
else:
info_dict = look_up_lib(args.keys)
# try to invoke the commands
try:
result = command_lib.get_pkg_attr_list(args.shell, info_dict,
args.package)
print('Output list: ' + ' '.join(result[0]))
print('Error messages: ' + result[1])
print('Number of elements: ' + str(len(result[0])))
except subprocess.CalledProcessError as error:
print(error.output)
# undo the mounts
rootfs.undo_mount()
rootfs.unmount_rootfs()
else:
print("Something when wrong in loading the image")
report.teardown()
report.clean_image_tars(image_obj)
report.clean_working_dir()
def analyze_docker_image(image_obj, redo=False, dockerfile=False): # pylint: disable=too-many-locals
'''Given a DockerImage object, for each layer, retrieve the packages, first
looking up in cache and if not there then looking up in the command
library. For looking up in command library first mount the filesystem
and then look up the command library for commands to run in chroot'''
# find the layers that are imported
if dockerfile:
dhelper.set_imported_layers(image_obj)
# add notices for each layer if it is imported
image_setup(image_obj)
shell = ''
# set up empty master list of packages
master_list = []
# find the binary by mounting the base layer
target = rootfs.mount_base_layer(image_obj.layers[0].tar_file)
binary = common.get_base_bin()
# set up a notice origin referring to the base command library listing
origin_command_lib = formats.invoking_base_commands
# set up a notice origin for the first layer
origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10]
# find the shell to invoke commands in
shell, _ = command_lib.get_image_shell(
command_lib.get_base_listing(binary))
if not shell:
# add a warning notice for no shell in the command library
logger.warning('No shell listing in command library. '
'Using default shell')
no_shell_message = errors.no_shell_listing.format(
binary=binary, default_shell=constants.shell)
image_obj.layers[0].origins.add_notice_to_origins(
origin_command_lib, Notice(no_shell_message, 'warning'))
# add a hint notice to add the shell to the command library
add_shell_message = errors.no_listing_for_base_key.format(
listing_key='shell')
image_obj.layers[0].origins.add_notice_to_origins(
origin_command_lib, Notice(add_shell_message, 'hint'))
shell = constants.shell
# only extract packages if there is a known binary and the layer is not
# cached
if binary:
if not common.load_from_cache(image_obj.layers[0], redo):
# get the packages of the first layer
rootfs.prep_rootfs(target)
common.add_base_packages(image_obj.layers[0], binary, shell)
# unmount proc, sys and dev
rootfs.undo_mount()
else:
no_base = errors.unrecognized_base.format(
image_name=image_obj.name, image_tag=image_obj.tag)
logger.warning(no_base)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(no_base, 'warning'))
# no binary means there is no shell so set to default shell
logger.warning('Unknown filesystem. Using default shell')
shell = constants.shell
# unmount the first layer
rootfs.unmount_rootfs()
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
# get packages for subsequent layers
curr_layer = 1
while curr_layer < len(image_obj.layers):
if not common.load_from_cache(image_obj.layers[curr_layer], redo):
# get commands that created the layer
# for docker images this is retrieved from the image history
command_list = dhelper.get_commands_from_history(
image_obj.layers[curr_layer])
if command_list:
# mount diff layers from 0 till the current layer
target = mount_overlay_fs(image_obj, curr_layer)
# mount dev, sys and proc after mounting diff layers
rootfs.prep_rootfs(target)
# for each command look up the snippet library
for command in command_list:
pkg_listing = command_lib.get_package_listing(command.name)
if isinstance(pkg_listing, str):
common.add_base_packages(
image_obj.layers[curr_layer], pkg_listing, shell)
else:
common.add_snippet_packages(
image_obj.layers[curr_layer], command, pkg_listing,
shell)
if command_list:
rootfs.undo_mount()
rootfs.unmount_rootfs()
# update the master list
common.update_master_list(master_list, image_obj.layers[curr_layer])
curr_layer = curr_layer + 1
common.save_to_cache(image_obj)
def analyze_subsequent_layers(
image_obj,
shell,
master_list,
redo,
dfobj=None, # noqa: R0912,R0913
dfile_lock=False,
driver=None):
# get packages for subsequent layers
curr_layer = 1
work_dir = None
while curr_layer < len(image_obj.layers): # pylint:disable=too-many-nested-blocks
# make a notice for each layer
origin_next_layer = 'Layer {}'.format(
image_obj.layers[curr_layer].layer_index)
# check if this is an empty layer
if common.is_empty_layer(image_obj.layers[curr_layer]):
# we continue to the next layer
logger.warning(errors.empty_layer)
image_obj.layers[curr_layer].origins.add_notice_to_origins(
origin_next_layer, Notice(errors.empty_layer, 'warning'))
curr_layer = curr_layer + 1
continue
# If workdir changes, update value accordingly
# so we can later execute base.yml commands from the workdir.
if image_obj.layers[curr_layer].get_layer_workdir() is not None:
work_dir = image_obj.layers[curr_layer].get_layer_workdir()
# if there is no shell, try to see if it exists in the current layer
if not shell:
shell = common.get_shell(image_obj.layers[curr_layer])
if not common.load_from_cache(image_obj.layers[curr_layer], redo):
# get commands that created the layer
# for docker images this is retrieved from the image history
command_list = dhelper.get_commands_from_history(
image_obj.layers[curr_layer])
if command_list:
# mount diff layers from 0 till the current layer
target = mount_overlay_fs(image_obj, curr_layer, driver)
# mount dev, sys and proc after mounting diff layers
rootfs.prep_rootfs(target)
# for each command look up the snippet library
for command in command_list:
pkg_listing = command_lib.get_package_listing(command.name)
# get list of environment variables
envs = dhelper.get_env_vars(image_obj)
if isinstance(pkg_listing, str):
try:
common.add_base_packages(image_obj.layers[curr_layer],
pkg_listing, shell, work_dir,
envs)
except KeyboardInterrupt:
logger.critical(errors.keyboard_interrupt)
abort_analysis()
else:
try:
common.add_snippet_packages(
image_obj.layers[curr_layer], command, pkg_listing,
shell, work_dir, envs)
except KeyboardInterrupt:
logger.critical(errors.keyboard_interrupt)
abort_analysis()
# pin any installed packages to a locked dockerfile.
if dfile_lock:
# collect list of RUN commands that could install pkgs
run_dict = d_file.get_run_layers(dfobj)
# use the run_dict to get list of packages being installed
install_list = d_file.get_install_packages(
run_dict[curr_layer - 1])
for install_pkg in install_list:
for layer_pkg in image_obj.layers[curr_layer].packages:
if install_pkg == layer_pkg.name:
# dockerfile package in layer, let's pin it
d_file.expand_package(
run_dict[curr_layer - 1], install_pkg,
layer_pkg.version,
command_lib.check_pinning_separator(
pkg_listing))
if command_list:
rootfs.undo_mount()
rootfs.unmount_rootfs()
# update the master list
common.update_master_list(master_list, image_obj.layers[curr_layer])
curr_layer = curr_layer + 1
