def analyze_first_layer(image_obj, master_list, redo):
# set up a notice origin for the first layer
origin_first_layer = 'Layer {}'.format(image_obj.layers[0].layer_index)
# find the shell from the first layer
shell = common.get_shell(image_obj.layers[0])
if not shell:
logger.warning(errors.no_shell)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_shell, 'warning'))
# find the binary from the first layer
binary = common.get_base_bin(image_obj.layers[0])
if not binary:
logger.warning(errors.no_package_manager)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_package_manager, 'warning'))
# try to load packages from cache
if not common.load_from_cache(image_obj.layers[0], redo):
# set a possible OS
common.get_os_style(image_obj.layers[0], binary)
# if there is a binary, extract packages
if shell and binary:
execute_base_layer(image_obj.layers[0], binary, shell)
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
return shell
def setUp(self):
self.notice_info = Notice("info")
self.notice_warn = Notice("warning", "warning")
self.notice_errr = Notice("error", "error")
self.notice_hint = Notice("hint", "hint")
self.origins = Origins()
def add_snippet_packages(image_layer, command, pkg_listing, shell):
'''Given an image layer object, a command object, the package listing
and the shell used to invoke commands, add package metadata to the layer
object. We assume the filesystem is already mounted and ready
1. Get the packages installed by the command
3. For each package get the dependencies
4. For each unique package name, find the metadata and add to the
layer'''
# set up a notice origin for the layer
origin_layer = 'Layer: ' + image_layer.fs_hash[:10]
# find packages for the command
cmd_msg = formats.invoke_for_snippets + '\n' + \
content.print_package_invoke(command.name)
image_layer.origins.add_notice_to_origins(origin_layer,
Notice(cmd_msg, 'info'))
pkg_list = get_installed_package_names(command)
# collect all the dependencies for each package name
all_pkgs = []
for pkg_name in pkg_list:
pkg_invoke = command_lib.check_for_unique_package(
pkg_listing, pkg_name)
deps, deps_msg = get_package_dependencies(pkg_invoke, pkg_name, shell)
if deps_msg:
logger.warning(deps_msg)
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(deps_msg, 'error'))
all_pkgs.append(pkg_name)
all_pkgs.extend(deps)
unique_pkgs = list(set(all_pkgs))
# get package metadata for each package name
for pkg_name in unique_pkgs:
pkg = Package(pkg_name)
fill_package_metadata(pkg, pkg_invoke, shell)
image_layer.add_package(pkg)
def get_commands_from_history(image_layer):
'''Given the image layer object and the shell, get the list of command
objects that created the layer'''
# set up notice origin for the layer
origin_layer = 'Layer {}'.format(image_layer.layer_index)
if image_layer.created_by:
instruction = created_to_instruction(image_layer.created_by)
image_layer.origins.add_notice_to_origins(
origin_layer,
Notice(
formats.dockerfile_line.format(
dockerfile_instruction=instruction), 'info'))
command_line = instruction.split(' ', 1)[1]
else:
instruction = ''
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(formats.no_created_by, 'warning'))
command_line = instruction
# Image layers are created with the directives RUN, ADD and COPY
# For ADD and COPY instructions, there is no information about the
# packages added
if 'ADD' in instruction or 'COPY' in instruction:
image_layer.origins.add_notice_to_origins(
origin_layer,
Notice(errors.unknown_content.format(files=command_line),
'warning'))
# return an empty list as we cannot find any commands
return []
# for RUN instructions we can return a list of commands
command_list, msg = common.filter_install_commands(command_line)
if msg:
image_layer.origins.add_notice_to_origins(origin_layer,
Notice(msg, 'warning'))
return command_list
def get_os_style(image_layer, binary):
'''Given an ImageLayer object and a binary package manager, check for the
OS identifier in the os-release file first. If the os-release file
is not available, make an educated guess as to what kind of OS the layer
might be based off of given the pkg_format + package manager. If the binary
provided does not exist in base.yml, add a warning notice'''
origin_layer = 'Layer {}'.format(image_layer.layer_index)
# see if we can find what OS this is
os_release = get_os_release(image_layer)
if os_release:
# We know with high degree of certainty what the OS is
image_layer.origins.add_notice_to_origins(origin_layer, Notice(
formats.os_release.format(os_style=os_release), 'info'))
# we can set the OS of the image layer
image_layer.os_guess = os_release
else:
# We will try looking for the possible OSs based on the binary
os_guess = command_lib.check_os_guess(binary)
if os_guess:
# We can make a guess
image_layer.origins.add_notice_to_origins(origin_layer, Notice(
formats.os_style_guess.format(
package_manager=binary, os_list=os_guess), 'info'))
image_layer.os_guess = os_guess
else:
# No binary and no os-release means we have no idea about base OS
image_layer.origins.add_notice_to_origins(origin_layer, Notice(
errors.no_etc_release, 'warning'))
# set a package format if there is one for this binary
image_layer.pkg_format = command_lib.check_pkg_format(binary)
def get_os_style(image_layer, binary):
'''Given an ImageLayer object and a binary package manager, check for the
OS identifier in the os-release file first. If the os-release file
is not available, make an educated guess as to what kind of OS the layer
might be based off of given the pkg_format + package manager. If the binary
provided does not exist in base.yml, add a warning notice'''
origin_command_lib = formats.invoking_base_commands
origin_layer = 'Layer: ' + image_layer.fs_hash[:10]
pkg_format = command_lib.check_pkg_format(binary)
os_guess = command_lib.check_os_guess(binary)
if get_os_release():
# We know with high degree of certainty what the OS is
image_layer.origins.add_notice_to_origins(origin_layer, Notice(
formats.os_release.format(os_style=get_os_release()), 'info'))
elif binary is None:
# No binary and no os-release means we have no idea about base OS
image_layer.origins.add_notice_to_origins(origin_layer, Notice(
errors.no_etc_release, 'warning'))
else:
# We make a guess about the OS based on pkg_format + binary
# First check that binary exists in base.yml
if not pkg_format or not os_guess:
image_layer.origins.add_notice_to_origins(
origin_command_lib, Notice(
errors.no_listing_for_base_key.format(listing_key=binary),
'warning'))
else:
# Assign image layer attributes and guess OS
image_layer.pkg_format = pkg_format
image_layer.os_guess = os_guess
image_layer.origins.add_notice_to_origins(origin_layer, Notice(
formats.os_style_guess.format(
package_manager=binary,
package_format=image_layer.pkg_format,
os_list=image_layer.os_guess), 'info'))
def load_base_image():
'''Create base image from dockerfile instructions and return the image'''
base_image, dockerfile_lines = dhelper.get_dockerfile_base()
# try to get image metadata
if not container.check_image(base_image.repotag):
# if no base image is found, give a warning and continue
if not container.pull_image(base_image.repotag):
logger.warning(
errors.cannot_find_image.format(imagetag=base_image.repotag))
try:
base_image.load_image()
except NameError as error:
logger.warning('Error in loading base image: ' + str(error))
base_image.origins.add_notice_to_origins(dockerfile_lines,
Notice(str(error), 'error'))
except subprocess.CalledProcessError as error:
logger.warning('Error in loading base image: ' +
str(error.output, 'utf-8'))
base_image.origins.add_notice_to_origins(
dockerfile_lines, Notice(str(error.output, 'utf-8'), 'error'))
except IOError as error:
logger.warning('Error in loading base image: ' + str(error))
base_image.origins.add_notice_to_origin(dockerfile_lines,
Notice(str(error), 'error'))
return base_image
def execute_snippets(layer_obj, command_obj, prereqs):
"""Given in ImageLayer object, shell and binary to look up, find packages
installed in the layer using the default method:
For snippets, we will get the packages installed by the command"""
# set up a notice origin for the layer
origin_layer = 'Layer {}'.format(layer_obj.layer_index)
# find packages for the command
cmd_msg = (formats.invoke_for_snippets + '\n' +
content.print_package_invoke(command_obj.name))
layer_obj.origins.add_notice_to_origins(origin_layer,
Notice(cmd_msg, 'info'))
pkg_list = filter.get_installed_package_names(command_obj)
# collect all the dependencies for each package name
all_pkgs = []
for pkg_name in pkg_list:
pkg_invoke = command_lib.check_for_unique_package(
prereqs.listing, pkg_name)
deps, deps_msg = com.get_package_dependencies(pkg_invoke, pkg_name,
prereqs.shell)
if deps_msg:
logger.warning(deps_msg)
layer_obj.origins.add_notice_to_origins(origin_layer,
Notice(deps_msg, 'error'))
all_pkgs.append(pkg_name)
all_pkgs.extend(deps)
unique_pkgs = list(set(all_pkgs))
# get package metadata for each package name
for pkg_name in unique_pkgs:
pkg = Package(pkg_name)
dcom.fill_package_metadata(pkg, pkg_invoke, prereqs.shell,
layer_obj.get_layer_workdir(), prereqs.envs)
layer_obj.add_package(pkg)
def analyze_first_layer(image_obj, master_list, options):
"""Analyze the first layer of an image. Return a Prereqs object for the
next layer.
1. Check if the layer is empty. If it is not, return None
2. See if we can load the layer from cache
3. If we can't load from cache
3.1 See if we can find any information about the rootfs
3.2 If we are able to find any information, use any prescribed methods
to extract package information
4. Process and bundle that information into the image object
5. Return a Prereqs object for subsequent layer processing"""
# set up a notice origin for the first layer
origin_first_layer = 'Layer {}'.format(image_obj.layers[0].layer_index)
# check if the layer is empty
if com.is_empty_layer(image_obj.layers[0]):
logger.warning(errors.empty_layer)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.empty_layer, 'warning'))
return None
# create a Prereqs object
prereqs = core.Prereqs()
# find the shell from the first layer
prereqs.fs_shell = dcom.get_shell(image_obj.layers[0])
# find a shell for the host
prereqs.host_shell = host.check_shell()
if not prereqs.fs_shell and not prereqs.host_shell:
logger.warning(errors.no_shell)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_shell, 'warning'))
# find the binary from the first layer
prereqs.binary = dcom.get_base_bin(image_obj.layers[0])
# try to load packages from cache
if not com.load_from_cache(image_obj.layers[0], options.redo):
# add a notice if there is a "created by"
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(formats.layer_created_by.format(
created_by=image_obj.layers[0].created_by), 'info'))
# set a possible OS and package format
get_os_style(image_obj.layers[0], prereqs.binary)
# if there is a binary, extract packages
if prereqs.binary:
# mount the first layer
target_dir = prep_first_layer(image_obj.layers[0])
# set the host path to the mount point
if target_dir:
prereqs.host_path = target_dir
# core default execution on the first layer
core.execute_base(image_obj.layers[0], prereqs)
else:
logger.warning(errors.no_package_manager)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(
errors.no_package_manager, 'warning'))
return None
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
return prereqs
def execute_base(layer_obj, prereqs):
"""Given an ImageLayer object, find packages installed in the layer
using the default method. The prereqisites required for this to work:
prereqs.shell: the shell to use
prereqs.binary: the binary to look up in the command library
optional prerequisites:
prereqs.envs: any environment variables to set before execution
1. Use command_lib's base to look up the binary to see if there
is a method to retrieve the metadata
2. If there is, invoke the scripts in a chroot environment and
process the results
3. Add the results to the ImageLayer object
It is assumed that the filesystem is prepped for execution by mounting
the filesystem in the working directory and /proc, /sys and /dev device
nodes are mounted"""
# Add notices to this layer object
origin_layer = 'Layer {}'.format(layer_obj.layer_index)
# find the binary listing
listing = command_lib.get_base_listing(prereqs.binary)
if listing:
# put info notice about what is going to be invoked
snippet_msg = (formats.invoke_for_base + '\n' +
content.print_base_invoke(prereqs.binary))
layer_obj.origins.add_notice_to_origins(origin_layer,
Notice(snippet_msg, 'info'))
# get list of metadata by invoking scripts in chroot
logger.debug("Collecting metadata for image layer...")
pkg_dict, invoke_msg, warnings = collect.collect_list_metadata(
prereqs.shell, listing, layer_obj.get_layer_workdir(),
prereqs.envs)
# more processing for debian copyrights to get licenses
if listing.get("pkg_format") == "deb":
logger.debug("Processing Debian copyrights...")
pkg_dict["pkg_licenses"] = com.get_deb_package_licenses(
pkg_dict["copyrights"])
# add any errors and warnings to the layer's origins object
if invoke_msg:
logger.error(
"Script invocation error. Unable to collect some metadata")
layer_obj.origins.add_notice_to_origins(
origin_layer, Notice(invoke_msg, 'error'))
if warnings:
logger.warning("Some metadata may be missing")
layer_obj.origins.add_notice_to_origins(
origin_layer, Notice(warnings, 'warning'))
# bundle the results into Package objects
bundle.fill_pkg_results(layer_obj, pkg_dict)
# remove extra FileData objects from the layer
com.remove_duplicate_layer_files(layer_obj)
# if there is no listing add a notice
else:
layer_obj.origins.add_notice_to_origins(
origin_layer,
Notice(
errors.no_listing_for_base_key.format(
listing_key=prereqs.binary), 'error'))
def add_base_packages(image_layer, binary, shell): # pylint: disable=too-many-locals
'''Given the image layer, the binary to invoke and shell:
1. get the listing from the base.yml
2. Invoke any commands against the base layer
3. Make a list of packages and add them to the layer'''
origin_layer = 'Layer: ' + image_layer.fs_hash[:10]
if image_layer.created_by:
image_layer.origins.add_notice_to_origins(
origin_layer,
Notice(
formats.layer_created_by.format(
created_by=image_layer.created_by), 'info'))
else:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(formats.no_created_by, 'warning'))
origin_command_lib = formats.invoking_base_commands
# find the binary
listing = command_lib.get_base_listing(binary)
if listing:
# put info notice about what is going to be invoked
snippet_msg = formats.invoke_for_base + '\n' + \
content.print_base_invoke(binary)
image_layer.origins.add_notice_to_origins(origin_layer,
Notice(snippet_msg, 'info'))
shell, _ = command_lib.get_image_shell(listing)
if not shell:
shell = constants.shell
# get all the packages in the base layer
names, n_msg = command_lib.get_pkg_attr_list(shell, listing['names'])
versions, v_msg = command_lib.get_pkg_attr_list(
shell, listing['versions'])
licenses, l_msg = command_lib.get_pkg_attr_list(
shell, listing['licenses'])
src_urls, u_msg = command_lib.get_pkg_attr_list(
shell, listing['src_urls'])
# add a notice to the image if something went wrong
invoke_msg = n_msg + v_msg + l_msg + u_msg
if invoke_msg:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(invoke_msg, 'error'))
if names and len(names) > 1:
for index, name in enumerate(names):
pkg = Package(name)
if len(versions) == len(names):
pkg.version = versions[index]
if len(licenses) == len(names):
pkg.license = licenses[index]
if len(src_urls) == len(names):
pkg.src_url = src_urls[index]
image_layer.add_package(pkg)
# if there is no listing add a notice
else:
image_layer.origins.add_notice_to_origins(
origin_command_lib,
Notice(errors.no_listing_for_base_key.format(listing_key=binary),
'error'))
def setUp(self):
self.notice_info = Notice("info")
self.notice_warn = Notice("warning", "warning")
self.notice_errr = Notice("error", "error")
self.notice_hint = Notice("hint", "hint")
self.notices = [
self.notice_info, self.notice_warn, self.notice_errr,
self.notice_hint
]
self.notice_origin = NoticeOrigin('origin_str')
def analyze_first_layer(image_obj, master_list, options):
"""Analyze the first layer of an image. Return the installed shell.
If there is no installed shell, return None
1. Check if the layer is empty. If it is then we can't find a shell
2. See if we can load the layer from cache
3. If we can't load from cache
3.1 See if we can find any information about the rootfs
3.2 If we are able to find any information, use any prescribed methods
to extract package information
4. Process and bundle that information into the image object
5. Return the shell for subsequent layer processing"""
# set up a notice origin for the first layer
origin_first_layer = 'Layer {}'.format(image_obj.layers[0].layer_index)
# check if the layer is empty
if com.is_empty_layer(image_obj.layers[0]):
logger.warning(errors.empty_layer)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.empty_layer, 'warning'))
return None
# create a Prereqs object
prereqs = core.Prereqs()
# find the shell from the first layer
prereqs.shell = dcom.get_shell(image_obj.layers[0])
if not prereqs.shell:
logger.warning(errors.no_shell)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_shell, 'warning'))
# find the binary from the first layer
prereqs.binary = dcom.get_base_bin(image_obj.layers[0])
if not prereqs.binary:
logger.warning(errors.no_package_manager)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_package_manager, 'warning'))
# try to load packages from cache
if not com.load_from_cache(image_obj.layers[0], options.redo):
# set a possible OS
get_os_style(image_obj.layers[0], prereqs.binary)
# if there is a binary, extract packages
if prereqs.shell and prereqs.binary:
# mount the first layer
mount_first_layer(image_obj.layers[0])
# core default execution on the first layer
core.execute_base(image_obj.layers[0], prereqs)
# unmount
rootfs.undo_mount()
rootfs.unmount_rootfs()
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
return prereqs.shell
class TestClassNotice(unittest.TestCase):
def setUp(self):
self.notice = Notice()
def tearDown(self):
del self.notice
def testInstance(self):
self.assertFalse(self.notice.message)
self.assertEqual(self.notice.level, 'info')
def testSetters(self):
self.notice.message = 'tag'
self.assertEqual(self.notice.message, 'tag')
with self.assertRaises(NoticeException):
self.notice.level = 'something'
self.notice.level = 'warning'
def testGetters(self):
self.notice.message = 'tag'
self.notice.level = 'warning'
self.assertEqual(self.notice.message, 'tag')
self.assertEqual(self.notice.level, 'warning')
def testToDict(self):
self.notice.message = 'tag'
self.notice.level = 'warning'
dict = self.notice.to_dict()
self.assertEqual(dict['message'], 'tag')
self.assertEqual(dict['level'], 'warning')
def load_files_from_cache(layer):
'''Given a layer object, populate file level information'''
loaded = False
raw_file_list = cache.get_files(layer.fs_hash)
if raw_file_list:
logger.debug('Loading files from cache: layer \"%s\"',
layer.fs_hash[:10])
for file_dict in raw_file_list:
f = FileData(file_dict['name'], file_dict['path'])
f.fill(file_dict)
# collect file origins
if 'origins' in file_dict.keys():
for origin_dict in file_dict['origins']:
for notice in origin_dict['notices']:
f.origins.add_notice_to_origins(
origin_dict['origin_str'],
Notice(notice['message'], notice['level']))
layer.add_file(f)
loaded = True
else:
# if there are no files, generate them from the pre-calculated
# hash file
logger.debug('Reading files in filesystem...')
layer.add_files()
return loaded
def collect_layer_data(layer_obj):
'''Use scancode to collect data from a layer filesystem. This function will
create FileData and Package objects for every File and Package found. After
scanning, it will return a tuple with a list of FileData and a list of
Package objects.
'''
files = []
packages = []
# run scancode against a directory
try:
processes = len(os.sched_getaffinity(0))
command = "scancode -ilpcu --quiet --timeout 300 -n {} --json -".format(processes)
except (AttributeError, NotImplementedError):
command = "scancode -ilpcu --quiet --timeout 300 --json -"
full_cmd = get_filesystem_command(layer_obj, command)
origin_layer = 'Layer {}'.format(layer_obj.layer_index)
result, error = rootfs.shell_command(True, full_cmd)
if not result:
logger.error(
"No scancode results for this layer: %s", str(error))
layer_obj.origins.add_notice_to_origins(
origin_layer, Notice(str(error), 'error'))
else:
# make FileData objects for each result
data = json.loads(result)
add_scancode_headers(layer_obj, data["headers"])
for f in data['files']:
if f['type'] == 'file' and f['size'] != 0:
files.append(get_scancode_file(f))
for package in f['packages']:
packages.append(get_scancode_package(package))
return files, packages
def analyze_first_layer(image_obj, master_list, redo):
# find the binary of the first layer
binary = common.get_base_bin(image_obj.layers[0])
# see if there is an associated shell
# if there is no binary, this will be set to the default shell
shell = get_shell(image_obj, binary)
# try to load packages from cache
if not common.load_from_cache(image_obj.layers[0], redo):
# set a possible OS
common.get_os_style(image_obj.layers[0], binary)
# set up a notice origin for the first layer
origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10]
# if there is a binary, extract packages
if binary:
try:
target = rootfs.mount_base_layer(image_obj.layers[0].tar_file)
rootfs.prep_rootfs(target)
common.add_base_packages(image_obj.layers[0], binary, shell)
# unmount proc, sys and dev
rootfs.undo_mount()
rootfs.unmount_rootfs()
except KeyboardInterrupt:
logger.critical(errors.keyboard_interrupt)
abort_analysis()
else:
logger.warning(errors.no_package_manager)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_package_manager,
'warning'))
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
return shell
def load_from_cache(layer, redo=False):
'''Given a layer object, check against cache to see if that layer id exists
if yes then get the package list and load it in the layer and return true.
If it doesn't exist return false. Default operation is to not redo the
cache. Add notices to the layer's origins matching the origin_str'''
loaded = False
origin_layer = 'Layer: ' + layer.fs_hash[:10]
if not layer.packages and not redo:
# there are no packages in this layer and we are not repopulating the
# cache, try to get it from the cache
raw_pkg_list = cache.get_packages(layer.fs_hash)
if raw_pkg_list:
logger.debug('Loaded from cache: layer {}'.format(
layer.fs_hash[:10]))
message = formats.loading_from_cache.format(
layer_id=layer.fs_hash[:10])
# add notice to the origin
layer.origins.add_notice_to_origins(origin_layer,
Notice(message, 'info'))
for pkg_dict in raw_pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
layer.add_package(pkg)
loaded = True
return loaded
def collect_layer_data(layer_obj):
'''Use scancode to collect data from a layer filesystem. This function will
create a FileData object for every file found. After scanning, it will
return a list of FileData objects.
'''
files = []
# run scancode against a directory
command = 'scancode -ilpcu --quiet --timeout 300 --json -'
full_cmd = get_filesystem_command(layer_obj, command)
origin_layer = 'Layer: ' + layer_obj.fs_hash[:10]
result, error = rootfs.shell_command(True, full_cmd)
if not result:
logger.error("No scancode results for this layer: %s", str(error))
layer_obj.origins.add_notice_to_origins(origin_layer,
Notice(str(error), 'error'))
else:
# make FileData objects for each result
data = json.loads(result)
notice = data.get("headers")[0].get("notice")
headers = layer_obj.extension_info.get("headers", set())
headers.add(notice)
layer_obj.extension_info["headers"] = headers
for f in data['files']:
if f['type'] == 'file' and f['size'] != 0:
files.append(get_scancode_file(f))
return files
def analyze_subsequent_layers(image_obj,
shell,
master_list,
redo,
driver=None):
"""Assuming that we have a shell and have completed analyzing the first
layer of the given image object, we now analyze the remaining layers.
While we have layers:
1. Check if the layer is empty. If it is, then we can't do anything and
we should continue
2. See if we can load the layer from cache. If we can't then do a
fresh analysis
package information and bundle it into the image object
3. Update the master list"""
curr_layer = 1
# get list of environment variables
envs = lock.get_env_vars(image_obj)
while curr_layer < len(image_obj.layers):
# make a notice for each layer
origin_next_layer = 'Layer {}'.format(
image_obj.layers[curr_layer].layer_index)
# check if this is an empty layer
if common.is_empty_layer(image_obj.layers[curr_layer]):
# we continue to the next layer
logger.warning(errors.empty_layer)
image_obj.layers[curr_layer].origins.add_notice_to_origins(
origin_next_layer, Notice(errors.empty_layer, 'warning'))
curr_layer = curr_layer + 1
continue
if not common.load_from_cache(image_obj.layers[curr_layer], redo):
fresh_analysis(image_obj, curr_layer, shell, driver, envs)
# update the master list
dcom.update_master_list(master_list, image_obj.layers[curr_layer])
curr_layer = curr_layer + 1
def analyze_first_layer(image_obj, master_list, redo):
# find the binary and shell by mounting the base layer
target = rootfs.mount_base_layer(image_obj.layers[0].tar_file)
binary = common.get_base_bin()
shell = get_shell(image_obj, binary)
# set up a notice origin for the first layer
origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10]
# only extract packages if there is a known binary and the layer is not
# cached
if binary:
if not common.load_from_cache(image_obj.layers[0], redo):
# Determine pacakge/os style from binary in the image layer
common.get_os_style(image_obj.layers[0], binary)
# get the packages of the first layer
rootfs.prep_rootfs(target)
common.add_base_packages(image_obj.layers[0], binary, shell)
# unmount proc, sys and dev
rootfs.undo_mount()
else:
logger.warning(errors.no_package_manager)
# /etc/os-release may still be present even if binary is not
common.get_os_style(image_obj.layers[0], None)
image_obj.layers[0].origins.add_notice_to_origins(
origin_first_layer, Notice(errors.no_package_manager, 'warning'))
# no binary means there is no shell so set to default shell
logger.warning('Unknown filesystem. Using default shell')
shell = constants.shell
# unmount the first layer
rootfs.unmount_rootfs()
# populate the master list with all packages found in the first layer
for p in image_obj.layers[0].packages:
master_list.append(p)
return shell
def get_dockerfile_packages():
'''Given a Dockerfile return an approximate image object. This is mosty
guess work and shouldn't be relied on for accurate information. Add
Notice messages indicating as such:
1. Create an image with a placeholder repotag
2. For each RUN command, create a package list
3. Create layer objects with incremental integers and add the package
list to that layer with a Notice about parsing
4. Return stub image'''
stub_image = Image('easteregg:cookie')
layer_count = 0
for cmd in dhelper.docker_commands:
if cmd['instruction'] == 'RUN':
layer_count = layer_count + 1
layer = ImageLayer(layer_count)
install_commands, msg = \
common.filter_install_commands(cmd['value'])
if msg:
layer.origins.add_notice_to_origins(cmd['value'],
Notice(msg, 'info'))
pkg_names = []
for command in install_commands:
pkg_names.append(common.get_installed_package_names(command))
for pkg_name in pkg_names:
pkg = Package(pkg_name)
# shell parser does not parse version pins yet
# when that is enabled, Notices for no versions need to be
# added here
layer.add_package(pkg)
return stub_image
def get_scancode_file(file_dict):
'''Given a file dictionary from the scancode results, return a FileData
object with the results'''
# scancode records paths from the target directory onwards
# which in tern's case is tern.utils.constants.untar_dir
# removing that portion of the file path
fspath = file_dict['path'].replace(
constants.untar_dir + os.path.sep, '')
fd = FileData(
file_dict['name'], fspath, file_dict['date'], file_dict['file_type'])
fd.short_file_type = get_file_type(file_dict)
fd.add_checksums({'sha1': file_dict['sha1'], 'md5': file_dict['md5']})
if file_dict['licenses']:
fd.licenses = [li['short_name'] for li in file_dict['licenses']]
fd.license_expressions = file_dict['license_expressions']
if file_dict['copyrights']:
fd.copyrights = [c['value'] for c in file_dict['copyrights']]
if file_dict['urls']:
fd.urls = [u['url'] for u in file_dict['urls']]
fd.packages = file_dict['packages']
fd.authors = [a['value'] for a in file_dict['authors']]
if file_dict['scan_errors']:
# for each scan error make a notice
for err in file_dict['scan_errors']:
fd.origins.add_notice_to_origins(
'File: ' + fd.path, Notice(err, 'error'))
return fd
def get_dockerfile_base():
'''Get the base image object from the dockerfile base instructions
1. get the instructions around FROM
2. get the base image and tag
3. Make notes based on what the image and tag rules are
4. Return an image object and the base instructions string
NOTE: Potential ARG values in the Dockerfile object have already been
expanded at this point. However, Dockerfile rules say that if no
--build-arg is passed during docker build and ARG has no default, the
build will fail. We assume for now that we will not be passing build
arguments in which case if there is no default ARG, we will raise an
exception indicating that since the build arguments are determined by
the user we will not be able to determine what the user wanted'''
try:
# Get the base image tag.
# NOTE: ARG values have already been expanded.
base_image_string, from_line = get_base_image_tag(docker_commands)
# check for scratch
if base_image_string == 'scratch':
# there is no base image to pull
raise ValueError("Cannot pull 'scratch' base image.")
# there should be some image object here
base_image = DockerImage(base_image_string)
base_image.origins.add_notice_origin(from_line)
base_image.name = base_image_string.split(':')[0]
# check if there is a tag
if not check_image_string(base_image_string):
message_string = errors.dockerfile_no_tag.format(
dockerfile_line=from_line)
base_image.origins.add_notice_to_origins(
docker_commands, Notice(message_string, 'warning'))
base_image.tag = 'latest'
else:
base_image.tag = base_image_string.split(':')[1]
# check if the tag is 'latest'
if base_image.tag == 'latest':
message_string = errors.dockerfile_using_latest.format(
dockerfile_line=from_line)
base_image.origins.add_notice_to_origins(
docker_commands, Notice(message_string, 'warning'))
return base_image, from_line
except ValueError as e:
logger.fatal(
"%s",
errors.cannot_parse_base_image.format(dockerfile=dockerfile_global,
error_msg=e))
sys.exit(1)
def load_full_image(image_tag_string):
'''Create image object from image name and tag and return the object'''
test_image = DockerImage(image_tag_string)
failure_origin = formats.image_load_failure.format(
testimage=test_image.repotag)
try:
test_image.load_image()
except NameError as error:
test_image.origins.add_notice_to_origins(failure_origin,
Notice(str(error), 'error'))
except subprocess.CalledProcessError as error:
test_image.origins.add_notice_to_origins(
failure_origin, Notice(str(error.output, 'utf-8'), 'error'))
except IOError as error:
test_image.origins.add_notice_to_origins(failure_origin,
Notice(str(error), 'error'))
return test_image
def image_setup(image_obj):
'''Add notices for each layer'''
for layer in image_obj.layers:
origin_str = 'Layer: ' + layer.fs_hash[:10]
layer.origins.add_notice_origin(origin_str)
if layer.import_str:
layer.origins.add_notice_to_origins(origin_str, Notice(
'Imported in Dockerfile using: ' + layer.import_str, 'info'))
def get_commands_from_metadata(image_layer):
"""Given the image layer object, get the list of command objects that
created the layer. Return an empty list of we can't do that"""
# set up notice origin for the layer
origin_layer = 'Layer {}'.format(image_layer.layer_index)
# check if there is a key containing the script that created the layer
if image_layer.created_by:
command_line = fltr.get_run_command(image_layer.created_by)
if command_line:
command_list, msg = fltr.filter_install_commands(
general.clean_command(command_line))
if msg:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(msg, 'warning'))
return command_list
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(errors.no_layer_created_by, 'warning'))
return []
def load_notices_from_cache(layer):
'''Given a layer object, populate the notices from the cache'''
origins_list = cache.get_origins(layer.fs_hash)
for origin_dict in origins_list:
layer.origins.add_notice_origin(origin_dict['origin_str'])
for notice in origin_dict['notices']:
layer.origins.add_notice_to_origins(
origin_dict['origin_str'],
Notice(notice['message'], notice['level']))
def get_dockerfile_base():
'''Get the base image object from the dockerfile base instructions
1. get the instructions around FROM
2. get the base image and tag
3. Make notes based on what the image and tag rules are
4. Return an image object and the base instructions string'''
try:
base_instructions = dockerfile.get_base_instructions(docker_commands)
base_image_tag = dockerfile.get_base_image_tag(base_instructions)
dockerfile_lines = print_dockerfile_base(base_instructions)
# check for scratch
if base_image_tag[0] == 'scratch':
# there is no base image - return no image object
return None
# there should be some image object here
repotag = base_image_tag[0] + dockerfile.tag_separator + \
base_image_tag[1]
from_line = 'FROM ' + repotag
base_image = DockerImage(repotag)
base_image.origins.add_notice_origin(dockerfile_lines)
base_image.name = base_image_tag[0]
# check if there is a tag
if not base_image_tag[1]:
message_string = errors.dockerfile_no_tag.format(
dockerfile_line=from_line)
base_image.origins.add_notice_to_origins(
dockerfile_lines, Notice(message_string, 'warning'))
base_image.tag = 'latest'
else:
base_image.tag = base_image_tag[1]
# check if the tag is 'latest'
if base_image_tag[1] == 'latest':
message_string = errors.dockerfile_using_latest.format(
dockerfile_line=from_line)
base_image.origins.add_notice_to_origins(
dockerfile_lines, Notice(message_string, 'warning'))
return base_image, dockerfile_lines
except ValueError as e:
logger.warning(
"%s",
errors.cannot_parse_base_image.format(dockerfile=dockerfile_global,
error_msg=e))
return None
def collect_layer_data(layer_obj):
'''Use scancode to collect data from a layer filesystem. This function will
create a FileData object for every file found. After scanning, it will
return a list of FileData objects.
'''
files = []
# run scancode against a directory
command = 'scancode -ilpcu --quiet --json -'
full_cmd = get_filesystem_command(layer_obj, command)
origin_layer = 'Layer: ' + layer_obj.fs_hash[:10]
result, error = rootfs.shell_command(True, full_cmd)
if not result:
logger.error("No scancode results for this layer: %s", str(error))
layer_obj.origins.add_notice_to_origins(origin_layer,
Notice(str(error), 'error'))
else:
# make FileData objects for each result
data = json.loads(result)
for f in data['files']:
if f['type'] == 'file':
# scancode records paths from the target directory onwards
# which in tern's case is tern.utils.constants.untar_dir
# removing that portion of the file path
fspath = f['path'].replace(constants.untar_dir + os.path.sep,
'')
fd = FileData(f['name'], fspath, f['date'], f['file_type'])
if f['licenses']:
fd.licenses = [l['short_name'] for l in f['licenses']]
fd.license_expressions = f['license_expressions']
if f['copyrights']:
fd.copyrights = [c['value'] for c in f['copyrights']]
if f['urls']:
fd.urls = [u['url'] for u in f['urls']]
fd.packages = f['packages']
fd.authors = [a['value'] for a in f['authors']]
if f['scan_errors']:
# for each scan error make a notice
for err in f['scan_errors']:
fd.origins.add_notice_to_origins(
'File: ' + fd.path, Notice(err, 'error'))
files.append(fd)
return files
