def analyze_first_layer(image_obj, master_list, redo): # set up a notice origin for the first layer origin_first_layer = 'Layer {}'.format(image_obj.layers[0].layer_index) # find the shell from the first layer shell = common.get_shell(image_obj.layers[0]) if not shell: logger.warning(errors.no_shell) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(errors.no_shell, 'warning')) # find the binary from the first layer binary = common.get_base_bin(image_obj.layers[0]) if not binary: logger.warning(errors.no_package_manager) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(errors.no_package_manager, 'warning')) # try to load packages from cache if not common.load_from_cache(image_obj.layers[0], redo): # set a possible OS common.get_os_style(image_obj.layers[0], binary) # if there is a binary, extract packages if shell and binary: execute_base_layer(image_obj.layers[0], binary, shell) # populate the master list with all packages found in the first layer for p in image_obj.layers[0].packages: master_list.append(p) return shell
def setUp(self): self.notice_info = Notice("info") self.notice_warn = Notice("warning", "warning") self.notice_errr = Notice("error", "error") self.notice_hint = Notice("hint", "hint") self.origins = Origins()
def add_snippet_packages(image_layer, command, pkg_listing, shell): '''Given an image layer object, a command object, the package listing and the shell used to invoke commands, add package metadata to the layer object. We assume the filesystem is already mounted and ready 1. Get the packages installed by the command 3. For each package get the dependencies 4. For each unique package name, find the metadata and add to the layer''' # set up a notice origin for the layer origin_layer = 'Layer: ' + image_layer.fs_hash[:10] # find packages for the command cmd_msg = formats.invoke_for_snippets + '\n' + \ content.print_package_invoke(command.name) image_layer.origins.add_notice_to_origins(origin_layer, Notice(cmd_msg, 'info')) pkg_list = get_installed_package_names(command) # collect all the dependencies for each package name all_pkgs = [] for pkg_name in pkg_list: pkg_invoke = command_lib.check_for_unique_package( pkg_listing, pkg_name) deps, deps_msg = get_package_dependencies(pkg_invoke, pkg_name, shell) if deps_msg: logger.warning(deps_msg) image_layer.origins.add_notice_to_origins( origin_layer, Notice(deps_msg, 'error')) all_pkgs.append(pkg_name) all_pkgs.extend(deps) unique_pkgs = list(set(all_pkgs)) # get package metadata for each package name for pkg_name in unique_pkgs: pkg = Package(pkg_name) fill_package_metadata(pkg, pkg_invoke, shell) image_layer.add_package(pkg)
def get_commands_from_history(image_layer): '''Given the image layer object and the shell, get the list of command objects that created the layer''' # set up notice origin for the layer origin_layer = 'Layer {}'.format(image_layer.layer_index) if image_layer.created_by: instruction = created_to_instruction(image_layer.created_by) image_layer.origins.add_notice_to_origins( origin_layer, Notice( formats.dockerfile_line.format( dockerfile_instruction=instruction), 'info')) command_line = instruction.split(' ', 1)[1] else: instruction = '' image_layer.origins.add_notice_to_origins( origin_layer, Notice(formats.no_created_by, 'warning')) command_line = instruction # Image layers are created with the directives RUN, ADD and COPY # For ADD and COPY instructions, there is no information about the # packages added if 'ADD' in instruction or 'COPY' in instruction: image_layer.origins.add_notice_to_origins( origin_layer, Notice(errors.unknown_content.format(files=command_line), 'warning')) # return an empty list as we cannot find any commands return [] # for RUN instructions we can return a list of commands command_list, msg = common.filter_install_commands(command_line) if msg: image_layer.origins.add_notice_to_origins(origin_layer, Notice(msg, 'warning')) return command_list
def get_os_style(image_layer, binary): '''Given an ImageLayer object and a binary package manager, check for the OS identifier in the os-release file first. If the os-release file is not available, make an educated guess as to what kind of OS the layer might be based off of given the pkg_format + package manager. If the binary provided does not exist in base.yml, add a warning notice''' origin_layer = 'Layer {}'.format(image_layer.layer_index) # see if we can find what OS this is os_release = get_os_release(image_layer) if os_release: # We know with high degree of certainty what the OS is image_layer.origins.add_notice_to_origins(origin_layer, Notice( formats.os_release.format(os_style=os_release), 'info')) # we can set the OS of the image layer image_layer.os_guess = os_release else: # We will try looking for the possible OSs based on the binary os_guess = command_lib.check_os_guess(binary) if os_guess: # We can make a guess image_layer.origins.add_notice_to_origins(origin_layer, Notice( formats.os_style_guess.format( package_manager=binary, os_list=os_guess), 'info')) image_layer.os_guess = os_guess else: # No binary and no os-release means we have no idea about base OS image_layer.origins.add_notice_to_origins(origin_layer, Notice( errors.no_etc_release, 'warning')) # set a package format if there is one for this binary image_layer.pkg_format = command_lib.check_pkg_format(binary)
def get_os_style(image_layer, binary): '''Given an ImageLayer object and a binary package manager, check for the OS identifier in the os-release file first. If the os-release file is not available, make an educated guess as to what kind of OS the layer might be based off of given the pkg_format + package manager. If the binary provided does not exist in base.yml, add a warning notice''' origin_command_lib = formats.invoking_base_commands origin_layer = 'Layer: ' + image_layer.fs_hash[:10] pkg_format = command_lib.check_pkg_format(binary) os_guess = command_lib.check_os_guess(binary) if get_os_release(): # We know with high degree of certainty what the OS is image_layer.origins.add_notice_to_origins(origin_layer, Notice( formats.os_release.format(os_style=get_os_release()), 'info')) elif binary is None: # No binary and no os-release means we have no idea about base OS image_layer.origins.add_notice_to_origins(origin_layer, Notice( errors.no_etc_release, 'warning')) else: # We make a guess about the OS based on pkg_format + binary # First check that binary exists in base.yml if not pkg_format or not os_guess: image_layer.origins.add_notice_to_origins( origin_command_lib, Notice( errors.no_listing_for_base_key.format(listing_key=binary), 'warning')) else: # Assign image layer attributes and guess OS image_layer.pkg_format = pkg_format image_layer.os_guess = os_guess image_layer.origins.add_notice_to_origins(origin_layer, Notice( formats.os_style_guess.format( package_manager=binary, package_format=image_layer.pkg_format, os_list=image_layer.os_guess), 'info'))
def load_base_image(): '''Create base image from dockerfile instructions and return the image''' base_image, dockerfile_lines = dhelper.get_dockerfile_base() # try to get image metadata if not container.check_image(base_image.repotag): # if no base image is found, give a warning and continue if not container.pull_image(base_image.repotag): logger.warning( errors.cannot_find_image.format(imagetag=base_image.repotag)) try: base_image.load_image() except NameError as error: logger.warning('Error in loading base image: ' + str(error)) base_image.origins.add_notice_to_origins(dockerfile_lines, Notice(str(error), 'error')) except subprocess.CalledProcessError as error: logger.warning('Error in loading base image: ' + str(error.output, 'utf-8')) base_image.origins.add_notice_to_origins( dockerfile_lines, Notice(str(error.output, 'utf-8'), 'error')) except IOError as error: logger.warning('Error in loading base image: ' + str(error)) base_image.origins.add_notice_to_origin(dockerfile_lines, Notice(str(error), 'error')) return base_image
def execute_snippets(layer_obj, command_obj, prereqs): """Given in ImageLayer object, shell and binary to look up, find packages installed in the layer using the default method: For snippets, we will get the packages installed by the command""" # set up a notice origin for the layer origin_layer = 'Layer {}'.format(layer_obj.layer_index) # find packages for the command cmd_msg = (formats.invoke_for_snippets + '\n' + content.print_package_invoke(command_obj.name)) layer_obj.origins.add_notice_to_origins(origin_layer, Notice(cmd_msg, 'info')) pkg_list = filter.get_installed_package_names(command_obj) # collect all the dependencies for each package name all_pkgs = [] for pkg_name in pkg_list: pkg_invoke = command_lib.check_for_unique_package( prereqs.listing, pkg_name) deps, deps_msg = com.get_package_dependencies(pkg_invoke, pkg_name, prereqs.shell) if deps_msg: logger.warning(deps_msg) layer_obj.origins.add_notice_to_origins(origin_layer, Notice(deps_msg, 'error')) all_pkgs.append(pkg_name) all_pkgs.extend(deps) unique_pkgs = list(set(all_pkgs)) # get package metadata for each package name for pkg_name in unique_pkgs: pkg = Package(pkg_name) dcom.fill_package_metadata(pkg, pkg_invoke, prereqs.shell, layer_obj.get_layer_workdir(), prereqs.envs) layer_obj.add_package(pkg)
def analyze_first_layer(image_obj, master_list, options): """Analyze the first layer of an image. Return a Prereqs object for the next layer. 1. Check if the layer is empty. If it is not, return None 2. See if we can load the layer from cache 3. If we can't load from cache 3.1 See if we can find any information about the rootfs 3.2 If we are able to find any information, use any prescribed methods to extract package information 4. Process and bundle that information into the image object 5. Return a Prereqs object for subsequent layer processing""" # set up a notice origin for the first layer origin_first_layer = 'Layer {}'.format(image_obj.layers[0].layer_index) # check if the layer is empty if com.is_empty_layer(image_obj.layers[0]): logger.warning(errors.empty_layer) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(errors.empty_layer, 'warning')) return None # create a Prereqs object prereqs = core.Prereqs() # find the shell from the first layer prereqs.fs_shell = dcom.get_shell(image_obj.layers[0]) # find a shell for the host prereqs.host_shell = host.check_shell() if not prereqs.fs_shell and not prereqs.host_shell: logger.warning(errors.no_shell) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(errors.no_shell, 'warning')) # find the binary from the first layer prereqs.binary = dcom.get_base_bin(image_obj.layers[0]) # try to load packages from cache if not com.load_from_cache(image_obj.layers[0], options.redo): # add a notice if there is a "created by" image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(formats.layer_created_by.format( created_by=image_obj.layers[0].created_by), 'info')) # set a possible OS and package format get_os_style(image_obj.layers[0], prereqs.binary) # if there is a binary, extract packages if prereqs.binary: # mount the first layer target_dir = prep_first_layer(image_obj.layers[0]) # set the host path to the mount point if target_dir: prereqs.host_path = target_dir # core default execution on the first layer core.execute_base(image_obj.layers[0], prereqs) else: logger.warning(errors.no_package_manager) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice( errors.no_package_manager, 'warning')) return None # populate the master list with all packages found in the first layer for p in image_obj.layers[0].packages: master_list.append(p) return prereqs
def execute_base(layer_obj, prereqs): """Given an ImageLayer object, find packages installed in the layer using the default method. The prereqisites required for this to work: prereqs.shell: the shell to use prereqs.binary: the binary to look up in the command library optional prerequisites: prereqs.envs: any environment variables to set before execution 1. Use command_lib's base to look up the binary to see if there is a method to retrieve the metadata 2. If there is, invoke the scripts in a chroot environment and process the results 3. Add the results to the ImageLayer object It is assumed that the filesystem is prepped for execution by mounting the filesystem in the working directory and /proc, /sys and /dev device nodes are mounted""" # Add notices to this layer object origin_layer = 'Layer {}'.format(layer_obj.layer_index) # find the binary listing listing = command_lib.get_base_listing(prereqs.binary) if listing: # put info notice about what is going to be invoked snippet_msg = (formats.invoke_for_base + '\n' + content.print_base_invoke(prereqs.binary)) layer_obj.origins.add_notice_to_origins(origin_layer, Notice(snippet_msg, 'info')) # get list of metadata by invoking scripts in chroot logger.debug("Collecting metadata for image layer...") pkg_dict, invoke_msg, warnings = collect.collect_list_metadata( prereqs.shell, listing, layer_obj.get_layer_workdir(), prereqs.envs) # more processing for debian copyrights to get licenses if listing.get("pkg_format") == "deb": logger.debug("Processing Debian copyrights...") pkg_dict["pkg_licenses"] = com.get_deb_package_licenses( pkg_dict["copyrights"]) # add any errors and warnings to the layer's origins object if invoke_msg: logger.error( "Script invocation error. Unable to collect some metadata") layer_obj.origins.add_notice_to_origins( origin_layer, Notice(invoke_msg, 'error')) if warnings: logger.warning("Some metadata may be missing") layer_obj.origins.add_notice_to_origins( origin_layer, Notice(warnings, 'warning')) # bundle the results into Package objects bundle.fill_pkg_results(layer_obj, pkg_dict) # remove extra FileData objects from the layer com.remove_duplicate_layer_files(layer_obj) # if there is no listing add a notice else: layer_obj.origins.add_notice_to_origins( origin_layer, Notice( errors.no_listing_for_base_key.format( listing_key=prereqs.binary), 'error'))
def add_base_packages(image_layer, binary, shell): # pylint: disable=too-many-locals '''Given the image layer, the binary to invoke and shell: 1. get the listing from the base.yml 2. Invoke any commands against the base layer 3. Make a list of packages and add them to the layer''' origin_layer = 'Layer: ' + image_layer.fs_hash[:10] if image_layer.created_by: image_layer.origins.add_notice_to_origins( origin_layer, Notice( formats.layer_created_by.format( created_by=image_layer.created_by), 'info')) else: image_layer.origins.add_notice_to_origins( origin_layer, Notice(formats.no_created_by, 'warning')) origin_command_lib = formats.invoking_base_commands # find the binary listing = command_lib.get_base_listing(binary) if listing: # put info notice about what is going to be invoked snippet_msg = formats.invoke_for_base + '\n' + \ content.print_base_invoke(binary) image_layer.origins.add_notice_to_origins(origin_layer, Notice(snippet_msg, 'info')) shell, _ = command_lib.get_image_shell(listing) if not shell: shell = constants.shell # get all the packages in the base layer names, n_msg = command_lib.get_pkg_attr_list(shell, listing['names']) versions, v_msg = command_lib.get_pkg_attr_list( shell, listing['versions']) licenses, l_msg = command_lib.get_pkg_attr_list( shell, listing['licenses']) src_urls, u_msg = command_lib.get_pkg_attr_list( shell, listing['src_urls']) # add a notice to the image if something went wrong invoke_msg = n_msg + v_msg + l_msg + u_msg if invoke_msg: image_layer.origins.add_notice_to_origins( origin_layer, Notice(invoke_msg, 'error')) if names and len(names) > 1: for index, name in enumerate(names): pkg = Package(name) if len(versions) == len(names): pkg.version = versions[index] if len(licenses) == len(names): pkg.license = licenses[index] if len(src_urls) == len(names): pkg.src_url = src_urls[index] image_layer.add_package(pkg) # if there is no listing add a notice else: image_layer.origins.add_notice_to_origins( origin_command_lib, Notice(errors.no_listing_for_base_key.format(listing_key=binary), 'error'))
def setUp(self): self.notice_info = Notice("info") self.notice_warn = Notice("warning", "warning") self.notice_errr = Notice("error", "error") self.notice_hint = Notice("hint", "hint") self.notices = [ self.notice_info, self.notice_warn, self.notice_errr, self.notice_hint ] self.notice_origin = NoticeOrigin('origin_str')
def analyze_first_layer(image_obj, master_list, options): """Analyze the first layer of an image. Return the installed shell. If there is no installed shell, return None 1. Check if the layer is empty. If it is then we can't find a shell 2. See if we can load the layer from cache 3. If we can't load from cache 3.1 See if we can find any information about the rootfs 3.2 If we are able to find any information, use any prescribed methods to extract package information 4. Process and bundle that information into the image object 5. Return the shell for subsequent layer processing""" # set up a notice origin for the first layer origin_first_layer = 'Layer {}'.format(image_obj.layers[0].layer_index) # check if the layer is empty if com.is_empty_layer(image_obj.layers[0]): logger.warning(errors.empty_layer) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(errors.empty_layer, 'warning')) return None # create a Prereqs object prereqs = core.Prereqs() # find the shell from the first layer prereqs.shell = dcom.get_shell(image_obj.layers[0]) if not prereqs.shell: logger.warning(errors.no_shell) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(errors.no_shell, 'warning')) # find the binary from the first layer prereqs.binary = dcom.get_base_bin(image_obj.layers[0]) if not prereqs.binary: logger.warning(errors.no_package_manager) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(errors.no_package_manager, 'warning')) # try to load packages from cache if not com.load_from_cache(image_obj.layers[0], options.redo): # set a possible OS get_os_style(image_obj.layers[0], prereqs.binary) # if there is a binary, extract packages if prereqs.shell and prereqs.binary: # mount the first layer mount_first_layer(image_obj.layers[0]) # core default execution on the first layer core.execute_base(image_obj.layers[0], prereqs) # unmount rootfs.undo_mount() rootfs.unmount_rootfs() # populate the master list with all packages found in the first layer for p in image_obj.layers[0].packages: master_list.append(p) return prereqs.shell
class TestClassNotice(unittest.TestCase): def setUp(self): self.notice = Notice() def tearDown(self): del self.notice def testInstance(self): self.assertFalse(self.notice.message) self.assertEqual(self.notice.level, 'info') def testSetters(self): self.notice.message = 'tag' self.assertEqual(self.notice.message, 'tag') with self.assertRaises(NoticeException): self.notice.level = 'something' self.notice.level = 'warning' def testGetters(self): self.notice.message = 'tag' self.notice.level = 'warning' self.assertEqual(self.notice.message, 'tag') self.assertEqual(self.notice.level, 'warning') def testToDict(self): self.notice.message = 'tag' self.notice.level = 'warning' dict = self.notice.to_dict() self.assertEqual(dict['message'], 'tag') self.assertEqual(dict['level'], 'warning')
def load_files_from_cache(layer): '''Given a layer object, populate file level information''' loaded = False raw_file_list = cache.get_files(layer.fs_hash) if raw_file_list: logger.debug('Loading files from cache: layer \"%s\"', layer.fs_hash[:10]) for file_dict in raw_file_list: f = FileData(file_dict['name'], file_dict['path']) f.fill(file_dict) # collect file origins if 'origins' in file_dict.keys(): for origin_dict in file_dict['origins']: for notice in origin_dict['notices']: f.origins.add_notice_to_origins( origin_dict['origin_str'], Notice(notice['message'], notice['level'])) layer.add_file(f) loaded = True else: # if there are no files, generate them from the pre-calculated # hash file logger.debug('Reading files in filesystem...') layer.add_files() return loaded
def collect_layer_data(layer_obj): '''Use scancode to collect data from a layer filesystem. This function will create FileData and Package objects for every File and Package found. After scanning, it will return a tuple with a list of FileData and a list of Package objects. ''' files = [] packages = [] # run scancode against a directory try: processes = len(os.sched_getaffinity(0)) command = "scancode -ilpcu --quiet --timeout 300 -n {} --json -".format(processes) except (AttributeError, NotImplementedError): command = "scancode -ilpcu --quiet --timeout 300 --json -" full_cmd = get_filesystem_command(layer_obj, command) origin_layer = 'Layer {}'.format(layer_obj.layer_index) result, error = rootfs.shell_command(True, full_cmd) if not result: logger.error( "No scancode results for this layer: %s", str(error)) layer_obj.origins.add_notice_to_origins( origin_layer, Notice(str(error), 'error')) else: # make FileData objects for each result data = json.loads(result) add_scancode_headers(layer_obj, data["headers"]) for f in data['files']: if f['type'] == 'file' and f['size'] != 0: files.append(get_scancode_file(f)) for package in f['packages']: packages.append(get_scancode_package(package)) return files, packages
def analyze_first_layer(image_obj, master_list, redo): # find the binary of the first layer binary = common.get_base_bin(image_obj.layers[0]) # see if there is an associated shell # if there is no binary, this will be set to the default shell shell = get_shell(image_obj, binary) # try to load packages from cache if not common.load_from_cache(image_obj.layers[0], redo): # set a possible OS common.get_os_style(image_obj.layers[0], binary) # set up a notice origin for the first layer origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10] # if there is a binary, extract packages if binary: try: target = rootfs.mount_base_layer(image_obj.layers[0].tar_file) rootfs.prep_rootfs(target) common.add_base_packages(image_obj.layers[0], binary, shell) # unmount proc, sys and dev rootfs.undo_mount() rootfs.unmount_rootfs() except KeyboardInterrupt: logger.critical(errors.keyboard_interrupt) abort_analysis() else: logger.warning(errors.no_package_manager) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(errors.no_package_manager, 'warning')) # populate the master list with all packages found in the first layer for p in image_obj.layers[0].packages: master_list.append(p) return shell
def load_from_cache(layer, redo=False): '''Given a layer object, check against cache to see if that layer id exists if yes then get the package list and load it in the layer and return true. If it doesn't exist return false. Default operation is to not redo the cache. Add notices to the layer's origins matching the origin_str''' loaded = False origin_layer = 'Layer: ' + layer.fs_hash[:10] if not layer.packages and not redo: # there are no packages in this layer and we are not repopulating the # cache, try to get it from the cache raw_pkg_list = cache.get_packages(layer.fs_hash) if raw_pkg_list: logger.debug('Loaded from cache: layer {}'.format( layer.fs_hash[:10])) message = formats.loading_from_cache.format( layer_id=layer.fs_hash[:10]) # add notice to the origin layer.origins.add_notice_to_origins(origin_layer, Notice(message, 'info')) for pkg_dict in raw_pkg_list: pkg = Package(pkg_dict['name']) pkg.fill(pkg_dict) layer.add_package(pkg) loaded = True return loaded
def collect_layer_data(layer_obj): '''Use scancode to collect data from a layer filesystem. This function will create a FileData object for every file found. After scanning, it will return a list of FileData objects. ''' files = [] # run scancode against a directory command = 'scancode -ilpcu --quiet --timeout 300 --json -' full_cmd = get_filesystem_command(layer_obj, command) origin_layer = 'Layer: ' + layer_obj.fs_hash[:10] result, error = rootfs.shell_command(True, full_cmd) if not result: logger.error("No scancode results for this layer: %s", str(error)) layer_obj.origins.add_notice_to_origins(origin_layer, Notice(str(error), 'error')) else: # make FileData objects for each result data = json.loads(result) notice = data.get("headers")[0].get("notice") headers = layer_obj.extension_info.get("headers", set()) headers.add(notice) layer_obj.extension_info["headers"] = headers for f in data['files']: if f['type'] == 'file' and f['size'] != 0: files.append(get_scancode_file(f)) return files
def analyze_subsequent_layers(image_obj, shell, master_list, redo, driver=None): """Assuming that we have a shell and have completed analyzing the first layer of the given image object, we now analyze the remaining layers. While we have layers: 1. Check if the layer is empty. If it is, then we can't do anything and we should continue 2. See if we can load the layer from cache. If we can't then do a fresh analysis package information and bundle it into the image object 3. Update the master list""" curr_layer = 1 # get list of environment variables envs = lock.get_env_vars(image_obj) while curr_layer < len(image_obj.layers): # make a notice for each layer origin_next_layer = 'Layer {}'.format( image_obj.layers[curr_layer].layer_index) # check if this is an empty layer if common.is_empty_layer(image_obj.layers[curr_layer]): # we continue to the next layer logger.warning(errors.empty_layer) image_obj.layers[curr_layer].origins.add_notice_to_origins( origin_next_layer, Notice(errors.empty_layer, 'warning')) curr_layer = curr_layer + 1 continue if not common.load_from_cache(image_obj.layers[curr_layer], redo): fresh_analysis(image_obj, curr_layer, shell, driver, envs) # update the master list dcom.update_master_list(master_list, image_obj.layers[curr_layer]) curr_layer = curr_layer + 1
def analyze_first_layer(image_obj, master_list, redo): # find the binary and shell by mounting the base layer target = rootfs.mount_base_layer(image_obj.layers[0].tar_file) binary = common.get_base_bin() shell = get_shell(image_obj, binary) # set up a notice origin for the first layer origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10] # only extract packages if there is a known binary and the layer is not # cached if binary: if not common.load_from_cache(image_obj.layers[0], redo): # Determine pacakge/os style from binary in the image layer common.get_os_style(image_obj.layers[0], binary) # get the packages of the first layer rootfs.prep_rootfs(target) common.add_base_packages(image_obj.layers[0], binary, shell) # unmount proc, sys and dev rootfs.undo_mount() else: logger.warning(errors.no_package_manager) # /etc/os-release may still be present even if binary is not common.get_os_style(image_obj.layers[0], None) image_obj.layers[0].origins.add_notice_to_origins( origin_first_layer, Notice(errors.no_package_manager, 'warning')) # no binary means there is no shell so set to default shell logger.warning('Unknown filesystem. Using default shell') shell = constants.shell # unmount the first layer rootfs.unmount_rootfs() # populate the master list with all packages found in the first layer for p in image_obj.layers[0].packages: master_list.append(p) return shell
def get_dockerfile_packages(): '''Given a Dockerfile return an approximate image object. This is mosty guess work and shouldn't be relied on for accurate information. Add Notice messages indicating as such: 1. Create an image with a placeholder repotag 2. For each RUN command, create a package list 3. Create layer objects with incremental integers and add the package list to that layer with a Notice about parsing 4. Return stub image''' stub_image = Image('easteregg:cookie') layer_count = 0 for cmd in dhelper.docker_commands: if cmd['instruction'] == 'RUN': layer_count = layer_count + 1 layer = ImageLayer(layer_count) install_commands, msg = \ common.filter_install_commands(cmd['value']) if msg: layer.origins.add_notice_to_origins(cmd['value'], Notice(msg, 'info')) pkg_names = [] for command in install_commands: pkg_names.append(common.get_installed_package_names(command)) for pkg_name in pkg_names: pkg = Package(pkg_name) # shell parser does not parse version pins yet # when that is enabled, Notices for no versions need to be # added here layer.add_package(pkg) return stub_image
def get_scancode_file(file_dict): '''Given a file dictionary from the scancode results, return a FileData object with the results''' # scancode records paths from the target directory onwards # which in tern's case is tern.utils.constants.untar_dir # removing that portion of the file path fspath = file_dict['path'].replace( constants.untar_dir + os.path.sep, '') fd = FileData( file_dict['name'], fspath, file_dict['date'], file_dict['file_type']) fd.short_file_type = get_file_type(file_dict) fd.add_checksums({'sha1': file_dict['sha1'], 'md5': file_dict['md5']}) if file_dict['licenses']: fd.licenses = [li['short_name'] for li in file_dict['licenses']] fd.license_expressions = file_dict['license_expressions'] if file_dict['copyrights']: fd.copyrights = [c['value'] for c in file_dict['copyrights']] if file_dict['urls']: fd.urls = [u['url'] for u in file_dict['urls']] fd.packages = file_dict['packages'] fd.authors = [a['value'] for a in file_dict['authors']] if file_dict['scan_errors']: # for each scan error make a notice for err in file_dict['scan_errors']: fd.origins.add_notice_to_origins( 'File: ' + fd.path, Notice(err, 'error')) return fd
def get_dockerfile_base(): '''Get the base image object from the dockerfile base instructions 1. get the instructions around FROM 2. get the base image and tag 3. Make notes based on what the image and tag rules are 4. Return an image object and the base instructions string NOTE: Potential ARG values in the Dockerfile object have already been expanded at this point. However, Dockerfile rules say that if no --build-arg is passed during docker build and ARG has no default, the build will fail. We assume for now that we will not be passing build arguments in which case if there is no default ARG, we will raise an exception indicating that since the build arguments are determined by the user we will not be able to determine what the user wanted''' try: # Get the base image tag. # NOTE: ARG values have already been expanded. base_image_string, from_line = get_base_image_tag(docker_commands) # check for scratch if base_image_string == 'scratch': # there is no base image to pull raise ValueError("Cannot pull 'scratch' base image.") # there should be some image object here base_image = DockerImage(base_image_string) base_image.origins.add_notice_origin(from_line) base_image.name = base_image_string.split(':')[0] # check if there is a tag if not check_image_string(base_image_string): message_string = errors.dockerfile_no_tag.format( dockerfile_line=from_line) base_image.origins.add_notice_to_origins( docker_commands, Notice(message_string, 'warning')) base_image.tag = 'latest' else: base_image.tag = base_image_string.split(':')[1] # check if the tag is 'latest' if base_image.tag == 'latest': message_string = errors.dockerfile_using_latest.format( dockerfile_line=from_line) base_image.origins.add_notice_to_origins( docker_commands, Notice(message_string, 'warning')) return base_image, from_line except ValueError as e: logger.fatal( "%s", errors.cannot_parse_base_image.format(dockerfile=dockerfile_global, error_msg=e)) sys.exit(1)
def load_full_image(image_tag_string): '''Create image object from image name and tag and return the object''' test_image = DockerImage(image_tag_string) failure_origin = formats.image_load_failure.format( testimage=test_image.repotag) try: test_image.load_image() except NameError as error: test_image.origins.add_notice_to_origins(failure_origin, Notice(str(error), 'error')) except subprocess.CalledProcessError as error: test_image.origins.add_notice_to_origins( failure_origin, Notice(str(error.output, 'utf-8'), 'error')) except IOError as error: test_image.origins.add_notice_to_origins(failure_origin, Notice(str(error), 'error')) return test_image
def image_setup(image_obj): '''Add notices for each layer''' for layer in image_obj.layers: origin_str = 'Layer: ' + layer.fs_hash[:10] layer.origins.add_notice_origin(origin_str) if layer.import_str: layer.origins.add_notice_to_origins(origin_str, Notice( 'Imported in Dockerfile using: ' + layer.import_str, 'info'))
def get_commands_from_metadata(image_layer): """Given the image layer object, get the list of command objects that created the layer. Return an empty list of we can't do that""" # set up notice origin for the layer origin_layer = 'Layer {}'.format(image_layer.layer_index) # check if there is a key containing the script that created the layer if image_layer.created_by: command_line = fltr.get_run_command(image_layer.created_by) if command_line: command_list, msg = fltr.filter_install_commands( general.clean_command(command_line)) if msg: image_layer.origins.add_notice_to_origins( origin_layer, Notice(msg, 'warning')) return command_list image_layer.origins.add_notice_to_origins( origin_layer, Notice(errors.no_layer_created_by, 'warning')) return []
def load_notices_from_cache(layer): '''Given a layer object, populate the notices from the cache''' origins_list = cache.get_origins(layer.fs_hash) for origin_dict in origins_list: layer.origins.add_notice_origin(origin_dict['origin_str']) for notice in origin_dict['notices']: layer.origins.add_notice_to_origins( origin_dict['origin_str'], Notice(notice['message'], notice['level']))
def get_dockerfile_base(): '''Get the base image object from the dockerfile base instructions 1. get the instructions around FROM 2. get the base image and tag 3. Make notes based on what the image and tag rules are 4. Return an image object and the base instructions string''' try: base_instructions = dockerfile.get_base_instructions(docker_commands) base_image_tag = dockerfile.get_base_image_tag(base_instructions) dockerfile_lines = print_dockerfile_base(base_instructions) # check for scratch if base_image_tag[0] == 'scratch': # there is no base image - return no image object return None # there should be some image object here repotag = base_image_tag[0] + dockerfile.tag_separator + \ base_image_tag[1] from_line = 'FROM ' + repotag base_image = DockerImage(repotag) base_image.origins.add_notice_origin(dockerfile_lines) base_image.name = base_image_tag[0] # check if there is a tag if not base_image_tag[1]: message_string = errors.dockerfile_no_tag.format( dockerfile_line=from_line) base_image.origins.add_notice_to_origins( dockerfile_lines, Notice(message_string, 'warning')) base_image.tag = 'latest' else: base_image.tag = base_image_tag[1] # check if the tag is 'latest' if base_image_tag[1] == 'latest': message_string = errors.dockerfile_using_latest.format( dockerfile_line=from_line) base_image.origins.add_notice_to_origins( dockerfile_lines, Notice(message_string, 'warning')) return base_image, dockerfile_lines except ValueError as e: logger.warning( "%s", errors.cannot_parse_base_image.format(dockerfile=dockerfile_global, error_msg=e)) return None
def collect_layer_data(layer_obj): '''Use scancode to collect data from a layer filesystem. This function will create a FileData object for every file found. After scanning, it will return a list of FileData objects. ''' files = [] # run scancode against a directory command = 'scancode -ilpcu --quiet --json -' full_cmd = get_filesystem_command(layer_obj, command) origin_layer = 'Layer: ' + layer_obj.fs_hash[:10] result, error = rootfs.shell_command(True, full_cmd) if not result: logger.error("No scancode results for this layer: %s", str(error)) layer_obj.origins.add_notice_to_origins(origin_layer, Notice(str(error), 'error')) else: # make FileData objects for each result data = json.loads(result) for f in data['files']: if f['type'] == 'file': # scancode records paths from the target directory onwards # which in tern's case is tern.utils.constants.untar_dir # removing that portion of the file path fspath = f['path'].replace(constants.untar_dir + os.path.sep, '') fd = FileData(f['name'], fspath, f['date'], f['file_type']) if f['licenses']: fd.licenses = [l['short_name'] for l in f['licenses']] fd.license_expressions = f['license_expressions'] if f['copyrights']: fd.copyrights = [c['value'] for c in f['copyrights']] if f['urls']: fd.urls = [u['url'] for u in f['urls']] fd.packages = f['packages'] fd.authors = [a['value'] for a in f['authors']] if f['scan_errors']: # for each scan error make a notice for err in f['scan_errors']: fd.origins.add_notice_to_origins( 'File: ' + fd.path, Notice(err, 'error')) files.append(fd) return files