def load_from_cache(layer): '''Given a layer object, check against cache to see if that layer id exists if yes then get the package list and load it in the layer and return true. If it doesn't exist return false Add notices to the layer's origins matching the origin_str''' loaded = False origin_layer = 'Layer: ' + layer.diff_id[:10] if not layer.packages: # there are no packages in this layer # try to get it from the cache raw_pkg_list = cache.get_packages(layer.diff_id) if raw_pkg_list: logger.debug('Loaded from cache: layer {}'.format( layer.diff_id[:10])) message = formats.loading_from_cache.format( layer_id=layer.diff_id[:10]) # add notice to the origin layer.origins.add_notice_to_origins(origin_layer, Notice( message, 'info')) for pkg_dict in raw_pkg_list: pkg = Package(pkg_dict['name']) pkg.fill(pkg_dict) layer.add_package(pkg) loaded = True return loaded
def get_packages_from_base(base_image_tag): '''Get a list of package objects from invoking the commands in the command library base section: 1. For the image and tag name find if there is a list of package names 2. If there is an invoke dictionary, invoke the commands 3. Create a list of packages''' pkg_list = [] # information under the base image tag in the command library info = cmds.get_base_info(base_image_tag) if info: names = cmds.get_pkg_attr_list(info['shell'], info['names']) versions = cmds.get_pkg_attr_list(info['shell'], info['versions']) licenses = cmds.get_pkg_attr_list(info['shell'], info['licenses']) src_urls = cmds.get_pkg_attr_list(info['shell'], info['src_urls']) if names and len(names) > 1: for index in range(0, len(names)): pkg = Package(names[index]) if len(versions) == len(names): pkg.version = versions[index] if len(licenses) == len(names): pkg.license = licenses[index] if len(src_urls) == len(names): pkg.src_url = src_urls[index] pkg_list.append(pkg) else: logger.warning( cannot_retrieve_base_packages.format(image=base_image_tag[0], tag=base_image_tag[1])) else: logger.warning( no_image_tag_listing.format(image=base_image_tag[0], tag=base_image_tag[1])) return pkg_list
def testRemovePackage(self): p1 = Package('x') p2 = Package('y') self.layer.add(p1) self.layer.add(p2) self.assertTrue(self.layer.remove('y')) self.assertFalse(self.layer.remove('y'))
def load_from_cache(image): '''Given an image object, check against cache to see if a layer id exists if yes then get the package list and load it in the image layer. If it doesn't exist continue. If not all the layers have packages, return False else return True''' is_full = True # check if we can use repotag origin_str = '' if image.repotag: origin_str = image.repotag else: origin_str = 'Image ID - ' + image.id[:10] for layer in image.layers: if not layer.packages: # create an origin for this layer origin_str = origin_str + ': ' + layer.diff_id[:10] # there are no packages in this layer # try to get it from the cache raw_pkg_list = cache.get_packages(layer.diff_id) if not raw_pkg_list: is_full = False else: logger.debug('Loaded from cache: layer {}'.format( layer.diff_id[:10])) message = formats.loading_from_cache.format( layer_id=layer.diff_id[:10]) # add notice to the origin layer.origins.add_notice_to_origins(origin_str, Notice(message, 'info')) for pkg_dict in raw_pkg_list: pkg = Package(pkg_dict['name']) pkg.fill(pkg_dict) layer.add_package(pkg) return is_full
def add_base_packages(image_layer, binary, shell): '''Given the image layer, the binary to invoke and shell: 1. get the listing from the base.yml 2. Invoke any commands against the base layer 3. Make a list of packages and add them to the layer''' origin_layer = 'Layer: ' + image_layer.fs_hash[:10] if image_layer.created_by: image_layer.origins.add_notice_to_origins( origin_layer, Notice( formats.layer_created_by.format( created_by=image_layer.created_by), 'info')) else: image_layer.origins.add_notice_to_origins( origin_layer, Notice(formats.no_created_by, 'warning')) origin_command_lib = formats.invoking_base_commands # find the binary listing = command_lib.get_base_listing(binary) if listing: # put info notice about what is going to be invoked snippet_msg = formats.invoke_for_base + '\n' + \ content.print_base_invoke(binary) image_layer.origins.add_notice_to_origins(origin_layer, Notice(snippet_msg, 'info')) shell, msg = command_lib.get_image_shell(listing) if not shell: shell = constants.shell # get all the packages in the base layer names, n_msg = command_lib.get_pkg_attr_list(shell, listing['names']) versions, v_msg = command_lib.get_pkg_attr_list( shell, listing['versions']) licenses, l_msg = command_lib.get_pkg_attr_list( shell, listing['licenses']) src_urls, u_msg = command_lib.get_pkg_attr_list( shell, listing['src_urls']) # add a notice to the image if something went wrong invoke_msg = n_msg + v_msg + l_msg + u_msg if invoke_msg: image_layer.origins.add_notice_to_origins( origin_layer, Notice(invoke_msg, 'error')) if names and len(names) > 1: for index in range(0, len(names)): pkg = Package(names[index]) if len(versions) == len(names): pkg.version = versions[index] if len(licenses) == len(names): pkg.license = licenses[index] if len(src_urls) == len(names): pkg.src_url = src_urls[index] image_layer.add_package(pkg) # if there is no listing add a notice else: image_layer.origins.add_notice_to_origins( origin_command_lib, Notice(errors.no_listing_for_base_key.format(listing_key=binary), 'error'))
def get_layer_obj(sha): '''Given the sha, retrieve the list of packages from the cache and return a layer object''' layer_obj = Layer(sha) packages = cache.get_packages(sha) for package in packages: pkg_obj = Package(package['name']) pkg_obj.fill(package) layer_obj.add(pkg_obj) return layer_obj
class TestClassPackage(unittest.TestCase): def setUp(self): self.package = Package('x') def tearDown(self): del self.package def testInstance(self): self.assertEqual(self.package.name, 'x') self.assertEqual(self.package.version, '') self.assertFalse(self.package.src_url) self.assertFalse(self.package.license) def testSetters(self): self.assertRaises(AttributeError, setattr, self.package, 'name', 'y') self.package.version = '1.0' self.assertEqual(self.package.version, '1.0') self.package.license = 'Apache 2.0' self.assertEqual(self.package.license, 'Apache 2.0') self.package.src_url = 'github.com' self.assertEqual(self.package.src_url, 'github.com') def testGetters(self): self.package.version = '1.0' self.package.license = 'Apache 2.0' self.package.src_url = 'github.com' self.assertEqual(self.package.name, 'x') self.assertEqual(self.package.version, '1.0') self.assertEqual(self.package.license, 'Apache 2.0') self.assertEqual(self.package.src_url, 'github.com') def testToDict(self): self.package.version = '1.0' self.package.license = 'Apache 2.0' self.package.src_url = 'github.com' a_dict = self.package.to_dict() self.assertEqual(a_dict['name'], 'x') self.assertEqual(a_dict['version'], '1.0') self.assertEqual(a_dict['license'], 'Apache 2.0') self.assertEqual(a_dict['src_url'], 'github.com') def testIsEqual(self): p = Package('x') p.license = 'TestLicense' p.version = '1.0' p.src_url = 'TestUrl' self.package.license = 'TestLicense' self.package.version = '2.0' self.package.src_url = 'TestUrl' self.assertFalse(self.package.is_equal(p)) p.version = '2.0' self.assertTrue(self.package.is_equal(p))
class TestClassPackage(unittest.TestCase): def setUp(self): self.package = Package('x') def tearDown(self): del self.package def testInstance(self): self.assertEqual(self.package.name, 'x') self.assertEqual(self.package.version, '') self.assertFalse(self.package.src_url) self.assertFalse(self.package.license) self.assertFalse(self.package.notices) def testSetters(self): self.assertRaises(AttributeError, setattr, self.package, 'name', 'y') self.package.version = '1.0' self.assertEqual(self.package.version, '1.0') self.package.license = 'Apache 2.0' self.assertEqual(self.package.license, 'Apache 2.0') self.package.src_url = 'github.com' self.assertEqual(self.package.src_url, 'github.com') def testGetters(self): self.package.version = '1.0' self.package.license = 'Apache 2.0' self.package.src_url = 'github.com' self.assertEqual(self.package.name, 'x') self.assertEqual(self.package.version, '1.0') self.assertEqual(self.package.license, 'Apache 2.0') self.assertEqual(self.package.src_url, 'github.com') def testAddNotice(self): n = Notice() n.origin = 'FROM' n.message = 'no image' n.level = 'error' self.package.add_notice(n) self.assertEqual(len(self.package.notices), 1) self.assertEqual(self.package.notices[0].origin, 'FROM') def testToDict(self): self.package.version = '1.0' self.package.license = 'Apache 2.0' self.package.src_url = 'github.com' a_dict = self.package.to_dict() self.assertEqual(a_dict['name'], 'x') self.assertEqual(a_dict['version'], '1.0') self.assertEqual(a_dict['license'], 'Apache 2.0') self.assertEqual(a_dict['src_url'], 'github.com')
def testAddPackage(self): err = "Object type String, should be Package" p1 = Package('x') self.layer.add_package(p1) self.assertEqual(len(self.layer.packages), 1) with self.assertRaises(TypeError, msg=err): self.layer.add_package("not_a_package")
def get_dockerfile_packages(): '''Given a Dockerfile return an approximate image object. This is mosty guess work and shouldn't be relied on for accurate information. Add Notice messages indicating as such: 1. Create an image with a placeholder repotag 2. For each RUN command, create a package list 3. Create layer objects with incremental integers and add the package list to that layer with a Notice about parsing 4. Return stub image''' stub_image = Image('easteregg:cookie') layer_count = 0 for inst in docker.docker_commands: if inst[0] == 'RUN': layer_count = layer_count + 1 layer = ImageLayer(layer_count) install_commands, msg = common.filter_install_commands(inst[1]) if msg: layer.origins.add_notice_to_origins( inst[1], Notice(msg, 'info')) pkg_names = [] for command in install_commands: pkg_names.append(common.get_installed_package_names(command)) for pkg_name in pkg_names: pkg = Package(pkg_name) # shell parser does not parse version pins yet # when that is enabled, Notices for no versions need to be # added here layer.add_package(pkg) return stub_image
def add_snippet_packages(image_layer, command, pkg_listing, shell): '''Given an image layer object, a command object, the package listing and the shell used to invoke commands, add package metadata to the layer object. We assume the filesystem is already mounted and ready 1. Get the packages installed by the command 3. For each package get the dependencies 4. For each unique package name, find the metadata and add to the layer''' # set up a notice origin for the layer origin_layer = 'Layer: ' + image_layer.fs_hash[:10] # find packages for the command cmd_msg = formats.invoke_for_snippets + '\n' + \ content.print_package_invoke(command.name) image_layer.origins.add_notice_to_origins(origin_layer, Notice( cmd_msg, 'info')) pkg_list = get_installed_package_names(command) # collect all the dependencies for each package name all_pkgs = [] for pkg_name in pkg_list: pkg_invoke = command_lib.check_for_unique_package( pkg_listing, pkg_name) deps, deps_msg = get_package_dependencies( pkg_invoke, pkg_name, shell) if deps_msg: logger.warning(deps_msg) image_layer.origins.add_notice_to_origins( origin_layer, Notice(deps_msg, 'error')) all_pkgs.append(pkg_name) all_pkgs.extend(deps) unique_pkgs = list(set(all_pkgs)) # get package metadata for each package name for pkg_name in unique_pkgs: pkg = Package(pkg_name) fill_package_metadata(pkg, pkg_invoke, shell) image_layer.add_package(pkg)
def testToDict(self): p1 = Package('x') self.layer.add(p1) a_dict = self.layer.to_dict() print(a_dict) self.assertTrue(a_dict['123abc']) self.assertEqual(len(a_dict['123abc']), 1) self.assertEqual(a_dict['123abc'][0]['name'], 'x')
def testToDict(self): p1 = Package('x') self.layer.add_package(p1) a_dict = self.layer.to_dict() self.assertTrue(a_dict['']) self.assertEqual(len(a_dict['']['packages']), 1) self.assertEqual(a_dict['']['packages'][0]['name'], 'x') self.assertEqual(a_dict['']['tar_file'], 'path/to/tar')
def get_package_obj(command_name, package_name, shell): '''Given the command name, and the package name, retrieve the package information, create an oject and return the package object''' # look up command name in snippet library if command_name in cmds.command_lib['snippets'].keys(): # get the unique or default information pkg_list = cmds.command_lib['snippets'][command_name]['packages'] pkg_info = check_for_unique_package(pkg_list, package_name) if pkg_info: pkg = Package(package_name) # get the information for values keys = pkg_info.keys() if 'version' in keys: try: pkg.version = cmds.get_pkg_attr_list( shell, pkg_info['version'], package_name=package_name)[0] except subprocess.CalledProcessError as error: logger.warning(error.output) if 'license' in keys: try: pkg.license = cmds.get_pkg_attr_list( shell, pkg_info['license'], package_name=package_name)[0] except subprocess.CalledProcessError as error: logger.warning(error.output) if 'src_url' in keys: try: pkg.src_url = cmds.get_pkg_attr_list( shell, pkg_info['src_url'], package_name=package_name)[0] except subprocess.CalledProcessError as error: logger.warning(error.output) return pkg else: print( 'No package named {} nor default listing'.format(package_name)) else: print('No command {} listed in snippet library'.format(command_name))
def add_packages_from_history(image_obj, shell): '''Given a DockerImage object, get package objects installed in each layer Assume that the imported images have already gone through this process and have their layer's packages populated. So collecting package object occurs from the last linked layer: 1. For each layer get a list of package names 2. For each package name get a list of dependencies 3. Create a list of package objects with metadata 4. Add this to the layer''' image_layers = image_obj.layers[image_obj.get_last_import_layer() + 1:] logger.debug('Retrieving metadata for remaining {} layers'.format( len(image_layers))) for layer in image_layers: instruction = created_to_instruction(layer.created_by) origin_layer = instruction + ' -> ' + layer.diff_id[:10] layer.origins.add_notice_origin(origin_layer) origin_info = formats.invoke_for_snippets layer.origins.add_notice_origin(origin_info) if 'RUN' in instruction: # for Docker the created_by comes from the instruction in the # dockerfile run_command_line = instruction.split(' ', 1)[1] cmd_list, msg = common.filter_install_commands(run_command_line) if msg: layer.origins.add_notice_to_origins(origin_info, Notice(msg, 'warning')) for command in cmd_list: origin_cmd = content.print_package_invoke(command.name) layer.origins.add_notice_origin(origin_cmd) pkg_list = common.get_installed_package_names(command) all_pkgs = [] for pkg_name in pkg_list: pkg_listing = command_lib.get_package_listing( command.name, pkg_name) deps, deps_msg = common.get_package_dependencies( pkg_listing, pkg_name, shell) if deps_msg: logger.warning(deps_msg) all_pkgs.append(pkg_name) all_pkgs.extend(deps) unique_pkgs = list(set(all_pkgs)) for pkg_name in unique_pkgs: pkg = Package(pkg_name) pkg_listing = command_lib.get_package_listing( command.name, pkg_name) common.fill_package_metadata(pkg, pkg_listing, shell) layer.add_package(pkg)
def testIsEqual(self): p = Package('x') p.license = 'TestLicense' p.version = '1.0' p.src_url = 'TestUrl' self.package.license = 'TestLicense' self.package.version = '2.0' self.package.src_url = 'TestUrl' self.assertFalse(self.package.is_equal(p)) p.version = '2.0' self.assertTrue(self.package.is_equal(p))
def add_diff_packages(diff_layer, command_line, shell): '''Given a layer object, command line string that created it, and the shell used to invoke commands, add package metadata to the layer object 1. Parse the command line to get individual install commands 2. For each command get the packages installed 3. For each package get the dependencies 4. For each unique package name, find the metadata and add to the layer''' origin_layer = 'Layer: ' + diff_layer.diff_id[:10] # parse all installed commands cmd_list, msg = filter_install_commands(command_line) if msg: diff_layer.origins.add_notice_to_origins( origin_layer, Notice(msg, 'warning')) # find packages for each command for command in cmd_list: cmd_msg = formats.invoke_for_snippets + '\n' + \ content.print_package_invoke(command.name) diff_layer.origins.add_notice_to_origins(origin_layer, Notice( cmd_msg, 'info')) pkg_list = get_installed_package_names(command) # collect all the dependencies for each package name all_pkgs = [] for pkg_name in pkg_list: pkg_listing = command_lib.get_package_listing( command.name, pkg_name) deps, deps_msg = get_package_dependencies( pkg_listing, pkg_name, shell) if deps_msg: logger.warning(deps_msg) diff_layer.origins.add_notice_to_origins( origin_layer, Notice(deps_msg, 'error')) all_pkgs.append(pkg_name) all_pkgs.extend(deps) unique_pkgs = list(set(all_pkgs)) # get package metadata for each package name for pkg_name in unique_pkgs: pkg = Package(pkg_name) pkg_listing = command_lib.get_package_listing( command.name, pkg_name) fill_package_metadata(pkg, pkg_listing, shell) diff_layer.add_package(pkg)
def setUp(self): self.package = Package('x')
def testAddingPackage(self): p1 = Package('x') self.layer.add(p1) self.assertEqual(len(self.layer.packages), 1)
def testGetPackageNames(self): p1 = Package('x') self.layer.add_package(p1) pkgs = self.layer.get_package_names() self.assertEqual(pkgs[0], 'x')
def add_base_packages(image): '''Given an image object, get a list of package objects from invoking the commands in the command library base section: 1. For the image and tag name find if there is a list of package names 2. If there is an invoke dictionary, invoke the commands 3. Create a list of packages 4. Add them to the image''' # information under the base image tag in the command library listing = command_lib.get_base_listing(image.name, image.tag) # create the origin for the base image origin_info = formats.invoking_base_commands + '\n' + \ content.print_base_invoke(image.name, image.tag) image.origins.add_notice_origin(origin_info) origin_str = 'command_lib/base.yml' if listing: shell, msg = command_lib.get_image_shell(listing) if not shell: # add a warning notice for no shell in the command library logger.warning('No shell listing in command library. ' 'Using default shell') no_shell_message = errors.no_shell_listing.format( image_name=image.name, image_tag=image.tag, default_shell=constants.shell) image.origins.add_notice_to_origins( origin_str, Notice(no_shell_message, 'warning')) # add a hint notice to add the shell to the command library add_shell_message = errors.no_listing_for_base_key.format( listing_key='shell') image.origins.add_notice_origins(origin_str, Notice(add_shell_message, 'hint')) shell = constants.shell # check if a container is running first # eventually this needs to change to use derivatives that have # more than 1 layer # for now, we add the list of packages to all the layers in a # starting base image if check_container(): names, n_msg = command_lib.get_pkg_attr_list( shell, listing['names']) versions, v_msg = command_lib.get_pkg_attr_list( shell, listing['versions']) licenses, l_msg = command_lib.get_pkg_attr_list( shell, listing['licenses']) src_urls, u_msg = command_lib.get_pkg_attr_list( shell, listing['src_urls']) # add a notice to the image if something went wrong invoke_msg = n_msg + v_msg + l_msg + u_msg if invoke_msg: image.origins.add_notice_to_origins( origin_str, Notice(invoke_msg, 'error')) if names and len(names) > 1: for index in range(0, len(names)): pkg = Package(names[index]) if len(versions) == len(names): pkg.version = versions[index] if len(licenses) == len(names): pkg.license = licenses[index] if len(src_urls) == len(names): pkg.src_url = src_urls[index] for layer in image.layers: layer.add_package(pkg) # if no container is running give a logging error else: logger.error(errors.no_running_docker_container) # if there is no listing add a notice else: image.origins.add_notice_to_origins( origin_str, Notice( errors.no_image_tag_listing.format(image_name=image.name, image_tag=image.tag), 'error'))
def add_base_packages(base_layer, binary): '''Given the base layer and the binary found in layer fs: 1. get the listing from the base.yml 2. Invoke any commands against the base layer 3. Make a list of packages and add them to the layer''' origin_layer = 'Layer: ' + base_layer.diff_id[:10] if base_layer.created_by: base_layer.origins.add_notice_to_origins(origin_layer, Notice( formats.layer_created_by.format(created_by=base_layer.created_by), 'info')) else: base_layer.origins.add_notice_to_origins(origin_layer, Notice( formats.no_created_by, 'warning')) origin_command_lib = formats.invoking_base_commands # find the binary listing = command_lib.get_base_listing(binary) if listing: # put info notice about what is going to be invoked snippet_msg = formats.invoke_for_base + '\n' + \ content.print_base_invoke(binary) base_layer.origins.add_notice_to_origins( origin_layer, Notice(snippet_msg, 'info')) shell, msg = command_lib.get_image_shell(listing) if not shell: # add a warning notice for no shell in the command library logger.warning('No shell listing in command library. ' 'Using default shell') no_shell_message = errors.no_shell_listing.format( binary, default_shell=constants.shell) base_layer.origins.add_notice_to_origins( origin_command_lib, Notice(no_shell_message, 'warning')) # add a hint notice to add the shell to the command library add_shell_message = errors.no_listing_for_base_key.format( listing_key='shell') base_layer.origins.add_notice_origins( origin_command_lib, Notice(add_shell_message, 'hint')) shell = constants.shell # get all the packages in the base layer names, n_msg = command_lib.get_pkg_attr_list(shell, listing['names']) versions, v_msg = command_lib.get_pkg_attr_list( shell, listing['versions']) licenses, l_msg = command_lib.get_pkg_attr_list( shell, listing['licenses']) src_urls, u_msg = command_lib.get_pkg_attr_list( shell, listing['src_urls']) # add a notice to the image if something went wrong invoke_msg = n_msg + v_msg + l_msg + u_msg if invoke_msg: base_layer.origins.add_notice_to_origins( origin_layer, Notice(invoke_msg, 'error')) if names and len(names) > 1: for index in range(0, len(names)): pkg = Package(names[index]) if len(versions) == len(names): pkg.version = versions[index] if len(licenses) == len(names): pkg.license = licenses[index] if len(src_urls) == len(names): pkg.src_url = src_urls[index] base_layer.add_package(pkg) # if there is no listing add a notice else: base_layer.origins.add_notice_to_origins( origin_command_lib, Notice(errors.no_listing_for_base_key.format( listing_key=binary), 'error'))