def load_from_cache(layer):
'''Given a layer object, check against cache to see if that layer id exists
if yes then get the package list and load it in the layer and return true.
If it doesn't exist return false
Add notices to the layer's origins matching the origin_str'''
loaded = False
origin_layer = 'Layer: ' + layer.diff_id[:10]
if not layer.packages:
# there are no packages in this layer
# try to get it from the cache
raw_pkg_list = cache.get_packages(layer.diff_id)
if raw_pkg_list:
logger.debug('Loaded from cache: layer {}'.format(
layer.diff_id[:10]))
message = formats.loading_from_cache.format(
layer_id=layer.diff_id[:10])
# add notice to the origin
layer.origins.add_notice_to_origins(origin_layer, Notice(
message, 'info'))
for pkg_dict in raw_pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
layer.add_package(pkg)
loaded = True
return loaded
def get_packages_from_base(base_image_tag):
'''Get a list of package objects from invoking the commands in the
command library base section:
1. For the image and tag name find if there is a list of package names
2. If there is an invoke dictionary, invoke the commands
3. Create a list of packages'''
pkg_list = []
# information under the base image tag in the command library
info = cmds.get_base_info(base_image_tag)
if info:
names = cmds.get_pkg_attr_list(info['shell'], info['names'])
versions = cmds.get_pkg_attr_list(info['shell'], info['versions'])
licenses = cmds.get_pkg_attr_list(info['shell'], info['licenses'])
src_urls = cmds.get_pkg_attr_list(info['shell'], info['src_urls'])
if names and len(names) > 1:
for index in range(0, len(names)):
pkg = Package(names[index])
if len(versions) == len(names):
pkg.version = versions[index]
if len(licenses) == len(names):
pkg.license = licenses[index]
if len(src_urls) == len(names):
pkg.src_url = src_urls[index]
pkg_list.append(pkg)
else:
logger.warning(
cannot_retrieve_base_packages.format(image=base_image_tag[0],
tag=base_image_tag[1]))
else:
logger.warning(
no_image_tag_listing.format(image=base_image_tag[0],
tag=base_image_tag[1]))
return pkg_list
def testRemovePackage(self):
p1 = Package('x')
p2 = Package('y')
self.layer.add(p1)
self.layer.add(p2)
self.assertTrue(self.layer.remove('y'))
self.assertFalse(self.layer.remove('y'))
def load_from_cache(image):
'''Given an image object, check against cache to see if a layer id exists
if yes then get the package list and load it in the image layer. If it
doesn't exist continue. If not all the layers have packages, return False
else return True'''
is_full = True
# check if we can use repotag
origin_str = ''
if image.repotag:
origin_str = image.repotag
else:
origin_str = 'Image ID - ' + image.id[:10]
for layer in image.layers:
if not layer.packages:
# create an origin for this layer
origin_str = origin_str + ': ' + layer.diff_id[:10]
# there are no packages in this layer
# try to get it from the cache
raw_pkg_list = cache.get_packages(layer.diff_id)
if not raw_pkg_list:
is_full = False
else:
logger.debug('Loaded from cache: layer {}'.format(
layer.diff_id[:10]))
message = formats.loading_from_cache.format(
layer_id=layer.diff_id[:10])
# add notice to the origin
layer.origins.add_notice_to_origins(origin_str,
Notice(message, 'info'))
for pkg_dict in raw_pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
layer.add_package(pkg)
return is_full
def add_base_packages(image_layer, binary, shell):
'''Given the image layer, the binary to invoke and shell:
1. get the listing from the base.yml
2. Invoke any commands against the base layer
3. Make a list of packages and add them to the layer'''
origin_layer = 'Layer: ' + image_layer.fs_hash[:10]
if image_layer.created_by:
image_layer.origins.add_notice_to_origins(
origin_layer,
Notice(
formats.layer_created_by.format(
created_by=image_layer.created_by), 'info'))
else:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(formats.no_created_by, 'warning'))
origin_command_lib = formats.invoking_base_commands
# find the binary
listing = command_lib.get_base_listing(binary)
if listing:
# put info notice about what is going to be invoked
snippet_msg = formats.invoke_for_base + '\n' + \
content.print_base_invoke(binary)
image_layer.origins.add_notice_to_origins(origin_layer,
Notice(snippet_msg, 'info'))
shell, msg = command_lib.get_image_shell(listing)
if not shell:
shell = constants.shell
# get all the packages in the base layer
names, n_msg = command_lib.get_pkg_attr_list(shell, listing['names'])
versions, v_msg = command_lib.get_pkg_attr_list(
shell, listing['versions'])
licenses, l_msg = command_lib.get_pkg_attr_list(
shell, listing['licenses'])
src_urls, u_msg = command_lib.get_pkg_attr_list(
shell, listing['src_urls'])
# add a notice to the image if something went wrong
invoke_msg = n_msg + v_msg + l_msg + u_msg
if invoke_msg:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(invoke_msg, 'error'))
if names and len(names) > 1:
for index in range(0, len(names)):
pkg = Package(names[index])
if len(versions) == len(names):
pkg.version = versions[index]
if len(licenses) == len(names):
pkg.license = licenses[index]
if len(src_urls) == len(names):
pkg.src_url = src_urls[index]
image_layer.add_package(pkg)
# if there is no listing add a notice
else:
image_layer.origins.add_notice_to_origins(
origin_command_lib,
Notice(errors.no_listing_for_base_key.format(listing_key=binary),
'error'))
def get_layer_obj(sha):
'''Given the sha, retrieve the list of packages from the cache and
return a layer object'''
layer_obj = Layer(sha)
packages = cache.get_packages(sha)
for package in packages:
pkg_obj = Package(package['name'])
pkg_obj.fill(package)
layer_obj.add(pkg_obj)
return layer_obj
class TestClassPackage(unittest.TestCase):
def setUp(self):
self.package = Package('x')
def tearDown(self):
del self.package
def testInstance(self):
self.assertEqual(self.package.name, 'x')
self.assertEqual(self.package.version, '')
self.assertFalse(self.package.src_url)
self.assertFalse(self.package.license)
def testSetters(self):
self.assertRaises(AttributeError, setattr, self.package, 'name', 'y')
self.package.version = '1.0'
self.assertEqual(self.package.version, '1.0')
self.package.license = 'Apache 2.0'
self.assertEqual(self.package.license, 'Apache 2.0')
self.package.src_url = 'github.com'
self.assertEqual(self.package.src_url, 'github.com')
def testGetters(self):
self.package.version = '1.0'
self.package.license = 'Apache 2.0'
self.package.src_url = 'github.com'
self.assertEqual(self.package.name, 'x')
self.assertEqual(self.package.version, '1.0')
self.assertEqual(self.package.license, 'Apache 2.0')
self.assertEqual(self.package.src_url, 'github.com')
def testToDict(self):
self.package.version = '1.0'
self.package.license = 'Apache 2.0'
self.package.src_url = 'github.com'
a_dict = self.package.to_dict()
self.assertEqual(a_dict['name'], 'x')
self.assertEqual(a_dict['version'], '1.0')
self.assertEqual(a_dict['license'], 'Apache 2.0')
self.assertEqual(a_dict['src_url'], 'github.com')
def testIsEqual(self):
p = Package('x')
p.license = 'TestLicense'
p.version = '1.0'
p.src_url = 'TestUrl'
self.package.license = 'TestLicense'
self.package.version = '2.0'
self.package.src_url = 'TestUrl'
self.assertFalse(self.package.is_equal(p))
p.version = '2.0'
self.assertTrue(self.package.is_equal(p))
class TestClassPackage(unittest.TestCase):
def setUp(self):
self.package = Package('x')
def tearDown(self):
del self.package
def testInstance(self):
self.assertEqual(self.package.name, 'x')
self.assertEqual(self.package.version, '')
self.assertFalse(self.package.src_url)
self.assertFalse(self.package.license)
self.assertFalse(self.package.notices)
def testSetters(self):
self.assertRaises(AttributeError, setattr, self.package, 'name', 'y')
self.package.version = '1.0'
self.assertEqual(self.package.version, '1.0')
self.package.license = 'Apache 2.0'
self.assertEqual(self.package.license, 'Apache 2.0')
self.package.src_url = 'github.com'
self.assertEqual(self.package.src_url, 'github.com')
def testGetters(self):
self.package.version = '1.0'
self.package.license = 'Apache 2.0'
self.package.src_url = 'github.com'
self.assertEqual(self.package.name, 'x')
self.assertEqual(self.package.version, '1.0')
self.assertEqual(self.package.license, 'Apache 2.0')
self.assertEqual(self.package.src_url, 'github.com')
def testAddNotice(self):
n = Notice()
n.origin = 'FROM'
n.message = 'no image'
n.level = 'error'
self.package.add_notice(n)
self.assertEqual(len(self.package.notices), 1)
self.assertEqual(self.package.notices[0].origin, 'FROM')
def testToDict(self):
self.package.version = '1.0'
self.package.license = 'Apache 2.0'
self.package.src_url = 'github.com'
a_dict = self.package.to_dict()
self.assertEqual(a_dict['name'], 'x')
self.assertEqual(a_dict['version'], '1.0')
self.assertEqual(a_dict['license'], 'Apache 2.0')
self.assertEqual(a_dict['src_url'], 'github.com')
def testAddPackage(self):
err = "Object type String, should be Package"
p1 = Package('x')
self.layer.add_package(p1)
self.assertEqual(len(self.layer.packages), 1)
with self.assertRaises(TypeError, msg=err):
self.layer.add_package("not_a_package")
def get_dockerfile_packages():
'''Given a Dockerfile return an approximate image object. This is mosty
guess work and shouldn't be relied on for accurate information. Add
Notice messages indicating as such:
1. Create an image with a placeholder repotag
2. For each RUN command, create a package list
3. Create layer objects with incremental integers and add the package
list to that layer with a Notice about parsing
4. Return stub image'''
stub_image = Image('easteregg:cookie')
layer_count = 0
for inst in docker.docker_commands:
if inst[0] == 'RUN':
layer_count = layer_count + 1
layer = ImageLayer(layer_count)
install_commands, msg = common.filter_install_commands(inst[1])
if msg:
layer.origins.add_notice_to_origins(
inst[1], Notice(msg, 'info'))
pkg_names = []
for command in install_commands:
pkg_names.append(common.get_installed_package_names(command))
for pkg_name in pkg_names:
pkg = Package(pkg_name)
# shell parser does not parse version pins yet
# when that is enabled, Notices for no versions need to be
# added here
layer.add_package(pkg)
return stub_image
def add_snippet_packages(image_layer, command, pkg_listing, shell):
'''Given an image layer object, a command object, the package listing
and the shell used to invoke commands, add package metadata to the layer
object. We assume the filesystem is already mounted and ready
1. Get the packages installed by the command
3. For each package get the dependencies
4. For each unique package name, find the metadata and add to the
layer'''
# set up a notice origin for the layer
origin_layer = 'Layer: ' + image_layer.fs_hash[:10]
# find packages for the command
cmd_msg = formats.invoke_for_snippets + '\n' + \
content.print_package_invoke(command.name)
image_layer.origins.add_notice_to_origins(origin_layer, Notice(
cmd_msg, 'info'))
pkg_list = get_installed_package_names(command)
# collect all the dependencies for each package name
all_pkgs = []
for pkg_name in pkg_list:
pkg_invoke = command_lib.check_for_unique_package(
pkg_listing, pkg_name)
deps, deps_msg = get_package_dependencies(
pkg_invoke, pkg_name, shell)
if deps_msg:
logger.warning(deps_msg)
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(deps_msg, 'error'))
all_pkgs.append(pkg_name)
all_pkgs.extend(deps)
unique_pkgs = list(set(all_pkgs))
# get package metadata for each package name
for pkg_name in unique_pkgs:
pkg = Package(pkg_name)
fill_package_metadata(pkg, pkg_invoke, shell)
image_layer.add_package(pkg)
def testToDict(self):
p1 = Package('x')
self.layer.add(p1)
a_dict = self.layer.to_dict()
print(a_dict)
self.assertTrue(a_dict['123abc'])
self.assertEqual(len(a_dict['123abc']), 1)
self.assertEqual(a_dict['123abc'][0]['name'], 'x')
def testToDict(self):
p1 = Package('x')
self.layer.add_package(p1)
a_dict = self.layer.to_dict()
self.assertTrue(a_dict[''])
self.assertEqual(len(a_dict['']['packages']), 1)
self.assertEqual(a_dict['']['packages'][0]['name'], 'x')
self.assertEqual(a_dict['']['tar_file'], 'path/to/tar')
def get_package_obj(command_name, package_name, shell):
'''Given the command name, and the package name, retrieve the package
information, create an oject and return the package object'''
# look up command name in snippet library
if command_name in cmds.command_lib['snippets'].keys():
# get the unique or default information
pkg_list = cmds.command_lib['snippets'][command_name]['packages']
pkg_info = check_for_unique_package(pkg_list, package_name)
if pkg_info:
pkg = Package(package_name)
# get the information for values
keys = pkg_info.keys()
if 'version' in keys:
try:
pkg.version = cmds.get_pkg_attr_list(
shell, pkg_info['version'],
package_name=package_name)[0]
except subprocess.CalledProcessError as error:
logger.warning(error.output)
if 'license' in keys:
try:
pkg.license = cmds.get_pkg_attr_list(
shell, pkg_info['license'],
package_name=package_name)[0]
except subprocess.CalledProcessError as error:
logger.warning(error.output)
if 'src_url' in keys:
try:
pkg.src_url = cmds.get_pkg_attr_list(
shell, pkg_info['src_url'],
package_name=package_name)[0]
except subprocess.CalledProcessError as error:
logger.warning(error.output)
return pkg
else:
print(
'No package named {} nor default listing'.format(package_name))
else:
print('No command {} listed in snippet library'.format(command_name))
def add_packages_from_history(image_obj, shell):
'''Given a DockerImage object, get package objects installed in each layer
Assume that the imported images have already gone through this process and
have their layer's packages populated. So collecting package object occurs
from the last linked layer:
1. For each layer get a list of package names
2. For each package name get a list of dependencies
3. Create a list of package objects with metadata
4. Add this to the layer'''
image_layers = image_obj.layers[image_obj.get_last_import_layer() + 1:]
logger.debug('Retrieving metadata for remaining {} layers'.format(
len(image_layers)))
for layer in image_layers:
instruction = created_to_instruction(layer.created_by)
origin_layer = instruction + ' -> ' + layer.diff_id[:10]
layer.origins.add_notice_origin(origin_layer)
origin_info = formats.invoke_for_snippets
layer.origins.add_notice_origin(origin_info)
if 'RUN' in instruction:
# for Docker the created_by comes from the instruction in the
# dockerfile
run_command_line = instruction.split(' ', 1)[1]
cmd_list, msg = common.filter_install_commands(run_command_line)
if msg:
layer.origins.add_notice_to_origins(origin_info,
Notice(msg, 'warning'))
for command in cmd_list:
origin_cmd = content.print_package_invoke(command.name)
layer.origins.add_notice_origin(origin_cmd)
pkg_list = common.get_installed_package_names(command)
all_pkgs = []
for pkg_name in pkg_list:
pkg_listing = command_lib.get_package_listing(
command.name, pkg_name)
deps, deps_msg = common.get_package_dependencies(
pkg_listing, pkg_name, shell)
if deps_msg:
logger.warning(deps_msg)
all_pkgs.append(pkg_name)
all_pkgs.extend(deps)
unique_pkgs = list(set(all_pkgs))
for pkg_name in unique_pkgs:
pkg = Package(pkg_name)
pkg_listing = command_lib.get_package_listing(
command.name, pkg_name)
common.fill_package_metadata(pkg, pkg_listing, shell)
layer.add_package(pkg)
def testIsEqual(self):
p = Package('x')
p.license = 'TestLicense'
p.version = '1.0'
p.src_url = 'TestUrl'
self.package.license = 'TestLicense'
self.package.version = '2.0'
self.package.src_url = 'TestUrl'
self.assertFalse(self.package.is_equal(p))
p.version = '2.0'
self.assertTrue(self.package.is_equal(p))
def add_diff_packages(diff_layer, command_line, shell):
'''Given a layer object, command line string that created it, and the
shell used to invoke commands, add package metadata to the layer object
1. Parse the command line to get individual install commands
2. For each command get the packages installed
3. For each package get the dependencies
4. For each unique package name, find the metadata and add to the
layer'''
origin_layer = 'Layer: ' + diff_layer.diff_id[:10]
# parse all installed commands
cmd_list, msg = filter_install_commands(command_line)
if msg:
diff_layer.origins.add_notice_to_origins(
origin_layer, Notice(msg, 'warning'))
# find packages for each command
for command in cmd_list:
cmd_msg = formats.invoke_for_snippets + '\n' + \
content.print_package_invoke(command.name)
diff_layer.origins.add_notice_to_origins(origin_layer, Notice(
cmd_msg, 'info'))
pkg_list = get_installed_package_names(command)
# collect all the dependencies for each package name
all_pkgs = []
for pkg_name in pkg_list:
pkg_listing = command_lib.get_package_listing(
command.name, pkg_name)
deps, deps_msg = get_package_dependencies(
pkg_listing, pkg_name, shell)
if deps_msg:
logger.warning(deps_msg)
diff_layer.origins.add_notice_to_origins(
origin_layer, Notice(deps_msg, 'error'))
all_pkgs.append(pkg_name)
all_pkgs.extend(deps)
unique_pkgs = list(set(all_pkgs))
# get package metadata for each package name
for pkg_name in unique_pkgs:
pkg = Package(pkg_name)
pkg_listing = command_lib.get_package_listing(
command.name, pkg_name)
fill_package_metadata(pkg, pkg_listing, shell)
diff_layer.add_package(pkg)
def setUp(self):
self.package = Package('x')
def testAddingPackage(self):
p1 = Package('x')
self.layer.add(p1)
self.assertEqual(len(self.layer.packages), 1)
def testGetPackageNames(self):
p1 = Package('x')
self.layer.add_package(p1)
pkgs = self.layer.get_package_names()
self.assertEqual(pkgs[0], 'x')
def add_base_packages(image):
'''Given an image object, get a list of package objects from
invoking the commands in the command library base section:
1. For the image and tag name find if there is a list of package names
2. If there is an invoke dictionary, invoke the commands
3. Create a list of packages
4. Add them to the image'''
# information under the base image tag in the command library
listing = command_lib.get_base_listing(image.name, image.tag)
# create the origin for the base image
origin_info = formats.invoking_base_commands + '\n' + \
content.print_base_invoke(image.name, image.tag)
image.origins.add_notice_origin(origin_info)
origin_str = 'command_lib/base.yml'
if listing:
shell, msg = command_lib.get_image_shell(listing)
if not shell:
# add a warning notice for no shell in the command library
logger.warning('No shell listing in command library. '
'Using default shell')
no_shell_message = errors.no_shell_listing.format(
image_name=image.name,
image_tag=image.tag,
default_shell=constants.shell)
image.origins.add_notice_to_origins(
origin_str, Notice(no_shell_message, 'warning'))
# add a hint notice to add the shell to the command library
add_shell_message = errors.no_listing_for_base_key.format(
listing_key='shell')
image.origins.add_notice_origins(origin_str,
Notice(add_shell_message, 'hint'))
shell = constants.shell
# check if a container is running first
# eventually this needs to change to use derivatives that have
# more than 1 layer
# for now, we add the list of packages to all the layers in a
# starting base image
if check_container():
names, n_msg = command_lib.get_pkg_attr_list(
shell, listing['names'])
versions, v_msg = command_lib.get_pkg_attr_list(
shell, listing['versions'])
licenses, l_msg = command_lib.get_pkg_attr_list(
shell, listing['licenses'])
src_urls, u_msg = command_lib.get_pkg_attr_list(
shell, listing['src_urls'])
# add a notice to the image if something went wrong
invoke_msg = n_msg + v_msg + l_msg + u_msg
if invoke_msg:
image.origins.add_notice_to_origins(
origin_str, Notice(invoke_msg, 'error'))
if names and len(names) > 1:
for index in range(0, len(names)):
pkg = Package(names[index])
if len(versions) == len(names):
pkg.version = versions[index]
if len(licenses) == len(names):
pkg.license = licenses[index]
if len(src_urls) == len(names):
pkg.src_url = src_urls[index]
for layer in image.layers:
layer.add_package(pkg)
# if no container is running give a logging error
else:
logger.error(errors.no_running_docker_container)
# if there is no listing add a notice
else:
image.origins.add_notice_to_origins(
origin_str,
Notice(
errors.no_image_tag_listing.format(image_name=image.name,
image_tag=image.tag),
'error'))
def add_base_packages(base_layer, binary):
'''Given the base layer and the binary found in layer fs:
1. get the listing from the base.yml
2. Invoke any commands against the base layer
3. Make a list of packages and add them to the layer'''
origin_layer = 'Layer: ' + base_layer.diff_id[:10]
if base_layer.created_by:
base_layer.origins.add_notice_to_origins(origin_layer, Notice(
formats.layer_created_by.format(created_by=base_layer.created_by),
'info'))
else:
base_layer.origins.add_notice_to_origins(origin_layer, Notice(
formats.no_created_by, 'warning'))
origin_command_lib = formats.invoking_base_commands
# find the binary
listing = command_lib.get_base_listing(binary)
if listing:
# put info notice about what is going to be invoked
snippet_msg = formats.invoke_for_base + '\n' + \
content.print_base_invoke(binary)
base_layer.origins.add_notice_to_origins(
origin_layer, Notice(snippet_msg, 'info'))
shell, msg = command_lib.get_image_shell(listing)
if not shell:
# add a warning notice for no shell in the command library
logger.warning('No shell listing in command library. '
'Using default shell')
no_shell_message = errors.no_shell_listing.format(
binary, default_shell=constants.shell)
base_layer.origins.add_notice_to_origins(
origin_command_lib, Notice(no_shell_message, 'warning'))
# add a hint notice to add the shell to the command library
add_shell_message = errors.no_listing_for_base_key.format(
listing_key='shell')
base_layer.origins.add_notice_origins(
origin_command_lib, Notice(add_shell_message, 'hint'))
shell = constants.shell
# get all the packages in the base layer
names, n_msg = command_lib.get_pkg_attr_list(shell, listing['names'])
versions, v_msg = command_lib.get_pkg_attr_list(
shell, listing['versions'])
licenses, l_msg = command_lib.get_pkg_attr_list(
shell, listing['licenses'])
src_urls, u_msg = command_lib.get_pkg_attr_list(
shell, listing['src_urls'])
# add a notice to the image if something went wrong
invoke_msg = n_msg + v_msg + l_msg + u_msg
if invoke_msg:
base_layer.origins.add_notice_to_origins(
origin_layer, Notice(invoke_msg, 'error'))
if names and len(names) > 1:
for index in range(0, len(names)):
pkg = Package(names[index])
if len(versions) == len(names):
pkg.version = versions[index]
if len(licenses) == len(names):
pkg.license = licenses[index]
if len(src_urls) == len(names):
pkg.src_url = src_urls[index]
base_layer.add_package(pkg)
# if there is no listing add a notice
else:
base_layer.origins.add_notice_to_origins(
origin_command_lib, Notice(errors.no_listing_for_base_key.format(
listing_key=binary), 'error'))
