def testFill(self):
p_dict = {'name': 'p1',
'version': '1.0',
'pkg_license': 'Apache 2.0',
'checksum': 'abcxyz',
'pkg_licenses': ['MIT', 'GPL'],
'files': [{'name': 'a.txt', 'path': '/usr/a.txt'},
{'name': 'b.txt', 'path': '/lib/b.txt'}],
'pkg_format': 'rpm',
'src_name': 'p1src',
'src_version': '1.0'
}
p = Package('p1')
p.fill(p_dict)
self.assertEqual(p.name, 'p1')
self.assertEqual(p.version, '1.0')
self.assertEqual(p.pkg_license, 'Apache 2.0')
self.assertEqual(p.checksum, 'abcxyz')
self.assertEqual(p.pkg_licenses, ['MIT', 'GPL'])
self.assertEqual(len(p.files), 2)
self.assertEqual(p.files[0].name, 'a.txt')
self.assertEqual(p.files[0].path, '/usr/a.txt')
self.assertFalse(p.copyright)
self.assertFalse(p.proj_url)
self.assertEqual(len(p.origins.origins), 1)
self.assertEqual(p.origins.origins[0].origin_str, 'p1')
self.assertEqual(len(p.origins.origins[0].notices), 3)
self.assertEqual(p.origins.origins[0].notices[0].message,
"No metadata for key: copyright")
self.assertEqual(p.origins.origins[0].notices[0].level, 'warning')
self.assertEqual(p.origins.origins[0].notices[1].message,
"No metadata for key: proj_url")
self.assertEqual(p.origins.origins[0].notices[2].message,
"No metadata for key: download_url")
def testFill(self):
f_dict1 = {'name': 'a.txt', 'path': '/usr/a.txt'}
f_dict2 = {'name': 'b.txt', 'path': '/lib/b.txt'}
fd1 = FileData(f_dict1['name'], f_dict1['path'])
fd2 = FileData(f_dict2['name'], f_dict2['path'])
fd1.fill(f_dict1)
fd2.fill(f_dict2)
p_dict = {'name': 'p1',
'version': '1.0',
'pkg_license': 'Apache 2.0',
'checksum': 'abcxyz',
'pkg_licenses': ['MIT', 'GPL'],
'files': '/usr/a.txt\n/lib/b.txt'}
p = Package('p1')
p.fill(p_dict)
self.assertEqual(p.name, 'p1')
self.assertEqual(p.version, '1.0')
self.assertEqual(p.pkg_license, 'Apache 2.0')
self.assertEqual(p.checksum, 'abcxyz')
self.assertEqual(p.pkg_licenses, ['MIT', 'GPL'])
self.assertTrue(fd1.is_equal(p.files[0]))
self.assertTrue(fd2.is_equal(p.files[1]))
self.assertFalse(p.copyright)
self.assertFalse(p.proj_url)
self.assertEqual(len(p.origins.origins), 1)
self.assertEqual(p.origins.origins[0].origin_str, 'p1')
self.assertEqual(len(p.origins.origins[0].notices), 3)
self.assertEqual(p.origins.origins[0].notices[0].message,
"No metadata for key: copyright")
self.assertEqual(p.origins.origins[0].notices[0].level, 'warning')
self.assertEqual(p.origins.origins[0].notices[1].message,
"No metadata for key: proj_url")
self.assertEqual(p.origins.origins[0].notices[2].message,
"No metadata for key: download_url")
def testFill(self):
p_dict = {
'name': 'p1',
'version': '1.0',
'pkg_license': 'Apache 2.0',
'checksum': 'abcxyz',
'pkg_licenses': ['MIT', 'GPL']
}
p = Package('p1')
p.fill(p_dict)
self.assertEqual(p.name, 'p1')
self.assertEqual(p.version, '1.0')
self.assertEqual(p.pkg_license, 'Apache 2.0')
self.assertEqual(p.checksum, 'abcxyz')
self.assertEqual(p.pkg_licenses, ['MIT', 'GPL'])
self.assertFalse(p.copyright)
self.assertFalse(p.proj_url)
self.assertEqual(len(p.origins.origins), 1)
self.assertEqual(p.origins.origins[0].origin_str, 'p1')
self.assertEqual(len(p.origins.origins[0].notices), 4)
self.assertEqual(p.origins.origins[0].notices[0].message,
"No metadata for key: copyright")
self.assertEqual(p.origins.origins[0].notices[0].level, 'warning')
self.assertEqual(p.origins.origins[0].notices[1].message,
"No metadata for key: proj_url")
self.assertEqual(p.origins.origins[0].notices[2].message,
"No metadata for key: download_url")
def create_image_layer(report):
"""Given a report file, create an ImageLayer object with the metadata"""
# expect a json input, raise an error if it is not
content = {}
try:
with open(os.path.abspath(report), encoding='utf-8') as f:
content = json.load(f)
except OSError as err:
logger.critical("Cannot access file %s: %s", report, err)
raise ConsumerError(f"Error with given report file: {report}") from err
except json.JSONDecodeError as err:
logger.critical("Cannot parse JSON in file %s: %s", report, err)
raise ConsumerError(f"Error with given report file: {report}") from err
# we should have some content but it may be empty
if not content:
raise ConsumerError("No content consumed from given report file")
# instantiate a layer and fill it
layer = ImageLayer("")
try:
layer.os_guess = content['os_guess']
for pkg in content['packages']:
pkg_obj = Package(pkg['name'])
pkg_obj.fill(pkg)
layer.add_package(pkg_obj)
for filedict in content['files']:
file_obj = FileData(filedict['name'], filedict['path'])
file_obj.fill(filedict)
layer.add_file(file_obj)
return layer
except ValueError as err:
logger.critical("Cannot find required data in report: %s", err)
return None
def load_from_cache(layer, redo=False):
'''Given a layer object, check against cache to see if that layer id exists
if yes then get the package list and load it in the layer and return true.
If it doesn't exist return false. Default operation is to not redo the
cache. Add notices to the layer's origins matching the origin_str'''
loaded = False
origin_layer = 'Layer: ' + layer.fs_hash[:10]
if not layer.packages and not redo:
# there are no packages in this layer and we are not repopulating the
# cache, try to get it from the cache
raw_pkg_list = cache.get_packages(layer.fs_hash)
if raw_pkg_list:
logger.debug('Loaded from cache: layer {}'.format(
layer.fs_hash[:10]))
message = formats.loading_from_cache.format(
layer_id=layer.fs_hash[:10])
# add notice to the origin
layer.origins.add_notice_to_origins(origin_layer,
Notice(message, 'info'))
for pkg_dict in raw_pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
layer.add_package(pkg)
loaded = True
return loaded
def fill_pkg_results(image_layer, pkg_list_dict):
"""Fill results from collecting package information into the image layer
object"""
if 'names' in pkg_list_dict and len(pkg_list_dict['names']) > 1:
pkg_list = convert_to_pkg_dicts(pkg_list_dict)
for pkg_dict in pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
image_layer.add_package(pkg)
def add_base_packages(image_layer, binary, shell, work_dir=None, envs=None):
'''Given the image layer, the binary to invoke and shell:
1. get the listing from the base.yml
2. Invoke any commands against the base layer
3. Make a list of packages and add them to the layer'''
origin_layer = 'Layer {}'.format(image_layer.layer_index)
if image_layer.created_by:
image_layer.origins.add_notice_to_origins(
origin_layer,
Notice(
formats.layer_created_by.format(
created_by=image_layer.created_by), 'info'))
else:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(formats.no_created_by, 'warning'))
origin_command_lib = formats.invoking_base_commands
# find the binary
listing = command_lib.get_base_listing(binary)
if listing:
# put info notice about what is going to be invoked
snippet_msg = (formats.invoke_for_base + '\n' +
content.print_base_invoke(binary))
image_layer.origins.add_notice_to_origins(origin_layer,
Notice(snippet_msg, 'info'))
# get all the packages in the base layer
pkg_dict, invoke_msg, warnings = collate_list_metadata(
shell, listing, work_dir, envs)
if listing.get("pkg_format") == "deb":
pkg_dict["pkg_licenses"] = get_deb_package_licenses(
pkg_dict["copyrights"])
if invoke_msg:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(invoke_msg, 'error'))
if warnings:
image_layer.origins.add_notice_to_origins(
origin_command_lib, Notice(warnings, 'warning'))
if 'names' in pkg_dict and len(pkg_dict['names']) > 1:
pkg_list = convert_to_pkg_dicts(pkg_dict)
for pkg_dict in pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
image_layer.add_package(pkg)
remove_duplicate_layer_files(image_layer)
# if there is no listing add a notice
else:
image_layer.origins.add_notice_to_origins(
origin_command_lib,
Notice(errors.no_listing_for_base_key.format(listing_key=binary),
'error'))
def add_base_packages(image_layer, binary, shell):
'''Given the image layer, the binary to invoke and shell:
1. get the listing from the base.yml
2. Invoke any commands against the base layer
3. Make a list of packages and add them to the layer'''
origin_layer = 'Layer: ' + image_layer.fs_hash[:10]
if image_layer.created_by:
image_layer.origins.add_notice_to_origins(
origin_layer,
Notice(
formats.layer_created_by.format(
created_by=image_layer.created_by), 'info'))
else:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(formats.no_created_by, 'warning'))
origin_command_lib = formats.invoking_base_commands
# find the binary
listing = command_lib.get_base_listing(binary)
if listing:
# put info notice about what is going to be invoked
snippet_msg = formats.invoke_for_base + '\n' + \
content.print_base_invoke(binary)
image_layer.origins.add_notice_to_origins(origin_layer,
Notice(snippet_msg, 'info'))
shell, _ = command_lib.get_image_shell(listing)
if not shell:
shell = constants.shell
# get all the packages in the base layer
pkg_dict, invoke_msg, warnings = collate_list_metadata(shell, listing)
if invoke_msg:
image_layer.origins.add_notice_to_origins(
origin_layer, Notice(invoke_msg, 'error'))
if warnings:
image_layer.origins.add_notice_to_origins(
origin_command_lib, Notice(warnings, 'warning'))
if 'names' in pkg_dict and len(pkg_dict['names']) > 1:
pkg_list = convert_to_pkg_dicts(pkg_dict)
for pkg_dict in pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
image_layer.add_package(pkg)
# if there is no listing add a notice
else:
image_layer.origins.add_notice_to_origins(
origin_command_lib,
Notice(errors.no_listing_for_base_key.format(listing_key=binary),
'error'))
def load_from_cache(layer, redo=False):
'''Given a layer object, check against cache to see if that layer id exists
if yes then get the package list and load it in the layer and return true.
If it doesn't exist return false. Default operation is to not redo the
cache. Add notices to the layer's origins matching the origin_str'''
loaded = False
if not layer.packages and not redo:
# there are no packages in this layer and we are not repopulating the
# cache, try to get it from the cache
raw_pkg_list = cache.get_packages(layer.fs_hash)
if raw_pkg_list:
logger.debug('Loaded from cache: layer \"%s\"', layer.fs_hash[:10])
for pkg_dict in raw_pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
layer.add_package(pkg)
load_notices_from_cache(layer)
loaded = True
return loaded
def load_packages_from_cache(layer):
'''Given a layer object, populate package level information'''
loaded = False
raw_pkg_list = cache.get_packages(layer.fs_hash)
if raw_pkg_list:
logger.debug('Loading packages from cache: layer \"%s\"',
layer.fs_hash[:10])
for pkg_dict in raw_pkg_list:
pkg = Package(pkg_dict['name'])
pkg.fill(pkg_dict)
# collect package origins
if 'origins' in pkg_dict.keys():
for origin_dict in pkg_dict['origins']:
for notice in origin_dict['notices']:
pkg.origins.add_notice_to_origins(
origin_dict['origin_str'],
Notice(notice['message'], notice['level']))
layer.add_package(pkg)
loaded = True
return loaded