def collect_layer_data(layer_obj):
'''Use scancode to collect data from a layer filesystem. This function will
create a FileData object for every file found. After scanning, it will
return a list of FileData objects.
'''
files = []
# run scancode against a directory
command = 'scancode -ilpcu --quiet --timeout 300 --json -'
full_cmd = get_filesystem_command(layer_obj, command)
origin_layer = 'Layer: ' + layer_obj.fs_hash[:10]
result, error = rootfs.shell_command(True, full_cmd)
if not result:
logger.error("No scancode results for this layer: %s", str(error))
layer_obj.origins.add_notice_to_origins(origin_layer,
Notice(str(error), 'error'))
else:
# make FileData objects for each result
data = json.loads(result)
notice = data.get("headers")[0].get("notice")
headers = layer_obj.extension_info.get("headers", set())
headers.add(notice)
layer_obj.extension_info["headers"] = headers
for f in data['files']:
if f['type'] == 'file' and f['size'] != 0:
files.append(get_scancode_file(f))
return files
def collect_layer_data(layer_obj):
'''Use scancode to collect data from a layer filesystem. This function will
create FileData and Package objects for every File and Package found. After
scanning, it will return a tuple with a list of FileData and a list of
Package objects.
'''
files = []
packages = []
# run scancode against a directory
try:
processes = len(os.sched_getaffinity(0))
command = "scancode -ilpcu --quiet --timeout 300 -n {} --json -".format(processes)
except (AttributeError, NotImplementedError):
command = "scancode -ilpcu --quiet --timeout 300 --json -"
full_cmd = get_filesystem_command(layer_obj, command)
origin_layer = 'Layer {}'.format(layer_obj.layer_index)
result, error = rootfs.shell_command(True, full_cmd)
if not result:
logger.error(
"No scancode results for this layer: %s", str(error))
layer_obj.origins.add_notice_to_origins(
origin_layer, Notice(str(error), 'error'))
else:
# make FileData objects for each result
data = json.loads(result)
add_scancode_headers(layer_obj, data["headers"])
for f in data['files']:
if f['type'] == 'file' and f['size'] != 0:
files.append(get_scancode_file(f))
for package in f['packages']:
packages.append(get_scancode_package(package))
return files, packages
def execute_and_pass(layer_obj, command, is_sudo=False):
'''Similar to execute_external_command, but the results and the errors
are stored together in layer_obj's analyzed_output property to be
post-processed. The result and error will be separated by two new line
characters \n\n'''
full_cmd = get_filesystem_command(layer_obj, command)
result, error = rootfs.shell_command(is_sudo, full_cmd)
layer_obj.analyzed_output = error.decode('utf-8') + '\n\n' + result.decode(
'utf-8')
def pull_image(image_tag_string, no_tls=False):
"""Use skopeo to pull a remote image into the working directory"""
# Check if skopeo is set up
check_skopeo_setup()
# we will assume the docker transport for now
remote = f'docker://{image_tag_string}'
local = f'dir:{rootfs.get_working_dir()}'
logger.debug("Attempting to pull image \"%s\"", image_tag_string)
if no_tls:
result, error = rootfs.shell_command(
False, ['skopeo', 'copy', '--src-tls-verify=false', remote, local])
else:
result, error = rootfs.shell_command(False,
['skopeo', 'copy', remote, local])
if error:
logger.error("Error when downloading image: \"%s\"", error)
return None
return result
def execute_external_command(layer_obj, command, is_sudo=False):
'''Given an Imagelayer object and a command in the form of a list, execute
the command and store the results in the ImageLayer object either as
results or as a Notice object'''
origin_layer = 'Layer: ' + layer_obj.fs_hash[:10]
result, error = rootfs.shell_command(is_sudo, command)
if error:
logger.error("Error in executing external command: %s", str(error))
layer_obj.origins.add_notice_to_origins(origin_layer,
Notice(str(error), 'error'))
return False
layer_obj.analyzed_output = result.decode()
return True
def execute_external_command(layer_obj, command, is_sudo=False):
'''Given an Imagelayer object and a command in the form of a list, execute
the command and store the results in the ImageLayer object either as
results or as a Notice object'''
origin_layer = 'Layer {}'.format(layer_obj.layer_index)
result, error = rootfs.shell_command(is_sudo, command)
if error:
msg = error.decode('utf-8')
logger.error("Error in executing external command: %s", msg)
layer_obj.origins.add_notice_to_origins(origin_layer,
Notice(msg, 'error'))
return False
layer_obj.analyzed_output = result.decode('utf-8')
return True
def invoke_live(snippet_list, prereqs, method):
"""Given a list of commands to run, invoke the commands and return
the result. The prereqs object should"""
# we first create a single command from the snippet list
command = snippets_to_script(snippet_list)
logger.debug("Invoking command: %s", command)
# we then insert this command into our unshare script
script_path = create_script(command, prereqs, method)
if method == 'container':
full_cmd = ['unshare', '-mpf', '-r', script_path]
if method == 'host':
full_cmd = ['unshare', '-pf', '-r', script_path]
# invoke the script and remove it
output, error = rootfs.shell_command(False, full_cmd)
os.remove(script_path)
return output, error
def get_image_digest(image_tag_string):
"""Use skopeo to get the remote image's digest"""
# check if skopeo is set up
check_skopeo_setup()
remote = f'docker://{image_tag_string}'
logger.debug("Inspecting remote image \"%s\"", image_tag_string)
result, error = rootfs.shell_command(False, ['skopeo', 'inspect', remote])
if error or not result:
logger.error("Unable to retrieve image digest")
return None, None
result_string = json.loads(result)
digest_string = result_string.get("Digest")
if not digest_string:
logger.error("No image digest available")
return None, None
digest_list = digest_string.split(":")
return digest_list[0], digest_list[1]
def collect_layer_data(layer_obj):
'''Use scancode to collect data from a layer filesystem. This function will
create a FileData object for every file found. After scanning, it will
return a list of FileData objects.
'''
files = []
# run scancode against a directory
command = 'scancode -ilpcu --quiet --json -'
full_cmd = get_filesystem_command(layer_obj, command)
origin_layer = 'Layer: ' + layer_obj.fs_hash[:10]
result, error = rootfs.shell_command(True, full_cmd)
if not result:
logger.error("No scancode results for this layer: %s", str(error))
layer_obj.origins.add_notice_to_origins(origin_layer,
Notice(str(error), 'error'))
else:
# make FileData objects for each result
data = json.loads(result)
for f in data['files']:
if f['type'] == 'file':
# scancode records paths from the target directory onwards
# which in tern's case is tern.utils.constants.untar_dir
# removing that portion of the file path
fspath = f['path'].replace(constants.untar_dir + os.path.sep,
'')
fd = FileData(f['name'], fspath, f['date'], f['file_type'])
if f['licenses']:
fd.licenses = [l['short_name'] for l in f['licenses']]
fd.license_expressions = f['license_expressions']
if f['copyrights']:
fd.copyrights = [c['value'] for c in f['copyrights']]
if f['urls']:
fd.urls = [u['url'] for u in f['urls']]
fd.packages = f['packages']
fd.authors = [a['value'] for a in f['authors']]
if f['scan_errors']:
# for each scan error make a notice
for err in f['scan_errors']:
fd.origins.add_notice_to_origins(
'File: ' + fd.path, Notice(err, 'error'))
files.append(fd)
return files
def run_on_image(image_obj, command):
'''Scancode errors out when it fails to scan any file it is given even
if it is successful with other files. Hence we cannot use the available
run_on_image function in the passthrough module. Instead we will check
if a json object was returned or not'''
if not command:
logger.error("No command to execute. No report will be generated")
return False
for layer in image_obj.layers:
layer.files_analyzed = True
full_cmd = get_filesystem_command(layer, command)
origin_layer = 'Layer: ' + layer.fs_hash[:10]
result, error = rootfs.shell_command(True, full_cmd)
if not result:
logger.error(
"No scancode results for this layer: %s", str(error))
layer.origins.add_notice_to_origins(
origin_layer, Notice(str(error), 'error'))
layer.analyzed_output = result.decode()
return True
