def collect_layer_data(layer_obj): '''Use scancode to collect data from a layer filesystem. This function will create a FileData object for every file found. After scanning, it will return a list of FileData objects. ''' files = [] # run scancode against a directory command = 'scancode -ilpcu --quiet --timeout 300 --json -' full_cmd = get_filesystem_command(layer_obj, command) origin_layer = 'Layer: ' + layer_obj.fs_hash[:10] result, error = rootfs.shell_command(True, full_cmd) if not result: logger.error("No scancode results for this layer: %s", str(error)) layer_obj.origins.add_notice_to_origins(origin_layer, Notice(str(error), 'error')) else: # make FileData objects for each result data = json.loads(result) notice = data.get("headers")[0].get("notice") headers = layer_obj.extension_info.get("headers", set()) headers.add(notice) layer_obj.extension_info["headers"] = headers for f in data['files']: if f['type'] == 'file' and f['size'] != 0: files.append(get_scancode_file(f)) return files
def collect_layer_data(layer_obj): '''Use scancode to collect data from a layer filesystem. This function will create FileData and Package objects for every File and Package found. After scanning, it will return a tuple with a list of FileData and a list of Package objects. ''' files = [] packages = [] # run scancode against a directory try: processes = len(os.sched_getaffinity(0)) command = "scancode -ilpcu --quiet --timeout 300 -n {} --json -".format(processes) except (AttributeError, NotImplementedError): command = "scancode -ilpcu --quiet --timeout 300 --json -" full_cmd = get_filesystem_command(layer_obj, command) origin_layer = 'Layer {}'.format(layer_obj.layer_index) result, error = rootfs.shell_command(True, full_cmd) if not result: logger.error( "No scancode results for this layer: %s", str(error)) layer_obj.origins.add_notice_to_origins( origin_layer, Notice(str(error), 'error')) else: # make FileData objects for each result data = json.loads(result) add_scancode_headers(layer_obj, data["headers"]) for f in data['files']: if f['type'] == 'file' and f['size'] != 0: files.append(get_scancode_file(f)) for package in f['packages']: packages.append(get_scancode_package(package)) return files, packages
def execute_and_pass(layer_obj, command, is_sudo=False): '''Similar to execute_external_command, but the results and the errors are stored together in layer_obj's analyzed_output property to be post-processed. The result and error will be separated by two new line characters \n\n''' full_cmd = get_filesystem_command(layer_obj, command) result, error = rootfs.shell_command(is_sudo, full_cmd) layer_obj.analyzed_output = error.decode('utf-8') + '\n\n' + result.decode( 'utf-8')
def pull_image(image_tag_string, no_tls=False): """Use skopeo to pull a remote image into the working directory""" # Check if skopeo is set up check_skopeo_setup() # we will assume the docker transport for now remote = f'docker://{image_tag_string}' local = f'dir:{rootfs.get_working_dir()}' logger.debug("Attempting to pull image \"%s\"", image_tag_string) if no_tls: result, error = rootfs.shell_command( False, ['skopeo', 'copy', '--src-tls-verify=false', remote, local]) else: result, error = rootfs.shell_command(False, ['skopeo', 'copy', remote, local]) if error: logger.error("Error when downloading image: \"%s\"", error) return None return result
def execute_external_command(layer_obj, command, is_sudo=False): '''Given an Imagelayer object and a command in the form of a list, execute the command and store the results in the ImageLayer object either as results or as a Notice object''' origin_layer = 'Layer: ' + layer_obj.fs_hash[:10] result, error = rootfs.shell_command(is_sudo, command) if error: logger.error("Error in executing external command: %s", str(error)) layer_obj.origins.add_notice_to_origins(origin_layer, Notice(str(error), 'error')) return False layer_obj.analyzed_output = result.decode() return True
def execute_external_command(layer_obj, command, is_sudo=False): '''Given an Imagelayer object and a command in the form of a list, execute the command and store the results in the ImageLayer object either as results or as a Notice object''' origin_layer = 'Layer {}'.format(layer_obj.layer_index) result, error = rootfs.shell_command(is_sudo, command) if error: msg = error.decode('utf-8') logger.error("Error in executing external command: %s", msg) layer_obj.origins.add_notice_to_origins(origin_layer, Notice(msg, 'error')) return False layer_obj.analyzed_output = result.decode('utf-8') return True
def invoke_live(snippet_list, prereqs, method): """Given a list of commands to run, invoke the commands and return the result. The prereqs object should""" # we first create a single command from the snippet list command = snippets_to_script(snippet_list) logger.debug("Invoking command: %s", command) # we then insert this command into our unshare script script_path = create_script(command, prereqs, method) if method == 'container': full_cmd = ['unshare', '-mpf', '-r', script_path] if method == 'host': full_cmd = ['unshare', '-pf', '-r', script_path] # invoke the script and remove it output, error = rootfs.shell_command(False, full_cmd) os.remove(script_path) return output, error
def get_image_digest(image_tag_string): """Use skopeo to get the remote image's digest""" # check if skopeo is set up check_skopeo_setup() remote = f'docker://{image_tag_string}' logger.debug("Inspecting remote image \"%s\"", image_tag_string) result, error = rootfs.shell_command(False, ['skopeo', 'inspect', remote]) if error or not result: logger.error("Unable to retrieve image digest") return None, None result_string = json.loads(result) digest_string = result_string.get("Digest") if not digest_string: logger.error("No image digest available") return None, None digest_list = digest_string.split(":") return digest_list[0], digest_list[1]
def collect_layer_data(layer_obj): '''Use scancode to collect data from a layer filesystem. This function will create a FileData object for every file found. After scanning, it will return a list of FileData objects. ''' files = [] # run scancode against a directory command = 'scancode -ilpcu --quiet --json -' full_cmd = get_filesystem_command(layer_obj, command) origin_layer = 'Layer: ' + layer_obj.fs_hash[:10] result, error = rootfs.shell_command(True, full_cmd) if not result: logger.error("No scancode results for this layer: %s", str(error)) layer_obj.origins.add_notice_to_origins(origin_layer, Notice(str(error), 'error')) else: # make FileData objects for each result data = json.loads(result) for f in data['files']: if f['type'] == 'file': # scancode records paths from the target directory onwards # which in tern's case is tern.utils.constants.untar_dir # removing that portion of the file path fspath = f['path'].replace(constants.untar_dir + os.path.sep, '') fd = FileData(f['name'], fspath, f['date'], f['file_type']) if f['licenses']: fd.licenses = [l['short_name'] for l in f['licenses']] fd.license_expressions = f['license_expressions'] if f['copyrights']: fd.copyrights = [c['value'] for c in f['copyrights']] if f['urls']: fd.urls = [u['url'] for u in f['urls']] fd.packages = f['packages'] fd.authors = [a['value'] for a in f['authors']] if f['scan_errors']: # for each scan error make a notice for err in f['scan_errors']: fd.origins.add_notice_to_origins( 'File: ' + fd.path, Notice(err, 'error')) files.append(fd) return files
def run_on_image(image_obj, command): '''Scancode errors out when it fails to scan any file it is given even if it is successful with other files. Hence we cannot use the available run_on_image function in the passthrough module. Instead we will check if a json object was returned or not''' if not command: logger.error("No command to execute. No report will be generated") return False for layer in image_obj.layers: layer.files_analyzed = True full_cmd = get_filesystem_command(layer, command) origin_layer = 'Layer: ' + layer.fs_hash[:10] result, error = rootfs.shell_command(True, full_cmd) if not result: logger.error( "No scancode results for this layer: %s", str(error)) layer.origins.add_notice_to_origins( origin_layer, Notice(str(error), 'error')) layer.analyzed_output = result.decode() return True