def should_suppress_output(type_str, command): """ Method to find if the tool's output should be suppressed """ if "credscan" in type_str: return True if "php" in type_str and not LOG.isEnabledFor(DEBUG): return True if command in ["gitleaks"]: return True if command in ["psalm"] and not LOG.isEnabledFor(DEBUG): return True return False
def exec_tool( # scan:ignore tool_name, args, cwd=None, env=utils.get_env(), stdout=subprocess.DEVNULL): """ Convenience method to invoke cli tools Args: tool_name Tool name args cli command and args cwd Current working directory env Environment variables stdout stdout configuration for run command Returns: CompletedProcess instance """ with Progress( console=console, redirect_stderr=False, redirect_stdout=False, refresh_per_second=1, ) as progress: task = None try: env = use_java(env) LOG.debug('⚡︎ Executing {} "{}"'.format(tool_name, " ".join(args))) stderr = subprocess.DEVNULL if LOG.isEnabledFor(DEBUG): stderr = subprocess.STDOUT tool_verb = "Scanning with" if "init" in tool_name: tool_verb = "Initializing" elif "build" in tool_name: tool_verb = "Building with" task = progress.add_task("[green]" + tool_verb + " " + tool_name, total=100, start=False) cp = subprocess.run( args, stdout=stdout, stderr=stderr, cwd=cwd, env=env, check=False, shell=False, encoding="utf-8", ) if cp and stdout == subprocess.PIPE: for line in cp.stdout: progress.update(task, completed=5) if (cp and LOG.isEnabledFor(DEBUG) and cp.returncode and cp.stdout is not None): LOG.debug(cp.stdout) progress.update(task, completed=100, total=100) return cp except Exception as e: if task: progress.update(task, completed=20, total=10, visible=False) if not LOG.isEnabledFor(DEBUG): LOG.info( f"{tool_name} has reported few errors. To view, pass the environment variable SCAN_DEBUG_MODE=debug" ) LOG.debug(e) return None
def execute_default_cmd( # scan:ignore cmd_map_list, type_str, tool_name, src, reports_dir, convert, scan_mode, repo_context, ): """ Method to execute default command for the given type Args: cmd_map_list Default commands in the form of a dict (multiple) or list type_str Project type tool_name Tool name src Project dir reports_dir Directory for output reports convert Boolean to enable normalisation of reports json scan_mode Scan mode string repo_context Repo context """ # Check if there is a default command specified for the given type # Create the reports dir report_fname_prefix = os.path.join(reports_dir, tool_name + "-report") # Look for any additional direct arguments for the tool and inject them if config.get(tool_name + "_direct_args"): direct_args = config.get(tool_name + "_direct_args").split(" ") if direct_args: cmd_map_list += direct_args src_or_file = src if config.get("SHIFTLEFT_ANALYZE_FILE"): src_or_file = config.get("SHIFTLEFT_ANALYZE_FILE") default_cmd = " ".join(cmd_map_list) % dict( src=src, src_or_file=src_or_file, reports_dir=reports_dir, report_fname_prefix=report_fname_prefix, type=type_str, scan_mode=scan_mode, ) # Try to detect if the output could be json outext = ".out" if "json" in default_cmd: outext = ".json" elif "csv" in default_cmd: outext = ".csv" elif "sarif" in default_cmd: outext = ".sarif" elif "xml" in default_cmd: outext = ".xml" report_fname = report_fname_prefix + outext # If the command doesn't support file output then redirect stdout automatically stdout = None if LOG.isEnabledFor(DEBUG): stdout = None if reports_dir and report_fname_prefix not in default_cmd: report_fname = report_fname_prefix + outext stdout = io.open(report_fname, "w") LOG.debug("Output will be written to {}".format(report_fname)) # If the command is requesting list of files then construct the argument filelist_prefix = "(filelist=" if default_cmd.find(filelist_prefix) > -1: si = default_cmd.find(filelist_prefix) ei = default_cmd.find(")", si + 10) ext = default_cmd[si + 10:ei] filelist = utils.find_files(src, ext) # Temporary fix for the yaml issue if ext == "yaml": yml_list = utils.find_files(src, "yml") if yml_list: filelist.extend(yml_list) delim = " " default_cmd = default_cmd.replace(filelist_prefix + ext + ")", delim.join(filelist)) cmd_with_args = default_cmd.split(" ") # Suppress psalm output if should_suppress_output(type_str, cmd_with_args[0]): stdout = subprocess.DEVNULL exec_tool(tool_name, cmd_with_args, cwd=src, stdout=stdout) # Should we attempt to convert the report to sarif format if should_convert(convert, tool_name, cmd_with_args[0], report_fname): crep_fname = utils.get_report_file(tool_name, reports_dir, convert, ext_name="sarif") if (cmd_with_args[0] == "java" or "pmd-bin" in cmd_with_args[0] or "php" in tool_name): convertLib.convert_file( tool_name, cmd_with_args, src, report_fname, crep_fname, ) else: convertLib.convert_file( cmd_with_args[0], cmd_with_args[1:], src, report_fname, crep_fname, ) try: if not LOG.isEnabledFor(DEBUG): os.remove(report_fname) except Exception: LOG.debug("Unable to remove file {}".format(report_fname)) elif type_str == "depscan": # Convert depscan and license scan files to html depscan_files = utils.find_files(reports_dir, "depscan", True) for df in depscan_files: if not df.endswith(".html"): depscan_data = grafeas.parse(df) if depscan_data and len(depscan_data): html_fname = df.replace(".json", ".html") grafeas.render_html(depscan_data, html_fname) track({ "id": config.get("run_uuid"), "depscan_summary": depscan_data }) LOG.debug( "Depscan and HTML report written to file: %s, %s :thumbsup:", df, html_fname, ) licence_files = utils.find_files(reports_dir, "license", True) for lf in licence_files: if not lf.endswith(".html"): licence_data = licence.parse(lf) if licence_data and len(licence_data): html_fname = lf.replace(".json", ".html") licence.render_html(licence_data, html_fname) track({ "id": config.get("run_uuid"), "license_summary": licence_data }) LOG.debug( "License check and HTML report written to file: %s, %s :thumbsup:", lf, html_fname, )