def inspect_scan(language, src, reports_dir, convert, repo_context): """ Method to perform inspect cloud scan Args: language Project language src Project dir reports_dir Directory for output reports convert Boolean to enable normalisation of reports json repo_context Repo context """ run_uuid = config.get("run_uuid") cpg_mode = config.get("SHIFTLEFT_CPG") env = os.environ.copy() env["SCAN_JAVA_HOME"] = os.environ.get("SCAN_JAVA_8_HOME") report_fname = utils.get_report_file( "ng-sast", reports_dir, convert, ext_name="json" ) sl_cmd = config.get("SHIFTLEFT_NGSAST_CMD") # Check if sl cli is available if not utils.check_command(sl_cmd): LOG.warning( "sl cli is not available. Please check if your build uses shiftleft/scan-java as the image" ) return analyze_files = config.get("SHIFTLEFT_ANALYZE_FILE") analyze_target_dir = config.get( "SHIFTLEFT_ANALYZE_DIR", os.path.join(src, "target") ) extra_args = None if not analyze_files: if language == "java": analyze_files = utils.find_java_artifacts(analyze_target_dir) elif language == "csharp": if not utils.check_dotnet(): LOG.warning( "dotnet is not available. Please check if your build uses shiftleft/scan-csharp as the image" ) return analyze_files = utils.find_csharp_artifacts(src) cpg_mode = True else: if language == "ts" or language == "nodejs": language = "js" extra_args = ["--", "--ts", "--babel"] analyze_files = [src] cpg_mode = True app_name = find_app_name(src, repo_context) branch = repo_context.get("revisionId") if not branch: branch = "master" if not analyze_files: LOG.warning( "Unable to find any build artifacts. Compile your project first before invoking scan or use the auto build feature." ) return if isinstance(analyze_files, list) and len(analyze_files) > 1: LOG.warning( "Multiple files found in {}. Only {} will be analyzed".format( analyze_target_dir, analyze_files[0] ) ) analyze_files = analyze_files[0] sl_args = [ sl_cmd, "analyze", "--no-auto-update" if language == "java" else None, "--wait", "--cpg" if cpg_mode else None, "--" + language, "--tag", "branch=" + branch, "--app", app_name, ] sl_args += [analyze_files] if extra_args: sl_args += extra_args sl_args = [arg for arg in sl_args if arg is not None] LOG.info( "About to perform ShiftLeft NG SAST cloud analysis. This might take a few minutes ..." ) LOG.debug(" ".join(sl_args)) LOG.debug(repo_context) cp = exec_tool("NG SAST", sl_args, src, env=env) if cp.returncode != 0: LOG.warning("NG SAST cloud analyze has failed with the below logs") LOG.debug(sl_args) LOG.info(cp.stderr) return findings_data = fetch_findings(app_name, branch, report_fname) if findings_data and convert: crep_fname = utils.get_report_file( "ng-sast", reports_dir, convert, ext_name="sarif" ) convertLib.convert_file("ng-sast", sl_args[1:], src, report_fname, crep_fname) track({"id": run_uuid, "scan_mode": "ng-sast", "sl_args": sl_args})
def execute_default_cmd( cmd_map_list, type_str, tool_name, src, reports_dir, convert, ): """ Method to execute default command for the given type Args: cmd_map_list Default commands in the form of a dict (multiple) or list type_str Project type tool_name Tool name src Project dir reports_dir Directory for output reports convert Boolean to enable normalisation of reports json """ # Check if there is a default command specified for the given type # Create the reports dir os.makedirs(reports_dir, exist_ok=True) report_fname_prefix = os.path.join(reports_dir, tool_name + "-report") default_cmd = " ".join(cmd_map_list) % dict( src=src, reports_dir=reports_dir, report_fname_prefix=report_fname_prefix, type=type_str, ) # Try to detect if the output could be json outext = ".out" if default_cmd.find("json") > -1: outext = ".json" if default_cmd.find("csv") > -1: outext = ".csv" if default_cmd.find("sarif") > -1: outext = ".sarif" report_fname = report_fname_prefix + outext # If the command doesn't support file output then redirect stdout automatically stdout = None if reports_dir and default_cmd.find(report_fname_prefix) == -1: report_fname = report_fname_prefix + outext stdout = io.open(report_fname, "w") LOG.debug("Output will be written to {}".format(report_fname)) # If the command is requesting list of files then construct the argument filelist_prefix = "(filelist=" if default_cmd.find(filelist_prefix) > -1: si = default_cmd.find(filelist_prefix) ei = default_cmd.find(")", si + 10) ext = default_cmd[si + 10:ei] filelist = utils.find_files(src, ext) default_cmd = default_cmd.replace(filelist_prefix + ext + ")", " ".join(filelist)) cmd_with_args = default_cmd.split(" ") exec_tool(cmd_with_args, cwd=src, stdout=stdout) # Should we attempt to convert the report to sarif format if (convert and config.tool_purpose_message.get(cmd_with_args[0]) and os.path.isfile(report_fname)): crep_fname = utils.get_report_file(tool_name, reports_dir, convert, ext_name="sarif") convertLib.convert_file( cmd_with_args[0], cmd_with_args[1:], src, report_fname, crep_fname, )
def execute_default_cmd( cmd_map_list, type_str, tool_name, src, reports_dir, convert, scan_mode, repo_context, ): """ Method to execute default command for the given type Args: cmd_map_list Default commands in the form of a dict (multiple) or list type_str Project type tool_name Tool name src Project dir reports_dir Directory for output reports convert Boolean to enable normalisation of reports json scan_mode Scan mode string repo_context Repo context """ # Check if there is a default command specified for the given type # Create the reports dir os.makedirs(reports_dir, exist_ok=True) report_fname_prefix = os.path.join(reports_dir, tool_name + "-report") default_cmd = " ".join(cmd_map_list) % dict( src=src, reports_dir=reports_dir, report_fname_prefix=report_fname_prefix, type=type_str, scan_mode=scan_mode, ) # Try to detect if the output could be json outext = ".out" if default_cmd.find("json") > -1: outext = ".json" if default_cmd.find("csv") > -1: outext = ".csv" if default_cmd.find("sarif") > -1: outext = ".sarif" report_fname = report_fname_prefix + outext # If the command doesn't support file output then redirect stdout automatically stdout = None if reports_dir and default_cmd.find(report_fname_prefix) == -1: report_fname = report_fname_prefix + outext stdout = io.open(report_fname, "w") LOG.debug("Output will be written to {}".format(report_fname)) # If the command is requesting list of files then construct the argument filelist_prefix = "(filelist=" if default_cmd.find(filelist_prefix) > -1: si = default_cmd.find(filelist_prefix) ei = default_cmd.find(")", si + 10) ext = default_cmd[si + 10 : ei] filelist = utils.find_files(src, ext) delim = " " default_cmd = default_cmd.replace( filelist_prefix + ext + ")", delim.join(filelist) ) cmd_with_args = default_cmd.split(" ") exec_tool(cmd_with_args, cwd=src, stdout=stdout) # Should we attempt to convert the report to sarif format if ( convert and config.tool_purpose_message.get(cmd_with_args[0]) and os.path.isfile(report_fname) ): crep_fname = utils.get_report_file( tool_name, reports_dir, convert, ext_name="sarif" ) convertLib.convert_file( cmd_with_args[0], cmd_with_args[1:], src, report_fname, crep_fname, ) try: if not os.environ.get("SCAN_DEBUG_MODE") == "debug": os.remove(report_fname) except Exception: LOG.debug("Unable to remove file {}".format(report_fname)) elif type_str == "depscan": # Convert depscan and license scan files to html depscan_files = utils.find_files(reports_dir, "depscan", True) for df in depscan_files: if not df.endswith(".html"): depscan_data = grafeas.parse(df) if depscan_data and len(depscan_data): html_fname = df.replace(".json", ".html") grafeas.render_html(depscan_data, html_fname) track( {"id": config.get("run_uuid"), "depscan_summary": depscan_data} ) LOG.debug( "Depscan and HTML report written to file: %s, %s 👍", df, html_fname, ) licence_files = utils.find_files(reports_dir, "license", True) for lf in licence_files: if not lf.endswith(".html"): licence_data = licence.parse(lf) if licence_data and len(licence_data): html_fname = lf.replace(".json", ".html") licence.render_html(licence_data, html_fname) track( {"id": config.get("run_uuid"), "license_summary": licence_data} ) LOG.debug( "License check and HTML report written to file: %s, %s 👍", lf, html_fname, )
def execute_default_cmd( # scan:ignore cmd_map_list, type_str, tool_name, src, reports_dir, convert, scan_mode, repo_context, ): """ Method to execute default command for the given type Args: cmd_map_list Default commands in the form of a dict (multiple) or list type_str Project type tool_name Tool name src Project dir reports_dir Directory for output reports convert Boolean to enable normalisation of reports json scan_mode Scan mode string repo_context Repo context """ # Check if there is a default command specified for the given type # Create the reports dir report_fname_prefix = os.path.join(reports_dir, tool_name + "-report") # Look for any additional direct arguments for the tool and inject them if config.get(tool_name + "_direct_args"): direct_args = config.get(tool_name + "_direct_args").split(" ") if direct_args: cmd_map_list += direct_args src_or_file = src if config.get("SHIFTLEFT_ANALYZE_FILE"): src_or_file = config.get("SHIFTLEFT_ANALYZE_FILE") default_cmd = " ".join(cmd_map_list) % dict( src=src, src_or_file=src_or_file, reports_dir=reports_dir, report_fname_prefix=report_fname_prefix, type=type_str, scan_mode=scan_mode, ) # Try to detect if the output could be json outext = ".out" if "json" in default_cmd: outext = ".json" elif "csv" in default_cmd: outext = ".csv" elif "sarif" in default_cmd: outext = ".sarif" elif "xml" in default_cmd: outext = ".xml" report_fname = report_fname_prefix + outext # If the command doesn't support file output then redirect stdout automatically stdout = None if LOG.isEnabledFor(DEBUG): stdout = None if reports_dir and report_fname_prefix not in default_cmd: report_fname = report_fname_prefix + outext stdout = io.open(report_fname, "w") LOG.debug("Output will be written to {}".format(report_fname)) # If the command is requesting list of files then construct the argument filelist_prefix = "(filelist=" if default_cmd.find(filelist_prefix) > -1: si = default_cmd.find(filelist_prefix) ei = default_cmd.find(")", si + 10) ext = default_cmd[si + 10:ei] filelist = utils.find_files(src, ext) # Temporary fix for the yaml issue if ext == "yaml": yml_list = utils.find_files(src, "yml") if yml_list: filelist.extend(yml_list) delim = " " default_cmd = default_cmd.replace(filelist_prefix + ext + ")", delim.join(filelist)) cmd_with_args = default_cmd.split(" ") # Suppress psalm output if should_suppress_output(type_str, cmd_with_args[0]): stdout = subprocess.DEVNULL exec_tool(tool_name, cmd_with_args, cwd=src, stdout=stdout) # Should we attempt to convert the report to sarif format if should_convert(convert, tool_name, cmd_with_args[0], report_fname): crep_fname = utils.get_report_file(tool_name, reports_dir, convert, ext_name="sarif") if (cmd_with_args[0] == "java" or "pmd-bin" in cmd_with_args[0] or "php" in tool_name): convertLib.convert_file( tool_name, cmd_with_args, src, report_fname, crep_fname, ) else: convertLib.convert_file( cmd_with_args[0], cmd_with_args[1:], src, report_fname, crep_fname, ) try: if not LOG.isEnabledFor(DEBUG): os.remove(report_fname) except Exception: LOG.debug("Unable to remove file {}".format(report_fname)) elif type_str == "depscan": # Convert depscan and license scan files to html depscan_files = utils.find_files(reports_dir, "depscan", True) for df in depscan_files: if not df.endswith(".html"): depscan_data = grafeas.parse(df) if depscan_data and len(depscan_data): html_fname = df.replace(".json", ".html") grafeas.render_html(depscan_data, html_fname) track({ "id": config.get("run_uuid"), "depscan_summary": depscan_data }) LOG.debug( "Depscan and HTML report written to file: %s, %s :thumbsup:", df, html_fname, ) licence_files = utils.find_files(reports_dir, "license", True) for lf in licence_files: if not lf.endswith(".html"): licence_data = licence.parse(lf) if licence_data and len(licence_data): html_fname = lf.replace(".json", ".html") licence.render_html(licence_data, html_fname) track({ "id": config.get("run_uuid"), "license_summary": licence_data }) LOG.debug( "License check and HTML report written to file: %s, %s :thumbsup:", lf, html_fname, )
def inspect_scan(language, src, reports_dir, convert, repo_context): """ Method to perform inspect cloud scan Args: language Project language src Project dir reports_dir Directory for output reports convert Boolean to enable normalisation of reports json repo_context Repo context """ report_fname = utils.get_report_file("inspect", reports_dir, convert, ext_name="json") sl_cmd = config.get("SHIFTLEFT_INSPECT_CMD") analyze_target_dir = config.get("SHIFTLEFT_ANALYZE_DIR", os.path.join(src, "target")) analyze_files = config.get("SHIFTLEFT_ANALYZE_FILE") if not analyze_files: if language == "java": analyze_files = utils.find_java_artifacts(analyze_target_dir) app_name = config.get("SHIFTLEFT_APP", repo_context.get("repositoryName")) no_cpg = config.get("SHIFTLEFT_NO_CPG") if not app_name: app_name = os.path.dirname(src) branch = repo_context.get("revisionId") if not branch: branch = "master" if not analyze_files: LOG.warning( "Unable to find any build artifacts in {}. Run mvn package or a similar command before invoking inspect scan" .format(analyze_target_dir)) return if len(analyze_files) > 1: LOG.warning( "Multiple jar files found in {}. Only {} will be analyzed".format( analyze_target_dir, analyze_files[0])) sl_args = [ sl_cmd, "analyze", "--no-auto-update", "--wait", "--cpg" if not no_cpg else "", "--java", "--tag", "branch=" + branch, "--app", app_name, ] sl_args += [analyze_files[0]] env = os.environ.copy() env["JAVA_HOME"] = os.environ.get("JAVA_8_HOME") LOG.info( "About to perform ShiftLeft Inspect cloud analysis. This might take a few minutes ..." ) cp = exec_tool(sl_args, src, env=env) if cp.returncode != 0: LOG.warning("Inspect cloud analyze has failed with the below logs") LOG.info(cp.stdout) LOG.info(cp.stderr) return findings_data = fetch_findings(app_name, branch, report_fname) if findings_data and convert: crep_fname = utils.get_report_file("inspect", reports_dir, convert, ext_name="sarif") convertLib.convert_file("inspect", sl_args[1:], src, report_fname, crep_fname)