def convert_to_findings(src_dir, repo_context, reports_dir, sarif_files):
"""
Convert the sarif files to Inspect findings format
:param src_dir: Source directory
:param repo_context: Repository context
:param reports_dir: Reports directory
:param sarif_files: SARIF files
:return: None
"""
app_name = find_app_name(src_dir, repo_context)
findings_fname = utils.get_report_file("inspect",
reports_dir,
True,
ext_name="findings.json")
# Exclude any inspect sarif files
sarif_files = [f for f in sarif_files if "inspect" not in f]
if sarif_files:
convert_sarif(app_name, repo_context, sarif_files, findings_fname)
def inspect_scan(language, src, reports_dir, convert, repo_context):
"""
Method to perform inspect cloud scan
Args:
language Project language
src Project dir
reports_dir Directory for output reports
convert Boolean to enable normalisation of reports json
repo_context Repo context
"""
run_uuid = config.get("run_uuid")
cpg_mode = config.get("SHIFTLEFT_CPG")
env = os.environ.copy()
env["SCAN_JAVA_HOME"] = os.environ.get("SCAN_JAVA_8_HOME")
report_fname = utils.get_report_file(
"ng-sast", reports_dir, convert, ext_name="json"
)
sl_cmd = config.get("SHIFTLEFT_NGSAST_CMD")
# Check if sl cli is available
if not utils.check_command(sl_cmd):
LOG.warning(
"sl cli is not available. Please check if your build uses shiftleft/scan-java as the image"
)
return
analyze_files = config.get("SHIFTLEFT_ANALYZE_FILE")
analyze_target_dir = config.get(
"SHIFTLEFT_ANALYZE_DIR", os.path.join(src, "target")
)
extra_args = None
if not analyze_files:
if language == "java":
analyze_files = utils.find_java_artifacts(analyze_target_dir)
elif language == "csharp":
if not utils.check_dotnet():
LOG.warning(
"dotnet is not available. Please check if your build uses shiftleft/scan-csharp as the image"
)
return
analyze_files = utils.find_csharp_artifacts(src)
cpg_mode = True
else:
if language == "ts" or language == "nodejs":
language = "js"
extra_args = ["--", "--ts", "--babel"]
analyze_files = [src]
cpg_mode = True
app_name = find_app_name(src, repo_context)
branch = repo_context.get("revisionId")
if not branch:
branch = "master"
if not analyze_files:
LOG.warning(
"Unable to find any build artifacts. Compile your project first before invoking scan or use the auto build feature."
)
return
if isinstance(analyze_files, list) and len(analyze_files) > 1:
LOG.warning(
"Multiple files found in {}. Only {} will be analyzed".format(
analyze_target_dir, analyze_files[0]
)
)
analyze_files = analyze_files[0]
sl_args = [
sl_cmd,
"analyze",
"--no-auto-update" if language == "java" else None,
"--wait",
"--cpg" if cpg_mode else None,
"--" + language,
"--tag",
"branch=" + branch,
"--app",
app_name,
]
sl_args += [analyze_files]
if extra_args:
sl_args += extra_args
sl_args = [arg for arg in sl_args if arg is not None]
LOG.info(
"About to perform ShiftLeft NG SAST cloud analysis. This might take a few minutes ..."
)
LOG.debug(" ".join(sl_args))
LOG.debug(repo_context)
cp = exec_tool("NG SAST", sl_args, src, env=env)
if cp.returncode != 0:
LOG.warning("NG SAST cloud analyze has failed with the below logs")
LOG.debug(sl_args)
LOG.info(cp.stderr)
return
findings_data = fetch_findings(app_name, branch, report_fname)
if findings_data and convert:
crep_fname = utils.get_report_file(
"ng-sast", reports_dir, convert, ext_name="sarif"
)
convertLib.convert_file("ng-sast", sl_args[1:], src, report_fname, crep_fname)
track({"id": run_uuid, "scan_mode": "ng-sast", "sl_args": sl_args})
def execute_default_cmd(
cmd_map_list,
type_str,
tool_name,
src,
reports_dir,
convert,
):
"""
Method to execute default command for the given type
Args:
cmd_map_list Default commands in the form of a dict (multiple) or list
type_str Project type
tool_name Tool name
src Project dir
reports_dir Directory for output reports
convert Boolean to enable normalisation of reports json
"""
# Check if there is a default command specified for the given type
# Create the reports dir
os.makedirs(reports_dir, exist_ok=True)
report_fname_prefix = os.path.join(reports_dir, tool_name + "-report")
default_cmd = " ".join(cmd_map_list) % dict(
src=src,
reports_dir=reports_dir,
report_fname_prefix=report_fname_prefix,
type=type_str,
)
# Try to detect if the output could be json
outext = ".out"
if default_cmd.find("json") > -1:
outext = ".json"
if default_cmd.find("csv") > -1:
outext = ".csv"
if default_cmd.find("sarif") > -1:
outext = ".sarif"
report_fname = report_fname_prefix + outext
# If the command doesn't support file output then redirect stdout automatically
stdout = None
if reports_dir and default_cmd.find(report_fname_prefix) == -1:
report_fname = report_fname_prefix + outext
stdout = io.open(report_fname, "w")
LOG.debug("Output will be written to {}".format(report_fname))
# If the command is requesting list of files then construct the argument
filelist_prefix = "(filelist="
if default_cmd.find(filelist_prefix) > -1:
si = default_cmd.find(filelist_prefix)
ei = default_cmd.find(")", si + 10)
ext = default_cmd[si + 10:ei]
filelist = utils.find_files(src, ext)
default_cmd = default_cmd.replace(filelist_prefix + ext + ")",
" ".join(filelist))
cmd_with_args = default_cmd.split(" ")
exec_tool(cmd_with_args, cwd=src, stdout=stdout)
# Should we attempt to convert the report to sarif format
if (convert and config.tool_purpose_message.get(cmd_with_args[0])
and os.path.isfile(report_fname)):
crep_fname = utils.get_report_file(tool_name,
reports_dir,
convert,
ext_name="sarif")
convertLib.convert_file(
cmd_with_args[0],
cmd_with_args[1:],
src,
report_fname,
crep_fname,
)
def execute_default_cmd( # scan:ignore
cmd_map_list,
type_str,
tool_name,
src,
reports_dir,
convert,
scan_mode,
repo_context,
):
"""
Method to execute default command for the given type
Args:
cmd_map_list Default commands in the form of a dict (multiple) or list
type_str Project type
tool_name Tool name
src Project dir
reports_dir Directory for output reports
convert Boolean to enable normalisation of reports json
scan_mode Scan mode string
repo_context Repo context
"""
# Check if there is a default command specified for the given type
# Create the reports dir
report_fname_prefix = os.path.join(reports_dir, tool_name + "-report")
# Look for any additional direct arguments for the tool and inject them
if config.get(tool_name + "_direct_args"):
direct_args = config.get(tool_name + "_direct_args").split(" ")
if direct_args:
cmd_map_list += direct_args
src_or_file = src
if config.get("SHIFTLEFT_ANALYZE_FILE"):
src_or_file = config.get("SHIFTLEFT_ANALYZE_FILE")
default_cmd = " ".join(cmd_map_list) % dict(
src=src,
src_or_file=src_or_file,
reports_dir=reports_dir,
report_fname_prefix=report_fname_prefix,
type=type_str,
scan_mode=scan_mode,
)
# Try to detect if the output could be json
outext = ".out"
if "json" in default_cmd:
outext = ".json"
elif "csv" in default_cmd:
outext = ".csv"
elif "sarif" in default_cmd:
outext = ".sarif"
elif "xml" in default_cmd:
outext = ".xml"
report_fname = report_fname_prefix + outext
# If the command doesn't support file output then redirect stdout automatically
stdout = None
if LOG.isEnabledFor(DEBUG):
stdout = None
if reports_dir and report_fname_prefix not in default_cmd:
report_fname = report_fname_prefix + outext
stdout = io.open(report_fname, "w")
LOG.debug("Output will be written to {}".format(report_fname))
# If the command is requesting list of files then construct the argument
filelist_prefix = "(filelist="
if default_cmd.find(filelist_prefix) > -1:
si = default_cmd.find(filelist_prefix)
ei = default_cmd.find(")", si + 10)
ext = default_cmd[si + 10:ei]
filelist = utils.find_files(src, ext)
# Temporary fix for the yaml issue
if ext == "yaml":
yml_list = utils.find_files(src, "yml")
if yml_list:
filelist.extend(yml_list)
delim = " "
default_cmd = default_cmd.replace(filelist_prefix + ext + ")",
delim.join(filelist))
cmd_with_args = default_cmd.split(" ")
# Suppress psalm output
if should_suppress_output(type_str, cmd_with_args[0]):
stdout = subprocess.DEVNULL
exec_tool(tool_name, cmd_with_args, cwd=src, stdout=stdout)
# Should we attempt to convert the report to sarif format
if should_convert(convert, tool_name, cmd_with_args[0], report_fname):
crep_fname = utils.get_report_file(tool_name,
reports_dir,
convert,
ext_name="sarif")
if (cmd_with_args[0] == "java" or "pmd-bin" in cmd_with_args[0]
or "php" in tool_name):
convertLib.convert_file(
tool_name,
cmd_with_args,
src,
report_fname,
crep_fname,
)
else:
convertLib.convert_file(
cmd_with_args[0],
cmd_with_args[1:],
src,
report_fname,
crep_fname,
)
try:
if not LOG.isEnabledFor(DEBUG):
os.remove(report_fname)
except Exception:
LOG.debug("Unable to remove file {}".format(report_fname))
elif type_str == "depscan":
# Convert depscan and license scan files to html
depscan_files = utils.find_files(reports_dir, "depscan", True)
for df in depscan_files:
if not df.endswith(".html"):
depscan_data = grafeas.parse(df)
if depscan_data and len(depscan_data):
html_fname = df.replace(".json", ".html")
grafeas.render_html(depscan_data, html_fname)
track({
"id": config.get("run_uuid"),
"depscan_summary": depscan_data
})
LOG.debug(
"Depscan and HTML report written to file: %s, %s :thumbsup:",
df,
html_fname,
)
licence_files = utils.find_files(reports_dir, "license", True)
for lf in licence_files:
if not lf.endswith(".html"):
licence_data = licence.parse(lf)
if licence_data and len(licence_data):
html_fname = lf.replace(".json", ".html")
licence.render_html(licence_data, html_fname)
track({
"id": config.get("run_uuid"),
"license_summary": licence_data
})
LOG.debug(
"License check and HTML report written to file: %s, %s :thumbsup:",
lf,
html_fname,
)
def execute_default_cmd(
cmd_map_list,
type_str,
tool_name,
src,
reports_dir,
convert,
scan_mode,
repo_context,
):
"""
Method to execute default command for the given type
Args:
cmd_map_list Default commands in the form of a dict (multiple) or list
type_str Project type
tool_name Tool name
src Project dir
reports_dir Directory for output reports
convert Boolean to enable normalisation of reports json
scan_mode Scan mode string
repo_context Repo context
"""
# Check if there is a default command specified for the given type
# Create the reports dir
os.makedirs(reports_dir, exist_ok=True)
report_fname_prefix = os.path.join(reports_dir, tool_name + "-report")
default_cmd = " ".join(cmd_map_list) % dict(
src=src,
reports_dir=reports_dir,
report_fname_prefix=report_fname_prefix,
type=type_str,
scan_mode=scan_mode,
)
# Try to detect if the output could be json
outext = ".out"
if default_cmd.find("json") > -1:
outext = ".json"
if default_cmd.find("csv") > -1:
outext = ".csv"
if default_cmd.find("sarif") > -1:
outext = ".sarif"
report_fname = report_fname_prefix + outext
# If the command doesn't support file output then redirect stdout automatically
stdout = None
if reports_dir and default_cmd.find(report_fname_prefix) == -1:
report_fname = report_fname_prefix + outext
stdout = io.open(report_fname, "w")
LOG.debug("Output will be written to {}".format(report_fname))
# If the command is requesting list of files then construct the argument
filelist_prefix = "(filelist="
if default_cmd.find(filelist_prefix) > -1:
si = default_cmd.find(filelist_prefix)
ei = default_cmd.find(")", si + 10)
ext = default_cmd[si + 10 : ei]
filelist = utils.find_files(src, ext)
delim = " "
default_cmd = default_cmd.replace(
filelist_prefix + ext + ")", delim.join(filelist)
)
cmd_with_args = default_cmd.split(" ")
exec_tool(cmd_with_args, cwd=src, stdout=stdout)
# Should we attempt to convert the report to sarif format
if (
convert
and config.tool_purpose_message.get(cmd_with_args[0])
and os.path.isfile(report_fname)
):
crep_fname = utils.get_report_file(
tool_name, reports_dir, convert, ext_name="sarif"
)
convertLib.convert_file(
cmd_with_args[0], cmd_with_args[1:], src, report_fname, crep_fname,
)
try:
if not os.environ.get("SCAN_DEBUG_MODE") == "debug":
os.remove(report_fname)
except Exception:
LOG.debug("Unable to remove file {}".format(report_fname))
elif type_str == "depscan":
# Convert depscan and license scan files to html
depscan_files = utils.find_files(reports_dir, "depscan", True)
for df in depscan_files:
if not df.endswith(".html"):
depscan_data = grafeas.parse(df)
if depscan_data and len(depscan_data):
html_fname = df.replace(".json", ".html")
grafeas.render_html(depscan_data, html_fname)
track(
{"id": config.get("run_uuid"), "depscan_summary": depscan_data}
)
LOG.debug(
"Depscan and HTML report written to file: %s, %s 👍",
df,
html_fname,
)
licence_files = utils.find_files(reports_dir, "license", True)
for lf in licence_files:
if not lf.endswith(".html"):
licence_data = licence.parse(lf)
if licence_data and len(licence_data):
html_fname = lf.replace(".json", ".html")
licence.render_html(licence_data, html_fname)
track(
{"id": config.get("run_uuid"), "license_summary": licence_data}
)
LOG.debug(
"License check and HTML report written to file: %s, %s 👍",
lf,
html_fname,
)
def inspect_scan(language, src, reports_dir, convert, repo_context):
"""
Method to perform inspect cloud scan
Args:
language Project language
src Project dir
reports_dir Directory for output reports
convert Boolean to enable normalisation of reports json
repo_context Repo context
"""
report_fname = utils.get_report_file("inspect",
reports_dir,
convert,
ext_name="json")
sl_cmd = config.get("SHIFTLEFT_INSPECT_CMD")
analyze_target_dir = config.get("SHIFTLEFT_ANALYZE_DIR",
os.path.join(src, "target"))
analyze_files = config.get("SHIFTLEFT_ANALYZE_FILE")
if not analyze_files:
if language == "java":
analyze_files = utils.find_java_artifacts(analyze_target_dir)
app_name = config.get("SHIFTLEFT_APP", repo_context.get("repositoryName"))
no_cpg = config.get("SHIFTLEFT_NO_CPG")
if not app_name:
app_name = os.path.dirname(src)
branch = repo_context.get("revisionId")
if not branch:
branch = "master"
if not analyze_files:
LOG.warning(
"Unable to find any build artifacts in {}. Run mvn package or a similar command before invoking inspect scan"
.format(analyze_target_dir))
return
if len(analyze_files) > 1:
LOG.warning(
"Multiple jar files found in {}. Only {} will be analyzed".format(
analyze_target_dir, analyze_files[0]))
sl_args = [
sl_cmd,
"analyze",
"--no-auto-update",
"--wait",
"--cpg" if not no_cpg else "",
"--java",
"--tag",
"branch=" + branch,
"--app",
app_name,
]
sl_args += [analyze_files[0]]
env = os.environ.copy()
env["JAVA_HOME"] = os.environ.get("JAVA_8_HOME")
LOG.info(
"About to perform ShiftLeft Inspect cloud analysis. This might take a few minutes ..."
)
cp = exec_tool(sl_args, src, env=env)
if cp.returncode != 0:
LOG.warning("Inspect cloud analyze has failed with the below logs")
LOG.info(cp.stdout)
LOG.info(cp.stderr)
return
findings_data = fetch_findings(app_name, branch, report_fname)
if findings_data and convert:
crep_fname = utils.get_report_file("inspect",
reports_dir,
convert,
ext_name="sarif")
convertLib.convert_file("inspect", sl_args[1:], src, report_fname,
crep_fname)
