def authenticate():
"""
Method to authenticate with ShiftLeft NG SAST cloud when the required tokens gets passed via
environment variables
"""
if is_authenticated():
return
sl_org = config.get("SHIFTLEFT_ORG_ID", config.get("SHIFTLEFT_ORGANIZATION_ID"))
sl_token = config.get("SHIFTLEFT_ACCESS_TOKEN")
sl_cmd = config.get("SHIFTLEFT_NGSAST_CMD")
run_uuid = config.get("run_uuid")
if sl_org and sl_token and sl_cmd and utils.check_command(sl_cmd):
inspect_login_args = [
sl_cmd,
"auth",
"--no-auto-update",
"--no-diagnostic",
"--org",
sl_org,
"--token",
sl_token,
]
cp = exec_tool("NG SAST", inspect_login_args)
if cp.returncode != 0:
LOG.warning(
"ShiftLeft NG SAST authentication has failed. Please check the credentials"
)
else:
LOG.info("Successfully authenticated with NG SAST cloud")
track({"id": run_uuid, "scan_mode": "ng-sast", "sl_org": sl_org})
def java_build(src, reports_dir, lang_tools):
"""
Automatically build java project
:param src: Source directory
:param reports_dir: Reports directory to store any logs
:param lang_tools: Language specific build tools
:return: boolean status from the build. True if the command executed successfully. False otherwise
"""
cmd_args = []
pom_files = [p.as_posix() for p in Path(src).glob("pom.xml")]
env = os.environ.copy()
if os.environ.get("USE_JAVA_8") or os.environ.get("WITH_JAVA_8"):
env["SCAN_JAVA_HOME"] = os.environ.get("SCAN_JAVA_8_HOME")
else:
env["SCAN_JAVA_HOME"] = os.environ.get("SCAN_JAVA_11_HOME")
if pom_files:
cmd_args = lang_tools.get("maven")
else:
gradle_files = [p.as_posix() for p in Path(src).glob("build.gradle")]
if gradle_files:
cmd_args = lang_tools.get("gradle")
if not cmd_args:
LOG.info(
"Java auto build is supported only for maven or gradle based projects"
)
return False
cp = exec_tool(cmd_args, src, env=env, stdout=subprocess.PIPE)
LOG.debug(cp.stdout)
return cp.returncode == 0
def convert_file(
tool_name,
tool_args,
working_dir,
report_file,
converted_file,
file_path_list=None,
):
"""Convert report file
:param tool_name: tool name
:param tool_args: tool args
:param working_dir: Working directory
:param report_file: Report file
:param converted_file: Converted file
:param file_path_list: Full file path for any manipulation
:return serialized_log: SARIF output data
"""
issues, metrics, skips = extract_from_file(
tool_name, tool_args, working_dir, report_file, file_path_list
)
issues, suppress_list = suppress_issues(issues)
if suppress_list:
LOG.info(f"Suppressed {len(suppress_list)} issues")
return report(
tool_name,
tool_args,
working_dir,
metrics,
skips,
issues,
converted_file,
file_path_list,
)
def exec_tool(args, cwd=None, env=os.environ.copy(), stdout=subprocess.DEVNULL):
"""
Convenience method to invoke cli tools
Args:
args cli command and args
cwd Current working directory
env Environment variables
stdout stdout configuration for run command
Returns:
CompletedProcess instance
"""
try:
env = use_java(env)
LOG.info("=" * 80)
LOG.debug('⚡︎ Executing "{}"'.format(" ".join(args)))
cp = subprocess.run(
args,
stdout=stdout,
stderr=subprocess.STDOUT,
cwd=cwd,
env=env,
check=False,
shell=False,
encoding="utf-8",
)
return cp
except Exception as e:
LOG.error(e)
return None
def nodejs_build(src, reports_dir, lang_tools):
"""
Automatically build nodejs project
:param src: Source directory
:param reports_dir: Reports directory to store any logs
:param lang_tools: Language specific build tools
:return: boolean status from the build. True if the command executed successfully. False otherwise
"""
cmd_args = lang_tools.get("npm")
yarn_mode = False
pjson_files = [p.as_posix() for p in Path(src).glob("package.json")]
ylock_files = [p.as_posix() for p in Path(src).glob("yarn.lock")]
if ylock_files:
cmd_args = lang_tools.get("yarn")
yarn_mode = True
elif not pjson_files:
LOG.info(
"Nodejs auto build is supported only for npm or yarn based projects"
)
return False
cp = exec_tool(cmd_args, src)
LOG.debug(cp.stdout)
ret = cp.returncode == 0
try:
cmd_args = ["npm"]
if yarn_mode:
cmd_args = ["yarn"]
cmd_args += ["run", "build"]
exec_tool(cmd_args, src)
except Exception:
LOG.debug("Automatic build has failed for the node.js project")
return ret
def java_build(src, reports_dir, lang_tools):
"""
Automatically build java project
:param src: Source directory
:param reports_dir: Reports directory to store any logs
:param lang_tools: Language specific build tools
:return: boolean status from the build. True if the command executed successfully. False otherwise
"""
cmd_args = []
pom_files = [p.as_posix() for p in Path(src).rglob("pom.xml")]
gradle_files = [p.as_posix() for p in Path(src).rglob("build.gradle")]
sbt_files = [p.as_posix() for p in Path(src).rglob("build.sbt")]
env = get_env()
if pom_files:
cmd_args = lang_tools.get("maven")
elif gradle_files:
cmd_args = get_gradle_cmd(src, lang_tools.get("gradle"))
elif sbt_files:
cmd_args = lang_tools.get("sbt")
if not cmd_args:
LOG.info(
"Java auto build is supported only for maven or gradle based projects"
)
return False
cp = exec_tool("auto-build",
cmd_args,
src,
env=env,
stdout=subprocess.PIPE)
if cp:
LOG.debug(cp.stdout)
return cp.returncode == 0
return False
def android_build(src, reports_dir, lang_tools):
"""
Automatically build android project
:param src: Source directory
:param reports_dir: Reports directory to store any logs
:param lang_tools: Language specific build tools
:return: boolean status from the build. True if the command executed successfully. False otherwise
"""
if not os.getenv("ANDROID_SDK_ROOT") and not os.getenv("ANDROID_HOME"):
LOG.info(
"ANDROID_SDK_ROOT or ANDROID_HOME should be set for automatically building android projects"
)
return False
lang_tools = build_tools_map.get("android")
env = get_env()
gradle_files = [p.as_posix() for p in Path(src).rglob("build.gradle")]
gradle_kts_files = [
p.as_posix() for p in Path(src).rglob("build.gradle.kts")
]
if gradle_files or gradle_kts_files:
cmd_args = get_gradle_cmd(src, lang_tools.get("gradle"))
cp = exec_tool("auto-build",
cmd_args,
src,
env=env,
stdout=subprocess.PIPE)
if cp:
LOG.debug(cp.stdout)
return cp.returncode == 0
return False
def read_algorithm_config(config_file):
"""Read config file and return a list of values"""
LOG.info("Reading config file ...")
config_data = ()
with open(config_file, "r") as f:
data = f.read()
out_re = data.replace("\r", "").replace(" ", "")
out_ind = out_re.split('\n')
config_data = (out_ind[0].split(':')[1], out_ind[1].split(':')[1],
out_ind[2].split(':')[1], out_ind[3].split(':')[1])
# config_data (Population_Range, Termination, Adaptive_Mutation_Step,
# Survivor_Selection)
LOG.info("{0}".format(config_data))
return config_data
def inspect_scan(language, src, reports_dir, convert, repo_context):
"""
Method to perform inspect cloud scan
Args:
language Project language
src Project dir
reports_dir Directory for output reports
convert Boolean to enable normalisation of reports json
repo_context Repo context
"""
run_uuid = config.get("run_uuid")
cpg_mode = config.get("SHIFTLEFT_CPG")
env = os.environ.copy()
env["SCAN_JAVA_HOME"] = os.environ.get("SCAN_JAVA_8_HOME")
report_fname = utils.get_report_file(
"ng-sast", reports_dir, convert, ext_name="json"
)
sl_cmd = config.get("SHIFTLEFT_NGSAST_CMD")
# Check if sl cli is available
if not utils.check_command(sl_cmd):
LOG.warning(
"sl cli is not available. Please check if your build uses shiftleft/scan-java as the image"
)
return
analyze_files = config.get("SHIFTLEFT_ANALYZE_FILE")
analyze_target_dir = config.get(
"SHIFTLEFT_ANALYZE_DIR", os.path.join(src, "target")
)
extra_args = None
if not analyze_files:
if language == "java":
analyze_files = utils.find_java_artifacts(analyze_target_dir)
elif language == "csharp":
if not utils.check_dotnet():
LOG.warning(
"dotnet is not available. Please check if your build uses shiftleft/scan-csharp as the image"
)
return
analyze_files = utils.find_csharp_artifacts(src)
cpg_mode = True
else:
if language == "ts" or language == "nodejs":
language = "js"
extra_args = ["--", "--ts", "--babel"]
analyze_files = [src]
cpg_mode = True
app_name = find_app_name(src, repo_context)
branch = repo_context.get("revisionId")
if not branch:
branch = "master"
if not analyze_files:
LOG.warning(
"Unable to find any build artifacts. Compile your project first before invoking scan or use the auto build feature."
)
return
if isinstance(analyze_files, list) and len(analyze_files) > 1:
LOG.warning(
"Multiple files found in {}. Only {} will be analyzed".format(
analyze_target_dir, analyze_files[0]
)
)
analyze_files = analyze_files[0]
sl_args = [
sl_cmd,
"analyze",
"--no-auto-update" if language == "java" else None,
"--wait",
"--cpg" if cpg_mode else None,
"--" + language,
"--tag",
"branch=" + branch,
"--app",
app_name,
]
sl_args += [analyze_files]
if extra_args:
sl_args += extra_args
sl_args = [arg for arg in sl_args if arg is not None]
LOG.info(
"About to perform ShiftLeft NG SAST cloud analysis. This might take a few minutes ..."
)
LOG.debug(" ".join(sl_args))
LOG.debug(repo_context)
cp = exec_tool("NG SAST", sl_args, src, env=env)
if cp.returncode != 0:
LOG.warning("NG SAST cloud analyze has failed with the below logs")
LOG.debug(sl_args)
LOG.info(cp.stderr)
return
findings_data = fetch_findings(app_name, branch, report_fname)
if findings_data and convert:
crep_fname = utils.get_report_file(
"ng-sast", reports_dir, convert, ext_name="sarif"
)
convertLib.convert_file("ng-sast", sl_args[1:], src, report_fname, crep_fname)
track({"id": run_uuid, "scan_mode": "ng-sast", "sl_args": sl_args})
def exec_tool( # scan:ignore
tool_name,
args,
cwd=None,
env=utils.get_env(),
stdout=subprocess.DEVNULL):
"""
Convenience method to invoke cli tools
Args:
tool_name Tool name
args cli command and args
cwd Current working directory
env Environment variables
stdout stdout configuration for run command
Returns:
CompletedProcess instance
"""
with Progress(
console=console,
redirect_stderr=False,
redirect_stdout=False,
refresh_per_second=1,
) as progress:
task = None
try:
env = use_java(env)
LOG.debug('⚡︎ Executing {} "{}"'.format(tool_name, " ".join(args)))
stderr = subprocess.DEVNULL
if LOG.isEnabledFor(DEBUG):
stderr = subprocess.STDOUT
tool_verb = "Scanning with"
if "init" in tool_name:
tool_verb = "Initializing"
elif "build" in tool_name:
tool_verb = "Building with"
task = progress.add_task("[green]" + tool_verb + " " + tool_name,
total=100,
start=False)
cp = subprocess.run(
args,
stdout=stdout,
stderr=stderr,
cwd=cwd,
env=env,
check=False,
shell=False,
encoding="utf-8",
)
if cp and stdout == subprocess.PIPE:
for line in cp.stdout:
progress.update(task, completed=5)
if (cp and LOG.isEnabledFor(DEBUG) and cp.returncode
and cp.stdout is not None):
LOG.debug(cp.stdout)
progress.update(task, completed=100, total=100)
return cp
except Exception as e:
if task:
progress.update(task, completed=20, total=10, visible=False)
if not LOG.isEnabledFor(DEBUG):
LOG.info(
f"{tool_name} has reported few errors. To view, pass the environment variable SCAN_DEBUG_MODE=debug"
)
LOG.debug(e)
return None
def annotate_pr(self, repo_context, findings_file, report_summary,
build_status):
if not findings_file:
return
with open(findings_file, mode="r") as fp:
try:
gitlab_context = self.get_context(repo_context)
findings_obj = json.load(fp)
findings = findings_obj.get("findings")
if not findings:
LOG.debug("No findings from scan available to report")
return
if not gitlab_context.get(
"mergeRequestIID") or not gitlab_context.get(
"mergeRequestProjectId"):
LOG.debug(
"Scan is not running as part of a merge request. Check if the pipeline is using only: [merge_requests] or rules syntax"
)
return
private_token = self.get_token()
if not private_token:
LOG.info(
"To create a merge request note, create a personal access token with api scope and set it as GITLAB_TOKEN environment variable"
)
return
summary = "| Tool | Critical | High | Medium | Low | Status |\n"
summary = (
summary +
"| ---- | ------- | ------ | ----- | ---- | ---- |\n")
for rk, rv in report_summary.items():
summary = f'{summary}| {rv.get("tool")} | {rv.get("critical")} | {rv.get("high")} | {rv.get("medium")} | {rv.get("low")} | {rv.get("status")} |\n'
template = config.get("PR_COMMENT_TEMPLATE")
recommendation = (
f"Please review the [scan reports]({gitlab_context.get('jobUrl')}/artifacts/browse/reports) before approving this merge request."
if build_status == "fail" else "Looks good")
apiUrl = (f"{gitlab_context.get('apiUrl')}")
mergeRequestIID = (f"{gitlab_context.get('mergeRequestIID')}")
mergeRequestProjectId = (
f"{gitlab_context.get('mergeRequestProjectId')}")
mergeRequestSourceBranch = (
f"{gitlab_context.get('mergeRequestSourceBranch')}")
mergeRequestTargetBranch = (
f"{gitlab_context.get('mergeRequestTargetBranch')}")
commitSHA = (f"{gitlab_context.get('commitSHA')}")
projectId = (f"{gitlab_context.get('projectId')}")
projectName = (f"{gitlab_context.get('projectName')}")
projectUrl = (f"{gitlab_context.get('projectUrl')}")
jobUrl = (f"{gitlab_context.get('jobUrl')}")
jobId = (f"{gitlab_context.get('jobId')}")
jobName = (f"{gitlab_context.get('jobName')}")
jobToken = (f"{gitlab_context.get('jobToken')}")
body = template % dict(
summary=summary,
recommendation=recommendation,
apiUrl=apiUrl,
mergeRequestIID=mergeRequestIID,
mergeRequestProjectId=mergeRequestProjectId,
mergeRequestSourceBranch=mergeRequestSourceBranch,
mergeRequestTargetBranch=mergeRequestTargetBranch,
commitSHA=commitSHA,
projectId=projectId,
projectName=projectName,
projectUrl=projectUrl,
jobUrl=jobUrl,
jobId=jobId,
jobName=jobName,
jobToken=jobToken)
rr = requests.post(
self.get_mr_notes_url(repo_context),
headers={
"Content-Type": "application/json",
"PRIVATE-TOKEN": self.get_token(),
},
json={"body": body},
)
if not rr.ok:
LOG.debug(rr.json())
except Exception as e:
LOG.debug(e)
def summary(
sarif_files,
depscan_files=None,
aggregate_file=None,
override_rules={},
baseline_file=None,
):
"""Generate overall scan summary based on the generated
SARIF file
:param sarif_files: List of generated sarif report files
:param depscan_files: Depscan result files
:param aggregate_file: Filename to store aggregate data
:param override_rules Build break rules to override for testing
:param baseline_file: Scan baseline file
:returns dict representing the summary
"""
report_summary = {}
baseline_fingerprints = {
"scanPrimaryLocationHash": [],
"scanTagsHash": [],
}
build_status = "pass"
# This is the list of all runs which will get stored as an aggregate
run_data_list = []
default_rules = config.get("build_break_rules").get("default")
depscan_default_rules = config.get("build_break_rules").get("depscan")
# Collect stats from depscan files if available
if depscan_files:
for df in depscan_files:
with open(df, mode="r") as drep_file:
dep_data = get_depscan_data(drep_file)
if not dep_data:
continue
# depscan-java or depscan-nodejs based on filename
dep_type = (os.path.basename(df).replace(".json", "").replace(
"-report", ""))
metrics, required_pkgs_found = calculate_depscan_metrics(
dep_data)
report_summary[dep_type] = {
"tool":
f"""Dependency Scan ({dep_type.replace("depscan-", "")})""",
"critical": metrics["critical"],
"high": metrics["high"],
"medium": metrics["medium"],
"low": metrics["low"],
"status": ":white_heavy_check_mark:",
}
report_summary[dep_type].pop("total", None)
# Compare against the build break rule to determine status
dep_tool_rules = config.get("build_break_rules").get(
dep_type, {})
build_break_rules = {**depscan_default_rules, **dep_tool_rules}
if override_rules and override_rules.get("depscan"):
build_break_rules = {
**build_break_rules,
**override_rules.get("depscan"),
}
# Default severity categories for build status
build_status_categories = (
"critical",
"required_critical",
"optional_critical",
"high",
"required_high",
"optional_high",
"medium",
"required_medium",
"optional_medium",
"low",
"required_low",
"optional_low",
)
# Issue 233 - Consider only required packages if available
if required_pkgs_found:
build_status_categories = (
"required_critical",
"required_high",
"required_medium",
"required_low",
)
for rsev in build_status_categories:
if build_break_rules.get("max_" + rsev) is not None:
if metrics.get(rsev) > build_break_rules["max_" +
rsev]:
report_summary[dep_type]["status"] = ":cross_mark:"
build_status = "fail"
for sf in sarif_files:
with open(sf, mode="r") as report_file:
report_data = json.load(report_file)
# skip this file if the data is empty
if not report_data or not report_data.get("runs"):
LOG.warn("Report file {} is invalid. Skipping ...".format(sf))
continue
# Iterate through all the runs
for run in report_data["runs"]:
# Add it to the run data list for aggregation
run_data_list.append(run)
tool_desc = run["tool"]["driver"]["name"]
tool_name = tool_desc
# Initialise
report_summary[tool_name] = {
"tool": tool_desc,
"critical": 0,
"high": 0,
"medium": 0,
"low": 0,
"status": ":white_heavy_check_mark:",
}
results = run.get("results", [])
metrics = run.get("properties", {}).get("metrics", None)
# If the result includes metrics use it. If not compute it
if metrics:
report_summary[tool_name].update(metrics)
report_summary[tool_name].pop("total", None)
for aresult in results:
if not metrics:
if aresult.get("properties"):
sev = aresult["properties"][
"issue_severity"].lower()
else:
sev = config.get("exttool_default_severity").get(
tool_name.lower(), "medium")
report_summary[tool_name][sev] += 1
# Track the fingerprints
if aresult.get("partialFingerprints"):
result_fingerprints = aresult.get(
"partialFingerprints")
for rfk, rfv in result_fingerprints.items():
if not rfv:
continue
# We are only interested in a small subset of hashes namely scanPrimaryLocationHash, scanTagsHash
if rfk in [
"scanPrimaryLocationHash", "scanTagsHash"
]:
baseline_fingerprints.setdefault(
rfk, []).append(rfv)
# Compare against the build break rule to determine status
tool_rules = config.get("build_break_rules").get(tool_name, {})
build_break_rules = {
**default_rules,
**tool_rules,
**override_rules
}
for rsev in ("critical", "high", "medium", "low"):
if build_break_rules.get("max_" + rsev) is not None:
if (report_summary.get(tool_name).get(rsev) >
build_break_rules["max_" + rsev]):
report_summary[tool_name][
"status"] = ":cross_mark:"
build_status = "fail"
# Should we store the aggregate data
if aggregate_file:
# agg_sarif_file = aggregate_file.replace(".json", ".sarif")
# aggregate.sarif_aggregate(run_data_list, agg_sarif_file)
aggregate.jsonl_aggregate(run_data_list, aggregate_file)
LOG.debug("Aggregate report written to {}\n".format(aggregate_file))
if baseline_file:
aggregate.store_baseline(baseline_fingerprints, baseline_file)
LOG.info("Baseline file written to {}".format(baseline_file))
return report_summary, build_status
parser.add_option("-g", "--ga-conf", dest="ga_conf",
action="store", help="Configuration file of the" \
"genetic algorithm")
parser.add_option("-e", "--es-conf", dest="es_conf", action="store",
help="Configuration file of the evolutionary strategy")
opts = parser.parse_args()[0]
try:
if len(sys.argv) < 2:
raise OptsError("Missing arguments")
if opts.ga_conf:
algorithm_name = 'GA behavior'
LOG.info("Starting GA")
pop_size, term, ad_mut_stp, mu_lambda = read_algorithm_config(opts.ga_conf)
search_ga(int(term), int(pop_size), ast.literal_eval(ad_mut_stp),
ast.literal_eval((mu_lambda)))
plot_data(algorithm_name, LOG_NAME)
LOG.info("Finish GA")
if opts.es_conf:
algorithm_name = 'ES behavior'
LOG.info("Starting ES")
pop_range, term, ad_mut_stp, mu_lambda = read_algorithm_config(opts.es_conf)
search_es(int(term), int(pop_range), ast.literal_eval(ad_mut_stp),
ast.literal_eval((mu_lambda)))
plot_data(algorithm_name, LOG_NAME)
LOG.info("Finish ES")