def test_pmd_parse():
with open(
os.path.join(
os.path.dirname(os.path.realpath(__file__)), "data", "pmd-report.csv",
)
) as rf:
headers, report_data = csv_parser.get_report_data(rf)
assert len(headers) == 8
assert len(report_data) == 2
def extract_from_file(tool_name, report_file, file_path_list=None):
"""Extract properties from reports
:param tool_name: tool name
:param report_file: Report file
:param file_path_list: Full file path for any manipulation
:return issues, metrics, skips information
"""
issues = []
metrics = None
skips = []
extn = pathlib.PurePosixPath(report_file).suffix
with io.open(report_file, "r") as rfile:
# Static check use jsonlines format, duh
if tool_name == "staticcheck":
contents = rfile.read()
issues = [
json.loads(str(item)) for item in contents.strip().split("\n")
]
return issues, metrics, skips
if extn == ".json":
try:
report_data = json.loads(rfile.read())
except json.decoder.JSONDecodeError:
return issues, metrics, skips
if isinstance(report_data, list):
issues = report_data
else:
# NodeJsScan uses sec_issues
if "sec_issues" in report_data:
sec_data = report_data["sec_issues"]
for key, value in sec_data.items():
if isinstance(value, list):
issues = issues + value
else:
issues.append(value)
if "Issues" in report_data or "results" in report_data:
for issue in report_data.get(
"Issues", report_data.get("results", [])):
issues.append(issue)
if extn == ".csv":
headers, issues = csv_parser.get_report_data(rfile)
if extn == ".xml":
issues, metrics = xml_parser.get_report_data(rfile, file_path_list)
return issues, metrics, skips
def extract_from_file(tool_name, report_file, file_path_list=None):
"""Extract properties from reports
:param tool_name: tool name
:param report_file: Report file
:param file_path_list: Full file path for any manipulation
:return issues, metrics, skips information
"""
issues = []
metrics = None
skips = []
extn = pathlib.PurePosixPath(report_file).suffix
with io.open(report_file, "r") as rfile:
if extn == ".json":
report_data = json.loads(rfile.read())
if isinstance(report_data, list):
issues = report_data
else:
# NodeJsScan uses sec_issues
if "sec_issues" in report_data:
sec_data = report_data["sec_issues"]
for key, value in sec_data.items():
if isinstance(value, list):
issues = issues + value
else:
issues.append(value)
if "Issues" in report_data or "results" in report_data:
for issue in report_data.get(
"Issues", report_data.get("results", [])):
issues.append(issue)
if extn == ".csv":
headers, issues = csv_parser.get_report_data(rfile)
if extn == ".xml":
issues, metrics = xml_parser.get_report_data(rfile, file_path_list)
return issues, metrics, skips
def extract_from_file(
tool_name, tool_args, working_dir, report_file, file_path_list=None
):
"""Extract properties from reports
:param tool_name: tool name
:param tool_args: tool args
:param working_dir: Working directory
:param report_file: Report file
:param file_path_list: Full file path for any manipulation
:return issues, metrics, skips information
"""
issues = []
metrics = None
skips = []
# If the tools did not produce any result do not crash
if not os.path.isfile(report_file):
return issues, metrics, skips
extn = pathlib.PurePosixPath(report_file).suffix
with io.open(report_file, "r") as rfile:
# Static check use jsonlines format, duh
if tool_name == "staticcheck":
contents = rfile.read()
try:
issues = [
json.loads(str(item)) for item in contents.strip().split("\n")
]
except json.decoder.JSONDecodeError:
LOG.warning(
"staticcheck produced no result since the project was not built before analysis!"
)
return issues, metrics, skips
if extn == ".json":
try:
report_data = json.loads(rfile.read())
except json.decoder.JSONDecodeError:
return issues, metrics, skips
# NG SAST (Formerly Inspect) uses vulnerabilities
if tool_name == "ng-sast":
for v in report_data.get("vulnerabilities"):
if not v:
continue
vuln = v["vulnerability"]
location_list = []
if vuln.get("dataFlow") and vuln.get("dataFlow", {}).get(
"dataFlow"
):
location_list = convert_dataflow(
working_dir, tool_args, vuln["dataFlow"]["dataFlow"]["list"]
)
for location in location_list:
issues.append(
{
"rule_id": vuln["category"],
"title": vuln["title"],
"description": vuln["description"],
"score": vuln["score"],
"severity": vuln["severity"],
"line_number": location.get("line_number"),
"filename": location.get("filename"),
"first_found": vuln["firstVersionDetected"],
"issue_confidence": "HIGH",
}
)
elif tool_name == "taint-php":
for entry in report_data:
taint_trace = entry.get("taint_trace")
labels = []
if taint_trace:
source, sink, labels = get_from_taints(taint_trace)
else:
source, _, _ = get_from_taints([entry])
issues.append(
{
"rule_id": entry.get("shortcode"),
"test_name": entry.get("type"),
"description": "{}: {}".format(
entry.get("message"), "\\n".join(labels)
),
"link": entry.get("link"),
"severity": entry.get("severity"),
"issue_confidence": "HIGH",
"line_number": source.get("line_number"),
"filename": source.get("filename"),
}
)
elif tool_name == "taint-python":
taint_list = report_data.get("vulnerabilities")
for taint in taint_list:
source = taint.get("source")
sink = taint.get("sink")
tags = {}
for taint_props in [
"source_trigger_word",
"source_label",
"source_type",
"sink_trigger_word",
"sink_label",
"sink_type",
]:
if taint.get(taint_props):
tags[taint_props] = taint.get(taint_props)
issues.append(
{
"rule_id": taint.get("rule_id"),
"test_name": taint.get("rule_name"),
"short_description": taint.get("short_description"),
"cwe_category": taint.get("cwe_category"),
"owasp_category": taint.get("owasp_category"),
"description": taint.get("description"),
"severity": taint.get("severity"),
"issue_confidence": "HIGH",
"line_from": source.get("line_number"),
"line_to": sink.get("line_number"),
"filename": source.get("path"),
"tags": tags,
}
)
elif tool_name == "phpstan" or tool_name == "source-php":
file_errors = report_data.get("files")
for filename, messageobj in file_errors.items():
messages = messageobj.get("messages")
for msg in messages:
# Create a rule id for phpstan
rule_word = msg.get("message", "").split(" ")[0]
rule_word = "phpstan-" + rule_word.lower()
issues.append(
{
"rule_id": rule_word,
"title": msg.get("message"),
"line_number": msg.get("line"),
"filename": filename,
"severity": "LOW",
"issue_confidence": "MEDIUM",
}
)
elif tool_name == "source-js":
njs_findings = report_data.get("nodejs", {})
njs_findings.update(report_data.get("templates", {}))
for k, v in njs_findings.items():
# Password detection by njsscan is full of false positives
if k == "node_password":
continue
files = v.get("files", [])
metadata = v.get("metadata", {})
if not files or not metadata:
continue
for afile in files:
line_number = 0
if afile.get("match_lines"):
line_number = afile.get("match_lines")[0]
issues.append(
{
"rule_id": metadata.get("owasp")
.replace(":", "-")
.replace(" ", "")
.lower(),
"title": metadata.get("cwe"),
"description": metadata.get("description"),
"severity": metadata.get("severity"),
"line_number": line_number,
"filename": afile.get("file_path"),
"issue_confidence": "HIGH",
}
)
elif tool_name == "checkov":
if isinstance(report_data, list):
for rd in report_data:
issues += rd.get("results", {}).get("failed_checks")
else:
issues = report_data.get("results", {}).get("failed_checks")
elif tool_name == "source-ruby":
issues = report_data.get("warnings", [])
issues += report_data.get("errors", [])
elif isinstance(report_data, list):
issues = report_data
else:
if "sec_issues" in report_data:
# NodeJsScan uses sec_issues
sec_data = report_data["sec_issues"]
for key, value in sec_data.items():
if isinstance(value, list):
issues = issues + value
else:
issues.append(value)
elif "Issues" in report_data:
tmpL = report_data.get("Issues", [])
if tmpL:
issues += tmpL
else:
LOG.debug("%s produced no result" % tool_name)
elif "results" in report_data:
tmpL = report_data.get("results", [])
if tmpL:
issues += tmpL
else:
LOG.debug("%s produced no result" % tool_name)
if extn == ".csv":
headers, issues = csv_parser.get_report_data(rfile)
if extn == ".xml":
issues, metrics = xml_parser.get_report_data(
rfile, file_path_list=file_path_list, working_dir=working_dir
)
return issues, metrics, skips
def extract_from_file(tool_name,
working_dir,
report_file,
file_path_list=None):
"""Extract properties from reports
:param tool_name: tool name
:param working_dir: Working directory
:param report_file: Report file
:param file_path_list: Full file path for any manipulation
:return issues, metrics, skips information
"""
issues = []
metrics = None
skips = []
# If the tools did not produce any result do not crash
if not os.path.isfile(report_file):
return issues, metrics, skips
extn = pathlib.PurePosixPath(report_file).suffix
with io.open(report_file, "r") as rfile:
# Static check use jsonlines format, duh
if tool_name == "staticcheck":
contents = rfile.read()
try:
issues = [
json.loads(str(item))
for item in contents.strip().split("\n")
]
except json.decoder.JSONDecodeError:
LOG.warning(
"staticcheck produced no result since the project was not built before analysis!"
)
return issues, metrics, skips
if extn == ".json":
try:
report_data = json.loads(rfile.read())
except json.decoder.JSONDecodeError:
return issues, metrics, skips
# Inspect uses vulnerabilities
if tool_name == "inspect":
file_name_prefix = ""
for v in report_data.get("vulnerabilities"):
if not v:
continue
vuln = v["vulnerability"]
location = {}
if vuln.get("dataFlow") and vuln.get("dataFlow").get(
"dataFlow"):
location = vuln["dataFlow"]["dataFlow"]["list"][0][
"location"]
fileName = location.get("fileName")
if not file_name_prefix:
file_name_prefix = find_path_prefix(
working_dir, fileName)
issues.append({
"rule_id":
vuln["category"],
"title":
vuln["title"],
"description":
vuln["description"],
"score":
vuln["score"],
"severity":
vuln["severity"],
"line_number":
location.get("lineNumber"),
"filename":
os.path.join(file_name_prefix, fileName),
"first_found":
vuln["firstVersionDetected"],
})
elif isinstance(report_data, list):
issues = report_data
else:
# NodeJsScan uses sec_issues
if "sec_issues" in report_data:
sec_data = report_data["sec_issues"]
for key, value in sec_data.items():
if isinstance(value, list):
issues = issues + value
else:
issues.append(value)
elif "Issues" in report_data:
tmpL = report_data.get("Issues", [])
if tmpL:
issues += tmpL
else:
LOG.warning("%s produced no result" % tool_name)
elif "results" in report_data:
tmpL = report_data.get("results", [])
if tmpL:
issues += tmpL
else:
LOG.warning("%s produced no result" % tool_name)
if extn == ".csv":
headers, issues = csv_parser.get_report_data(rfile)
if extn == ".xml":
issues, metrics = xml_parser.get_report_data(rfile, file_path_list)
return issues, metrics, skips
def extract_from_file(tool_name, report_file, file_path_list=None):
"""Extract properties from reports
:param tool_name: tool name
:param report_file: Report file
:param file_path_list: Full file path for any manipulation
:return issues, metrics, skips information
"""
issues = []
metrics = None
skips = []
# If the tools did not produce any result do not crash
if not os.path.isfile(report_file):
return issues, metrics, skips
extn = pathlib.PurePosixPath(report_file).suffix
with io.open(report_file, "r") as rfile:
# Static check use jsonlines format, duh
if tool_name == "staticcheck":
contents = rfile.read()
try:
issues = [
json.loads(str(item))
for item in contents.strip().split("\n")
]
except json.decoder.JSONDecodeError:
LOG.warning(
"staticcheck produced no result since the project was not built before analysis!"
)
return issues, metrics, skips
if extn == ".json":
try:
report_data = json.loads(rfile.read())
except json.decoder.JSONDecodeError:
return issues, metrics, skips
if isinstance(report_data, list):
issues = report_data
else:
# NodeJsScan uses sec_issues
if "sec_issues" in report_data:
sec_data = report_data["sec_issues"]
for key, value in sec_data.items():
if isinstance(value, list):
issues = issues + value
else:
issues.append(value)
elif "Issues" in report_data:
tmpL = report_data.get("Issues", [])
if tmpL:
issues += tmpL
else:
LOG.warning("%s produced no result" % tool_name)
elif "results" in report_data:
tmpL = report_data.get("results", [])
if tmpL:
issues += tmpL
else:
LOG.warning("%s produced no result" % tool_name)
if extn == ".csv":
headers, issues = csv_parser.get_report_data(rfile)
if extn == ".xml":
issues, metrics = xml_parser.get_report_data(rfile, file_path_list)
return issues, metrics, skips