def test_pmd_parse(): with open( os.path.join( os.path.dirname(os.path.realpath(__file__)), "data", "pmd-report.csv", ) ) as rf: headers, report_data = csv_parser.get_report_data(rf) assert len(headers) == 8 assert len(report_data) == 2
def extract_from_file(tool_name, report_file, file_path_list=None): """Extract properties from reports :param tool_name: tool name :param report_file: Report file :param file_path_list: Full file path for any manipulation :return issues, metrics, skips information """ issues = [] metrics = None skips = [] extn = pathlib.PurePosixPath(report_file).suffix with io.open(report_file, "r") as rfile: # Static check use jsonlines format, duh if tool_name == "staticcheck": contents = rfile.read() issues = [ json.loads(str(item)) for item in contents.strip().split("\n") ] return issues, metrics, skips if extn == ".json": try: report_data = json.loads(rfile.read()) except json.decoder.JSONDecodeError: return issues, metrics, skips if isinstance(report_data, list): issues = report_data else: # NodeJsScan uses sec_issues if "sec_issues" in report_data: sec_data = report_data["sec_issues"] for key, value in sec_data.items(): if isinstance(value, list): issues = issues + value else: issues.append(value) if "Issues" in report_data or "results" in report_data: for issue in report_data.get( "Issues", report_data.get("results", [])): issues.append(issue) if extn == ".csv": headers, issues = csv_parser.get_report_data(rfile) if extn == ".xml": issues, metrics = xml_parser.get_report_data(rfile, file_path_list) return issues, metrics, skips
def extract_from_file(tool_name, report_file, file_path_list=None): """Extract properties from reports :param tool_name: tool name :param report_file: Report file :param file_path_list: Full file path for any manipulation :return issues, metrics, skips information """ issues = [] metrics = None skips = [] extn = pathlib.PurePosixPath(report_file).suffix with io.open(report_file, "r") as rfile: if extn == ".json": report_data = json.loads(rfile.read()) if isinstance(report_data, list): issues = report_data else: # NodeJsScan uses sec_issues if "sec_issues" in report_data: sec_data = report_data["sec_issues"] for key, value in sec_data.items(): if isinstance(value, list): issues = issues + value else: issues.append(value) if "Issues" in report_data or "results" in report_data: for issue in report_data.get( "Issues", report_data.get("results", [])): issues.append(issue) if extn == ".csv": headers, issues = csv_parser.get_report_data(rfile) if extn == ".xml": issues, metrics = xml_parser.get_report_data(rfile, file_path_list) return issues, metrics, skips
def extract_from_file( tool_name, tool_args, working_dir, report_file, file_path_list=None ): """Extract properties from reports :param tool_name: tool name :param tool_args: tool args :param working_dir: Working directory :param report_file: Report file :param file_path_list: Full file path for any manipulation :return issues, metrics, skips information """ issues = [] metrics = None skips = [] # If the tools did not produce any result do not crash if not os.path.isfile(report_file): return issues, metrics, skips extn = pathlib.PurePosixPath(report_file).suffix with io.open(report_file, "r") as rfile: # Static check use jsonlines format, duh if tool_name == "staticcheck": contents = rfile.read() try: issues = [ json.loads(str(item)) for item in contents.strip().split("\n") ] except json.decoder.JSONDecodeError: LOG.warning( "staticcheck produced no result since the project was not built before analysis!" ) return issues, metrics, skips if extn == ".json": try: report_data = json.loads(rfile.read()) except json.decoder.JSONDecodeError: return issues, metrics, skips # NG SAST (Formerly Inspect) uses vulnerabilities if tool_name == "ng-sast": for v in report_data.get("vulnerabilities"): if not v: continue vuln = v["vulnerability"] location_list = [] if vuln.get("dataFlow") and vuln.get("dataFlow", {}).get( "dataFlow" ): location_list = convert_dataflow( working_dir, tool_args, vuln["dataFlow"]["dataFlow"]["list"] ) for location in location_list: issues.append( { "rule_id": vuln["category"], "title": vuln["title"], "description": vuln["description"], "score": vuln["score"], "severity": vuln["severity"], "line_number": location.get("line_number"), "filename": location.get("filename"), "first_found": vuln["firstVersionDetected"], "issue_confidence": "HIGH", } ) elif tool_name == "taint-php": for entry in report_data: taint_trace = entry.get("taint_trace") labels = [] if taint_trace: source, sink, labels = get_from_taints(taint_trace) else: source, _, _ = get_from_taints([entry]) issues.append( { "rule_id": entry.get("shortcode"), "test_name": entry.get("type"), "description": "{}: {}".format( entry.get("message"), "\\n".join(labels) ), "link": entry.get("link"), "severity": entry.get("severity"), "issue_confidence": "HIGH", "line_number": source.get("line_number"), "filename": source.get("filename"), } ) elif tool_name == "taint-python": taint_list = report_data.get("vulnerabilities") for taint in taint_list: source = taint.get("source") sink = taint.get("sink") tags = {} for taint_props in [ "source_trigger_word", "source_label", "source_type", "sink_trigger_word", "sink_label", "sink_type", ]: if taint.get(taint_props): tags[taint_props] = taint.get(taint_props) issues.append( { "rule_id": taint.get("rule_id"), "test_name": taint.get("rule_name"), "short_description": taint.get("short_description"), "cwe_category": taint.get("cwe_category"), "owasp_category": taint.get("owasp_category"), "description": taint.get("description"), "severity": taint.get("severity"), "issue_confidence": "HIGH", "line_from": source.get("line_number"), "line_to": sink.get("line_number"), "filename": source.get("path"), "tags": tags, } ) elif tool_name == "phpstan" or tool_name == "source-php": file_errors = report_data.get("files") for filename, messageobj in file_errors.items(): messages = messageobj.get("messages") for msg in messages: # Create a rule id for phpstan rule_word = msg.get("message", "").split(" ")[0] rule_word = "phpstan-" + rule_word.lower() issues.append( { "rule_id": rule_word, "title": msg.get("message"), "line_number": msg.get("line"), "filename": filename, "severity": "LOW", "issue_confidence": "MEDIUM", } ) elif tool_name == "source-js": njs_findings = report_data.get("nodejs", {}) njs_findings.update(report_data.get("templates", {})) for k, v in njs_findings.items(): # Password detection by njsscan is full of false positives if k == "node_password": continue files = v.get("files", []) metadata = v.get("metadata", {}) if not files or not metadata: continue for afile in files: line_number = 0 if afile.get("match_lines"): line_number = afile.get("match_lines")[0] issues.append( { "rule_id": metadata.get("owasp") .replace(":", "-") .replace(" ", "") .lower(), "title": metadata.get("cwe"), "description": metadata.get("description"), "severity": metadata.get("severity"), "line_number": line_number, "filename": afile.get("file_path"), "issue_confidence": "HIGH", } ) elif tool_name == "checkov": if isinstance(report_data, list): for rd in report_data: issues += rd.get("results", {}).get("failed_checks") else: issues = report_data.get("results", {}).get("failed_checks") elif tool_name == "source-ruby": issues = report_data.get("warnings", []) issues += report_data.get("errors", []) elif isinstance(report_data, list): issues = report_data else: if "sec_issues" in report_data: # NodeJsScan uses sec_issues sec_data = report_data["sec_issues"] for key, value in sec_data.items(): if isinstance(value, list): issues = issues + value else: issues.append(value) elif "Issues" in report_data: tmpL = report_data.get("Issues", []) if tmpL: issues += tmpL else: LOG.debug("%s produced no result" % tool_name) elif "results" in report_data: tmpL = report_data.get("results", []) if tmpL: issues += tmpL else: LOG.debug("%s produced no result" % tool_name) if extn == ".csv": headers, issues = csv_parser.get_report_data(rfile) if extn == ".xml": issues, metrics = xml_parser.get_report_data( rfile, file_path_list=file_path_list, working_dir=working_dir ) return issues, metrics, skips
def extract_from_file(tool_name, working_dir, report_file, file_path_list=None): """Extract properties from reports :param tool_name: tool name :param working_dir: Working directory :param report_file: Report file :param file_path_list: Full file path for any manipulation :return issues, metrics, skips information """ issues = [] metrics = None skips = [] # If the tools did not produce any result do not crash if not os.path.isfile(report_file): return issues, metrics, skips extn = pathlib.PurePosixPath(report_file).suffix with io.open(report_file, "r") as rfile: # Static check use jsonlines format, duh if tool_name == "staticcheck": contents = rfile.read() try: issues = [ json.loads(str(item)) for item in contents.strip().split("\n") ] except json.decoder.JSONDecodeError: LOG.warning( "staticcheck produced no result since the project was not built before analysis!" ) return issues, metrics, skips if extn == ".json": try: report_data = json.loads(rfile.read()) except json.decoder.JSONDecodeError: return issues, metrics, skips # Inspect uses vulnerabilities if tool_name == "inspect": file_name_prefix = "" for v in report_data.get("vulnerabilities"): if not v: continue vuln = v["vulnerability"] location = {} if vuln.get("dataFlow") and vuln.get("dataFlow").get( "dataFlow"): location = vuln["dataFlow"]["dataFlow"]["list"][0][ "location"] fileName = location.get("fileName") if not file_name_prefix: file_name_prefix = find_path_prefix( working_dir, fileName) issues.append({ "rule_id": vuln["category"], "title": vuln["title"], "description": vuln["description"], "score": vuln["score"], "severity": vuln["severity"], "line_number": location.get("lineNumber"), "filename": os.path.join(file_name_prefix, fileName), "first_found": vuln["firstVersionDetected"], }) elif isinstance(report_data, list): issues = report_data else: # NodeJsScan uses sec_issues if "sec_issues" in report_data: sec_data = report_data["sec_issues"] for key, value in sec_data.items(): if isinstance(value, list): issues = issues + value else: issues.append(value) elif "Issues" in report_data: tmpL = report_data.get("Issues", []) if tmpL: issues += tmpL else: LOG.warning("%s produced no result" % tool_name) elif "results" in report_data: tmpL = report_data.get("results", []) if tmpL: issues += tmpL else: LOG.warning("%s produced no result" % tool_name) if extn == ".csv": headers, issues = csv_parser.get_report_data(rfile) if extn == ".xml": issues, metrics = xml_parser.get_report_data(rfile, file_path_list) return issues, metrics, skips
def extract_from_file(tool_name, report_file, file_path_list=None): """Extract properties from reports :param tool_name: tool name :param report_file: Report file :param file_path_list: Full file path for any manipulation :return issues, metrics, skips information """ issues = [] metrics = None skips = [] # If the tools did not produce any result do not crash if not os.path.isfile(report_file): return issues, metrics, skips extn = pathlib.PurePosixPath(report_file).suffix with io.open(report_file, "r") as rfile: # Static check use jsonlines format, duh if tool_name == "staticcheck": contents = rfile.read() try: issues = [ json.loads(str(item)) for item in contents.strip().split("\n") ] except json.decoder.JSONDecodeError: LOG.warning( "staticcheck produced no result since the project was not built before analysis!" ) return issues, metrics, skips if extn == ".json": try: report_data = json.loads(rfile.read()) except json.decoder.JSONDecodeError: return issues, metrics, skips if isinstance(report_data, list): issues = report_data else: # NodeJsScan uses sec_issues if "sec_issues" in report_data: sec_data = report_data["sec_issues"] for key, value in sec_data.items(): if isinstance(value, list): issues = issues + value else: issues.append(value) elif "Issues" in report_data: tmpL = report_data.get("Issues", []) if tmpL: issues += tmpL else: LOG.warning("%s produced no result" % tool_name) elif "results" in report_data: tmpL = report_data.get("results", []) if tmpL: issues += tmpL else: LOG.warning("%s produced no result" % tool_name) if extn == ".csv": headers, issues = csv_parser.get_report_data(rfile) if extn == ".xml": issues, metrics = xml_parser.get_report_data(rfile, file_path_list) return issues, metrics, skips