def test_get_pkg(): test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom.xml") pkg_list = get_pkg_list(test_bom) assert len(pkg_list) == 157 for pkg in pkg_list: assert pkg["vendor"] != "maven" assert " " not in pkg["name"] assert pkg["version"] test_py_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-py.xml") pkg_list = get_pkg_list(test_py_bom) assert len(pkg_list) == 31 for pkg in pkg_list: assert pkg["vendor"] == "pypi" assert " " not in pkg["name"] assert pkg["version"] test_dn_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-dotnet.xml") pkg_list = get_pkg_list(test_dn_bom) assert len(pkg_list) == 38 for pkg in pkg_list: assert pkg["vendor"] assert " " not in pkg["name"] assert pkg["version"] test_dn_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-dotnet2.xml") pkg_list = get_pkg_list(test_dn_bom) assert len(pkg_list) == 6 for pkg in pkg_list: assert pkg["vendor"] assert " " not in pkg["name"] assert pkg["version"]
def test_search_webgoat_json(test_db): test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom.json") pkg_list = get_pkg_list(test_bom) assert len(pkg_list) == 157 search_res, pkg_aliases = search_pkgs(test_db, pkg_list) assert not len(search_res)
def test_get_pkg_by_type(test_db): test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-docker.json") pkg_list = get_pkg_list(test_bom) assert len(pkg_list) == 1824 filtered_list = get_pkg_by_type(pkg_list, "npm") assert len(filtered_list) == 1823
def test_large_lookup(test_license_data): test_bom = os.path.join( os.path.dirname(os.path.realpath(__file__)), "data", "bom-docker.json" ) pkg_list = get_pkg_list(test_bom) pkg_lic_dict = bulk_lookup(test_license_data, pkg_list) assert pkg_lic_dict analysis.analyse_licenses("nodejs", pkg_lic_dict)
def test_search(): test_bom = os.path.join( os.path.dirname(os.path.realpath(__file__)), "data", "bom.xml" ) pkg_list = get_pkg_list(test_bom) db = dbLib.get() search_res = search_pkgs(db, pkg_list) assert len(search_res)
def test_lookup(test_license_data): test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom.xml") pkg_list = get_pkg_list(test_bom) pkg_lic_dict = bulk_lookup(test_license_data, pkg_list) assert pkg_lic_dict test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-dotnet.xml") pkg_list = get_pkg_list(test_bom) pkg_lic_dict = bulk_lookup(test_license_data, pkg_list) assert pkg_lic_dict violations_list = [] for pkg, ll in pkg_lic_dict.items(): for lic in ll: if lic["condition_flag"]: violations_list.append(lic) assert len(violations_list) == 1
def test_get_pkg(): test_bom = os.path.join( os.path.dirname(os.path.realpath(__file__)), "data", "bom.xml" ) pkg_list = get_pkg_list(test_bom) assert len(pkg_list) == 157 for pkg in pkg_list: assert pkg["group"] assert pkg["name"] assert pkg["version"] test_py_bom = os.path.join( os.path.dirname(os.path.realpath(__file__)), "data", "bom-py.xml" ) pkg_list = get_pkg_list(test_py_bom) assert len(pkg_list) == 31 for pkg in pkg_list: assert not pkg["group"] assert pkg["name"] assert pkg["version"]
def test_dual_license(test_license_data): pkg_lic_dict = bulk_lookup( test_license_data, [{ "vendor": "npm", "name": "jszip", "version": "3.2.2", "licenses": ["(MIT OR GPL-3.0)"], }], ) assert pkg_lic_dict == { "npm:[email protected]": [{ "title": "MIT License", "spdx-id": "MIT", "featured": True, "hidden": False, "description": "A short and simple permissive license with conditions only requiring preservation of copyright and license notices. Licensed works, modifications, and larger works may be distributed under different terms and without source code.", "how": "Create a text file (typically named LICENSE or LICENSE.txt) in the root of your source code and copy the text of the license into the file. Replace [year] with the current year and [fullname] with the name (or names) of the copyright holders.", "using": [ { "Babel": "https://github.com/babel/babel/blob/master/LICENSE" }, { ".NET Core": "https://github.com/dotnet/runtime/blob/master/LICENSE.TXT" }, { "Rails": "https://github.com/rails/rails/blob/master/MIT-LICENSE" }, ], "permissions": [ "commercial-use", "modifications", "distribution", "private-use", ], "conditions": ["include-copyright"], "limitations": ["liability", "warranty"], "condition_flag": False, }] } test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-node2.xml") pkg_list = get_pkg_list(test_bom) pkg_lic_dict = bulk_lookup(test_license_data, pkg_list) assert pkg_lic_dict
def main(): args = build_args() print(at_logo, flush=True) db = dbLib.get() run_cacher = args.cache summary = None if not dbLib.index_count(db["index_file"]): run_cacher = True else: LOG.info("Vulnerability database loaded from {}".format(config.vulndb_bin_file)) sources_list = [NvdSource()] if os.environ.get("GITHUB_TOKEN"): sources_list.insert(0, GitHubSource()) else: LOG.info( "To use GitHub advisory source please set the environment variable GITHUB_TOKEN!" ) if run_cacher: for s in sources_list: LOG.info("Refreshing {}".format(s.__class__.__name__)) s.refresh() elif args.sync: for s in sources_list: LOG.info("Syncing {}".format(s.__class__.__name__)) s.download_recent() LOG.debug( "Vulnerability database contains {} records".format( dbLib.index_count(db["index_file"]) ) ) if args.bom: if not os.path.isfile(args.bom): LOG.error("Invalid bom file specified: {}".format(args.bom)) return LOG.debug("Scanning using the bom file {}".format(args.bom)) pkg_list = get_pkg_list(args.bom) summary = scan(db, pkg_list, args.report_file) # proj_type = utils.detect_project_type(args.src_dir) if summary and not args.noerror: # Hard coded build break logic for now if summary.get("CRITICAL") > 0: sys.exit(1)
def main(): args = build_args() if not args.no_banner: print(at_logo) src_dir = args.src_dir_image if not src_dir: src_dir = os.getcwd() reports_base_dir = src_dir # Detect the project types and perform the right type of scan if args.project_type: project_types_list = args.project_type.split(",") elif args.bom: project_types_list = ["bom"] else: project_types_list = utils.detect_project_type(src_dir) if ("docker" in project_types_list or "podman" in project_types_list or "container" in project_types_list or "binary" in project_types_list): reports_base_dir = os.getcwd() db = dbLib.get() run_cacher = args.cache areport_file = (args.report_file if args.report_file else os.path.join( reports_base_dir, "reports", "depscan.json")) reports_dir = os.path.dirname(areport_file) # Create reports directory if not os.path.exists(reports_dir): os.makedirs(reports_dir) if len(project_types_list) > 1: LOG.debug( "Multiple project types found: {}".format(project_types_list)) # Enable license scanning if "license" in project_types_list: os.environ["FETCH_LICENSE"] = "true" project_types_list.remove("license") console.print( Panel( "License audit is enabled for this scan. This would increase the time by up to 10 minutes.", title="License Audit", expand=False, )) for project_type in project_types_list: sug_version_dict = {} pkg_aliases = {} results = [] report_file = areport_file.replace(".json", "-{}.json".format(project_type)) risk_report_file = areport_file.replace( ".json", "-risk.{}.json".format(project_type)) LOG.info("=" * 80) creation_status = False if args.bom and os.path.exists(args.bom): bom_file = args.bom creation_status = True else: bom_file = os.path.join(reports_dir, "bom-" + project_type + ".json") creation_status = create_bom(project_type, bom_file, src_dir) if not creation_status: LOG.debug( "Bom file {} was not created successfully".format(bom_file)) continue LOG.debug("Scanning using the bom file {}".format(bom_file)) pkg_list = get_pkg_list(bom_file) if not pkg_list: LOG.debug("No packages found in the project!") continue scoped_pkgs = {} if project_type in ["python"]: all_imports = utils.get_all_imports(src_dir) LOG.debug(f"Identified {len(all_imports)} imports in your project") scoped_pkgs = utils.get_scope_from_imports(project_type, pkg_list, all_imports) else: scoped_pkgs = utils.get_pkgs_by_scope(project_type, pkg_list) if os.getenv("FETCH_LICENSE", "") in (True, "1", "true"): licenses_results = bulk_lookup( build_license_data(license_data_dir, spdx_license_list), pkg_list=pkg_list, ) license_report_file = os.path.join( reports_dir, "license-" + project_type + ".json") analyse_licenses(project_type, licenses_results, license_report_file) if project_type in risk_audit_map.keys(): if args.risk_audit: console.print( Panel( f"Performing OSS Risk Audit for packages from {src_dir}\nNo of packages [bold]{len(pkg_list)}[/bold]. This will take a while ...", title="OSS Risk Audit", expand=False, )) try: risk_results = risk_audit( project_type, scoped_pkgs, args.private_ns, pkg_list, risk_report_file, ) analyse_pkg_risks( project_type, scoped_pkgs, args.private_ns, risk_results, risk_report_file, ) except Exception as e: LOG.error(e) LOG.error("Risk audit was not successful") else: console.print( Panel( "Depscan supports OSS Risk audit for this project.\nTo enable set the environment variable [bold]ENABLE_OSS_RISK=true[/bold]", title="New Feature", expand=False, )) if project_type in type_audit_map.keys(): LOG.info("Performing remote audit for {} of type {}".format( src_dir, project_type)) LOG.debug(f"No of packages {len(pkg_list)}") try: audit_results = audit(project_type, pkg_list, report_file) if audit_results: LOG.debug( f"Remote audit yielded {len(audit_results)} results") results = results + audit_results except Exception as e: LOG.error("Remote audit was not successful") LOG.error(e) results = None # In case of docker, check if there are any npm packages that can be audited remotely if project_type in ("podman", "docker"): npm_pkg_list = get_pkg_by_type(pkg_list, "npm") if npm_pkg_list: LOG.debug(f"No of packages {len(npm_pkg_list)}") try: audit_results = audit("nodejs", npm_pkg_list, report_file) if audit_results: LOG.debug( f"Remote audit yielded {len(audit_results)} results" ) results = results + audit_results except Exception as e: LOG.error("Remote audit was not successful") LOG.error(e) if not dbLib.index_count(db["index_file"]): run_cacher = True else: LOG.debug("Vulnerability database loaded from {}".format( config.vdb_bin_file)) sources_list = [OSVSource(), NvdSource()] if os.environ.get("GITHUB_TOKEN"): sources_list.insert(0, GitHubSource()) if run_cacher: for s in sources_list: LOG.debug("Refreshing {}".format(s.__class__.__name__)) s.refresh() run_cacher = False elif args.sync: for s in sources_list: LOG.debug("Syncing {}".format(s.__class__.__name__)) s.download_recent() run_cacher = False LOG.debug("Vulnerability database contains {} records".format( dbLib.index_count(db["index_file"]))) LOG.info("Performing regular scan for {} using plugin {}".format( src_dir, project_type)) vdb_results, pkg_aliases, sug_version_dict = scan( db, project_type, pkg_list, args.suggest) if vdb_results: results = results + vdb_results # Summarise and print results summary = summarise( project_type, results, pkg_aliases, sug_version_dict, scoped_pkgs, report_file, True, ) if summary and not args.noerror and len(project_types_list) == 1: # Hard coded build break logic for now if summary.get("CRITICAL") > 0: sys.exit(1)
def main(): args = build_args() if not args.no_banner: print(at_logo, flush=True) # Set logging level if os.environ.get("SCAN_DEBUG_MODE") == "debug": LOG.setLevel(logging.DEBUG) src_dir = args.src_dir if not args.src_dir: src_dir = os.getcwd() db = dbLib.get() run_cacher = args.cache areport_file = ( args.report_file if args.report_file else os.path.join(src_dir, "reports", "depscan.json") ) reports_dir = os.path.dirname(areport_file) # Create reports directory if not os.path.exists(reports_dir): os.makedirs(reports_dir) # Detect the project types and perform the right type of scan if args.project_type: project_types_list = args.project_type.split(",") else: project_types_list = utils.detect_project_type(src_dir) if len(project_types_list) > 1: LOG.debug("Multiple project types found: {}".format(project_types_list)) for project_type in project_types_list: sug_version_dict = {} pkg_aliases = {} report_file = areport_file.replace(".json", "-{}.json".format(project_type)) LOG.info("=" * 80) bom_file = os.path.join(reports_dir, "bom-" + project_type + ".json") creation_status = create_bom(project_type, bom_file, src_dir) if not creation_status: LOG.debug("Bom file {} was not created successfully".format(bom_file)) continue LOG.debug("Scanning using the bom file {}".format(bom_file)) pkg_list = get_pkg_list(bom_file) if not pkg_list: LOG.debug("No packages found in the project!") continue if not args.no_license_scan: licenses_results = bulk_lookup( build_license_data(license_data_dir), pkg_list=pkg_list ) license_report_file = os.path.join( reports_dir, "license-" + project_type + ".json" ) analyse_licenses( project_type, licenses_results, license_report_file ) if project_type in type_audit_map.keys(): LOG.info( "Performing remote audit for {} of type {}".format( src_dir, project_type ) ) LOG.debug(f"No of packages {len(pkg_list)}") results = audit(project_type, pkg_list, report_file) else: if not dbLib.index_count(db["index_file"]): run_cacher = True else: LOG.debug( "Vulnerability database loaded from {}".format(config.vdb_bin_file) ) sources_list = [NvdSource()] if os.environ.get("GITHUB_TOKEN"): sources_list.insert(0, GitHubSource()) else: LOG.info( "To use GitHub advisory source please set the environment variable GITHUB_TOKEN!" ) if run_cacher: for s in sources_list: LOG.debug("Refreshing {}".format(s.__class__.__name__)) s.refresh() elif args.sync: for s in sources_list: LOG.debug("Syncing {}".format(s.__class__.__name__)) s.download_recent() LOG.debug( "Vulnerability database contains {} records".format( dbLib.index_count(db["index_file"]) ) ) LOG.info( "Performing regular scan for {} using plugin {}".format( src_dir, project_type ) ) results, pkg_aliases, sug_version_dict = scan(db, pkg_list, args.suggest) # Summarise and print results summary = summarise( project_type, results, pkg_aliases, sug_version_dict, report_file, True ) if summary and not args.noerror and len(project_types_list) == 1: # Hard coded build break logic for now if summary.get("CRITICAL") > 0: sys.exit(1)
def main(): args = build_args() if not args.no_banner: print(at_logo) src_dir = args.src_dir if not args.src_dir: src_dir = os.getcwd() db = dbLib.get() run_cacher = args.cache areport_file = ( args.report_file if args.report_file else os.path.join(src_dir, "reports", "depscan.json") ) reports_dir = os.path.dirname(areport_file) # Create reports directory if not os.path.exists(reports_dir): os.makedirs(reports_dir) # Detect the project types and perform the right type of scan if args.project_type: project_types_list = args.project_type.split(",") else: project_types_list = utils.detect_project_type(src_dir) if len(project_types_list) > 1: LOG.debug("Multiple project types found: {}".format(project_types_list)) for project_type in project_types_list: sug_version_dict = {} pkg_aliases = {} report_file = areport_file.replace(".json", "-{}.json".format(project_type)) risk_report_file = areport_file.replace( ".json", "-risk.{}.json".format(project_type) ) LOG.info("=" * 80) creation_status = False if args.bom and os.path.exists(args.bom): bom_file = args.bom creation_status = True else: bom_file = os.path.join(reports_dir, "bom-" + project_type + ".json") creation_status = create_bom(project_type, bom_file, src_dir) if not creation_status: LOG.debug("Bom file {} was not created successfully".format(bom_file)) continue LOG.debug("Scanning using the bom file {}".format(bom_file)) pkg_list = get_pkg_list(bom_file) if not pkg_list: LOG.debug("No packages found in the project!") continue scoped_pkgs = utils.get_pkgs_by_scope(pkg_list) if not args.no_license_scan: licenses_results = bulk_lookup( build_license_data(license_data_dir), pkg_list=pkg_list ) license_report_file = os.path.join( reports_dir, "license-" + project_type + ".json" ) analyse_licenses(project_type, licenses_results, license_report_file) if project_type in risk_audit_map.keys(): if args.risk_audit: console.print( Panel( f"Performing OSS Risk Audit for packages from {src_dir}\nNo of packages [bold]{len(pkg_list)}[/bold]. This will take a while ...", title="OSS Risk Audit", expand=False, ) ) try: risk_results = risk_audit( project_type, args.private_ns, pkg_list, risk_report_file ) analyse_pkg_risks( project_type, args.private_ns, risk_results, risk_report_file ) except Exception as e: LOG.error(e) LOG.error("Risk audit was not successful") risk_results = None else: console.print( Panel( "Depscan supports OSS Risk audit for this project.\nTo enable set the environment variable [bold]ENABLE_OSS_RISK=true[/bold]", title="New Feature", expand=False, ) ) if project_type in type_audit_map.keys(): LOG.info( "Performing remote audit for {} of type {}".format( src_dir, project_type ) ) LOG.debug(f"No of packages {len(pkg_list)}") try: results = audit(project_type, pkg_list, report_file) except Exception as e: LOG.error("Remote audit was not successful") LOG.error(e) results = None else: if not dbLib.index_count(db["index_file"]): run_cacher = True else: LOG.debug( "Vulnerability database loaded from {}".format(config.vdb_bin_file) ) sources_list = [NvdSource()] if os.environ.get("GITHUB_TOKEN"): sources_list.insert(0, GitHubSource()) else: LOG.info( "To use GitHub advisory source please set the environment variable GITHUB_TOKEN!" ) if run_cacher: for s in sources_list: LOG.debug("Refreshing {}".format(s.__class__.__name__)) s.refresh() elif args.sync: for s in sources_list: LOG.debug("Syncing {}".format(s.__class__.__name__)) s.download_recent() LOG.debug( "Vulnerability database contains {} records".format( dbLib.index_count(db["index_file"]) ) ) LOG.info( "Performing regular scan for {} using plugin {}".format( src_dir, project_type ) ) results, pkg_aliases, sug_version_dict = scan( db, project_type, pkg_list, args.suggest ) # Summarise and print results summary = summarise( project_type, results, pkg_aliases, sug_version_dict, scoped_pkgs, report_file, True, ) if summary and not args.noerror and len(project_types_list) == 1: # Hard coded build break logic for now if summary.get("CRITICAL") > 0: sys.exit(1)
def test_query_metadata2(): test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-py.xml") pkg_list = get_pkg_list(test_bom) metadata_dict = pypi_metadata(pkg_list, None) assert metadata_dict
def test_query_metadata1(): test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-goof.json") pkg_list = get_pkg_list(test_bom) metadata_dict = npm_metadata(pkg_list, "snyk") assert metadata_dict
def test_go_search(test_db): test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-go.xml") pkg_list = get_pkg_list(test_bom) search_res, pkg_aliases = search_pkgs(test_db, pkg_list) assert not len(search_res)
def test_query_metadata(): test_bom = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "bom-node.xml") pkg_list = get_pkg_list(test_bom) metadata_dict = npm_metadata({}, pkg_list, None) assert metadata_dict