コード例 #1
0
ファイル: cli.py プロジェクト: AppThreat/dep-scan
def scan(db, project_type, pkg_list, suggest_mode):
    """
    Method to search packages in our vulnerability database

    :param db: Reference to db
    :param project_type: Project Type
    :param pkg_list: List of packages
    :param suggest_mode: True if package fix version should be normalized across findings
    """
    if not pkg_list:
        LOG.debug("Empty package search attempted!")
    else:
        LOG.info("Scanning {} oss dependencies for issues".format(
            len(pkg_list)))
    results, pkg_aliases = utils.search_pkgs(db, project_type, pkg_list)
    # pkg_aliases is a dict that can be used to find the original vendor and package name
    # This way we consistently use the same names used by the caller irrespective of how
    # the result was obtained
    sug_version_dict = {}
    if suggest_mode:
        # From the results identify optimal max version
        sug_version_dict = suggest_version(results, pkg_aliases)
        if sug_version_dict:
            LOG.debug(
                "Adjusting fix version based on the initial suggestion {}".
                format(sug_version_dict))
            # Recheck packages
            sug_pkg_list = []
            for k, v in sug_version_dict.items():
                if not v:
                    continue
                vendor = ""
                name = None
                version = v
                tmpA = k.split(":")
                if len(tmpA) == 2:
                    vendor = tmpA[0]
                    name = tmpA[1]
                else:
                    name = tmpA[0]
                # De-alias the vendor and package name
                full_pkg = "{}:{}".format(vendor, name)
                full_pkg = pkg_aliases.get(full_pkg, full_pkg)
                vendor, name = full_pkg.split(":")
                sug_pkg_list.append({
                    "vendor": vendor,
                    "name": name,
                    "version": version
                })
            LOG.debug(
                "Re-checking our suggestion to ensure there are no further vulnerabilities"
            )
            override_results, _ = utils.search_pkgs(db, project_type,
                                                    sug_pkg_list)
            if override_results:
                new_sug_dict = suggest_version(override_results)
                LOG.debug("Received override results: {}".format(new_sug_dict))
                for nk, nv in new_sug_dict.items():
                    sug_version_dict[nk] = nv
    return results, pkg_aliases, sug_version_dict
コード例 #2
0
ファイル: analysis.py プロジェクト: AppThreat/dep-scan
def analyse(project_type, results):
    if not results:
        LOG.info("No oss vulnerabilities detected ✅")
        return None
    summary = {
        "UNSPECIFIED": 0,
        "LOW": 0,
        "MEDIUM": 0,
        "HIGH": 0,
        "CRITICAL": 0
    }
    for res in results:
        summary[res.severity] += 1
    return summary
コード例 #3
0
ファイル: analysis.py プロジェクト: AppThreat/dep-scan
def analyse_licenses(project_type, licenses_results, license_report_file=None):
    if not licenses_results:
        return
    table = Table(
        title=f"License Scan Summary ({project_type})",
        box=box.DOUBLE_EDGE,
        header_style="bold magenta",
    )
    headers = ["Package", "Version", "License Id", "License conditions"]
    for h in headers:
        table.add_column(header=h)
    report_data = []
    for pkg, ll in licenses_results.items():
        pkg_ver = pkg.split("@")
        for lic in ll:
            if not lic:
                data = [*pkg_ver, "Unknown license"]
                table.add_row(*data)
                report_data.append(dict(zip(headers, data)))
            elif lic["condition_flag"]:
                conditions_str = ", ".join(lic["conditions"])
                if "http" not in conditions_str:
                    conditions_str = (conditions_str.replace(
                        "--", " for ").replace("-", " ").title())
                data = [
                    *pkg_ver,
                    "{}{}".format(
                        "[cyan]" if "GPL" in lic["spdx-id"] or "CC-BY-"
                        in lic["spdx-id"] or "Facebook" in lic["spdx-id"]
                        or "WTFPL" in lic["spdx-id"] else "",
                        lic["spdx-id"],
                    ),
                    conditions_str,
                ]
                table.add_row(*data)
                report_data.append(dict(zip(headers, data)))
    if report_data:
        console.print(table)
        # Store the license scan findings in jsonl format
        if license_report_file:
            with open(license_report_file, "w") as outfile:
                for row in report_data:
                    json.dump(row, outfile)
                    outfile.write("\n")
    else:
        LOG.info("No license violation detected ✅")
コード例 #4
0
ファイル: bom.py プロジェクト: AppThreat/dep-scan
def create_bom(project_type, bom_file, src_dir="."):
    """Method to create BOM file by executing cdxgen command

    :param project_type: Project type
    :param bom_file: BOM file
    :param src_dir: Source directory

    :returns True if the command was executed. False if the executable was not found.
    """
    cdxgen_cmd = os.environ.get("CDXGEN_CMD", "cdxgen")
    if not shutil.which(cdxgen_cmd):
        LOG.warning(
            "{} command not found. Please install using npm install @appthreat/cdxgen or set PATH variable"
            .format(cdxgen_cmd))
        return False
    if project_type in ("docker"):
        LOG.info(
            f"Generating Software Bill-of-Materials for container image {src_dir}. This might take a few mins ..."
        )
    args = [cdxgen_cmd, "-r", "-t", project_type, "-o", bom_file, src_dir]
    exec_tool(args)
    return os.path.exists(bom_file)
コード例 #5
0
ファイル: cli.py プロジェクト: AppThreat/dep-scan
def summarise(
    project_type,
    results,
    pkg_aliases,
    sug_version_dict,
    scoped_pkgs={},
    report_file=None,
    console_print=True,
):
    """
    Method to summarise the results
    :param project_type: Project type
    :param results: Scan or audit results
    :param pkg_aliases: Package aliases used
    :param sug_version_dict: Dictionary containing version suggestions
    :param scoped_pkgs: Dict containing package scopes
    :param report_file: Output report file
    :param print: Boolean to indicate if the results should get printed to the console
    :return: Summary of the results
    """
    if not results:
        LOG.info(f"No oss vulnerabilities detected for type {project_type} ✅")
        return None
    if report_file:
        jsonl_report(
            project_type,
            results,
            pkg_aliases,
            sug_version_dict,
            scoped_pkgs,
            report_file,
        )
    if console_print:
        print_results(project_type, results, pkg_aliases, sug_version_dict,
                      scoped_pkgs)
    summary = analyse(project_type, results)
    return summary
コード例 #6
0
ファイル: cli.py プロジェクト: AppThreat/dep-scan
def main():
    args = build_args()
    if not args.no_banner:
        print(at_logo)
    src_dir = args.src_dir_image
    if not src_dir:
        src_dir = os.getcwd()
    reports_base_dir = src_dir
    # Detect the project types and perform the right type of scan
    if args.project_type:
        project_types_list = args.project_type.split(",")
    elif args.bom:
        project_types_list = ["bom"]
    else:
        project_types_list = utils.detect_project_type(src_dir)
    if ("docker" in project_types_list or "podman" in project_types_list
            or "container" in project_types_list
            or "binary" in project_types_list):
        reports_base_dir = os.getcwd()
    db = dbLib.get()
    run_cacher = args.cache
    areport_file = (args.report_file if args.report_file else os.path.join(
        reports_base_dir, "reports", "depscan.json"))
    reports_dir = os.path.dirname(areport_file)
    # Create reports directory
    if not os.path.exists(reports_dir):
        os.makedirs(reports_dir)
    if len(project_types_list) > 1:
        LOG.debug(
            "Multiple project types found: {}".format(project_types_list))
    # Enable license scanning
    if "license" in project_types_list:
        os.environ["FETCH_LICENSE"] = "true"
        project_types_list.remove("license")
        console.print(
            Panel(
                "License audit is enabled for this scan. This would increase the time by up to 10 minutes.",
                title="License Audit",
                expand=False,
            ))
    for project_type in project_types_list:
        sug_version_dict = {}
        pkg_aliases = {}
        results = []
        report_file = areport_file.replace(".json",
                                           "-{}.json".format(project_type))
        risk_report_file = areport_file.replace(
            ".json", "-risk.{}.json".format(project_type))
        LOG.info("=" * 80)
        creation_status = False
        if args.bom and os.path.exists(args.bom):
            bom_file = args.bom
            creation_status = True
        else:
            bom_file = os.path.join(reports_dir,
                                    "bom-" + project_type + ".json")
            creation_status = create_bom(project_type, bom_file, src_dir)
        if not creation_status:
            LOG.debug(
                "Bom file {} was not created successfully".format(bom_file))
            continue
        LOG.debug("Scanning using the bom file {}".format(bom_file))
        pkg_list = get_pkg_list(bom_file)
        if not pkg_list:
            LOG.debug("No packages found in the project!")
            continue
        scoped_pkgs = {}
        if project_type in ["python"]:
            all_imports = utils.get_all_imports(src_dir)
            LOG.debug(f"Identified {len(all_imports)} imports in your project")
            scoped_pkgs = utils.get_scope_from_imports(project_type, pkg_list,
                                                       all_imports)
        else:
            scoped_pkgs = utils.get_pkgs_by_scope(project_type, pkg_list)
        if os.getenv("FETCH_LICENSE", "") in (True, "1", "true"):
            licenses_results = bulk_lookup(
                build_license_data(license_data_dir, spdx_license_list),
                pkg_list=pkg_list,
            )
            license_report_file = os.path.join(
                reports_dir, "license-" + project_type + ".json")
            analyse_licenses(project_type, licenses_results,
                             license_report_file)
        if project_type in risk_audit_map.keys():
            if args.risk_audit:
                console.print(
                    Panel(
                        f"Performing OSS Risk Audit for packages from {src_dir}\nNo of packages [bold]{len(pkg_list)}[/bold]. This will take a while ...",
                        title="OSS Risk Audit",
                        expand=False,
                    ))
                try:
                    risk_results = risk_audit(
                        project_type,
                        scoped_pkgs,
                        args.private_ns,
                        pkg_list,
                        risk_report_file,
                    )
                    analyse_pkg_risks(
                        project_type,
                        scoped_pkgs,
                        args.private_ns,
                        risk_results,
                        risk_report_file,
                    )
                except Exception as e:
                    LOG.error(e)
                    LOG.error("Risk audit was not successful")
            else:
                console.print(
                    Panel(
                        "Depscan supports OSS Risk audit for this project.\nTo enable set the environment variable [bold]ENABLE_OSS_RISK=true[/bold]",
                        title="New Feature",
                        expand=False,
                    ))
        if project_type in type_audit_map.keys():
            LOG.info("Performing remote audit for {} of type {}".format(
                src_dir, project_type))
            LOG.debug(f"No of packages {len(pkg_list)}")
            try:
                audit_results = audit(project_type, pkg_list, report_file)
                if audit_results:
                    LOG.debug(
                        f"Remote audit yielded {len(audit_results)} results")
                    results = results + audit_results
            except Exception as e:
                LOG.error("Remote audit was not successful")
                LOG.error(e)
                results = None
        # In case of docker, check if there are any npm packages that can be audited remotely
        if project_type in ("podman", "docker"):
            npm_pkg_list = get_pkg_by_type(pkg_list, "npm")
            if npm_pkg_list:
                LOG.debug(f"No of packages {len(npm_pkg_list)}")
                try:
                    audit_results = audit("nodejs", npm_pkg_list, report_file)
                    if audit_results:
                        LOG.debug(
                            f"Remote audit yielded {len(audit_results)} results"
                        )
                        results = results + audit_results
                except Exception as e:
                    LOG.error("Remote audit was not successful")
                    LOG.error(e)
        if not dbLib.index_count(db["index_file"]):
            run_cacher = True
        else:
            LOG.debug("Vulnerability database loaded from {}".format(
                config.vdb_bin_file))
        sources_list = [OSVSource(), NvdSource()]
        if os.environ.get("GITHUB_TOKEN"):
            sources_list.insert(0, GitHubSource())
        if run_cacher:
            for s in sources_list:
                LOG.debug("Refreshing {}".format(s.__class__.__name__))
                s.refresh()
                run_cacher = False
        elif args.sync:
            for s in sources_list:
                LOG.debug("Syncing {}".format(s.__class__.__name__))
                s.download_recent()
                run_cacher = False
        LOG.debug("Vulnerability database contains {} records".format(
            dbLib.index_count(db["index_file"])))
        LOG.info("Performing regular scan for {} using plugin {}".format(
            src_dir, project_type))
        vdb_results, pkg_aliases, sug_version_dict = scan(
            db, project_type, pkg_list, args.suggest)
        if vdb_results:
            results = results + vdb_results
        # Summarise and print results
        summary = summarise(
            project_type,
            results,
            pkg_aliases,
            sug_version_dict,
            scoped_pkgs,
            report_file,
            True,
        )
        if summary and not args.noerror and len(project_types_list) == 1:
            # Hard coded build break logic for now
            if summary.get("CRITICAL") > 0:
                sys.exit(1)
コード例 #7
0
def main():
    args = build_args()
    if not args.no_banner:
        print(at_logo)
    src_dir = args.src_dir
    if not args.src_dir:
        src_dir = os.getcwd()
    db = dbLib.get()
    run_cacher = args.cache
    areport_file = (
        args.report_file
        if args.report_file
        else os.path.join(src_dir, "reports", "depscan.json")
    )
    reports_dir = os.path.dirname(areport_file)
    # Create reports directory
    if not os.path.exists(reports_dir):
        os.makedirs(reports_dir)
    # Detect the project types and perform the right type of scan
    if args.project_type:
        project_types_list = args.project_type.split(",")
    else:
        project_types_list = utils.detect_project_type(src_dir)
    if len(project_types_list) > 1:
        LOG.debug("Multiple project types found: {}".format(project_types_list))
    for project_type in project_types_list:
        sug_version_dict = {}
        pkg_aliases = {}
        report_file = areport_file.replace(".json", "-{}.json".format(project_type))
        risk_report_file = areport_file.replace(
            ".json", "-risk.{}.json".format(project_type)
        )
        LOG.info("=" * 80)
        creation_status = False
        if args.bom and os.path.exists(args.bom):
            bom_file = args.bom
            creation_status = True
        else:
            bom_file = os.path.join(reports_dir, "bom-" + project_type + ".json")
            creation_status = create_bom(project_type, bom_file, src_dir)
        if not creation_status:
            LOG.debug("Bom file {} was not created successfully".format(bom_file))
            continue
        LOG.debug("Scanning using the bom file {}".format(bom_file))
        pkg_list = get_pkg_list(bom_file)
        if not pkg_list:
            LOG.debug("No packages found in the project!")
            continue
        scoped_pkgs = utils.get_pkgs_by_scope(pkg_list)
        if not args.no_license_scan:
            licenses_results = bulk_lookup(
                build_license_data(license_data_dir), pkg_list=pkg_list
            )
            license_report_file = os.path.join(
                reports_dir, "license-" + project_type + ".json"
            )
            analyse_licenses(project_type, licenses_results, license_report_file)
        if project_type in risk_audit_map.keys():
            if args.risk_audit:
                console.print(
                    Panel(
                        f"Performing OSS Risk Audit for packages from {src_dir}\nNo of packages [bold]{len(pkg_list)}[/bold]. This will take a while ...",
                        title="OSS Risk Audit",
                        expand=False,
                    )
                )
                try:
                    risk_results = risk_audit(
                        project_type, args.private_ns, pkg_list, risk_report_file
                    )
                    analyse_pkg_risks(
                        project_type, args.private_ns, risk_results, risk_report_file
                    )
                except Exception as e:
                    LOG.error(e)
                    LOG.error("Risk audit was not successful")
                    risk_results = None
            else:
                console.print(
                    Panel(
                        "Depscan supports OSS Risk audit for this project.\nTo enable set the environment variable [bold]ENABLE_OSS_RISK=true[/bold]",
                        title="New Feature",
                        expand=False,
                    )
                )
        if project_type in type_audit_map.keys():
            LOG.info(
                "Performing remote audit for {} of type {}".format(
                    src_dir, project_type
                )
            )
            LOG.debug(f"No of packages {len(pkg_list)}")
            try:
                results = audit(project_type, pkg_list, report_file)
            except Exception as e:
                LOG.error("Remote audit was not successful")
                LOG.error(e)
                results = None
        else:
            if not dbLib.index_count(db["index_file"]):
                run_cacher = True
            else:
                LOG.debug(
                    "Vulnerability database loaded from {}".format(config.vdb_bin_file)
                )
            sources_list = [NvdSource()]
            if os.environ.get("GITHUB_TOKEN"):
                sources_list.insert(0, GitHubSource())
            else:
                LOG.info(
                    "To use GitHub advisory source please set the environment variable GITHUB_TOKEN!"
                )
            if run_cacher:
                for s in sources_list:
                    LOG.debug("Refreshing {}".format(s.__class__.__name__))
                    s.refresh()
            elif args.sync:
                for s in sources_list:
                    LOG.debug("Syncing {}".format(s.__class__.__name__))
                    s.download_recent()
            LOG.debug(
                "Vulnerability database contains {} records".format(
                    dbLib.index_count(db["index_file"])
                )
            )
            LOG.info(
                "Performing regular scan for {} using plugin {}".format(
                    src_dir, project_type
                )
            )
            results, pkg_aliases, sug_version_dict = scan(
                db, project_type, pkg_list, args.suggest
            )
        # Summarise and print results
        summary = summarise(
            project_type,
            results,
            pkg_aliases,
            sug_version_dict,
            scoped_pkgs,
            report_file,
            True,
        )
        if summary and not args.noerror and len(project_types_list) == 1:
            # Hard coded build break logic for now
            if summary.get("CRITICAL") > 0:
                sys.exit(1)
コード例 #8
0
ファイル: analysis.py プロジェクト: AppThreat/dep-scan
def analyse_pkg_risks(project_type,
                      scoped_pkgs,
                      private_ns,
                      risk_results,
                      risk_report_file=None):
    if not risk_results:
        return
    table = Table(
        title=f"Risk Audit Summary ({project_type})",
        box=box.DOUBLE_EDGE,
        header_style="bold magenta",
    )
    report_data = []
    required_pkgs = scoped_pkgs.get("required", [])
    optional_pkgs = scoped_pkgs.get("optional", [])
    excluded_pkgs = scoped_pkgs.get("excluded", [])
    headers = ["Package", "Used?", "Risk Score", "Identified Risks"]
    for h in headers:
        justify = "left"
        if h == "Risk Score":
            justify = "right"
        table.add_column(header=h, justify=justify)
    for pkg, risk_obj in risk_results.items():
        if not risk_obj:
            continue
        risk_metrics = risk_obj.get("risk_metrics")
        scope = risk_obj.get("scope")
        project_type_pkg = "{}:{}".format(project_type, pkg).lower()
        if project_type_pkg in required_pkgs:
            scope = "required"
        elif project_type_pkg in optional_pkgs:
            scope = "optional"
        elif project_type_pkg in excluded_pkgs:
            scope = "excluded"
        package_usage = "N/A"
        package_usage_simple = "N/A"
        if scope == "required":
            package_usage = "[bright_green][bold]Yes"
            package_usage_simple = "Yes"
        if scope == "optional":
            package_usage = "[magenta]No"
            package_usage_simple = "No"
        if not risk_metrics:
            continue
        if risk_metrics.get("risk_score") and (
                risk_metrics.get("risk_score") > config.pkg_max_risk_score
                or risk_metrics.get("pkg_private_on_public_registry_risk")):
            risk_score = f"""{round(risk_metrics.get("risk_score"), 2)}"""
            data = [
                pkg,
                package_usage,
                risk_score,
            ]
            edata = [
                pkg,
                package_usage_simple,
                risk_score,
            ]
            risk_categories = []
            risk_categories_simple = []
            for rk, rv in risk_metrics.items():
                if rk.endswith("_risk") and rv is True:
                    rcat = rk.replace("_risk", "")
                    help_text = config.risk_help_text.get(rcat)
                    # Only add texts that are available.
                    if help_text:
                        if rcat in ("pkg_deprecated",
                                    "pkg_private_on_public_registry"):
                            risk_categories.append(f":cross_mark: {help_text}")
                        else:
                            risk_categories.append(f":warning: {help_text}")
                        risk_categories_simple.append(help_text)
            data.append("\n".join(risk_categories))
            edata.append(", ".join(risk_categories_simple))
            table.add_row(*data)
            report_data.append(dict(zip(headers, edata)))
    if report_data:
        console.print(table)
        # Store the risk audit findings in jsonl format
        if risk_report_file:
            with open(risk_report_file, "w") as outfile:
                for row in report_data:
                    json.dump(row, outfile)
                    outfile.write("\n")
    else:
        LOG.info("No package risks detected ✅")
コード例 #9
0
ファイル: pkg_query.py プロジェクト: pihart/dep-scan
def metadata_from_registry(registry_type, pkg_list, private_ns=None):
    """
    Method to query registry for the package metadata

    :param pkg_list: List of packages
    :param private_ns: Private namespace
    """
    metadata_dict = {}
    task = None
    # Circuit breaker flag to break the risk audit in case of many api errors
    circuit_breaker = False
    # Track the api failures count
    failure_count = 0
    done_count = 0
    with Progress(
            console=console,
            transient=True,
            redirect_stderr=False,
            redirect_stdout=False,
            refresh_per_second=1,
    ) as progress:
        task = progress.add_task("[green] Auditing packages",
                                 total=len(pkg_list),
                                 start=True)
        for pkg in pkg_list:
            if circuit_breaker:
                LOG.info(
                    "Risk audited has been interrupted due to frequent api errors. Please try again later."
                )
                progress.stop()
                return {}
            scope = pkg.get("scope", "").lower()
            key, lookup_url = get_lookup_url(registry_type, pkg)
            if not key or key.startswith("https://"):
                progress.advance(task)
                continue
            progress.update(task, description=f"Checking {key}")
            try:
                r = requests.get(url=lookup_url,
                                 timeout=config.request_timeout_sec)
                json_data = r.json()
                # Npm returns this error if the package is not found
                if (json_data.get("code") == "MethodNotAllowedError"
                        or r.status_code > 400):
                    continue
                is_private_pkg = False
                if private_ns:
                    namespace_prefixes = private_ns.split(",")
                    for ns in namespace_prefixes:
                        if key.lower().startswith(ns.lower()) or key.lower(
                        ).startswith("@" + ns.lower()):
                            is_private_pkg = True
                            break
                risk_metrics = {}
                if registry_type == "npm":
                    risk_metrics = npm_pkg_risk(json_data, is_private_pkg,
                                                scope)
                elif registry_type == "pypi":
                    risk_metrics = pypi_pkg_risk(json_data, is_private_pkg,
                                                 scope)
                metadata_dict[key] = {
                    "scope": scope,
                    "pkg_metadata": json_data,
                    "risk_metrics": risk_metrics,
                    "is_private_pkg": is_private_pkg,
                }
            except Exception as e:
                LOG.debug(e)
                failure_count = failure_count + 1
            progress.advance(task)
            done_count = done_count + 1
            if failure_count >= config.max_request_failures:
                circuit_breaker = True
    LOG.debug(
        f"Retrieved package metadata for {done_count}/{len(pkg_list)} packages. Failures count {failure_count}"
    )
    return metadata_dict