Пример #1
0
def authenticate():
    """
    Method to authenticate with ShiftLeft NG SAST cloud when the required tokens gets passed via
    environment variables
    """
    if is_authenticated():
        return
    sl_org = config.get("SHIFTLEFT_ORG_ID", config.get("SHIFTLEFT_ORGANIZATION_ID"))
    sl_token = config.get("SHIFTLEFT_ACCESS_TOKEN")
    sl_cmd = config.get("SHIFTLEFT_NGSAST_CMD")
    run_uuid = config.get("run_uuid")
    if sl_org and sl_token and sl_cmd and utils.check_command(sl_cmd):
        inspect_login_args = [
            sl_cmd,
            "auth",
            "--no-auto-update",
            "--no-diagnostic",
            "--org",
            sl_org,
            "--token",
            sl_token,
        ]
        cp = exec_tool("NG SAST", inspect_login_args)
        if cp.returncode != 0:
            LOG.warning(
                "ShiftLeft NG SAST authentication has failed. Please check the credentials"
            )
        else:
            LOG.info("Successfully authenticated with NG SAST cloud")
        track({"id": run_uuid, "scan_mode": "ng-sast", "sl_org": sl_org})
Пример #2
0
def java_build(src, reports_dir, lang_tools):
    """
    Automatically build java project

    :param src: Source directory
    :param reports_dir: Reports directory to store any logs
    :param lang_tools: Language specific build tools

    :return: boolean status from the build. True if the command executed successfully. False otherwise
    """
    cmd_args = []
    pom_files = [p.as_posix() for p in Path(src).glob("pom.xml")]
    env = os.environ.copy()
    if os.environ.get("USE_JAVA_8") or os.environ.get("WITH_JAVA_8"):
        env["SCAN_JAVA_HOME"] = os.environ.get("SCAN_JAVA_8_HOME")
    else:
        env["SCAN_JAVA_HOME"] = os.environ.get("SCAN_JAVA_11_HOME")
    if pom_files:
        cmd_args = lang_tools.get("maven")
    else:
        gradle_files = [p.as_posix() for p in Path(src).glob("build.gradle")]
        if gradle_files:
            cmd_args = lang_tools.get("gradle")
    if not cmd_args:
        LOG.info(
            "Java auto build is supported only for maven or gradle based projects"
        )
        return False
    cp = exec_tool(cmd_args, src, env=env, stdout=subprocess.PIPE)
    LOG.debug(cp.stdout)
    return cp.returncode == 0
Пример #3
0
def convert_file(
    tool_name,
    tool_args,
    working_dir,
    report_file,
    converted_file,
    file_path_list=None,
):
    """Convert report file

    :param tool_name: tool name
    :param tool_args: tool args
    :param working_dir: Working directory
    :param report_file: Report file
    :param converted_file: Converted file
    :param file_path_list: Full file path for any manipulation

    :return serialized_log: SARIF output data
    """
    issues, metrics, skips = extract_from_file(
        tool_name, tool_args, working_dir, report_file, file_path_list
    )
    issues, suppress_list = suppress_issues(issues)
    if suppress_list:
        LOG.info(f"Suppressed {len(suppress_list)} issues")
    return report(
        tool_name,
        tool_args,
        working_dir,
        metrics,
        skips,
        issues,
        converted_file,
        file_path_list,
    )
Пример #4
0
def exec_tool(args, cwd=None, env=os.environ.copy(), stdout=subprocess.DEVNULL):
    """
    Convenience method to invoke cli tools

    Args:
      args cli command and args
      cwd Current working directory
      env Environment variables
      stdout stdout configuration for run command

    Returns:
      CompletedProcess instance
    """
    try:
        env = use_java(env)
        LOG.info("=" * 80)
        LOG.debug('⚡︎ Executing "{}"'.format(" ".join(args)))
        cp = subprocess.run(
            args,
            stdout=stdout,
            stderr=subprocess.STDOUT,
            cwd=cwd,
            env=env,
            check=False,
            shell=False,
            encoding="utf-8",
        )
        return cp
    except Exception as e:
        LOG.error(e)
        return None
Пример #5
0
def nodejs_build(src, reports_dir, lang_tools):
    """
    Automatically build nodejs project

    :param src: Source directory
    :param reports_dir: Reports directory to store any logs
    :param lang_tools: Language specific build tools

    :return: boolean status from the build. True if the command executed successfully. False otherwise
    """
    cmd_args = lang_tools.get("npm")
    yarn_mode = False
    pjson_files = [p.as_posix() for p in Path(src).glob("package.json")]
    ylock_files = [p.as_posix() for p in Path(src).glob("yarn.lock")]
    if ylock_files:
        cmd_args = lang_tools.get("yarn")
        yarn_mode = True
    elif not pjson_files:
        LOG.info(
            "Nodejs auto build is supported only for npm or yarn based projects"
        )
        return False
    cp = exec_tool(cmd_args, src)
    LOG.debug(cp.stdout)
    ret = cp.returncode == 0
    try:
        cmd_args = ["npm"]
        if yarn_mode:
            cmd_args = ["yarn"]
        cmd_args += ["run", "build"]
        exec_tool(cmd_args, src)
    except Exception:
        LOG.debug("Automatic build has failed for the node.js project")
    return ret
Пример #6
0
def java_build(src, reports_dir, lang_tools):
    """
    Automatically build java project

    :param src: Source directory
    :param reports_dir: Reports directory to store any logs
    :param lang_tools: Language specific build tools

    :return: boolean status from the build. True if the command executed successfully. False otherwise
    """
    cmd_args = []
    pom_files = [p.as_posix() for p in Path(src).rglob("pom.xml")]
    gradle_files = [p.as_posix() for p in Path(src).rglob("build.gradle")]
    sbt_files = [p.as_posix() for p in Path(src).rglob("build.sbt")]
    env = get_env()
    if pom_files:
        cmd_args = lang_tools.get("maven")
    elif gradle_files:
        cmd_args = get_gradle_cmd(src, lang_tools.get("gradle"))
    elif sbt_files:
        cmd_args = lang_tools.get("sbt")
    if not cmd_args:
        LOG.info(
            "Java auto build is supported only for maven or gradle based projects"
        )
        return False
    cp = exec_tool("auto-build",
                   cmd_args,
                   src,
                   env=env,
                   stdout=subprocess.PIPE)
    if cp:
        LOG.debug(cp.stdout)
        return cp.returncode == 0
    return False
Пример #7
0
def android_build(src, reports_dir, lang_tools):
    """
    Automatically build android project

    :param src: Source directory
    :param reports_dir: Reports directory to store any logs
    :param lang_tools: Language specific build tools

    :return: boolean status from the build. True if the command executed successfully. False otherwise
    """
    if not os.getenv("ANDROID_SDK_ROOT") and not os.getenv("ANDROID_HOME"):
        LOG.info(
            "ANDROID_SDK_ROOT or ANDROID_HOME should be set for automatically building android projects"
        )
        return False
    lang_tools = build_tools_map.get("android")
    env = get_env()
    gradle_files = [p.as_posix() for p in Path(src).rglob("build.gradle")]
    gradle_kts_files = [
        p.as_posix() for p in Path(src).rglob("build.gradle.kts")
    ]
    if gradle_files or gradle_kts_files:
        cmd_args = get_gradle_cmd(src, lang_tools.get("gradle"))
    cp = exec_tool("auto-build",
                   cmd_args,
                   src,
                   env=env,
                   stdout=subprocess.PIPE)
    if cp:
        LOG.debug(cp.stdout)
        return cp.returncode == 0
    return False
Пример #8
0
def read_algorithm_config(config_file):
    """Read config file and return a list of values"""
    LOG.info("Reading config file ...")
    config_data = ()
    with open(config_file, "r") as f:
        data = f.read()
    out_re = data.replace("\r", "").replace(" ", "")
    out_ind = out_re.split('\n')
    config_data = (out_ind[0].split(':')[1], out_ind[1].split(':')[1],
                   out_ind[2].split(':')[1], out_ind[3].split(':')[1])
    # config_data (Population_Range, Termination, Adaptive_Mutation_Step,
    #              Survivor_Selection)
    LOG.info("{0}".format(config_data))
    return config_data
Пример #9
0
def inspect_scan(language, src, reports_dir, convert, repo_context):
    """
    Method to perform inspect cloud scan

    Args:
      language Project language
      src Project dir
      reports_dir Directory for output reports
      convert Boolean to enable normalisation of reports json
      repo_context Repo context
    """
    run_uuid = config.get("run_uuid")
    cpg_mode = config.get("SHIFTLEFT_CPG")
    env = os.environ.copy()
    env["SCAN_JAVA_HOME"] = os.environ.get("SCAN_JAVA_8_HOME")
    report_fname = utils.get_report_file(
        "ng-sast", reports_dir, convert, ext_name="json"
    )
    sl_cmd = config.get("SHIFTLEFT_NGSAST_CMD")
    # Check if sl cli is available
    if not utils.check_command(sl_cmd):
        LOG.warning(
            "sl cli is not available. Please check if your build uses shiftleft/scan-java as the image"
        )
        return
    analyze_files = config.get("SHIFTLEFT_ANALYZE_FILE")
    analyze_target_dir = config.get(
        "SHIFTLEFT_ANALYZE_DIR", os.path.join(src, "target")
    )
    extra_args = None
    if not analyze_files:
        if language == "java":
            analyze_files = utils.find_java_artifacts(analyze_target_dir)
        elif language == "csharp":
            if not utils.check_dotnet():
                LOG.warning(
                    "dotnet is not available. Please check if your build uses shiftleft/scan-csharp as the image"
                )
                return
            analyze_files = utils.find_csharp_artifacts(src)
            cpg_mode = True
        else:
            if language == "ts" or language == "nodejs":
                language = "js"
                extra_args = ["--", "--ts", "--babel"]
            analyze_files = [src]
            cpg_mode = True
    app_name = find_app_name(src, repo_context)
    branch = repo_context.get("revisionId")
    if not branch:
        branch = "master"
    if not analyze_files:
        LOG.warning(
            "Unable to find any build artifacts. Compile your project first before invoking scan or use the auto build feature."
        )
        return
    if isinstance(analyze_files, list) and len(analyze_files) > 1:
        LOG.warning(
            "Multiple files found in {}. Only {} will be analyzed".format(
                analyze_target_dir, analyze_files[0]
            )
        )
        analyze_files = analyze_files[0]
    sl_args = [
        sl_cmd,
        "analyze",
        "--no-auto-update" if language == "java" else None,
        "--wait",
        "--cpg" if cpg_mode else None,
        "--" + language,
        "--tag",
        "branch=" + branch,
        "--app",
        app_name,
    ]
    sl_args += [analyze_files]
    if extra_args:
        sl_args += extra_args
    sl_args = [arg for arg in sl_args if arg is not None]
    LOG.info(
        "About to perform ShiftLeft NG SAST cloud analysis. This might take a few minutes ..."
    )
    LOG.debug(" ".join(sl_args))
    LOG.debug(repo_context)
    cp = exec_tool("NG SAST", sl_args, src, env=env)
    if cp.returncode != 0:
        LOG.warning("NG SAST cloud analyze has failed with the below logs")
        LOG.debug(sl_args)
        LOG.info(cp.stderr)
        return
    findings_data = fetch_findings(app_name, branch, report_fname)
    if findings_data and convert:
        crep_fname = utils.get_report_file(
            "ng-sast", reports_dir, convert, ext_name="sarif"
        )
        convertLib.convert_file("ng-sast", sl_args[1:], src, report_fname, crep_fname)
    track({"id": run_uuid, "scan_mode": "ng-sast", "sl_args": sl_args})
Пример #10
0
def exec_tool(  # scan:ignore
        tool_name,
        args,
        cwd=None,
        env=utils.get_env(),
        stdout=subprocess.DEVNULL):
    """
    Convenience method to invoke cli tools

    Args:
      tool_name Tool name
      args cli command and args
      cwd Current working directory
      env Environment variables
      stdout stdout configuration for run command

    Returns:
      CompletedProcess instance
    """
    with Progress(
            console=console,
            redirect_stderr=False,
            redirect_stdout=False,
            refresh_per_second=1,
    ) as progress:
        task = None
        try:
            env = use_java(env)
            LOG.debug('⚡︎ Executing {} "{}"'.format(tool_name, " ".join(args)))
            stderr = subprocess.DEVNULL
            if LOG.isEnabledFor(DEBUG):
                stderr = subprocess.STDOUT
            tool_verb = "Scanning with"
            if "init" in tool_name:
                tool_verb = "Initializing"
            elif "build" in tool_name:
                tool_verb = "Building with"
            task = progress.add_task("[green]" + tool_verb + " " + tool_name,
                                     total=100,
                                     start=False)
            cp = subprocess.run(
                args,
                stdout=stdout,
                stderr=stderr,
                cwd=cwd,
                env=env,
                check=False,
                shell=False,
                encoding="utf-8",
            )
            if cp and stdout == subprocess.PIPE:
                for line in cp.stdout:
                    progress.update(task, completed=5)
            if (cp and LOG.isEnabledFor(DEBUG) and cp.returncode
                    and cp.stdout is not None):
                LOG.debug(cp.stdout)
            progress.update(task, completed=100, total=100)
            return cp
        except Exception as e:
            if task:
                progress.update(task, completed=20, total=10, visible=False)
            if not LOG.isEnabledFor(DEBUG):
                LOG.info(
                    f"{tool_name} has reported few errors. To view, pass the environment variable SCAN_DEBUG_MODE=debug"
                )
            LOG.debug(e)
            return None
Пример #11
0
    def annotate_pr(self, repo_context, findings_file, report_summary,
                    build_status):
        if not findings_file:
            return
        with open(findings_file, mode="r") as fp:
            try:
                gitlab_context = self.get_context(repo_context)
                findings_obj = json.load(fp)
                findings = findings_obj.get("findings")
                if not findings:
                    LOG.debug("No findings from scan available to report")
                    return
                if not gitlab_context.get(
                        "mergeRequestIID") or not gitlab_context.get(
                            "mergeRequestProjectId"):
                    LOG.debug(
                        "Scan is not running as part of a merge request. Check if the pipeline is using only: [merge_requests] or rules syntax"
                    )
                    return
                private_token = self.get_token()
                if not private_token:
                    LOG.info(
                        "To create a merge request note, create a personal access token with api scope and set it as GITLAB_TOKEN environment variable"
                    )
                    return
                summary = "| Tool | Critical | High | Medium | Low | Status |\n"
                summary = (
                    summary +
                    "| ---- | ------- | ------ | ----- | ---- | ---- |\n")
                for rk, rv in report_summary.items():
                    summary = f'{summary}| {rv.get("tool")} | {rv.get("critical")} | {rv.get("high")} | {rv.get("medium")} | {rv.get("low")} | {rv.get("status")} |\n'
                template = config.get("PR_COMMENT_TEMPLATE")
                recommendation = (
                    f"Please review the [scan reports]({gitlab_context.get('jobUrl')}/artifacts/browse/reports) before approving this merge request."
                    if build_status == "fail" else "Looks good")
                apiUrl = (f"{gitlab_context.get('apiUrl')}")
                mergeRequestIID = (f"{gitlab_context.get('mergeRequestIID')}")
                mergeRequestProjectId = (
                    f"{gitlab_context.get('mergeRequestProjectId')}")
                mergeRequestSourceBranch = (
                    f"{gitlab_context.get('mergeRequestSourceBranch')}")
                mergeRequestTargetBranch = (
                    f"{gitlab_context.get('mergeRequestTargetBranch')}")
                commitSHA = (f"{gitlab_context.get('commitSHA')}")
                projectId = (f"{gitlab_context.get('projectId')}")
                projectName = (f"{gitlab_context.get('projectName')}")
                projectUrl = (f"{gitlab_context.get('projectUrl')}")
                jobUrl = (f"{gitlab_context.get('jobUrl')}")
                jobId = (f"{gitlab_context.get('jobId')}")
                jobName = (f"{gitlab_context.get('jobName')}")
                jobToken = (f"{gitlab_context.get('jobToken')}")

                body = template % dict(
                    summary=summary,
                    recommendation=recommendation,
                    apiUrl=apiUrl,
                    mergeRequestIID=mergeRequestIID,
                    mergeRequestProjectId=mergeRequestProjectId,
                    mergeRequestSourceBranch=mergeRequestSourceBranch,
                    mergeRequestTargetBranch=mergeRequestTargetBranch,
                    commitSHA=commitSHA,
                    projectId=projectId,
                    projectName=projectName,
                    projectUrl=projectUrl,
                    jobUrl=jobUrl,
                    jobId=jobId,
                    jobName=jobName,
                    jobToken=jobToken)
                rr = requests.post(
                    self.get_mr_notes_url(repo_context),
                    headers={
                        "Content-Type": "application/json",
                        "PRIVATE-TOKEN": self.get_token(),
                    },
                    json={"body": body},
                )
                if not rr.ok:
                    LOG.debug(rr.json())
            except Exception as e:
                LOG.debug(e)
Пример #12
0
def summary(
    sarif_files,
    depscan_files=None,
    aggregate_file=None,
    override_rules={},
    baseline_file=None,
):
    """Generate overall scan summary based on the generated
    SARIF file

    :param sarif_files: List of generated sarif report files
    :param depscan_files: Depscan result files
    :param aggregate_file: Filename to store aggregate data
    :param override_rules Build break rules to override for testing
    :param baseline_file: Scan baseline file
    :returns dict representing the summary
    """
    report_summary = {}
    baseline_fingerprints = {
        "scanPrimaryLocationHash": [],
        "scanTagsHash": [],
    }
    build_status = "pass"
    # This is the list of all runs which will get stored as an aggregate
    run_data_list = []
    default_rules = config.get("build_break_rules").get("default")
    depscan_default_rules = config.get("build_break_rules").get("depscan")
    # Collect stats from depscan files if available
    if depscan_files:
        for df in depscan_files:
            with open(df, mode="r") as drep_file:
                dep_data = get_depscan_data(drep_file)
                if not dep_data:
                    continue
                # depscan-java or depscan-nodejs based on filename
                dep_type = (os.path.basename(df).replace(".json", "").replace(
                    "-report", ""))
                metrics, required_pkgs_found = calculate_depscan_metrics(
                    dep_data)
                report_summary[dep_type] = {
                    "tool":
                    f"""Dependency Scan ({dep_type.replace("depscan-", "")})""",
                    "critical": metrics["critical"],
                    "high": metrics["high"],
                    "medium": metrics["medium"],
                    "low": metrics["low"],
                    "status": ":white_heavy_check_mark:",
                }
                report_summary[dep_type].pop("total", None)
                # Compare against the build break rule to determine status
                dep_tool_rules = config.get("build_break_rules").get(
                    dep_type, {})
                build_break_rules = {**depscan_default_rules, **dep_tool_rules}
                if override_rules and override_rules.get("depscan"):
                    build_break_rules = {
                        **build_break_rules,
                        **override_rules.get("depscan"),
                    }
                # Default severity categories for build status
                build_status_categories = (
                    "critical",
                    "required_critical",
                    "optional_critical",
                    "high",
                    "required_high",
                    "optional_high",
                    "medium",
                    "required_medium",
                    "optional_medium",
                    "low",
                    "required_low",
                    "optional_low",
                )
                # Issue 233 - Consider only required packages if available
                if required_pkgs_found:
                    build_status_categories = (
                        "required_critical",
                        "required_high",
                        "required_medium",
                        "required_low",
                    )
                for rsev in build_status_categories:
                    if build_break_rules.get("max_" + rsev) is not None:
                        if metrics.get(rsev) > build_break_rules["max_" +
                                                                 rsev]:
                            report_summary[dep_type]["status"] = ":cross_mark:"
                            build_status = "fail"

    for sf in sarif_files:
        with open(sf, mode="r") as report_file:
            report_data = json.load(report_file)
            # skip this file if the data is empty
            if not report_data or not report_data.get("runs"):
                LOG.warn("Report file {} is invalid. Skipping ...".format(sf))
                continue
            # Iterate through all the runs
            for run in report_data["runs"]:
                # Add it to the run data list for aggregation
                run_data_list.append(run)
                tool_desc = run["tool"]["driver"]["name"]
                tool_name = tool_desc
                # Initialise
                report_summary[tool_name] = {
                    "tool": tool_desc,
                    "critical": 0,
                    "high": 0,
                    "medium": 0,
                    "low": 0,
                    "status": ":white_heavy_check_mark:",
                }
                results = run.get("results", [])
                metrics = run.get("properties", {}).get("metrics", None)
                # If the result includes metrics use it. If not compute it
                if metrics:
                    report_summary[tool_name].update(metrics)
                    report_summary[tool_name].pop("total", None)
                for aresult in results:
                    if not metrics:
                        if aresult.get("properties"):
                            sev = aresult["properties"][
                                "issue_severity"].lower()
                        else:
                            sev = config.get("exttool_default_severity").get(
                                tool_name.lower(), "medium")
                        report_summary[tool_name][sev] += 1
                    # Track the fingerprints
                    if aresult.get("partialFingerprints"):
                        result_fingerprints = aresult.get(
                            "partialFingerprints")
                        for rfk, rfv in result_fingerprints.items():
                            if not rfv:
                                continue
                            # We are only interested in a small subset of hashes namely scanPrimaryLocationHash, scanTagsHash
                            if rfk in [
                                    "scanPrimaryLocationHash", "scanTagsHash"
                            ]:
                                baseline_fingerprints.setdefault(
                                    rfk, []).append(rfv)
                # Compare against the build break rule to determine status
                tool_rules = config.get("build_break_rules").get(tool_name, {})
                build_break_rules = {
                    **default_rules,
                    **tool_rules,
                    **override_rules
                }
                for rsev in ("critical", "high", "medium", "low"):
                    if build_break_rules.get("max_" + rsev) is not None:
                        if (report_summary.get(tool_name).get(rsev) >
                                build_break_rules["max_" + rsev]):
                            report_summary[tool_name][
                                "status"] = ":cross_mark:"
                            build_status = "fail"

    # Should we store the aggregate data
    if aggregate_file:
        # agg_sarif_file = aggregate_file.replace(".json", ".sarif")
        # aggregate.sarif_aggregate(run_data_list, agg_sarif_file)
        aggregate.jsonl_aggregate(run_data_list, aggregate_file)
        LOG.debug("Aggregate report written to {}\n".format(aggregate_file))
    if baseline_file:
        aggregate.store_baseline(baseline_fingerprints, baseline_file)
        LOG.info("Baseline file written to {}".format(baseline_file))
    return report_summary, build_status
Пример #13
0
    parser.add_option("-g", "--ga-conf", dest="ga_conf",
                      action="store", help="Configuration file of the" \
                      "genetic algorithm")

    parser.add_option("-e", "--es-conf", dest="es_conf", action="store",
                      help="Configuration file of the evolutionary strategy")

    opts = parser.parse_args()[0]

    try:
        if len(sys.argv) < 2:
            raise OptsError("Missing arguments")

        if opts.ga_conf:
            algorithm_name = 'GA behavior'
            LOG.info("Starting GA")
            pop_size, term, ad_mut_stp, mu_lambda = read_algorithm_config(opts.ga_conf)
            search_ga(int(term), int(pop_size), ast.literal_eval(ad_mut_stp),
                      ast.literal_eval((mu_lambda)))
            plot_data(algorithm_name, LOG_NAME)
            LOG.info("Finish GA")

        if opts.es_conf:
            algorithm_name = 'ES behavior'
            LOG.info("Starting ES")
            pop_range, term, ad_mut_stp, mu_lambda = read_algorithm_config(opts.es_conf)
            search_es(int(term), int(pop_range), ast.literal_eval(ad_mut_stp),
                      ast.literal_eval((mu_lambda)))
            plot_data(algorithm_name, LOG_NAME)
            LOG.info("Finish ES")