def _calculate_head_comparison(
target_file_manager: TargetFileManager,
tools: Iterable[Tool]) -> Tuple[Baseline, float]:
"""
Calculates a baseline consisting of all findings from the branch head
If no HEAD branch exists return empty baseline
:param paths: Which paths are being checked
:param tools: Which tools to check
:return: The branch head baseline
"""
try:
with target_file_manager.run_context(True,
RunStep.BASELINE) as target_paths:
runner = Runner(paths=target_paths,
use_cache=True,
skip_setup=True)
if len(runner.paths) > 0:
before = time.time()
comparison_results = runner.parallel_results(tools, {},
keep_bars=False)
baseline = {
tool_id: {f.syntactic_identifier_str()
for f in findings}
for tool_id, findings in comparison_results
if isinstance(findings, list)
}
elapsed = time.time() - before
return baseline, elapsed
else:
return {}, 0.0
except NoGitHeadException:
logging.debug("No git head found so defaulting to empty head baseline")
return {}, 0.0
def orchestrate(
baseline: Baseline,
target_file_manager: TargetFileManager,
staged: bool,
tools: Iterable[Tool],
) -> Tuple[Collection[RunResults], float]:
"""
Manages interactions between TargetFileManager, Runner and Tools
Uses passed target_file_manager, staged flag, and tool list to setup
Runner and runs tools on relevant files, returning aggregated output
of tool running and time to run all tools in parallel
"""
elapsed = 0.0
if staged:
head_baseline, elapsed = _calculate_head_comparison(
target_file_manager, tools)
for t in tools:
tool_id = t.tool_id()
if tool_id not in baseline:
baseline[tool_id] = head_baseline.get(tool_id, set())
else:
baseline[tool_id].update(head_baseline.get(tool_id, set()))
with target_file_manager.run_context(staged,
RunStep.CHECK) as target_paths:
use_cache = not staged # if --all then can use cache
skip_setup = staged # if check --all then include setup
runner = Runner(paths=target_paths,
use_cache=use_cache,
skip_setup=skip_setup)
if len(runner.paths) == 0:
echo_warning(
f"Nothing to check or archive. Please confirm that changes are staged and not excluded by `{IGNORE_FILE_NAME}`. To check all Git tracked files, use `--all`."
)
click.secho("", err=True)
all_results: Collection[RunResults] = []
elapsed = 0.0
else:
before = time.time()
all_results = runner.parallel_results(tools, baseline)
elapsed += time.time() - before
return all_results, elapsed
def _install_config_if_not_exists(self) -> bool:
"""
Installs .bento.yml if one does not already exist
:return: whether a config was installed
"""
config_path = self.context.config_path
pretty_path = self.context.pretty_path(config_path)
if not config_path.exists():
on_done = content.InstallConfig.install.echo(pretty_path)
with (open(
os.path.join(os.path.dirname(__file__),
"../configs/default.yml"))) as template:
yml = yaml.safe_load(template)
target_file_manager = TargetFileManager(
self.context.base_path,
[self.context.base_path],
staged=False,
ignore_rules_file_path=self.context.ignore_file_path,
)
for tid, tool in self.context.tool_inventory.items():
if (not tool(self.context).matches_project(
target_file_manager._target_paths)
and tid in yml["tools"]):
del yml["tools"][tid]
logging.debug(
f"Matching tools for this project: {', '.join(yml['tools'].keys())}"
)
if not yml["tools"]:
logging.warning("No tools match this project")
with config_path.open("w") as config_file:
yaml.safe_dump(yml, stream=config_file)
on_done()
logging.info(f"Created {pretty_path}.")
return True
else:
content.InstallConfig.install.echo(pretty_path, skip=True)
return False
def archive(context: Context, all_: bool, paths: Tuple[Path, ...]) -> None:
"""
Suppress current findings.
By default, only results introduced by currently staged changes will be
added to the archive (`.bento/archive.json`). Archived findings will
not appear in future `bento check` output and will not block commits if
`autorun` is enabled.
Use `--all` to archive findings in all Git tracked files, not just those
that are staged:
$ bento archive --all [PATHS]
Optional PATHS can be specified to archive results from specific directories
or files.
Archived findings are viewable in `.bento/archive.json`.
"""
# Default to no path filter
if len(paths) < 1:
path_list = [context.base_path]
else:
path_list = list(paths)
if not context.is_init:
if all_:
click.echo(f"Running Bento archive on all tracked files...\n", err=True)
else:
click.echo(f"Running Bento archive on staged files...\n", err=True)
if not context.config_path.exists():
echo_error("No Bento configuration found. Please run `bento init`.")
sys.exit(3)
if context.baseline_file_path.exists():
with context.baseline_file_path.open() as json_file:
old_baseline = bento.result.load_baseline(json_file)
old_hashes = {
h
for findings in old_baseline.values()
for h in findings.get(VIOLATIONS_KEY, {}).keys()
}
else:
old_baseline = {}
old_hashes = set()
new_baseline: Dict[str, Dict[str, Dict[str, Any]]] = {}
tools = context.tools.values()
target_file_manager = TargetFileManager(
context.base_path, path_list, not all_, context.ignore_file_path
)
target_paths = target_file_manager.get_target_files()
all_findings, elapsed = bento.orchestrator.orchestrate(
context, target_paths, not all_, tools
)
n_found = 0
n_existing = 0
found_hashes: Set[str] = set()
for tool_id, vv in all_findings:
if isinstance(vv, Exception):
raise vv
# Remove filtered
vv = [f for f in vv if not f.filtered]
n_found += len(vv)
new_baseline[tool_id] = bento.result.dump_results(vv)
if tool_id in old_baseline:
new_baseline[tool_id][VIOLATIONS_KEY].update(
old_baseline[tool_id][VIOLATIONS_KEY]
)
for v in vv:
h = v.syntactic_identifier_str()
found_hashes.add(h)
if h in old_hashes:
n_existing += 1
n_new = n_found - n_existing
context.baseline_file_path.parent.mkdir(exist_ok=True, parents=True)
with context.baseline_file_path.open("w") as json_file:
bento.result.write_tool_results(json_file, new_baseline)
finding_source_text = "in this project" if all_ else "due to staged changes"
success_str = f"{n_new} finding(s) {finding_source_text} were archived, and will be hidden in future Bento runs."
if n_existing > 0:
success_str += f"\nBento also kept {n_existing} existing finding(s)."
click.echo(success_str, err=True)
if not context.is_init:
echo_newline()
echo_next_step("To view archived results", "cat .bento/archive.json")
def check(
context: Context,
all_: bool = False,
formatter: Tuple[str, ...] = (),
pager: bool = True,
tool: Optional[str] = None,
staged_only: bool = False, # Should not be used. Legacy support for old pre-commit hooks
paths: Tuple[Path, ...] = (),
) -> None:
"""
Checks for new findings.
By default, only staged files are checked. New findings introduced by
these staged changes AND that are not in the archive (`.bento/archive.json`)
will be shown.
Use `--all` to check all Git tracked files, not just those that are staged:
$ bento check --all [PATHS]
Optional PATHS can be specified to check specific directories or files.
See `bento archive --help` to learn about suppressing findings.
"""
# Fail out if not configured
if not context.config_path.exists():
raise NoConfigurationException()
# Fail out if no .bentoignore
if not context.ignore_file_path.exists():
raise NoIgnoreFileException(context)
# Default to no path filter
if len(paths) < 1:
path_list = [context.base_path]
else:
path_list = list(paths)
# Handle specified tool that is not configured
if tool and tool not in context.configured_tools:
click.echo(
f"{tool} has not been configured. Adding default configuration for tool to {bento.constants.CONFIG_FILE_NAME}"
)
update_tool_run(context, tool, False)
# Set configured_tools to None so that future calls will
# update and include newly added tool
context._configured_tools = None
# Handle specified formatters
if formatter:
context.config["formatter"] = [{f: {}} for f in formatter]
if all_:
click.echo(f"Running Bento checks on all tracked files...\n", err=True)
else:
click.echo(f"Running Bento checks on staged files...\n", err=True)
tools: Iterable[Tool[Any]] = context.tools.values()
if tool:
tools = [context.configured_tools[tool]]
baseline: Baseline = {}
if context.baseline_file_path.exists():
with context.baseline_file_path.open() as json_file:
baseline = bento.result.json_to_violation_hashes(json_file)
target_file_manager = TargetFileManager(
context.base_path, path_list, not all_, context.ignore_file_path
)
all_results, elapsed = bento.orchestrator.orchestrate(
baseline, target_file_manager, not all_, tools
)
fmts = context.formatters
findings_to_log: List[Any] = []
n_all = 0
n_all_filtered = 0
filtered_findings: Dict[str, List[Violation]] = {}
for tool_id, findings in all_results:
if isinstance(findings, Exception):
logging.error(findings)
echo_error(f"Error while running {tool_id}: {findings}")
if isinstance(findings, BentoException):
click.secho(findings.msg, err=True)
else:
if isinstance(findings, subprocess.CalledProcessError):
click.secho(findings.stderr, err=True)
click.secho(findings.stdout, err=True)
if isinstance(findings, NodeError):
echo_warning(
f"Node.js not found or version is not compatible with ESLint v6."
)
click.secho(
f"""-------------------------------------------------------------------------------------------------
This may be due to a corrupted tool installation. You might be able to fix this issue by running:
bento init --clean
You can also view full details of this error in `{bento.constants.DEFAULT_LOG_PATH}`.
-------------------------------------------------------------------------------------------------
""",
err=True,
)
context.error_on_exit(ToolRunException())
elif isinstance(findings, list) and findings:
findings_to_log += bento.metrics.violations_to_metrics(
tool_id,
context.timestamp,
findings,
__get_ignores_for_tool(tool_id, context.config),
)
filtered = [f for f in findings if not f.filtered]
filtered_findings[tool_id] = filtered
n_all += len(findings)
n_filtered = len(filtered)
n_all_filtered += n_filtered
logging.debug(f"{tool_id}: {n_filtered} findings passed filter")
def post_metrics() -> None:
bento.network.post_metrics(findings_to_log, is_finding=True)
stats_thread = threading.Thread(name="stats", target=post_metrics)
stats_thread.start()
dumped = [f.dump(filtered_findings) for f in fmts]
context.start_user_timer()
bento.util.less(dumped, pager=pager, overrun_pages=OVERRUN_PAGES)
context.stop_user_timer()
finding_source_text = "in this project" if all_ else "due to staged changes"
if n_all_filtered > 0:
echo_warning(
f"{n_all_filtered} finding(s) {finding_source_text} in {elapsed:.2f} s"
)
click.secho("\nPlease fix these issues, or:\n", err=True)
echo_next_step("To archive findings as tech debt", f"bento archive")
echo_next_step("To disable a specific check", f"bento disable check TOOL CHECK")
else:
echo_success(f"0 findings {finding_source_text} in {elapsed:.2f} s\n")
n_archived = n_all - n_all_filtered
if n_archived > 0:
echo_next_step(
f"Not showing {n_archived} archived finding(s). To view",
"cat .bento/archive.json",
)
if not all_ and not context.autorun_is_blocking:
return
elif context.on_exit_exception:
raise context.on_exit_exception
elif n_all_filtered > 0:
sys.exit(2)