class LintOpts:
rules: LintRuleCollectionT
success_report: Type[LintSuccessReportBase]
failure_report: Type[LintFailureReportBase]
config: LintConfig = get_lint_config()
full_repo_metadata_config: Optional[FullRepoMetadataConfig] = None
extra: Dict[str, object] = field(default_factory=dict)
def lint_file_and_apply_patches(
file_path: Path,
source: bytes,
*,
use_ignore_byte_markers: bool = True,
use_ignore_comments: bool = True,
config: Optional[LintConfig] = None,
rules: LintRuleCollectionT,
max_iter: int = 100,
cst_wrapper: Optional[MetadataWrapper] = None,
find_unused_suppressions: bool = False,
) -> LintRuleReportsWithAppliedPatches:
"""
Runs `lint_file` in a loop, patching one auto-fixable report on each iteration.
Applying a single fix at a time prevents the scenario where multiple autofixes
combine in a way that results in invalid code.
"""
# lint_file will fetch this if we don't, but it requires disk I/O, so let's fetch it
# here to avoid hitting the disk inside our autofixer loop.
config = config if config is not None else get_lint_config()
reports = []
fixed_reports = []
# Avoid getting stuck in an infinite loop, cap the number of iterations at `max_iter`.
for i in range(max_iter):
reports = lint_file(
file_path,
source,
use_ignore_byte_markers=use_ignore_byte_markers,
use_ignore_comments=use_ignore_comments,
config=config,
rules=rules,
cst_wrapper=cst_wrapper,
find_unused_suppressions=find_unused_suppressions,
)
try:
first_fixable_report = next(r for r in reports if r.patch is not None)
except StopIteration:
# No reports with autofix were found.
break
else:
# We found a fixable report. Patch and re-run the linter on this file.
patch = first_fixable_report.patch
assert patch is not None
# TODO: This is really inefficient because we're forced to decode/reencode
# the source representation, just so that lint_file can decode the file
# again.
#
# We probably need to rethink how we're representing the source code.
encoding = _detect_encoding(source)
source = patch.apply(source.decode(encoding)).encode(encoding)
fixed_reports.append(first_fixable_report)
# `reports` shouldn't contain any fixable reports at this point, so there should be
# no overlap between `fixed_reports` and `reports`.
return LintRuleReportsWithAppliedPatches(
reports=(*fixed_reports, *reports), patched_source=source
)
def get_pyre_fixture_dir_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument(
"--fixture-dir",
type=(lambda p: Path(p).resolve(strict=True)),
help=("Main fixture file directory for integration testing."),
default=get_lint_config().fixture_dir,
)
return parser
def test_find_and_import_rule(self) -> None:
rules_packages = get_lint_config().packages
# Test with existing dummy rule. Should get the first one it finds, from dummy_1 module.
imported_rule = find_and_import_rule("DummyRule1", rules_packages)
self.assertEqual(imported_rule.__module__, f"{DUMMY_PACKAGE}.dummy_1")
with self.assertRaises(LintRuleNotFoundError):
imported_rule = find_and_import_rule("DummyRule1000",
rules_packages)
def relative_to_repo_root(_path: str) -> Path:
repo_root = get_lint_config().repo_root
try:
return Path(_path).resolve(strict=True).relative_to(repo_root)
except ValueError:
raise argparse.ArgumentTypeError(
f"Invalid value {_path}.\n"
+ f"Paths must be relative to the repo root `{repo_root}`"
+ " from the `repo_root` setting in the `.fixit.config.yaml` file."
)
def apply_fix_operation(
path: Path,
opts: LintOpts,
metadata_cache: Optional[Mapping["ProviderT", object]] = None,
) -> Iterable[str]:
with open(path, "rb") as f:
source = f.read()
patched_files_list = opts.patched_files_list
try:
if patched_files_list is None:
lint_result = lint_file_and_apply_patches(
path,
source,
rules=opts.rules,
use_ignore_byte_markers=opts.use_ignore_byte_markers,
use_ignore_comments=opts.use_ignore_comments,
find_unused_suppressions=True,
)
raw_reports = lint_result.reports
updated_source = lint_result.patched_source
if updated_source != source:
if not opts.skip_autoformatter:
# Format the code using the config file's formatter.
updated_source = invoke_formatter(
get_lint_config().formatter, updated_source)
with open(path, "wb") as f:
f.write(updated_source)
else:
lint_result = get_one_patchable_report_for_path(
path,
source,
opts.rules,
opts.use_ignore_byte_markers,
opts.use_ignore_comments,
metadata_cache,
)
# Will either be a single patchable report, or a collection of non-patchable reports.
raw_reports = lint_result.reports
updated_source = lint_result.patched_source
if updated_source != source:
# We don't do any formatting here as it's wasteful. The caller should handle formatting all files at the end.
with open(path, "wb") as f:
f.write(updated_source)
patched_files_list.append(str(path))
# Return only the report that was used in the patched source.
return [next(opts.formatter.format(rr) for rr in raw_reports)]
except (SyntaxError, ParserSyntaxError) as e:
print_red(
f"Encountered the following error while parsing source code in file {path}:"
)
print(e)
return []
return [opts.formatter.format(rr) for rr in raw_reports]
def get_formatted_reports_for_path(
path: Path,
opts: InsertSuppressionsOpts,
metadata_cache: Optional[Mapping["ProviderT", object]] = None,
) -> Iterable[str]:
with open(path, "rb") as f:
source = f.read()
try:
cst_wrapper = None
if metadata_cache is not None:
cst_wrapper = MetadataWrapper(
parse_module(source),
True,
metadata_cache,
)
raw_reports = lint_file(
path, source, rules={opts.rule}, cst_wrapper=cst_wrapper
)
except (SyntaxError, ParserSyntaxError) as e:
print_red(
f"Encountered the following error while parsing source code in file {path}:"
)
print(e)
return []
opts_message = opts.message
comments = []
for rr in raw_reports:
if isinstance(opts_message, str):
message = opts_message
elif opts_message == MessageKind.USE_LINT_REPORT:
message = rr.message
else: # opts_message == MessageKind.NO_MESSAGE
message = None
comments.append(
SuppressionComment(opts.kind, rr.line, rr.code, message, opts.max_lines)
)
insert_suppressions_result = insert_suppressions(source, comments)
updated_source = insert_suppressions_result.updated_source
assert (
not insert_suppressions_result.failed_insertions
), "Failed to insert some comments. This should not be possible."
if updated_source != source:
if not opts.skip_autoformatter:
# Format the code using the config file's formatter.
updated_source = invoke_formatter(
get_lint_config().formatter, updated_source
)
with open(path, "wb") as f:
f.write(updated_source)
# linter completed successfully
return [opts.formatter.format(rr) for rr in raw_reports]
def create_rule_file(file_path: Path) -> None:
"""Create a new rule file."""
context = _LICENCE + _IMPORTS + _TO_DOS + _RULE_CLASS
updated_context = invoke_formatter(get_lint_config().formatter, context)
with open(file_path, "w") as f:
f.write(updated_context)
print(
f"Successfully created {file_path.name} rule file at {file_path.parent}"
)
def import_rule(rule_name: str) -> LintRuleT:
# Using the rule_name or full dotted name, attempt to import the rule.
rule_module_path, _, rule_class_name = rule_name.rpartition(".")
if rule_module_path:
# If user provided a dotted path, we assume it's valid and import the rule directly.
imported_rule = getattr(
importlib.import_module(rule_module_path),
rule_class_name,
)
return imported_rule
# Otherwise, only a class name was provided, so try to find the rule by searching each package specified in the config.
return find_and_import_rule(rule_class_name, get_lint_config().packages)
def test_import_rule_from_package(self) -> None:
rules_package = get_lint_config().packages
self.assertEqual(rules_package, [DUMMY_PACKAGE])
# Test with an existing dummy rule.
imported_rule = import_rule_from_package(rules_package[0],
"DummyRule2")
self.assertTrue(imported_rule is not None)
self.assertEqual(imported_rule.__name__, "DummyRule2")
self.assertEqual(imported_rule.__module__, f"{DUMMY_PACKAGE}.dummy_2")
# Test with non-existent rule.
imported_rule = import_rule_from_package(rules_package[0],
"DummyRule1000")
self.assertIsNone(imported_rule)
def get_paths_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument(
"paths",
nargs="*",
type=relative_to_repo_root,
default=(Path(get_lint_config().repo_root),),
help=(
"The name of a directory (e.g. media) or file (e.g. media/views.py) "
+ "relative to the `repo_root` specified in the `fixit.config.yaml` file"
+ " on which to run this script. "
+ "If not specified, the `repo_root` value is used."
),
)
return parser
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
from pathlib import Path
from fixit import add_lint_rule_tests_to_module
from fixit.common.base import LintConfig
from fixit.common.config import get_lint_config, get_rules_from_config
# Add all the CstLintRules from `fixit.rules` package to this module as unit tests.
CONFIG: LintConfig = get_lint_config()
add_lint_rule_tests_to_module(
globals(),
get_rules_from_config(),
fixture_dir=Path(CONFIG.fixture_dir),
rules_package="fixit.rules",
)
def lint_file(
file_path: Path,
source: bytes,
*,
use_ignore_byte_markers: bool = True,
use_ignore_comments: bool = True,
config: Optional[LintConfig] = None,
rules: LintRuleCollectionT,
cst_wrapper: Optional[MetadataWrapper] = None,
find_unused_suppressions: bool = False,
) -> Collection[BaseLintRuleReport]:
"""
May raise a SyntaxError, which should be handled by the
caller.
"""
# Get settings from the nearest `.fixit.config.yaml` file if necessary.
config: LintConfig = config if config is not None else get_lint_config()
if use_ignore_byte_markers and any(
pattern.encode() in source for pattern in config.block_list_patterns
):
return []
tokens = None
if use_ignore_comments:
# Don't compute tokens unless we have to, it slows down
# `fixit.cli.run_rules`.
#
# `tokenize` is actually much more expensive than generating the whole AST,
# since AST parsing is heavily optimized C, and tokenize is pure python.
tokens = _get_tokens(source)
ignore_info = IgnoreInfo.compute(
comment_info=CommentInfo.compute(tokens=tokens),
line_mapping_info=LineMappingInfo.compute(tokens=tokens),
)
else:
ignore_info = None
# Don't waste time evaluating rules that are globally ignored.
evaluated_rules = [
r for r in rules if ignore_info is None or ignore_info.should_evaluate_rule(r)
]
# Categorize lint rules.
cst_rules: List[Type[CstLintRule]] = []
pseudo_rules: List[Type[PseudoLintRule]] = []
for r in evaluated_rules:
if issubclass(r, CstLintRule):
cst_rules.append(cast(Type[CstLintRule], r))
elif issubclass(r, PseudoLintRule):
pseudo_rules.append(cast(Type[PseudoLintRule], r))
# `self.context.report()` accumulates reports into the context object, we'll copy
# those into our local `reports` list.
ast_tree = None
reports = []
if cst_wrapper is None:
cst_wrapper = MetadataWrapper(cst.parse_module(source), unsafe_skip_copy=True)
if cst_rules:
cst_context = CstContext(cst_wrapper, source, file_path, config)
_visit_cst_rules_with_context(cst_wrapper, cst_rules, cst_context)
reports.extend(cst_context.reports)
if pseudo_rules:
psuedo_context = PseudoContext(file_path, source, tokens, ast_tree)
for pr_cls in pseudo_rules:
reports.extend(pr_cls(psuedo_context).lint_file())
if ignore_info is not None:
# filter the accumulated errors that should be suppressed and report unused suppressions
reports = [r for r in reports if not ignore_info.should_ignore_report(r)]
if find_unused_suppressions and cst_rules:
# We had to make sure to call ignore_info.should_ignore_report before running our
# RemoveUnusedSuppressionsRule because ignore_info needs to be up to date for it to work.
# We can construct a new context since we want a fresh set of reports to append to reports.
config.rule_config[RemoveUnusedSuppressionsRule.__name__] = {
"ignore_info": ignore_info,
"rules": cst_rules,
}
unused_suppressions_context = CstContext(
cst_wrapper, source, file_path, config
)
_visit_cst_rules_with_context(
cst_wrapper, [RemoveUnusedSuppressionsRule], unused_suppressions_context
)
reports.extend(unused_suppressions_context.reports)
return reports
def _main(args: argparse.Namespace) -> int:
width = shutil.get_terminal_size(fallback=(80, 24)).columns
rules = args.rules
use_ignore_byte_markers = args.use_ignore_byte_markers
use_ignore_comments = args.use_ignore_comments
skip_autoformatter = args.skip_autoformatter
formatter = AutofixingLintRuleReportFormatter(width, args.compact)
workers = args.workers
# Find files if directory was provided.
file_paths = tuple(find_files(args.paths))
if not args.compact:
print(f"Scanning {len(file_paths)} files")
print("\n".join(file_paths))
print()
start_time = time.time()
total_reports_count = 0
if rules_require_metadata_cache(rules):
touched_files = set()
next_files = file_paths
with Manager() as manager:
# Avoid getting stuck in an infinite loop.
for _ in range(MAX_ITER):
if not next_files:
break
patched_files = manager.list()
metadata_caches = get_metadata_caches(args.cache_timeout,
next_files)
next_files = []
# opts is a more type-safe version of args that we pass around
opts = LintOpts(
rules=rules,
use_ignore_byte_markers=use_ignore_byte_markers,
use_ignore_comments=use_ignore_comments,
skip_autoformatter=skip_autoformatter,
formatter=formatter,
patched_files_list=patched_files,
)
total_reports_count += call_map_paths_and_print_reports(
next_files, opts, workers, metadata_caches)
next_files = list(patched_files)
touched_files.update(patched_files)
# Finally, format all the touched files.
if not skip_autoformatter:
for path in touched_files:
with open(path, "rb") as f:
source = f.read()
# Format the code using the config file's formatter.
formatted_source = invoke_formatter(
get_lint_config().formatter, source)
with open(path, "wb") as f:
f.write(formatted_source)
else:
# opts is a more type-safe version of args that we pass around
opts = LintOpts(
rules=rules,
use_ignore_byte_markers=use_ignore_byte_markers,
use_ignore_comments=use_ignore_comments,
skip_autoformatter=skip_autoformatter,
formatter=formatter,
)
total_reports_count = call_map_paths_and_print_reports(
file_paths, opts, workers, None)
if not args.compact:
print()
print(
f"Found {total_reports_count} reports in {len(file_paths)} files in "
+ f"{time.time() - start_time :.2f} seconds.")
return 0
def main(raw_args: Sequence[str]) -> int:
parser = argparse.ArgumentParser(
description=(
"Runs a lint rule's autofixer over all of over a set of " +
"files or directories.\n" + "\n" +
"This is similar to the functionality provided by LibCST codemods "
+
"(https://libcst.readthedocs.io/en/latest/codemods_tutorial.html), "
+ "but limited to the small subset of APIs provided by Fixit."),
parents=[
get_rules_parser(),
get_metadata_cache_parser(),
get_paths_parser(),
get_skip_ignore_comments_parser(),
get_skip_ignore_byte_marker_parser(),
get_skip_autoformatter_parser(),
get_compact_parser(),
get_multiprocessing_parser(),
],
)
args = parser.parse_args(raw_args)
width = shutil.get_terminal_size(fallback=(80, 24)).columns
rules = args.rules
use_ignore_byte_markers = args.use_ignore_byte_markers
use_ignore_comments = args.use_ignore_comments
skip_autoformatter = args.skip_autoformatter
formatter = AutofixingLintRuleReportFormatter(width, args.compact)
workers = args.workers
# Find files if directory was provided.
file_paths = tuple(find_files(args.paths))
if not args.compact:
print(f"Scanning {len(file_paths)} files")
print("\n".join(file_paths))
print()
start_time = time.time()
total_reports_count = 0
if rules_require_metadata_cache(rules):
touched_files = set()
next_files = file_paths
with Manager() as manager:
# Avoid getting stuck in an infinite loop.
for _ in range(MAX_ITER):
if not next_files:
break
patched_files = manager.list()
metadata_caches = get_metadata_caches(args.cache_timeout,
next_files)
next_files = []
# opts is a more type-safe version of args that we pass around
opts = LintOpts(
rules=rules,
use_ignore_byte_markers=use_ignore_byte_markers,
use_ignore_comments=use_ignore_comments,
skip_autoformatter=skip_autoformatter,
formatter=formatter,
patched_files_list=patched_files,
)
total_reports_count += call_map_paths_and_print_reports(
next_files, opts, workers, metadata_caches)
next_files = list(patched_files)
touched_files.update(patched_files)
# Finally, format all the touched files.
if not skip_autoformatter:
for path in touched_files:
with open(path, "rb") as f:
source = f.read()
# Format the code using the config file's formatter.
formatted_source = invoke_formatter(
get_lint_config().formatter, source)
with open(path, "wb") as f:
f.write(formatted_source)
else:
# opts is a more type-safe version of args that we pass around
opts = LintOpts(
rules=rules,
use_ignore_byte_markers=use_ignore_byte_markers,
use_ignore_comments=use_ignore_comments,
skip_autoformatter=skip_autoformatter,
formatter=formatter,
)
total_reports_count = call_map_paths_and_print_reports(
file_paths, opts, workers, None)
if not args.compact:
print()
print(
f"Found {total_reports_count} reports in {len(file_paths)} files in "
+ f"{time.time() - start_time :.2f} seconds.")
return 0
