class LintOpts: rules: LintRuleCollectionT success_report: Type[LintSuccessReportBase] failure_report: Type[LintFailureReportBase] config: LintConfig = get_lint_config() full_repo_metadata_config: Optional[FullRepoMetadataConfig] = None extra: Dict[str, object] = field(default_factory=dict)
def lint_file_and_apply_patches( file_path: Path, source: bytes, *, use_ignore_byte_markers: bool = True, use_ignore_comments: bool = True, config: Optional[LintConfig] = None, rules: LintRuleCollectionT, max_iter: int = 100, cst_wrapper: Optional[MetadataWrapper] = None, find_unused_suppressions: bool = False, ) -> LintRuleReportsWithAppliedPatches: """ Runs `lint_file` in a loop, patching one auto-fixable report on each iteration. Applying a single fix at a time prevents the scenario where multiple autofixes combine in a way that results in invalid code. """ # lint_file will fetch this if we don't, but it requires disk I/O, so let's fetch it # here to avoid hitting the disk inside our autofixer loop. config = config if config is not None else get_lint_config() reports = [] fixed_reports = [] # Avoid getting stuck in an infinite loop, cap the number of iterations at `max_iter`. for i in range(max_iter): reports = lint_file( file_path, source, use_ignore_byte_markers=use_ignore_byte_markers, use_ignore_comments=use_ignore_comments, config=config, rules=rules, cst_wrapper=cst_wrapper, find_unused_suppressions=find_unused_suppressions, ) try: first_fixable_report = next(r for r in reports if r.patch is not None) except StopIteration: # No reports with autofix were found. break else: # We found a fixable report. Patch and re-run the linter on this file. patch = first_fixable_report.patch assert patch is not None # TODO: This is really inefficient because we're forced to decode/reencode # the source representation, just so that lint_file can decode the file # again. # # We probably need to rethink how we're representing the source code. encoding = _detect_encoding(source) source = patch.apply(source.decode(encoding)).encode(encoding) fixed_reports.append(first_fixable_report) # `reports` shouldn't contain any fixable reports at this point, so there should be # no overlap between `fixed_reports` and `reports`. return LintRuleReportsWithAppliedPatches( reports=(*fixed_reports, *reports), patched_source=source )
def get_pyre_fixture_dir_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(add_help=False) parser.add_argument( "--fixture-dir", type=(lambda p: Path(p).resolve(strict=True)), help=("Main fixture file directory for integration testing."), default=get_lint_config().fixture_dir, ) return parser
def test_find_and_import_rule(self) -> None: rules_packages = get_lint_config().packages # Test with existing dummy rule. Should get the first one it finds, from dummy_1 module. imported_rule = find_and_import_rule("DummyRule1", rules_packages) self.assertEqual(imported_rule.__module__, f"{DUMMY_PACKAGE}.dummy_1") with self.assertRaises(LintRuleNotFoundError): imported_rule = find_and_import_rule("DummyRule1000", rules_packages)
def relative_to_repo_root(_path: str) -> Path: repo_root = get_lint_config().repo_root try: return Path(_path).resolve(strict=True).relative_to(repo_root) except ValueError: raise argparse.ArgumentTypeError( f"Invalid value {_path}.\n" + f"Paths must be relative to the repo root `{repo_root}`" + " from the `repo_root` setting in the `.fixit.config.yaml` file." )
def apply_fix_operation( path: Path, opts: LintOpts, metadata_cache: Optional[Mapping["ProviderT", object]] = None, ) -> Iterable[str]: with open(path, "rb") as f: source = f.read() patched_files_list = opts.patched_files_list try: if patched_files_list is None: lint_result = lint_file_and_apply_patches( path, source, rules=opts.rules, use_ignore_byte_markers=opts.use_ignore_byte_markers, use_ignore_comments=opts.use_ignore_comments, find_unused_suppressions=True, ) raw_reports = lint_result.reports updated_source = lint_result.patched_source if updated_source != source: if not opts.skip_autoformatter: # Format the code using the config file's formatter. updated_source = invoke_formatter( get_lint_config().formatter, updated_source) with open(path, "wb") as f: f.write(updated_source) else: lint_result = get_one_patchable_report_for_path( path, source, opts.rules, opts.use_ignore_byte_markers, opts.use_ignore_comments, metadata_cache, ) # Will either be a single patchable report, or a collection of non-patchable reports. raw_reports = lint_result.reports updated_source = lint_result.patched_source if updated_source != source: # We don't do any formatting here as it's wasteful. The caller should handle formatting all files at the end. with open(path, "wb") as f: f.write(updated_source) patched_files_list.append(str(path)) # Return only the report that was used in the patched source. return [next(opts.formatter.format(rr) for rr in raw_reports)] except (SyntaxError, ParserSyntaxError) as e: print_red( f"Encountered the following error while parsing source code in file {path}:" ) print(e) return [] return [opts.formatter.format(rr) for rr in raw_reports]
def get_formatted_reports_for_path( path: Path, opts: InsertSuppressionsOpts, metadata_cache: Optional[Mapping["ProviderT", object]] = None, ) -> Iterable[str]: with open(path, "rb") as f: source = f.read() try: cst_wrapper = None if metadata_cache is not None: cst_wrapper = MetadataWrapper( parse_module(source), True, metadata_cache, ) raw_reports = lint_file( path, source, rules={opts.rule}, cst_wrapper=cst_wrapper ) except (SyntaxError, ParserSyntaxError) as e: print_red( f"Encountered the following error while parsing source code in file {path}:" ) print(e) return [] opts_message = opts.message comments = [] for rr in raw_reports: if isinstance(opts_message, str): message = opts_message elif opts_message == MessageKind.USE_LINT_REPORT: message = rr.message else: # opts_message == MessageKind.NO_MESSAGE message = None comments.append( SuppressionComment(opts.kind, rr.line, rr.code, message, opts.max_lines) ) insert_suppressions_result = insert_suppressions(source, comments) updated_source = insert_suppressions_result.updated_source assert ( not insert_suppressions_result.failed_insertions ), "Failed to insert some comments. This should not be possible." if updated_source != source: if not opts.skip_autoformatter: # Format the code using the config file's formatter. updated_source = invoke_formatter( get_lint_config().formatter, updated_source ) with open(path, "wb") as f: f.write(updated_source) # linter completed successfully return [opts.formatter.format(rr) for rr in raw_reports]
def create_rule_file(file_path: Path) -> None: """Create a new rule file.""" context = _LICENCE + _IMPORTS + _TO_DOS + _RULE_CLASS updated_context = invoke_formatter(get_lint_config().formatter, context) with open(file_path, "w") as f: f.write(updated_context) print( f"Successfully created {file_path.name} rule file at {file_path.parent}" )
def import_rule(rule_name: str) -> LintRuleT: # Using the rule_name or full dotted name, attempt to import the rule. rule_module_path, _, rule_class_name = rule_name.rpartition(".") if rule_module_path: # If user provided a dotted path, we assume it's valid and import the rule directly. imported_rule = getattr( importlib.import_module(rule_module_path), rule_class_name, ) return imported_rule # Otherwise, only a class name was provided, so try to find the rule by searching each package specified in the config. return find_and_import_rule(rule_class_name, get_lint_config().packages)
def test_import_rule_from_package(self) -> None: rules_package = get_lint_config().packages self.assertEqual(rules_package, [DUMMY_PACKAGE]) # Test with an existing dummy rule. imported_rule = import_rule_from_package(rules_package[0], "DummyRule2") self.assertTrue(imported_rule is not None) self.assertEqual(imported_rule.__name__, "DummyRule2") self.assertEqual(imported_rule.__module__, f"{DUMMY_PACKAGE}.dummy_2") # Test with non-existent rule. imported_rule = import_rule_from_package(rules_package[0], "DummyRule1000") self.assertIsNone(imported_rule)
def get_paths_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(add_help=False) parser.add_argument( "paths", nargs="*", type=relative_to_repo_root, default=(Path(get_lint_config().repo_root),), help=( "The name of a directory (e.g. media) or file (e.g. media/views.py) " + "relative to the `repo_root` specified in the `fixit.config.yaml` file" + " on which to run this script. " + "If not specified, the `repo_root` value is used." ), ) return parser
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. from pathlib import Path from fixit import add_lint_rule_tests_to_module from fixit.common.base import LintConfig from fixit.common.config import get_lint_config, get_rules_from_config # Add all the CstLintRules from `fixit.rules` package to this module as unit tests. CONFIG: LintConfig = get_lint_config() add_lint_rule_tests_to_module( globals(), get_rules_from_config(), fixture_dir=Path(CONFIG.fixture_dir), rules_package="fixit.rules", )
def lint_file( file_path: Path, source: bytes, *, use_ignore_byte_markers: bool = True, use_ignore_comments: bool = True, config: Optional[LintConfig] = None, rules: LintRuleCollectionT, cst_wrapper: Optional[MetadataWrapper] = None, find_unused_suppressions: bool = False, ) -> Collection[BaseLintRuleReport]: """ May raise a SyntaxError, which should be handled by the caller. """ # Get settings from the nearest `.fixit.config.yaml` file if necessary. config: LintConfig = config if config is not None else get_lint_config() if use_ignore_byte_markers and any( pattern.encode() in source for pattern in config.block_list_patterns ): return [] tokens = None if use_ignore_comments: # Don't compute tokens unless we have to, it slows down # `fixit.cli.run_rules`. # # `tokenize` is actually much more expensive than generating the whole AST, # since AST parsing is heavily optimized C, and tokenize is pure python. tokens = _get_tokens(source) ignore_info = IgnoreInfo.compute( comment_info=CommentInfo.compute(tokens=tokens), line_mapping_info=LineMappingInfo.compute(tokens=tokens), ) else: ignore_info = None # Don't waste time evaluating rules that are globally ignored. evaluated_rules = [ r for r in rules if ignore_info is None or ignore_info.should_evaluate_rule(r) ] # Categorize lint rules. cst_rules: List[Type[CstLintRule]] = [] pseudo_rules: List[Type[PseudoLintRule]] = [] for r in evaluated_rules: if issubclass(r, CstLintRule): cst_rules.append(cast(Type[CstLintRule], r)) elif issubclass(r, PseudoLintRule): pseudo_rules.append(cast(Type[PseudoLintRule], r)) # `self.context.report()` accumulates reports into the context object, we'll copy # those into our local `reports` list. ast_tree = None reports = [] if cst_wrapper is None: cst_wrapper = MetadataWrapper(cst.parse_module(source), unsafe_skip_copy=True) if cst_rules: cst_context = CstContext(cst_wrapper, source, file_path, config) _visit_cst_rules_with_context(cst_wrapper, cst_rules, cst_context) reports.extend(cst_context.reports) if pseudo_rules: psuedo_context = PseudoContext(file_path, source, tokens, ast_tree) for pr_cls in pseudo_rules: reports.extend(pr_cls(psuedo_context).lint_file()) if ignore_info is not None: # filter the accumulated errors that should be suppressed and report unused suppressions reports = [r for r in reports if not ignore_info.should_ignore_report(r)] if find_unused_suppressions and cst_rules: # We had to make sure to call ignore_info.should_ignore_report before running our # RemoveUnusedSuppressionsRule because ignore_info needs to be up to date for it to work. # We can construct a new context since we want a fresh set of reports to append to reports. config.rule_config[RemoveUnusedSuppressionsRule.__name__] = { "ignore_info": ignore_info, "rules": cst_rules, } unused_suppressions_context = CstContext( cst_wrapper, source, file_path, config ) _visit_cst_rules_with_context( cst_wrapper, [RemoveUnusedSuppressionsRule], unused_suppressions_context ) reports.extend(unused_suppressions_context.reports) return reports
def _main(args: argparse.Namespace) -> int: width = shutil.get_terminal_size(fallback=(80, 24)).columns rules = args.rules use_ignore_byte_markers = args.use_ignore_byte_markers use_ignore_comments = args.use_ignore_comments skip_autoformatter = args.skip_autoformatter formatter = AutofixingLintRuleReportFormatter(width, args.compact) workers = args.workers # Find files if directory was provided. file_paths = tuple(find_files(args.paths)) if not args.compact: print(f"Scanning {len(file_paths)} files") print("\n".join(file_paths)) print() start_time = time.time() total_reports_count = 0 if rules_require_metadata_cache(rules): touched_files = set() next_files = file_paths with Manager() as manager: # Avoid getting stuck in an infinite loop. for _ in range(MAX_ITER): if not next_files: break patched_files = manager.list() metadata_caches = get_metadata_caches(args.cache_timeout, next_files) next_files = [] # opts is a more type-safe version of args that we pass around opts = LintOpts( rules=rules, use_ignore_byte_markers=use_ignore_byte_markers, use_ignore_comments=use_ignore_comments, skip_autoformatter=skip_autoformatter, formatter=formatter, patched_files_list=patched_files, ) total_reports_count += call_map_paths_and_print_reports( next_files, opts, workers, metadata_caches) next_files = list(patched_files) touched_files.update(patched_files) # Finally, format all the touched files. if not skip_autoformatter: for path in touched_files: with open(path, "rb") as f: source = f.read() # Format the code using the config file's formatter. formatted_source = invoke_formatter( get_lint_config().formatter, source) with open(path, "wb") as f: f.write(formatted_source) else: # opts is a more type-safe version of args that we pass around opts = LintOpts( rules=rules, use_ignore_byte_markers=use_ignore_byte_markers, use_ignore_comments=use_ignore_comments, skip_autoformatter=skip_autoformatter, formatter=formatter, ) total_reports_count = call_map_paths_and_print_reports( file_paths, opts, workers, None) if not args.compact: print() print( f"Found {total_reports_count} reports in {len(file_paths)} files in " + f"{time.time() - start_time :.2f} seconds.") return 0
def main(raw_args: Sequence[str]) -> int: parser = argparse.ArgumentParser( description=( "Runs a lint rule's autofixer over all of over a set of " + "files or directories.\n" + "\n" + "This is similar to the functionality provided by LibCST codemods " + "(https://libcst.readthedocs.io/en/latest/codemods_tutorial.html), " + "but limited to the small subset of APIs provided by Fixit."), parents=[ get_rules_parser(), get_metadata_cache_parser(), get_paths_parser(), get_skip_ignore_comments_parser(), get_skip_ignore_byte_marker_parser(), get_skip_autoformatter_parser(), get_compact_parser(), get_multiprocessing_parser(), ], ) args = parser.parse_args(raw_args) width = shutil.get_terminal_size(fallback=(80, 24)).columns rules = args.rules use_ignore_byte_markers = args.use_ignore_byte_markers use_ignore_comments = args.use_ignore_comments skip_autoformatter = args.skip_autoformatter formatter = AutofixingLintRuleReportFormatter(width, args.compact) workers = args.workers # Find files if directory was provided. file_paths = tuple(find_files(args.paths)) if not args.compact: print(f"Scanning {len(file_paths)} files") print("\n".join(file_paths)) print() start_time = time.time() total_reports_count = 0 if rules_require_metadata_cache(rules): touched_files = set() next_files = file_paths with Manager() as manager: # Avoid getting stuck in an infinite loop. for _ in range(MAX_ITER): if not next_files: break patched_files = manager.list() metadata_caches = get_metadata_caches(args.cache_timeout, next_files) next_files = [] # opts is a more type-safe version of args that we pass around opts = LintOpts( rules=rules, use_ignore_byte_markers=use_ignore_byte_markers, use_ignore_comments=use_ignore_comments, skip_autoformatter=skip_autoformatter, formatter=formatter, patched_files_list=patched_files, ) total_reports_count += call_map_paths_and_print_reports( next_files, opts, workers, metadata_caches) next_files = list(patched_files) touched_files.update(patched_files) # Finally, format all the touched files. if not skip_autoformatter: for path in touched_files: with open(path, "rb") as f: source = f.read() # Format the code using the config file's formatter. formatted_source = invoke_formatter( get_lint_config().formatter, source) with open(path, "wb") as f: f.write(formatted_source) else: # opts is a more type-safe version of args that we pass around opts = LintOpts( rules=rules, use_ignore_byte_markers=use_ignore_byte_markers, use_ignore_comments=use_ignore_comments, skip_autoformatter=skip_autoformatter, formatter=formatter, ) total_reports_count = call_map_paths_and_print_reports( file_paths, opts, workers, None) if not args.compact: print() print( f"Found {total_reports_count} reports in {len(file_paths)} files in " + f"{time.time() - start_time :.2f} seconds.") return 0