def test_ignored_lines(self, *, source: str, ignored_code: str, ignored_lines: Container[int]) -> None: tokens = tuple( tokenize.tokenize(BytesIO(source.encode("utf-8")).readline)) ignore_info = IgnoreInfo.compute( comment_info=CommentInfo.compute(tokens=tokens), line_mapping_info=LineMappingInfo.compute(tokens=tokens), ) lines = range(1, tokens[-1].end[0] + 1) actual_ignored_lines = [] for line in lines: ignored = ignore_info.should_ignore_report( CstLintRuleReport( file_path=Path("fake/path.py"), node=cst.EmptyLine(), code=ignored_code, message="message", line=line, column=0, module=cst.MetadataWrapper(cst.parse_module(source)), module_bytes=source.encode("utf-8"), )) if ignored: actual_ignored_lines.append(line) # pyre-fixme[6]: Expected `Iterable[Variable[_T]]` for 1st param but got # `Container[int]`. self.assertEqual(actual_ignored_lines, list(ignored_lines))
def target_suggestions(self): with ctx_inliner.set(self): globls = Tracer(self.module, globls=self.base_globls).trace().globls collector = CollectTargetSuggestions(self, globls) cst.MetadataWrapper(self.module).visit(collector) return collector.suggestions
def test_lambda_metadata_matcher(self) -> None: # Match on qualified name provider module = cst.parse_module( "from typing import List\n\ndef foo() -> None: pass\n") wrapper = cst.MetadataWrapper(module) functiondef = cst.ensure_type(wrapper.module.body[1], cst.FunctionDef) self.assertTrue( matches( functiondef, m.FunctionDef(name=m.MatchMetadataIfTrue( meta.QualifiedNameProvider, lambda qualnames: any(n.name in {"foo", "bar", "baz"} for n in qualnames), )), metadata_resolver=wrapper, )) self.assertFalse( matches( functiondef, m.FunctionDef(name=m.MatchMetadataIfTrue( meta.QualifiedNameProvider, lambda qualnames: any(n.name in {"bar", "baz"} for n in qualnames), )), metadata_resolver=wrapper, ))
def main(): options = get_arg_parser().parse_args() root_file_path = 'delinter/test/input/test_unused_imports.py' msg_template = r'{path}:{line}:[{msg_id}({symbol}),{obj}]{msg}' pylint_command = f"{root_file_path} --enable=W --disable=C,R,E,F --msg-template={msg_template} --score=n" out, _ = lint.py_run(pylint_command, return_std=True) result = "".join(out.readlines()).split('\n') result = [r.strip() for r in result if r.strip() and not r.strip(). startswith('************* Module ')] parsed_warnings = Delinter.parse_linter_warnings(result, options.msg_id) if os.path.isdir(root_file_path): from pathlib import Path files = Path(root_file_path).glob('**/*.py') else: files = [root_file_path] for file_path in files: with open(file_path) as f: source_code = "".join(f.readlines()) source_tree = cst.parse_module(source_code) wrapper = cst.MetadataWrapper(source_tree) local_warnings = [p for p in parsed_warnings if p.file_path == str(file_path)] fixed_module = wrapper.visit( SUPPORTED_LINTER_MAP[options.msg_id][1](local_warnings)) a_file_path = 'a/' + str(file_path) b_file_path = 'b/' + str(file_path) print("".join(difflib.unified_diff( source_code.splitlines(1), fixed_module.code.splitlines(1), fromfile=a_file_path, tofile=b_file_path )))
def get_refs(mod): analyzer = AnalyzeCalls() cst.MetadataWrapper(mod).visit(analyzer) return [ {'start': span.start, 'end': span.start + span.length, 'name': name} for (name, span) in analyzer.calls ]
def code_folding(self): tracer = Tracer(self.module, globls=self.base_globls, args=TracerArgs(trace_lines=True)).trace() finder = FindUnexecutedBlocks(tracer) cst.MetadataWrapper(self.module, unsafe_skip_copy=True).visit(finder) return sorted(finder.unexecuted)
class LintRuleReportTest(UnitTest): @data_provider( { "AstLintRuleReport": [ AstLintRuleReport( file_path=Path("fake/path.py"), node=ast.parse(""), code="SomeFakeRule", message="some message", line=1, column=1, ) ], "CstLintRuleReport": [ CstLintRuleReport( file_path=Path("fake/path.py"), node=cst.parse_statement("pass\n"), code="SomeFakeRule", message="some message", line=1, column=1, module=cst.MetadataWrapper(cst.parse_module(b"pass\n")), module_bytes=b"pass\n", ) ], } ) def test_is_not_pickleable(self, report: BaseLintRuleReport) -> None: with pytest.raises(pickle.PicklingError): pickle.dumps(report)
def test( self, *, source: bytes, rules_in_lint_run: Collection[Type[CstLintRule]], rules_without_report: Collection[Type[CstLintRule]], suppressed_line: int, expected_unused_suppressions_report_messages: Collection[str], expected_replacements: Optional[List[str]] = None, ) -> None: reports = [ CstLintRuleReport( file_path=FILE_PATH, node=cst.EmptyLine(), code=rule.__name__, message="message", line=suppressed_line, column=0, module=cst.MetadataWrapper(cst.parse_module(source)), module_bytes=source, ) for rule in rules_in_lint_run if rule not in rules_without_report ] tokens = _get_tokens(source) ignore_info = IgnoreInfo.compute( comment_info=CommentInfo.compute(tokens=tokens), line_mapping_info=LineMappingInfo.compute(tokens=tokens), ) cst_wrapper = MetadataWrapper(cst.parse_module(source), unsafe_skip_copy=True) config = LintConfig( rule_config={ RemoveUnusedSuppressionsRule.__name__: { "ignore_info": ignore_info, "rules": rules_in_lint_run, } }) unused_suppressions_context = CstContext(cst_wrapper, source, FILE_PATH, config) for report in reports: ignore_info.should_ignore_report(report) _visit_cst_rules_with_context(cst_wrapper, [RemoveUnusedSuppressionsRule], unused_suppressions_context) messages = [] patches = [] for report in unused_suppressions_context.reports: messages.append(report.message) patches.append(report.patch) self.assertEqual(messages, expected_unused_suppressions_report_messages) if expected_replacements is None: self.assertEqual(len(patches), 0) else: self.assertEqual(len(patches), len(expected_replacements)) for idx, patch in enumerate(patches): replacement = patch.apply(source.decode()) self.assertEqual(replacement, expected_replacements[idx])
def test_replace_metadata(self) -> None: def _rename_foo( node: cst.CSTNode, extraction: Dict[str, Union[cst.CSTNode, Sequence[cst.CSTNode]]], ) -> cst.CSTNode: return cst.ensure_type(node, cst.Name).with_changes(value="replaced") original = cst.parse_module( "foo: int = 37\ndef bar(foo: int) -> int:\n return foo\n\nbiz: int = bar(42)\n" ) wrapper = cst.MetadataWrapper(original) replaced = cst.ensure_type( m.replace( wrapper, m.Name(metadata=m.MatchMetadataIfTrue( meta.QualifiedNameProvider, lambda qualnames: any(n.name == "foo" for n in qualnames), )), _rename_foo, ), cst.Module, ).code self.assertEqual( replaced, "replaced: int = 37\ndef bar(foo: int) -> int:\n return foo\n\nbiz: int = bar(42)\n", )
def collect_names(tree, ctx=()): """ Convenience wrapper for NameCollector """ visitor = NameCollector(ctx) wrapper = cst.MetadataWrapper(tree, unsafe_skip_copy=True) wrapper.visit(visitor) return visitor.names
def _parse(self, raw): try: cst = libcst.parse_module(raw) annotated_cst = libcst.MetadataWrapper(cst) except libcst._exceptions.ParserSyntaxError as error: # TODO: log something? What's the wanted behavior here? raise error return annotated_cst
def refactor_string( source: str, unused_imports: List[Union[Import, ImportFrom]], ) -> str: if unused_imports: wrapper = cst.MetadataWrapper(cst.parse_module(source)) fixed_module = wrapper.visit( RemoveUnusedImportTransformer(unused_imports)) return fixed_module.code return source
def test_assertEquals_02(): source = """ def test_1(): assertEquals(True, False) """ module = cst.parse_module(source) wrapper = cst.MetadataWrapper(module) checker = Checker(Path("(test)")) wrapper.visit(checker) assert not checker.errors
def ambiguous_tuple_equality_example(): codes = ["True, True, True == (True, True, True)", "True, True, (True == (True, True, True))", "(True, True, True) == (True, True, True)", ] for code in codes: print(code) node = cst.parse_module(code) node_with_metadata = cst.MetadataWrapper(node) # print(dump(node.body[0].body[0].value, show_syntax=True, indent=" " * 4, show_whitespace=True, show_defaults=True)) node_with_metadata.visit(Visitor())
def test_assertEquals_01(): source = """ class MyTestCase(unittest.TestCase): def test_1(): self.assertEquals(True, False) """ module = cst.parse_module(source) wrapper = cst.MetadataWrapper(module) checker = Checker(Path("(test)")) wrapper.visit(checker) assert checker.errors
def _make_fixture( self, code: str) -> Tuple[cst.BaseExpression, meta.MetadataWrapper]: module = cst.parse_module(dedent(code)) wrapper = cst.MetadataWrapper(module) return ( cst.ensure_type( cst.ensure_type(wrapper.module.body[0], cst.SimpleStatementLine).body[0], cst.Expr, ).value, wrapper, )
def check_result(source, expected): module = cst.parse_module(source) wrapper = cst.MetadataWrapper(module) modernizer = Modernizer(Path("(test)")) modified_tree = wrapper.visit(modernizer) diff = "".join( difflib.unified_diff( expected.splitlines(True), modified_tree.code.splitlines(True), fromfile="expected", tofile="actual", ) ) assert modified_tree.code == expected, diff
def transform_module_impl( self, tree: cst.Module, ) -> cst.Module: """ Collect type annotations from all stubs and apply them to ``tree``. Gather existing imports from ``tree`` so that we don't add duplicate imports. """ import_gatherer = GatherImportsVisitor(CodemodContext()) tree.visit(import_gatherer) existing_import_names = _get_imported_names( import_gatherer.all_imports) context_contents = self.context.scratch.get( ApplyTypeAnnotationsVisitor.CONTEXT_KEY) if context_contents is not None: ( stub, overwrite_existing_annotations, use_future_annotations, strict_posargs_matching, strict_annotation_matching, ) = context_contents self.overwrite_existing_annotations = ( self.overwrite_existing_annotations or overwrite_existing_annotations) self.use_future_annotations = (self.use_future_annotations or use_future_annotations) self.strict_posargs_matching = (self.strict_posargs_matching and strict_posargs_matching) self.strict_annotation_matching = (self.strict_annotation_matching or strict_annotation_matching) visitor = TypeCollector(existing_import_names, self.context) cst.MetadataWrapper(stub).visit(visitor) self.annotations.update(visitor.annotations) if self.use_future_annotations: AddImportsVisitor.add_needed_import(self.context, "__future__", "annotations") tree_with_imports = AddImportsVisitor( self.context).transform_module(tree) tree_with_changes = tree_with_imports.visit(self) # don't modify the imports if we didn't actually add any type information if self.annotation_counts.any_changes_applied(): return tree_with_changes else: return tree
def get_fully_qualified_names(file_path: str, module_str: str) -> Set[QualifiedName]: wrapper = cst.MetadataWrapper( cst.parse_module(dedent(module_str)), cache={ FullyQualifiedNameProvider: FullyQualifiedNameProvider.gen_cache( Path(""), [file_path], None ).get(file_path, "") }, ) return { qname for qnames in wrapper.resolve(FullyQualifiedNameProvider).values() for qname in qnames }
def generate_trial(self, program, cond): metadata = json.loads(program.header[0].comment.value[1:]) name = program.body[0].name.value program = cst.MetadataWrapper(program).visit(Preprocessor()) if cond == self.Condition.Random: program = cst.MetadataWrapper(program).visit(RandomRenamer()) call = f'mystery({metadata["input"]})' globls = {} exec(program.code, globls, globls) answer = eval(call, globls, globls) return { 'program': program.code, 'call': call, 'function': name, 'cond': str(cond), 'answer': str(answer), 'schema': metadata['schema'] }
def test_batchable_provider(self) -> None: test = self class SomeVisitor(cst.BatchableCSTVisitor): METADATA_DEPENDENCIES = (ByteSpanPositionProvider,) def visit_Pass(self, node: cst.Pass) -> None: test.assertEqual( self.get_metadata(ByteSpanPositionProvider, node), CodeSpan(start=0, length=4), ) wrapper = cst.MetadataWrapper(cst.parse_module("pass")) wrapper.visit_batched([SomeVisitor()])
def unused_vars(self): assert self.trace_reads, "Tracer was not executed with trace_reads=True" visitor = UnusedVarsVisitor(self) # unsafe_skip_copy to ensure that nodes in map are pointer-equivalent # to input mod wrapper = cst.MetadataWrapper(self.transformed_module, unsafe_skip_copy=True) wrapper.visit(visitor) unused_vars = visitor.unused_vars return { self.node_map[k]: v for k, v in unused_vars.items() if k in self.node_map }
def setUp(self) -> None: self.fake_filepath = Path("fake/path.py") self.report = CstLintRuleReport( file_path=self.fake_filepath, node=cst.parse_statement("pass\n"), code="SomeFakeRule", message=( "Some long message that should span multiple lines.\n" + "\n" + "Another paragraph with more information about the lint rule." ), line=1, column=1, module=cst.MetadataWrapper(cst.parse_module(b"pass\n")), module_bytes=b"pass\n", )
def coverage_collector_for_module(relative_path: str, module: cst.Module, strict_default: bool) -> CoverageCollector: module_with_metadata = cst.MetadataWrapper(module) strict_count_collector = StrictCountCollector(strict_default) try: module_with_metadata.visit(strict_count_collector) except RecursionError: LOG.warning(f"LibCST encountered recursion error in `{relative_path}`") coverage_collector = CoverageCollector( strict_count_collector.is_strict_module()) try: module_with_metadata.visit(coverage_collector) except RecursionError: LOG.warning(f"LibCST encountered recursion error in `{relative_path}`") return coverage_collector
def main() -> Optional[int]: parser = argparse.ArgumentParser(description="Test things.") parser.add_argument("file", nargs="+") parser.add_argument("-v", "--verbose", action="store_true", help="verbose output.") parser.add_argument("-q", "--quiet", action="store_true", help="no output.") parser.add_argument( "-x", "--exitfirst", action="store_true", help="exit instantly on first error or failed test.", ) parser.add_argument("--ignore", nargs="*", help="errors to ignore.") args = parser.parse_args() paths = expand_paths([Path(name).expanduser() for name in args.file]) errors = False for path in paths: if path.is_dir() or path.suffix != ".py": continue if args.verbose: print(f"Checking {path}") py_source = path.read_text() module = cst.parse_module(py_source) wrapper = cst.MetadataWrapper(module) checker = Checker(path, args.verbose, args.ignore) wrapper.visit(checker) if checker.errors: if args.exitfirst: return 1 errors = True modernizer = Modernizer(path, args.verbose, args.ignore) modified_tree = wrapper.visit(modernizer) if modernizer.errors: if args.exitfirst: return 1 errors = True if not args.quiet: diff = "".join( difflib.unified_diff( py_source.splitlines(True), modified_tree.code.splitlines(True), fromfile=f"a{path}", tofile=f"b{path}", ) ) if diff: print(diff) if errors: return 1
def test_extract_metadata(self) -> None: # Verify true behavior module = cst.parse_module("a + b[c], d(e, f * g)") wrapper = cst.MetadataWrapper(module) expression = cst.ensure_type( cst.ensure_type(wrapper.module.body[0], cst.SimpleStatementLine).body[0], cst.Expr, ).value nodes = m.extract( expression, m.Tuple(elements=[ m.Element( m.BinaryOperation(left=m.Name(metadata=m.SaveMatchedNode( m.MatchMetadata( meta.PositionProvider, self._make_coderange((1, 0), (1, 1)), ), "left", )))), m.Element(m.Call()), ]), metadata_resolver=wrapper, ) extracted_node = cst.ensure_type( cst.ensure_type(expression, cst.Tuple).elements[0].value, cst.BinaryOperation, ).left self.assertEqual(nodes, {"left": extracted_node}) # Verify false behavior nodes = m.extract( expression, m.Tuple(elements=[ m.Element( m.BinaryOperation(left=m.Name(metadata=m.SaveMatchedNode( m.MatchMetadata( meta.PositionProvider, self._make_coderange((1, 0), (1, 2)), ), "left", )))), m.Element(m.Call()), ]), metadata_resolver=wrapper, ) self.assertIsNone(nodes)
def _run_delinter(options): ''' Run the delinter and produce the diff. ''' root_file_path = options.file_path_or_folder # TODO: Handle Windows paths if Path(root_file_path).is_absolute(): msg_template = r'{abspath}:{line}:[{msg_id}({symbol}),{obj}]{msg}' sep = '' else: msg_template = r'{path}:{line}:[{msg_id}({symbol}),{obj}]{msg}' sep = '/' pylint_command = f"{root_file_path} --enable=W --disable=C,R,E,F --msg-template={msg_template} --score=n" out, _ = lint.py_run(pylint_command, return_std=True) orig_result = "".join(out.readlines()).split('\n') result = [ r.strip() for r in orig_result if r.strip() and not r.strip().startswith('************* Module ') ] parsed_warnings = Delinter.parse_linter_warnings(result, options.msg_id) if os.path.isdir(root_file_path): files = list(Path(root_file_path).glob('**/*.py')) else: files = [root_file_path] for file_path in files: with open(file_path) as f: source_code = "".join(f.readlines()) if not source_code: continue source_tree = cst.parse_module(source_code) wrapper = cst.MetadataWrapper(source_tree) local_warnings = [ p for p in parsed_warnings if p.file_path == str(file_path) ] fixed_module = wrapper.visit( SUPPORTED_LINTER_MAP[options.msg_id][1](local_warnings)) a_file_path = f'a{sep}{file_path}' b_file_path = f'b{sep}{file_path}' result = "".join( difflib.unified_diff(source_code.splitlines(1), fixed_module.code.splitlines(1), fromfile=a_file_path, tofile=b_file_path)) if result: print(result)
def exec_counts(self) -> ExecCounts: assert self.trace_lines, "Tracer was not executed with trace_lines=True" visitor = ExecCountsVisitor(self) # unsafe_skip_copy to ensure that nodes in map are pointer-equivalent # to input mod wrapper = cst.MetadataWrapper(self.transformed_module, unsafe_skip_copy=True) wrapper.visit(visitor) exec_counts = visitor.exec_counts return { self.node_map[k]: v for k, v in exec_counts.items() if k in self.node_map }
def test_pylint_warning(self): warnings = unused_import_warnings.split('\n') warnings = [w for w in warnings if w] parsed_warnings = Delinter.parse_linter_warnings(warnings) source_tree = cst.parse_module(source_code) wrapper = cst.MetadataWrapper(source_tree) fixed_module = wrapper.visit( unused_imports.RemoveUnusedImportTransformer(parsed_warnings)) diff = "".join( difflib.unified_diff(source_code.splitlines(1), fixed_module.code.splitlines(1))) diff = diff.replace('+++ ', '+++').replace('--- ', '---').replace('\n ', '\n') new_expected_diff = expected_diff.replace('\n ', '\n') self.assertEqual(diff, new_expected_diff)
def refactor_string( source: str, unused_imports: List[Union[Import, ImportFrom]], show_error: bool = False, ) -> str: try: wrapper = cst.MetadataWrapper(cst.parse_module(source)) except cst.ParserSyntaxError as err: if show_error: print(Color(str(err)).red) else: if unused_imports: fixed_module = wrapper.visit( RemoveUnusedImportTransformer(unused_imports) ) return fixed_module.code return source