def _run_codemods(code: str, min_version: Tuple[int, int]) -> str: """Run all Shed fixers on a code string.""" context = cst.codemod.CodemodContext() # We want LibCST to parse the code as if running on our target minimum version, # but fall back to the latest version it supports (currently 3.8) if our target # version is newer than that. Or jump *forward* if Black got the version wrong! try: v = _pick_compatible_python_version(".".join(map(str, min_version))) config = cst.PartialParserConfig(python_version=f"{v.major}.{v.minor}") mod = cst.parse_module(code, config) except cst.ParserSyntaxError as err: # missed f-string version check https://github.com/Zac-HD/shed/issues/31 msg = "Incomplete input. Encountered '=', but expected '!', ':', or '}'." if min_version >= (3, 8) or "=}" not in code or msg not in str(err): raise # pragma: no cover # no *known* cases trigger this, but... mod = cst.parse_module(code, cst.PartialParserConfig(python_version="3.8")) # Right here, we know that the minimum version was too low. We could in # principle `return shed.shed(code, ...)` to fully account for that, but # I don't want to pass around so many extra args just to cover an edge # case which should be fixed soon regardless. if imports_hypothesis(code): # pragma: no cover mod = attempt_hypothesis_codemods(context, mod) mod = ShedFixers(context).transform_module(mod) return mod.code
def _parse_statement_force_38(code: str) -> cst.BaseCompoundStatement: statement = cst.parse_statement( code, config=cst.PartialParserConfig(python_version="3.8")) if not isinstance(statement, cst.BaseCompoundStatement): raise Exception( "This function is expecting to parse compound statements only!") return statement
def main(filters=()): # dict[filename][version] results = {} for v in VERSIONS: # TODO as there get to be many more of these, should run them in # parallel, or at least streaming. output = subprocess.check_output([v, "run.py"], encoding="utf-8") for line in output.splitlines(): filename, result = line.split() results.setdefault(filename, {})[v] = result == "YES" # Now print a pretty table. TODO should write json or something consumable # as well. max_filename = max(len(f) for f in results) # TODO mark versions that will be libcst-checked too buf = [" " * max_filename] for v in VERSIONS: short_version = v[-3:] if short_version in LIBCST_VERSIONS: buf.append(f" \x1b[32m{short_version.replace('.', '')}\x1b[0m") else: buf.append(f" {short_version.replace('.', '')}") print("".join(buf)) for f in sorted(results): if filters and not any(filt in f for filt in filters): continue sys.stdout.write(f.ljust(max_filename + 1)) for v in VERSIONS: libcst_result = None if v[-3:] in LIBCST_VERSIONS: try: with open(f) as fo: data = fo.read() cst.parse_module( data, cst.PartialParserConfig(python_version=v[-3:])) libcst_result = True except cst.ParserSyntaxError: libcst_result = False if libcst_result != results[f][v]: sys.stdout.write( "\x1b[31m" f"{'o' if results[f][v] else '.'}{'o' if libcst_result else '.'} " "\x1b[0m") else: sys.stdout.write( f"{'o' if results[f][v] else '.'}{'o' if libcst_result else '.'} " ) else: sys.stdout.write(f"{'o' if results[f][v] else '.'} ") sys.stdout.write("\n") print("Legend:") print(" green header means will test with libcst") print() print(" first result is python, second [optional] result is libcst") print(" o parses") print(" . does not parse")
def try_parse(path: Path, data: Optional[bytes] = None) -> cst.Module: """ Attempts to parse the file with all syntax versions known by LibCST. If parsing fails on all supported grammar versions, then raises the parser error from the first/newest version attempted. """ if data is None: data = path.read_bytes() with timed(f"parsing {path}"): parse_error: Optional[cst.ParserSyntaxError] = None for version in cst.KNOWN_PYTHON_VERSION_STRINGS[::-1]: try: mod = cst.parse_module( data, cst.PartialParserConfig(python_version=version)) return mod except cst.ParserSyntaxError as e: # keep the first error we see in case parsing fails on all versions if parse_error is None: parse_error = e # not caring about existing traceback here because it's not useful for parse # errors, and usort_path is already going to wrap it in a custom class raise parse_error or Exception("unknown parse failure")
def parse_src(src, python_version): """Parses a string of source code into a LibCST tree.""" version_str = utils.format_version(python_version) if python_version >= (3, 9): log.warning("LibCST does not support Python %s; parsing with 3.8 instead.", version_str) version_str = "3.8" config = libcst.PartialParserConfig(python_version=version_str) return libcst.parse_module(src, config)
def test_byte_conversion(self, ) -> None: module_bytes = "fn()\n".encode("utf-16") mw = MetadataWrapper( cst.parse_module("fn()\n", cst.PartialParserConfig(encoding="utf-16"))) codegen_partial = mw.resolve(ExperimentalReentrantCodegenProvider)[ mw.module.body[0]] self.assertEqual(codegen_partial.get_original_module_bytes(), module_bytes) self.assertEqual( codegen_partial.get_modified_module_bytes( cst.parse_statement("fn2()\n")), "fn2()\n".encode("utf-16"), )
def test_walrus(self) -> None: code = """ if x := y: pass """ wrapper = MetadataWrapper( parse_module( dedent(code), config=cst.PartialParserConfig(python_version="3.8") ) ) wrapper.visit( DependentVisitor( test=self, name_to_context={ "x": ExpressionContext.STORE, "y": ExpressionContext.LOAD, }, ) )
def inner(code: str) -> cst.BaseStatement: return cst.parse_statement(code, config=cst.PartialParserConfig(**config))
def inner(code: str) -> cst.BaseExpression: return cst.parse_expression(code, config=cst.PartialParserConfig(**config))
def _parse_expression_force_38(code: str) -> cst.BaseExpression: return cst.parse_expression( code, config=cst.PartialParserConfig(python_version="3.8"))
def visit_Name(self, node: cst.Name) -> None: for var in self.template_vars: if node.value == mangled_name(var): raise Exception( f'Template variable "{var}" was not replaced properly') def unmangle_nodes( tree: cst.CSTNode, template_replacements: Mapping[str, ValidReplacementType], ) -> cst.CSTNode: unmangler = TemplateTransformer(template_replacements) return ensure_type(tree.visit(unmangler), cst.CSTNode) _DEFAULT_PARTIAL_PARSER_CONFIG: cst.PartialParserConfig = cst.PartialParserConfig( ) def parse_template_module( template: str, config: cst.PartialParserConfig = _DEFAULT_PARTIAL_PARSER_CONFIG, **template_replacements: ValidReplacementType, ) -> cst.Module: """ Accepts an entire python module template, including all leading and trailing whitespace. Any :class:`~libcst.CSTNode` provided as a keyword argument to this function will be inserted into the template at the appropriate location similar to an f-string expansion. For example:: module = parse_template_module("from {mod} import Foo\\n", mod=Name("bar"))
def main(version, filename): with open(filename) as fo: data = fo.read() mod = cst.parse_module(data, cst.PartialParserConfig(python_version=version)) print(mod)
import itertools import unittest import libcst import parameterized from craftier import parenthesize EXAMPLES = ( [ libcst.ensure_type( libcst.parse_statement( "if a := 1:\n pass", config=libcst.PartialParserConfig(python_version="3.8"), ), libcst.If, ).test, ], [ libcst.parse_expression("lambda x: x + 1"), ], [ libcst.parse_expression("x if x else y"), ], [ libcst.parse_expression("x or y"), ], [ libcst.parse_expression("x and y"), ], [
class Signal: '' def value(self): '' pass class Foo: """ """ def __init__(self): '' pass ''' config = cst.PartialParserConfig(python_version="3.5") source_tree = cst.parse_module(py_source, config) print(source_tree.get_docstring()) source_tree.code_for_node(source_tree.children[3]) ## select a class definition - Q&D classdef = source_tree.children[2] assert isinstance(classdef, cst.ClassDef), "Not a classdef" ## class docstring 1: '' # body=IndentedBlock( # body=[ # SimpleStatementLine( # body=[
"BottomOutput", "MethodDescriptorOutput", "parser_config", "NumpyUfuncOutput", ] # If there are more than this amount in a union, just use any MAX_UNION_ITEMS = 5 # If there are more than this amount in a string literal, just use any MAX_STRING_ITEMS = 5 # More than this and it will be tuple of arbitrary length MAX_TUPLE_ITEMS = 2 parser_config = cst.PartialParserConfig( python_version="3.8", encoding="utf-8", default_indent=" ", default_newline="\n", ) def annotation(tp: OutputType) -> typing.Optional[cst.BaseExpression]: if is_unknown(tp): return None return tp.annotation def create_type(o: object) -> OutputType: try: tp = pydantic.parse_obj_as(InputType, o) # type: ignore except pydantic.error_wrappers.ValidationError: raise ValueError(f"Could not parse JSON as input type: {o}")