def _run_codemods(code: str, min_version: Tuple[int, int]) -> str:
"""Run all Shed fixers on a code string."""
context = cst.codemod.CodemodContext()
# We want LibCST to parse the code as if running on our target minimum version,
# but fall back to the latest version it supports (currently 3.8) if our target
# version is newer than that. Or jump *forward* if Black got the version wrong!
try:
v = _pick_compatible_python_version(".".join(map(str, min_version)))
config = cst.PartialParserConfig(python_version=f"{v.major}.{v.minor}")
mod = cst.parse_module(code, config)
except cst.ParserSyntaxError as err:
# missed f-string version check https://github.com/Zac-HD/shed/issues/31
msg = "Incomplete input. Encountered '=', but expected '!', ':', or '}'."
if min_version >= (3, 8) or "=}" not in code or msg not in str(err):
raise # pragma: no cover # no *known* cases trigger this, but...
mod = cst.parse_module(code,
cst.PartialParserConfig(python_version="3.8"))
# Right here, we know that the minimum version was too low. We could in
# principle `return shed.shed(code, ...)` to fully account for that, but
# I don't want to pass around so many extra args just to cover an edge
# case which should be fixed soon regardless.
if imports_hypothesis(code): # pragma: no cover
mod = attempt_hypothesis_codemods(context, mod)
mod = ShedFixers(context).transform_module(mod)
return mod.code
def _parse_statement_force_38(code: str) -> cst.BaseCompoundStatement:
statement = cst.parse_statement(
code, config=cst.PartialParserConfig(python_version="3.8"))
if not isinstance(statement, cst.BaseCompoundStatement):
raise Exception(
"This function is expecting to parse compound statements only!")
return statement
def main(filters=()):
# dict[filename][version]
results = {}
for v in VERSIONS:
# TODO as there get to be many more of these, should run them in
# parallel, or at least streaming.
output = subprocess.check_output([v, "run.py"], encoding="utf-8")
for line in output.splitlines():
filename, result = line.split()
results.setdefault(filename, {})[v] = result == "YES"
# Now print a pretty table. TODO should write json or something consumable
# as well.
max_filename = max(len(f) for f in results)
# TODO mark versions that will be libcst-checked too
buf = [" " * max_filename]
for v in VERSIONS:
short_version = v[-3:]
if short_version in LIBCST_VERSIONS:
buf.append(f" \x1b[32m{short_version.replace('.', '')}\x1b[0m")
else:
buf.append(f" {short_version.replace('.', '')}")
print("".join(buf))
for f in sorted(results):
if filters and not any(filt in f for filt in filters):
continue
sys.stdout.write(f.ljust(max_filename + 1))
for v in VERSIONS:
libcst_result = None
if v[-3:] in LIBCST_VERSIONS:
try:
with open(f) as fo:
data = fo.read()
cst.parse_module(
data, cst.PartialParserConfig(python_version=v[-3:]))
libcst_result = True
except cst.ParserSyntaxError:
libcst_result = False
if libcst_result != results[f][v]:
sys.stdout.write(
"\x1b[31m"
f"{'o' if results[f][v] else '.'}{'o' if libcst_result else '.'} "
"\x1b[0m")
else:
sys.stdout.write(
f"{'o' if results[f][v] else '.'}{'o' if libcst_result else '.'} "
)
else:
sys.stdout.write(f"{'o' if results[f][v] else '.'} ")
sys.stdout.write("\n")
print("Legend:")
print(" green header means will test with libcst")
print()
print(" first result is python, second [optional] result is libcst")
print(" o parses")
print(" . does not parse")
def try_parse(path: Path, data: Optional[bytes] = None) -> cst.Module:
"""
Attempts to parse the file with all syntax versions known by LibCST.
If parsing fails on all supported grammar versions, then raises the parser error
from the first/newest version attempted.
"""
if data is None:
data = path.read_bytes()
with timed(f"parsing {path}"):
parse_error: Optional[cst.ParserSyntaxError] = None
for version in cst.KNOWN_PYTHON_VERSION_STRINGS[::-1]:
try:
mod = cst.parse_module(
data, cst.PartialParserConfig(python_version=version))
return mod
except cst.ParserSyntaxError as e:
# keep the first error we see in case parsing fails on all versions
if parse_error is None:
parse_error = e
# not caring about existing traceback here because it's not useful for parse
# errors, and usort_path is already going to wrap it in a custom class
raise parse_error or Exception("unknown parse failure")
def parse_src(src, python_version):
"""Parses a string of source code into a LibCST tree."""
version_str = utils.format_version(python_version)
if python_version >= (3, 9):
log.warning("LibCST does not support Python %s; parsing with 3.8 instead.",
version_str)
version_str = "3.8"
config = libcst.PartialParserConfig(python_version=version_str)
return libcst.parse_module(src, config)
def test_byte_conversion(self, ) -> None:
module_bytes = "fn()\n".encode("utf-16")
mw = MetadataWrapper(
cst.parse_module("fn()\n",
cst.PartialParserConfig(encoding="utf-16")))
codegen_partial = mw.resolve(ExperimentalReentrantCodegenProvider)[
mw.module.body[0]]
self.assertEqual(codegen_partial.get_original_module_bytes(),
module_bytes)
self.assertEqual(
codegen_partial.get_modified_module_bytes(
cst.parse_statement("fn2()\n")),
"fn2()\n".encode("utf-16"),
)
def test_walrus(self) -> None:
code = """
if x := y:
pass
"""
wrapper = MetadataWrapper(
parse_module(
dedent(code), config=cst.PartialParserConfig(python_version="3.8")
)
)
wrapper.visit(
DependentVisitor(
test=self,
name_to_context={
"x": ExpressionContext.STORE,
"y": ExpressionContext.LOAD,
},
)
)
def inner(code: str) -> cst.BaseStatement:
return cst.parse_statement(code,
config=cst.PartialParserConfig(**config))
def inner(code: str) -> cst.BaseExpression:
return cst.parse_expression(code,
config=cst.PartialParserConfig(**config))
def _parse_expression_force_38(code: str) -> cst.BaseExpression:
return cst.parse_expression(
code, config=cst.PartialParserConfig(python_version="3.8"))
def visit_Name(self, node: cst.Name) -> None:
for var in self.template_vars:
if node.value == mangled_name(var):
raise Exception(
f'Template variable "{var}" was not replaced properly')
def unmangle_nodes(
tree: cst.CSTNode,
template_replacements: Mapping[str, ValidReplacementType],
) -> cst.CSTNode:
unmangler = TemplateTransformer(template_replacements)
return ensure_type(tree.visit(unmangler), cst.CSTNode)
_DEFAULT_PARTIAL_PARSER_CONFIG: cst.PartialParserConfig = cst.PartialParserConfig(
)
def parse_template_module(
template: str,
config: cst.PartialParserConfig = _DEFAULT_PARTIAL_PARSER_CONFIG,
**template_replacements: ValidReplacementType,
) -> cst.Module:
"""
Accepts an entire python module template, including all leading and trailing
whitespace. Any :class:`~libcst.CSTNode` provided as a keyword argument to
this function will be inserted into the template at the appropriate location
similar to an f-string expansion. For example::
module = parse_template_module("from {mod} import Foo\\n", mod=Name("bar"))
def main(version, filename):
with open(filename) as fo:
data = fo.read()
mod = cst.parse_module(data,
cst.PartialParserConfig(python_version=version))
print(mod)
import itertools
import unittest
import libcst
import parameterized
from craftier import parenthesize
EXAMPLES = (
[
libcst.ensure_type(
libcst.parse_statement(
"if a := 1:\n pass",
config=libcst.PartialParserConfig(python_version="3.8"),
),
libcst.If,
).test,
],
[
libcst.parse_expression("lambda x: x + 1"),
],
[
libcst.parse_expression("x if x else y"),
],
[
libcst.parse_expression("x or y"),
],
[
libcst.parse_expression("x and y"),
],
[
class Signal:
''
def value(self):
''
pass
class Foo:
"""
"""
def __init__(self):
''
pass
'''
config = cst.PartialParserConfig(python_version="3.5")
source_tree = cst.parse_module(py_source, config)
print(source_tree.get_docstring())
source_tree.code_for_node(source_tree.children[3])
## select a class definition - Q&D
classdef = source_tree.children[2]
assert isinstance(classdef, cst.ClassDef), "Not a classdef"
## class docstring 1: ''
# body=IndentedBlock(
# body=[
# SimpleStatementLine(
# body=[
"BottomOutput",
"MethodDescriptorOutput",
"parser_config",
"NumpyUfuncOutput",
]
# If there are more than this amount in a union, just use any
MAX_UNION_ITEMS = 5
# If there are more than this amount in a string literal, just use any
MAX_STRING_ITEMS = 5
# More than this and it will be tuple of arbitrary length
MAX_TUPLE_ITEMS = 2
parser_config = cst.PartialParserConfig(
python_version="3.8",
encoding="utf-8",
default_indent=" ",
default_newline="\n",
)
def annotation(tp: OutputType) -> typing.Optional[cst.BaseExpression]:
if is_unknown(tp):
return None
return tp.annotation
def create_type(o: object) -> OutputType:
try:
tp = pydantic.parse_obj_as(InputType, o) # type: ignore
except pydantic.error_wrappers.ValidationError:
raise ValueError(f"Could not parse JSON as input type: {o}")
