def assertCodemod(
self,
before: str,
after: str,
*args: object,
context_override: Optional[CodemodContext] = None,
python_version: Optional[str] = None,
expected_warnings: Optional[Sequence[str]] = None,
expected_skip: bool = False,
**kwargs: object,
) -> None:
"""
Given a before and after code string, and any args/kwargs that should
be passed to the codemod constructor specified in
:attr:`~CodemodTest.TRANSFORM`, validate that the codemod executes as
expected. Verify that the codemod completes successfully, unless the
``expected_skip`` option is set to ``True``, in which case verify that
the codemod skips. Optionally, a :class:`CodemodContext` can be provided.
If none is specified, a default, empty context is created for you.
Additionally, the python version for the code parser can be overridden
to a valid python version string such as `"3.6"`. If none is specified,
the version of the interpreter running your tests will be used. Also, a
list of warning strings can be specified and :meth:`~CodemodTest.assertCodemod`
will verify that the codemod generates those warnings in the order
specified. If it is left out, warnings are not checked.
"""
context = context_override if context_override is not None else CodemodContext(
)
# pyre-fixme[45]: Cannot instantiate abstract class `Codemod`.
transform_instance = self.TRANSFORM(context, *args, **kwargs)
input_tree = parse_module(
CodemodTest.make_fixture_data(before),
config=(PartialParserConfig(python_version=python_version)
if python_version is not None else PartialParserConfig()),
)
try:
output_tree = transform_instance.transform_module(input_tree)
except SkipFile:
if not expected_skip:
raise
output_tree = input_tree
else:
if expected_skip:
# pyre-ignore This mixin needs to be used with a UnitTest subclass.
self.fail("Expected SkipFile but was not raised")
# pyre-ignore This mixin needs to be used with a UnitTest subclass.
self.assertEqual(
CodemodTest.make_fixture_data(after),
CodemodTest.make_fixture_data(output_tree.code),
)
if expected_warnings is not None:
# pyre-ignore This mixin needs to be used with a UnitTest subclass.
self.assertSequenceEqual(expected_warnings, context.warnings)
def transform_module(
transformer: Codemod, code: str, *, python_version: Optional[str] = None
) -> TransformResult:
"""
Given a module as represented by a string and a :class:`~libcst.codemod.Codemod`
that we wish to run, execute the codemod on the code and return a
:class:`~libcst.codemod.TransformResult`. This should never raise an exception.
On success, this returns a :class:`~libcst.codemod.TransformSuccess` containing
any generated warnings as well as the transformed code. If the codemod is
interrupted with a Ctrl+C, this returns a :class:`~libcst.codemod.TransformExit`.
If the codemod elected to skip by throwing a :class:`~libcst.codemod.SkipFile`
exception, this will return a :class:`~libcst.codemod.TransformSkip` containing
the reason for skipping as well as any warnings that were generated before
the codemod decided to skip. If the codemod throws an unexpected exception,
this will return a :class:`~libcst.codemod.TransformFailure` containing the
exception that occured as well as any warnings that were generated before the
codemod crashed.
"""
try:
input_tree = parse_module(
code,
config=(
PartialParserConfig(python_version=python_version)
if python_version is not None
else PartialParserConfig()
),
)
output_tree = transformer.transform_module(input_tree)
return TransformSuccess(
code=output_tree.code, warning_messages=transformer.context.warnings
)
except KeyboardInterrupt:
return TransformExit()
except SkipFile as ex:
return TransformSkip(
skip_description=str(ex),
skip_reason=SkipReason.OTHER,
warning_messages=transformer.context.warnings,
)
except Exception as ex:
return TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
)
def lenient_parse_module(
source: Union[str, bytes],
config: PartialParserConfig = PartialParserConfig(),
) -> Module:
try:
result = _lenient_parse(
"file_input",
source,
config,
detect_trailing_newline=True,
detect_default_newline=True,
)
assert isinstance(result, Module)
return result
except ParseTokenError as pse:
token, idx = pse.get_erroneous_token()
return lenient_parse_module(source[:idx] + '' + source[idx + 1:],
config)
def assertCodemod(
self,
before: str,
after: str,
*args: object,
context_override: Optional[CodemodContext] = None,
python_version: str = "3.7",
expected_warnings: Optional[Sequence[str]] = None,
expected_skip: bool = False,
**kwargs: object,
) -> None:
"""
Given a before and after string, and optionally any args/kwargs that
should be passed to the codemod visitor constructor, validate that
the codemod executes as expected.
"""
context = context_override if context_override is not None else CodemodContext(
)
transform_instance = self.TRANSFORM(context, *args, **kwargs)
input_tree = parse_module(
CodemodTest.make_fixture_data(before),
config=PartialParserConfig(python_version=python_version),
)
try:
output_tree = transform_instance.transform_module(input_tree)
except SkipFile:
if not expected_skip:
raise
output_tree = input_tree
else:
if expected_skip:
# pyre-ignore This mixin needs to be used with a UnitTest subclass.
self.fail("Expected SkipFile but was not raised")
# pyre-ignore This mixin needs to be used with a UnitTest subclass.
self.assertEqual(CodemodTest.make_fixture_data(after),
output_tree.code)
if expected_warnings is not None:
# pyre-ignore This mixin needs to be used with a UnitTest subclass.
self.assertSequenceEqual(expected_warnings, context.warnings)
def parallel_exec_transform_with_prettyprint( # noqa: C901
transform: Codemod,
files: Sequence[str],
*,
jobs: Optional[int] = None,
unified_diff: Optional[int] = None,
include_generated: bool = False,
generated_code_marker: str = _DEFAULT_GENERATED_CODE_MARKER,
format_code: bool = False,
formatter_args: Sequence[str] = (),
show_successes: bool = False,
hide_generated: bool = False,
hide_blacklisted: bool = False,
hide_progress: bool = False,
blacklist_patterns: Sequence[str] = (),
python_version: Optional[str] = None,
repo_root: Optional[str] = None,
) -> ParallelTransformResult:
"""
Given a list of files and an instantiated codemod we should apply to them,
fork and apply the codemod in parallel to all of the files, including any
configured formatter. The ``jobs`` parameter controls the maximum number of
in-flight transforms, and needs to be at least 1. If not included, the number
of jobs will automatically be set to the number of CPU cores. If ``unified_diff``
is set to a number, changes to files will be printed to stdout with
``unified_diff`` lines of context. If it is set to ``None`` or left out, files
themselves will be updated with changes and formatting. If a
``python_version`` is provided, then we will parse each source file using
this version. Otherwise, we will use the version of the currently executing python
binary.
A progress indicator as well as any generated warnings will be printed to stderr.
To supress the interactive progress indicator, set ``hide_progress`` to ``True``.
Files that include the generated code marker will be skipped unless the
``include_generated`` parameter is set to ``True``. Similarly, files that match
a supplied blacklist of regex patterns will be skipped. Warnings for skipping
both blacklisted and generated files will be printed to stderr along with
warnings generated by the codemod unless ``hide_blacklisted`` and
``hide_generated`` are set to ``True``. Files that were successfully codemodded
will not be printed to stderr unless ``show_successes`` is set to ``True``.
To make this API possible, we take an instantiated transform. This is due to
the fact that lambdas are not pickleable and pickling functions is undefined.
This means we're implicitly relying on fork behavior on UNIX-like systems, and
this function will not work on Windows systems. To create a command-line utility
that runs on Windows, please instead see
:func:`~libcst.codemod.exec_transform_with_prettyprint`.
"""
# Ensure that we have no duplicates, otherwise we might get race conditions
# on write.
files = sorted(list({os.path.abspath(f) for f in files}))
total = len(files)
progress = Progress(enabled=not hide_progress, total=total)
# Grab number of cores if we need to
jobs: int = jobs if jobs is not None else cpu_count()
if jobs < 1:
raise Exception("Must have at least one job to process!")
if total == 0:
return ParallelTransformResult(successes=0,
failures=0,
skips=0,
warnings=0)
if repo_root is not None:
# Make sure if there is a root that we have the absolute path to it.
repo_root = os.path.abspath(repo_root)
# Spin up a full repo metadata manager so that we can provide metadata
# like type inference to individual forked processes.
print("Calculating full-repo metadata...", file=sys.stderr)
metadata_manager = FullRepoManager(
repo_root,
files,
transform.get_inherited_dependencies(),
)
metadata_manager.resolve_cache()
transform.context = replace(
transform.context,
metadata_manager=metadata_manager,
)
print("Executing codemod...", file=sys.stderr)
config = ExecutionConfig(
repo_root=repo_root,
unified_diff=unified_diff,
include_generated=include_generated,
generated_code_marker=generated_code_marker,
format_code=format_code,
formatter_args=formatter_args,
blacklist_patterns=blacklist_patterns,
python_version=python_version,
)
if total == 1:
# Simple case, we should not pay for process overhead.
# Let's just use a dummy synchronous pool.
jobs = 1
pool_impl = DummyPool
else:
pool_impl = Pool
# Warm the parser, pre-fork.
parse_module(
"",
config=(PartialParserConfig(python_version=python_version)
if python_version is not None else PartialParserConfig()),
)
successes: int = 0
failures: int = 0
warnings: int = 0
skips: int = 0
with pool_impl(processes=jobs) as p: # type: ignore
args = [{
"transformer": transform,
"filename": filename,
"config": config,
} for filename in files]
try:
for result in p.imap_unordered(_execute_transform_wrap,
args,
chunksize=4):
# Print an execution result, keep track of failures
_print_parallel_result(
result,
progress,
unified_diff=bool(unified_diff),
show_successes=show_successes,
hide_generated=hide_generated,
hide_blacklisted=hide_blacklisted,
)
progress.print(successes + failures + skips)
if isinstance(result.transform_result, TransformFailure):
failures += 1
elif isinstance(result.transform_result, TransformSuccess):
successes += 1
elif isinstance(result.transform_result,
(TransformExit, TransformSkip)):
skips += 1
warnings += len(result.transform_result.warning_messages)
finally:
progress.clear()
# Return whether there was one or more failure.
return ParallelTransformResult(successes=successes,
failures=failures,
skips=skips,
warnings=warnings)
def _execute_transform( # noqa: C901
transformer: Codemod,
filename: str,
config: ExecutionConfig,
) -> ExecutionResult:
for pattern in config.blacklist_patterns:
if re.fullmatch(pattern, filename):
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformSkip(
skip_reason=SkipReason.BLACKLISTED,
skip_description=f"Blacklisted by pattern {pattern}.",
),
)
try:
with open(filename, "rb") as fp:
oldcode = fp.read()
# Skip generated files
if (not config.include_generated
and config.generated_code_marker.encode("utf-8") in oldcode):
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformSkip(
skip_reason=SkipReason.GENERATED,
skip_description="Generated file.",
),
)
# Somewhat gross hack to provide the filename in the transform's context.
# We do this after the fork so that a context that was initialized with
# some defaults before calling parallel_exec_transform_with_prettyprint
# will be updated per-file.
transformer.context = replace(
transformer.context,
filename=filename,
full_module_name=_calculate_module(config.repo_root, filename),
)
# Run the transform, bail if we failed or if we aren't formatting code
try:
input_tree = parse_module(
oldcode,
config=(PartialParserConfig(
python_version=str(config.python_version))
if config.python_version is not None else
PartialParserConfig()),
)
output_tree = transformer.transform_module(input_tree)
newcode = output_tree.bytes
encoding = output_tree.encoding
except KeyboardInterrupt:
return ExecutionResult(filename=filename,
changed=False,
transform_result=TransformExit())
except SkipFile as ex:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformSkip(
skip_reason=SkipReason.OTHER,
skip_description=str(ex),
warning_messages=transformer.context.warnings,
),
)
except Exception as ex:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
),
)
# Call formatter if needed, but only if we actually changed something in this
# file
if config.format_code and newcode != oldcode:
try:
newcode = invoke_formatter(config.formatter_args, newcode)
except KeyboardInterrupt:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformExit(),
)
except Exception as ex:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
),
)
# Format as unified diff if needed, otherwise save it back
changed = oldcode != newcode
if config.unified_diff:
newcode = diff_code(
oldcode.decode(encoding),
newcode.decode(encoding),
config.unified_diff,
filename=filename,
)
else:
# Write back if we changed
if changed:
with open(filename, "wb") as fp:
fp.write(newcode)
# Not strictly necessary, but saves space in pickle since we won't use it
newcode = ""
# Inform success
return ExecutionResult(
filename=filename,
changed=changed,
transform_result=TransformSuccess(
warning_messages=transformer.context.warnings, code=newcode),
)
except KeyboardInterrupt:
return ExecutionResult(filename=filename,
changed=False,
transform_result=TransformExit())
except Exception as ex:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
),
)
def parallel_exec_transform_with_prettyprint( # noqa: C901
transform: Codemod,
files: Sequence[str],
*,
jobs: Optional[int] = None,
unified_diff: Optional[int] = None,
include_generated: bool = False,
generated_code_marker: str = _DEFAULT_GENERATED_CODE_MARKER,
format_code: bool = False,
formatter_args: Sequence[str] = (),
show_successes: bool = False,
hide_generated: bool = False,
hide_blacklisted: bool = False,
hide_progress: bool = False,
blacklist_patterns: Sequence[str] = (),
python_version: Optional[str] = None,
repo_root: Optional[str] = None,
) -> ParallelTransformResult:
"""
Given a list of files and an instantiated codemod we should apply to them,
fork and apply the codemod in parallel to all of the files, including any
configured formatter. The ``jobs`` parameter controls the maximum number of
in-flight transforms, and needs to be at least 1. If not included, the number
of jobs will automatically be set to the number of CPU cores. If ``unified_diff``
is set to a number, changes to files will be printed to stdout with
``unified_diff`` lines of context. If it is set to ``None`` or left out, files
themselves will be updated with changes and formatting. If a
``python_version`` is provided, then we will parse each source file using
this version. Otherwise, we will use the version of the currently executing python
binary.
A progress indicator as well as any generated warnings will be printed to stderr.
To supress the interactive progress indicator, set ``hide_progress`` to ``True``.
Files that include the generated code marker will be skipped unless the
``include_generated`` parameter is set to ``True``. Similarly, files that match
a supplied blacklist of regex patterns will be skipped. Warnings for skipping
both blacklisted and generated files will be printed to stderr along with
warnings generated by the codemod unless ``hide_blacklisted`` and
``hide_generated`` are set to ``True``. Files that were successfully codemodded
will not be printed to stderr unless ``show_successes`` is set to ``True``.
To make this API possible, we take an instantiated transform. This is due to
the fact that lambdas are not pickleable and pickling functions is undefined.
This means we're implicitly relying on fork behavior on UNIX-like systems, and
this function will not work on Windows systems. To create a command-line utility
that runs on Windows, please instead see
:func:`~libcst.codemod.exec_transform_with_prettyprint`.
"""
# Ensure that we have no duplicates, otherwise we might get race conditions
# on write.
files = sorted(list(set(os.path.abspath(f) for f in files)))
total = len(files)
progress = Progress(enabled=not hide_progress, total=total)
# Grab number of cores if we need to
jobs: int = jobs or cpu_count()
if jobs < 1:
raise Exception("Must have at least one job to process!")
if total == 0:
return ParallelTransformResult(successes=0,
failures=0,
skips=0,
warnings=0)
if repo_root:
# Make sure if there is a root that we have the absolute path to it.
repo_root = os.path.abspath(repo_root)
# Spin up a full repo metadata manager so that we can provide metadata
# like type inference to individual forked processes.
print("Calculating full-repo metadata...", file=sys.stderr)
metadata_manager = FullRepoManager(
repo_root,
files,
transform.get_inherited_dependencies(),
)
metadata_manager.resolve_cache()
transform.context = replace(
transform.context,
metadata_manager=metadata_manager,
)
print("Executing codemod...", file=sys.stderr)
# We place results in this queue inside _parallel_exec_process_stub
# so that we can control when things get printed to the console.
queue = Queue()
if total == 1:
# Simple case, we should not pay for process overhead. Lets still
# use the exec stub however, so we can share code.
progress.print(0)
_parallel_exec_process_stub(
queue,
transform,
files[0],
repo_root,
unified_diff=unified_diff,
include_generated=include_generated,
generated_code_marker=generated_code_marker,
format_code=format_code,
formatter_args=formatter_args,
blacklist_patterns=blacklist_patterns,
python_version=python_version,
)
result = queue.get()
_print_parallel_result(
result,
progress,
unified_diff=bool(unified_diff),
show_successes=show_successes,
hide_generated=hide_generated,
hide_blacklisted=hide_blacklisted,
)
if isinstance(result.transform_result, TransformFailure):
return ParallelTransformResult(
successes=0,
failures=1,
skips=0,
warnings=len(result.transform_result.warning_messages),
)
elif isinstance(result.transform_result,
(TransformSkip, TransformExit)):
return ParallelTransformResult(
successes=0,
failures=0,
skips=1,
warnings=len(result.transform_result.warning_messages),
)
elif isinstance(result.transform_result, TransformSuccess):
return ParallelTransformResult(
successes=1,
failures=0,
skips=0,
warnings=len(result.transform_result.warning_messages),
)
else:
raise Exception("Logic error, unaccounted for result!")
# Warm the parser, pre-fork.
parse_module(
"",
config=(PartialParserConfig(python_version=python_version)
if python_version is not None else PartialParserConfig()),
)
# Complex case, more than one file
successes: int = 0
failures: int = 0
warnings: int = 0
skips: int = 0
pending_processes: List[Process] = []
# Start processes
filename_to_process: Dict[str, Process] = {}
for f in files:
process = Process(
target=_parallel_exec_process_stub,
args=(
queue,
transform,
f,
repo_root,
unified_diff,
include_generated,
generated_code_marker,
format_code,
formatter_args,
blacklist_patterns,
python_version,
),
)
pending_processes.append(process)
filename_to_process[f] = process
# Start the processes, allowing no more than num_processes to be running
# at once.
results_left = len(pending_processes)
joinable_processes: Set[Process] = set()
processes_started = 0
interrupted = False
while results_left > 0 and not interrupted:
while processes_started < jobs and pending_processes:
try:
# Move this process to the joinables
process = pending_processes.pop(0)
joinable_processes.add(process)
# Start it, bookkeep that we did
process.start()
processes_started += 1
except KeyboardInterrupt:
interrupted = True
continue
try:
result = queue.get(block=True, timeout=0.005)
except KeyboardInterrupt:
interrupted = True
continue
except Empty:
progress.print(successes + failures + skips)
continue
# Bookkeep the result, since we know the process that returned this is done.
results_left -= 1
processes_started -= 1
# Print an execution result, keep track of failures
_print_parallel_result(
result,
progress,
unified_diff=bool(unified_diff),
show_successes=show_successes,
hide_generated=hide_generated,
hide_blacklisted=hide_blacklisted,
)
progress.print(successes + failures + skips)
if isinstance(result.transform_result, TransformFailure):
failures += 1
elif isinstance(result.transform_result, TransformSuccess):
successes += 1
elif isinstance(result.transform_result,
(TransformExit, TransformSkip)):
skips += 1
warnings += len(result.transform_result.warning_messages)
# Join the process to free any related resources.
# Remove all references to the process to allow the GC to
# clean up any file handles.
process = filename_to_process.pop(result.filename, None)
if process:
process.join()
joinable_processes.discard(process)
# Now, join on all of them so we don't leave zombies or hang
for p in joinable_processes:
p.join()
# Return whether there was one or more failure.
progress.clear()
# If we caught an interrupt, raise that
if interrupted:
raise KeyboardInterrupt()
return ParallelTransformResult(successes=successes,
failures=failures,
skips=skips,
warnings=warnings)
class AwaitTest(CSTNodeTest):
@data_provider((
# Some simple calls
{
"node":
cst.Await(cst.Name("test")),
"code":
"await test",
"parser":
lambda code: parse_expression(
code, config=PartialParserConfig(python_version="3.7")),
"expected_position":
None,
},
{
"node":
cst.Await(cst.Call(cst.Name("test"))),
"code":
"await test()",
"parser":
lambda code: parse_expression(
code, config=PartialParserConfig(python_version="3.7")),
"expected_position":
None,
},
# Whitespace
{
"node":
cst.Await(
cst.Name("test"),
whitespace_after_await=cst.SimpleWhitespace(" "),
lpar=(cst.LeftParen(
whitespace_after=cst.SimpleWhitespace(" ")), ),
rpar=(cst.RightParen(
whitespace_before=cst.SimpleWhitespace(" ")), ),
),
"code":
"( await test )",
"parser":
lambda code: parse_expression(
code, config=PartialParserConfig(python_version="3.7")),
"expected_position":
CodeRange((1, 2), (1, 13)),
},
))
def test_valid_py37(self, **kwargs: Any) -> None:
# We don't have sentinel nodes for atoms, so we know that 100% of atoms
# can be parsed identically to their creation.
self.validate_node(**kwargs)
@data_provider((
# Some simple calls
{
"node":
cst.FunctionDef(
cst.Name("foo"),
cst.Parameters(),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Expr(cst.Await(cst.Name("test"))), )), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async def foo():\n await test\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.6")),
"expected_position":
None,
},
{
"node":
cst.FunctionDef(
cst.Name("foo"),
cst.Parameters(),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Expr(cst.Await(cst.Call(cst.Name("test")))), )), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async def foo():\n await test()\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.6")),
"expected_position":
None,
},
# Whitespace
{
"node":
cst.FunctionDef(
cst.Name("foo"),
cst.Parameters(),
cst.IndentedBlock((cst.SimpleStatementLine((cst.Expr(
cst.Await(
cst.Name("test"),
whitespace_after_await=cst.SimpleWhitespace(" "),
lpar=(cst.LeftParen(
whitespace_after=cst.SimpleWhitespace(" ")), ),
rpar=(cst.RightParen(
whitespace_before=cst.SimpleWhitespace(" ")), ),
)), )), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async def foo():\n ( await test )\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.6")),
"expected_position":
None,
},
))
def test_valid_py36(self, **kwargs: Any) -> None:
# We don't have sentinel nodes for atoms, so we know that 100% of atoms
# can be parsed identically to their creation.
self.validate_node(**kwargs)
@data_provider((
# Expression wrapping parenthesis rules
{
"get_node":
(lambda: cst.Await(cst.Name("foo"), lpar=(cst.LeftParen(), ))),
"expected_re":
"left paren without right paren",
},
{
"get_node":
(lambda: cst.Await(cst.Name("foo"), rpar=(cst.RightParen(), ))),
"expected_re":
"right paren without left paren",
},
{
"get_node":
(lambda: cst.Await(cst.Name("foo"),
whitespace_after_await=cst.SimpleWhitespace(""))
),
"expected_re":
"at least one space after await",
},
))
def test_invalid(self, **kwargs: Any) -> None:
self.assert_invalid(**kwargs)
def _print_tree_impl(proc_name: str, command_args: List[str]) -> int:
parser = argparse.ArgumentParser(
description="Print the LibCST tree representation of a file.",
prog=f"{proc_name} print",
fromfile_prefix_chars="@",
)
parser.add_argument(
"infile",
metavar="INFILE",
help='File to print tree for. Use "-" for stdin',
type=str,
)
parser.add_argument(
"--show-whitespace",
action="store_true",
help="Show whitespace nodes in printed tree",
)
parser.add_argument(
"--show-defaults",
action="store_true",
help="Show values that are unchanged from the default",
)
parser.add_argument(
"--show-syntax",
action="store_true",
help=
"Show values that exist only for syntax, like commas or semicolons",
)
parser.add_argument(
"-p",
"--python-version",
metavar="VERSION",
help=
("Override the version string used for parsing Python source files. Defaults "
+ "to the version of python used to run this tool."),
type=str,
default=None,
)
args = parser.parse_args(command_args)
infile = args.infile
# Grab input file
if infile == "-":
code = sys.stdin.read()
else:
with open(infile, "rb") as fp:
code = fp.read()
tree = parse_module(
code,
config=(PartialParserConfig(python_version=args.python_version)
if args.python_version is not None else PartialParserConfig()),
)
print(
dump(
tree,
show_defaults=args.show_defaults,
show_syntax=args.show_syntax,
show_whitespace=args.show_whitespace,
))
return 0
class ForTest(CSTNodeTest):
@data_provider((
# Simple for block
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"for target in iter(): pass\n",
"parser":
parse_statement,
},
# Simple async for block
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async for target in iter(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.7")),
},
# Python 3.6 async for block
{
"node":
cst.FunctionDef(
cst.Name("foo"),
cst.Parameters(),
cst.IndentedBlock((cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async def foo():\n async for target in iter(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.6")),
},
# For block with else
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
cst.Else(cst.SimpleStatementSuite((cst.Pass(), ))),
),
"code":
"for target in iter(): pass\nelse: pass\n",
"parser":
parse_statement,
},
# indentation
{
"node":
DummyIndentedBlock(
" ",
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
),
),
"code":
" for target in iter(): pass\n",
"parser":
None,
},
# for an indented body
{
"node":
DummyIndentedBlock(
" ",
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
),
),
"code":
" for target in iter():\n pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (2, 12)),
},
# leading_lines
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(), )), )),
cst.Else(
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# else comment")), ),
),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment")), ),
),
"code":
"# leading comment\nfor target in iter():\n pass\n# else comment\nelse:\n pass\n",
"parser":
None,
"expected_position":
CodeRange((2, 0), (6, 8)),
},
# Weird spacing rules
{
"node":
cst.For(
cst.Name("target",
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), )),
cst.Call(
cst.Name("iter"),
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), ),
),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_for=cst.SimpleWhitespace(""),
whitespace_before_in=cst.SimpleWhitespace(""),
whitespace_after_in=cst.SimpleWhitespace(""),
),
"code":
"for(target)in(iter()): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 27)),
},
# Whitespace
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_for=cst.SimpleWhitespace(" "),
whitespace_before_in=cst.SimpleWhitespace(" "),
whitespace_after_in=cst.SimpleWhitespace(" "),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
"code":
"for target in iter() : pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 31)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
@data_provider((
{
"get_node":
lambda: cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_for=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space after 'for' keyword",
},
{
"get_node":
lambda: cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_before_in=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space before 'in' keyword",
},
{
"get_node":
lambda: cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_in=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space after 'in' keyword",
},
))
def test_invalid(self, **kwargs: Any) -> None:
self.assert_invalid(**kwargs)
class WithTest(CSTNodeTest):
@data_provider((
# Simple with block
{
"node":
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with context_mgr(): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 24)),
},
# Simple async with block
{
"node":
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async with context_mgr(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.7")),
},
# Python 3.6 async with block
{
"node":
cst.FunctionDef(
cst.Name("foo"),
cst.Parameters(),
cst.IndentedBlock((cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async def foo():\n async with context_mgr(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.6")),
},
# Multiple context managers
{
"node":
cst.With(
(
cst.WithItem(cst.Call(cst.Name("foo"))),
cst.WithItem(cst.Call(cst.Name("bar"))),
),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with foo(), bar(): pass\n",
"parser":
None,
},
{
"node":
cst.With(
(
cst.WithItem(
cst.Call(cst.Name("foo")),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")),
),
cst.WithItem(cst.Call(cst.Name("bar"))),
),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with foo(), bar(): pass\n",
"parser":
parse_statement,
},
# With block containing variable for context manager.
{
"node":
cst.With(
(cst.WithItem(
cst.Call(cst.Name("context_mgr")),
cst.AsName(cst.Name("ctx")),
), ),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with context_mgr() as ctx: pass\n",
"parser":
parse_statement,
},
# indentation
{
"node":
DummyIndentedBlock(
" ",
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
),
),
"code":
" with context_mgr(): pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (1, 28)),
},
# with an indented body
{
"node":
DummyIndentedBlock(
" ",
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
),
),
"code":
" with context_mgr():\n pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (2, 12)),
},
# leading_lines
{
"node":
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment")), ),
),
"code":
"# leading comment\nwith context_mgr(): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((2, 0), (2, 24)),
},
# Weird spacing rules
{
"node":
cst.With(
(cst.WithItem(
cst.Call(
cst.Name("context_mgr"),
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), ),
)), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(""),
),
"code":
"with(context_mgr()): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 25)),
},
# Whitespace
{
"node":
cst.With(
(cst.WithItem(
cst.Call(cst.Name("context_mgr")),
cst.AsName(
cst.Name("ctx"),
whitespace_before_as=cst.SimpleWhitespace(" "),
whitespace_after_as=cst.SimpleWhitespace(" "),
),
), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(" "),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
"code":
"with context_mgr() as ctx : pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 36)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
@data_provider((
{
"get_node":
lambda: cst.With((),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), ))),
"expected_re":
"A With statement must have at least one WithItem",
},
{
"get_node":
lambda: cst.With(
(cst.WithItem(
cst.Call(cst.Name("foo")),
comma=cst.Comma(whitespace_after=cst.SimpleWhitespace(" ")
),
), ),
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(), )), )),
),
"expected_re":
"The last WithItem in a With cannot have a trailing comma",
},
{
"get_node":
lambda: cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space after with keyword",
},
))
def test_invalid(self, **kwargs: Any) -> None:
self.assert_invalid(**kwargs)
@data_provider((
{
"code": "with a, b: pass",
"parser": parse_statement_as(python_version="3.1"),
"expect_success": True,
},
{
"code": "with a, b: pass",
"parser": parse_statement_as(python_version="3.0"),
"expect_success": False,
},
))
def test_versions(self, **kwargs: Any) -> None:
self.assert_parses(**kwargs)
class SimpleCompTest(CSTNodeTest):
@data_provider([
# simple GeneratorExp
{
"node":
cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c"))),
"code":
"(a for b in c)",
"parser":
parse_expression,
"expected_position":
CodeRange((1, 1), (1, 13)),
},
# simple ListComp
{
"node":
cst.ListComp(cst.Name("a"),
cst.CompFor(target=cst.Name("b"),
iter=cst.Name("c"))),
"code":
"[a for b in c]",
"parser":
parse_expression,
"expected_position":
CodeRange((1, 0), (1, 14)),
},
# simple SetComp
{
"node":
cst.SetComp(cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c"))),
"code":
"{a for b in c}",
"parser":
parse_expression,
},
# async GeneratorExp
{
"node":
cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
asynchronous=cst.Asynchronous(),
),
),
"code":
"(a async for b in c)",
"parser":
lambda code: parse_expression(
code, config=PartialParserConfig(python_version="3.7")),
},
# Python 3.6 async GeneratorExp
{
"node":
cst.FunctionDef(
cst.Name("foo"),
cst.Parameters(),
cst.IndentedBlock((cst.SimpleStatementLine((cst.Expr(
cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
asynchronous=cst.Asynchronous(),
),
)), )), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async def foo():\n (a async for b in c)\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.6")),
},
# a generator doesn't have to own it's own parenthesis
{
"node":
cst.Call(
cst.Name("func"),
[
cst.Arg(
cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"),
iter=cst.Name("c")),
lpar=[],
rpar=[],
))
],
),
"code":
"func(a for b in c)",
"parser":
parse_expression,
},
# add a few 'if' clauses
{
"node":
cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
ifs=[
cst.CompIf(cst.Name("d")),
cst.CompIf(cst.Name("e")),
cst.CompIf(cst.Name("f")),
],
),
),
"code":
"(a for b in c if d if e if f)",
"parser":
parse_expression,
"expected_position":
CodeRange((1, 1), (1, 28)),
},
# nested/inner for-in clause
{
"node":
cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
inner_for_in=cst.CompFor(target=cst.Name("d"),
iter=cst.Name("e")),
),
),
"code":
"(a for b in c for d in e)",
"parser":
parse_expression,
},
# nested/inner for-in clause with an 'if' clause
{
"node":
cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
ifs=[cst.CompIf(cst.Name("d"))],
inner_for_in=cst.CompFor(target=cst.Name("e"),
iter=cst.Name("f")),
),
),
"code":
"(a for b in c if d for e in f)",
"parser":
parse_expression,
},
# custom whitespace
{
"node":
cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
ifs=[
cst.CompIf(
cst.Name("d"),
whitespace_before=cst.SimpleWhitespace("\t"),
whitespace_before_test=cst.SimpleWhitespace(
"\t\t"),
)
],
whitespace_before=cst.SimpleWhitespace(" "),
whitespace_after_for=cst.SimpleWhitespace(" "),
whitespace_before_in=cst.SimpleWhitespace(" "),
whitespace_after_in=cst.SimpleWhitespace(" "),
),
lpar=[
cst.LeftParen(whitespace_after=cst.SimpleWhitespace("\f"))
],
rpar=[
cst.RightParen(
whitespace_before=cst.SimpleWhitespace("\f\f"))
],
),
"code":
"(\fa for b in c\tif\t\td\f\f)",
"parser":
parse_expression,
"expected_position":
CodeRange((1, 2), (1, 30)),
},
# custom whitespace around ListComp's brackets
{
"node":
cst.ListComp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
lbracket=cst.LeftSquareBracket(
whitespace_after=cst.SimpleWhitespace("\t")),
rbracket=cst.RightSquareBracket(
whitespace_before=cst.SimpleWhitespace("\t\t")),
lpar=[
cst.LeftParen(whitespace_after=cst.SimpleWhitespace("\f"))
],
rpar=[
cst.RightParen(
whitespace_before=cst.SimpleWhitespace("\f\f"))
],
),
"code":
"(\f[\ta for b in c\t\t]\f\f)",
"parser":
parse_expression,
"expected_position":
CodeRange((1, 2), (1, 19)),
},
# custom whitespace around SetComp's braces
{
"node":
cst.SetComp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
lbrace=cst.LeftCurlyBrace(
whitespace_after=cst.SimpleWhitespace("\t")),
rbrace=cst.RightCurlyBrace(
whitespace_before=cst.SimpleWhitespace("\t\t")),
lpar=[
cst.LeftParen(whitespace_after=cst.SimpleWhitespace("\f"))
],
rpar=[
cst.RightParen(
whitespace_before=cst.SimpleWhitespace("\f\f"))
],
),
"code":
"(\f{\ta for b in c\t\t}\f\f)",
"parser":
parse_expression,
},
# no whitespace between elements
{
"node":
cst.GeneratorExp(
cst.Name("a", lpar=[cst.LeftParen()], rpar=[cst.RightParen()]),
cst.CompFor(
target=cst.Name("b",
lpar=[cst.LeftParen()],
rpar=[cst.RightParen()]),
iter=cst.Name("c",
lpar=[cst.LeftParen()],
rpar=[cst.RightParen()]),
ifs=[
cst.CompIf(
cst.Name("d",
lpar=[cst.LeftParen()],
rpar=[cst.RightParen()]),
whitespace_before=cst.SimpleWhitespace(""),
whitespace_before_test=cst.SimpleWhitespace(""),
)
],
inner_for_in=cst.CompFor(
target=cst.Name("e",
lpar=[cst.LeftParen()],
rpar=[cst.RightParen()]),
iter=cst.Name("f",
lpar=[cst.LeftParen()],
rpar=[cst.RightParen()]),
whitespace_before=cst.SimpleWhitespace(""),
whitespace_after_for=cst.SimpleWhitespace(""),
whitespace_before_in=cst.SimpleWhitespace(""),
whitespace_after_in=cst.SimpleWhitespace(""),
),
whitespace_before=cst.SimpleWhitespace(""),
whitespace_after_for=cst.SimpleWhitespace(""),
whitespace_before_in=cst.SimpleWhitespace(""),
whitespace_after_in=cst.SimpleWhitespace(""),
),
lpar=[cst.LeftParen()],
rpar=[cst.RightParen()],
),
"code":
"((a)for(b)in(c)if(d)for(e)in(f))",
"parser":
parse_expression,
"expected_position":
CodeRange((1, 1), (1, 31)),
},
# no whitespace before/after GeneratorExp is valid
{
"node":
cst.Comparison(
cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
),
[
cst.ComparisonTarget(
cst.Is(
whitespace_before=cst.SimpleWhitespace(""),
whitespace_after=cst.SimpleWhitespace(""),
),
cst.GeneratorExp(
cst.Name("d"),
cst.CompFor(target=cst.Name("e"),
iter=cst.Name("f")),
),
)
],
),
"code":
"(a for b in c)is(d for e in f)",
"parser":
parse_expression,
},
# no whitespace before/after ListComp is valid
{
"node":
cst.Comparison(
cst.ListComp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
),
[
cst.ComparisonTarget(
cst.Is(
whitespace_before=cst.SimpleWhitespace(""),
whitespace_after=cst.SimpleWhitespace(""),
),
cst.ListComp(
cst.Name("d"),
cst.CompFor(target=cst.Name("e"),
iter=cst.Name("f")),
),
)
],
),
"code":
"[a for b in c]is[d for e in f]",
"parser":
parse_expression,
},
# no whitespace before/after SetComp is valid
{
"node":
cst.Comparison(
cst.SetComp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
),
[
cst.ComparisonTarget(
cst.Is(
whitespace_before=cst.SimpleWhitespace(""),
whitespace_after=cst.SimpleWhitespace(""),
),
cst.SetComp(
cst.Name("d"),
cst.CompFor(target=cst.Name("e"),
iter=cst.Name("f")),
),
)
],
),
"code":
"{a for b in c}is{d for e in f}",
"parser":
parse_expression,
},
])
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
@data_provider((
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
lpar=[cst.LeftParen(), cst.LeftParen()],
rpar=[cst.RightParen()],
),
"unbalanced parens",
),
(
lambda: cst.ListComp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
lpar=[cst.LeftParen(), cst.LeftParen()],
rpar=[cst.RightParen()],
),
"unbalanced parens",
),
(
lambda: cst.SetComp(
cst.Name("a"),
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
lpar=[cst.LeftParen(), cst.LeftParen()],
rpar=[cst.RightParen()],
),
"unbalanced parens",
),
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
whitespace_before=cst.SimpleWhitespace(""),
),
),
"Must have at least one space before 'for' keyword.",
),
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
asynchronous=cst.Asynchronous(),
whitespace_before=cst.SimpleWhitespace(""),
),
),
"Must have at least one space before 'async' keyword.",
),
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
whitespace_after_for=cst.SimpleWhitespace(""),
),
),
"Must have at least one space after 'for' keyword.",
),
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
whitespace_before_in=cst.SimpleWhitespace(""),
),
),
"Must have at least one space before 'in' keyword.",
),
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
whitespace_after_in=cst.SimpleWhitespace(""),
),
),
"Must have at least one space after 'in' keyword.",
),
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
ifs=[
cst.CompIf(
cst.Name("d"),
whitespace_before=cst.SimpleWhitespace(""),
)
],
),
),
"Must have at least one space before 'if' keyword.",
),
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
ifs=[
cst.CompIf(
cst.Name("d"),
whitespace_before_test=cst.SimpleWhitespace(""),
)
],
),
),
"Must have at least one space after 'if' keyword.",
),
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
inner_for_in=cst.CompFor(
target=cst.Name("d"),
iter=cst.Name("e"),
whitespace_before=cst.SimpleWhitespace(""),
),
),
),
"Must have at least one space before 'for' keyword.",
),
(
lambda: cst.GeneratorExp(
cst.Name("a"),
cst.CompFor(
target=cst.Name("b"),
iter=cst.Name("c"),
inner_for_in=cst.CompFor(
target=cst.Name("d"),
iter=cst.Name("e"),
asynchronous=cst.Asynchronous(),
whitespace_before=cst.SimpleWhitespace(""),
),
),
),
"Must have at least one space before 'async' keyword.",
),
))
def test_invalid(self, get_node: Callable[[], cst.CSTNode],
expected_re: str) -> None:
self.assert_invalid(get_node, expected_re)
class WithTest(CSTNodeTest):
maxDiff: int = 2000
@data_provider((
# Simple with block
{
"node":
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with context_mgr(): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 24)),
},
# Simple async with block
{
"node":
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async with context_mgr(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.7")),
},
# Python 3.6 async with block
{
"node":
cst.FunctionDef(
cst.Name("foo"),
cst.Parameters(),
cst.IndentedBlock((cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async def foo():\n async with context_mgr(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.6")),
},
# Multiple context managers
{
"node":
cst.With(
(
cst.WithItem(cst.Call(cst.Name("foo"))),
cst.WithItem(cst.Call(cst.Name("bar"))),
),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with foo(), bar(): pass\n",
"parser":
None,
},
{
"node":
cst.With(
(
cst.WithItem(
cst.Call(cst.Name("foo")),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")),
),
cst.WithItem(cst.Call(cst.Name("bar"))),
),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with foo(), bar(): pass\n",
"parser":
parse_statement,
},
# With block containing variable for context manager.
{
"node":
cst.With(
(cst.WithItem(
cst.Call(cst.Name("context_mgr")),
cst.AsName(cst.Name("ctx")),
), ),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with context_mgr() as ctx: pass\n",
"parser":
parse_statement,
},
# indentation
{
"node":
DummyIndentedBlock(
" ",
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
),
),
"code":
" with context_mgr(): pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (1, 28)),
},
# with an indented body
{
"node":
DummyIndentedBlock(
" ",
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
),
),
"code":
" with context_mgr():\n pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (2, 12)),
},
# leading_lines
{
"node":
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment")), ),
),
"code":
"# leading comment\nwith context_mgr(): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((2, 0), (2, 24)),
},
# Whitespace
{
"node":
cst.With(
(cst.WithItem(
cst.Call(cst.Name("context_mgr")),
cst.AsName(
cst.Name("ctx"),
whitespace_before_as=cst.SimpleWhitespace(" "),
whitespace_after_as=cst.SimpleWhitespace(" "),
),
), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(" "),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
"code":
"with context_mgr() as ctx : pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 36)),
},
# Weird spacing rules, that parse differently depending on whether
# we are using a grammar that included parenthesized with statements.
{
"node":
cst.With(
(cst.WithItem(
cst.Call(
cst.Name("context_mgr"),
lpar=() if is_native() else (cst.LeftParen(), ),
rpar=() if is_native() else (cst.RightParen(), ),
)), ),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=(cst.LeftParen()
if is_native() else MaybeSentinel.DEFAULT),
rpar=(cst.RightParen()
if is_native() else MaybeSentinel.DEFAULT),
whitespace_after_with=cst.SimpleWhitespace(""),
),
"code":
"with(context_mgr()): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 25)),
},
# Multi-line parenthesized with.
{
"node":
cst.With(
(
cst.WithItem(
cst.Call(cst.Name("foo")),
comma=cst.
Comma(whitespace_after=cst.ParenthesizedWhitespace(
first_line=cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(value="", ),
comment=None,
newline=cst.Newline(value=None, ),
),
empty_lines=[],
indent=True,
last_line=cst.SimpleWhitespace(value=" ", ),
)),
),
cst.WithItem(cst.Call(cst.Name("bar")), comma=cst.Comma()),
),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(whitespace_after=cst.SimpleWhitespace(" ")),
rpar=cst.RightParen(
whitespace_before=cst.SimpleWhitespace(" ")),
),
"code": ("with ( foo(),\n"
" bar(), ): pass\n"), # noqa
"parser":
parse_statement if is_native() else None,
"expected_position":
CodeRange((1, 0), (2, 21)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
@data_provider((
{
"get_node":
lambda: cst.With((),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), ))),
"expected_re":
"A With statement must have at least one WithItem",
},
{
"get_node":
lambda: cst.With(
(cst.WithItem(
cst.Call(cst.Name("foo")),
comma=cst.Comma(whitespace_after=cst.SimpleWhitespace(" ")
),
), ),
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(), )), )),
),
"expected_re":
"The last WithItem in an unparenthesized With cannot " +
"have a trailing comma.",
},
{
"get_node":
lambda: cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space after with keyword",
},
{
"get_node":
lambda: cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(""),
lpar=cst.LeftParen(),
),
"expected_re":
"Do not mix concrete LeftParen/RightParen with " + "MaybeSentinel",
},
{
"get_node":
lambda: cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(""),
rpar=cst.RightParen(),
),
"expected_re":
"Do not mix concrete LeftParen/RightParen with " + "MaybeSentinel",
},
))
def test_invalid(self, **kwargs: Any) -> None:
self.assert_invalid(**kwargs)
@data_provider((
{
"code": "with a, b: pass",
"parser": parse_statement_as(python_version="3.1"),
"expect_success": True,
},
{
"code": "with a, b: pass",
"parser": parse_statement_as(python_version="3.0"),
"expect_success": False,
},
))
def test_versions(self, **kwargs: Any) -> None:
if is_native() and not kwargs.get("expect_success", True):
self.skipTest("parse errors are disabled for native parser")
self.assert_parses(**kwargs)
def test_adding_parens(self) -> None:
node = cst.With(
(
cst.WithItem(
cst.Call(cst.Name("foo")),
comma=cst.Comma(
whitespace_after=cst.ParenthesizedWhitespace(), ),
),
cst.WithItem(cst.Call(cst.Name("bar")), comma=cst.Comma()),
),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(whitespace_after=cst.SimpleWhitespace(" ")),
rpar=cst.RightParen(whitespace_before=cst.SimpleWhitespace(" ")),
)
module = cst.Module([])
self.assertEqual(
module.code_for_node(node),
("with ( foo(),\n"
"bar(), ): pass\n") # noqa
)
