def _insert_empty_line(
self,
statements: List[Union[libcst.SimpleStatementLine,
libcst.BaseCompoundStatement]],
) -> List[Union[libcst.SimpleStatementLine, libcst.BaseCompoundStatement]]:
if len(statements) < 1:
# No statements, nothing to add to
return statements
if len(statements[0].leading_lines) == 0:
# Statement has no leading lines, add one!
return [
statements[0].with_changes(
leading_lines=(libcst.EmptyLine(), )),
*statements[1:],
]
if statements[0].leading_lines[0].comment is None:
# First line is empty, so its safe to leave as-is
return statements
# Statement has a comment first line, so lets add one more empty line
return [
statements[0].with_changes(
leading_lines=(libcst.EmptyLine(),
*statements[0].leading_lines)),
*statements[1:],
]
def import_to_node_single(imp: SortableImport,
module: cst.Module) -> cst.BaseStatement:
leading_lines = [
cst.EmptyLine(indent=True, comment=cst.Comment(line))
if line.startswith("#") else cst.EmptyLine(indent=False)
for line in imp.comments.before
]
trailing_whitespace = cst.TrailingWhitespace()
trailing_comments = list(imp.comments.first_inline)
names: List[cst.ImportAlias] = []
for item in imp.items:
name = name_to_node(item.name)
asname = cst.AsName(
name=cst.Name(item.asname)) if item.asname else None
node = cst.ImportAlias(name=name, asname=asname)
names.append(node)
trailing_comments += item.comments.before
trailing_comments += item.comments.inline
trailing_comments += item.comments.following
trailing_comments += imp.comments.final
trailing_comments += imp.comments.last_inline
if trailing_comments:
text = COMMENT_INDENT.join(trailing_comments)
trailing_whitespace = cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(COMMENT_INDENT),
comment=cst.Comment(text))
if imp.stem:
stem, ndots = split_relative(imp.stem)
if not stem:
module_name = None
else:
module_name = name_to_node(stem)
relative = (cst.Dot(), ) * ndots
line = cst.SimpleStatementLine(
body=[
cst.ImportFrom(module=module_name,
names=names,
relative=relative)
],
leading_lines=leading_lines,
trailing_whitespace=trailing_whitespace,
)
else:
line = cst.SimpleStatementLine(
body=[cst.Import(names=names)],
leading_lines=leading_lines,
trailing_whitespace=trailing_whitespace,
)
return line
def _create_empty_line():
return cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(value='', ),
comment=None,
newline=cst.Newline(value=None, ),
)
def test(
self,
*,
source: bytes,
rules_in_lint_run: Collection[Type[CstLintRule]],
rules_without_report: Collection[Type[CstLintRule]],
suppressed_line: int,
expected_unused_suppressions_report_messages: Collection[str],
expected_replacements: Optional[List[str]] = None,
) -> None:
reports = [
CstLintRuleReport(
file_path=FILE_PATH,
node=cst.EmptyLine(),
code=rule.__name__,
message="message",
line=suppressed_line,
column=0,
module=cst.MetadataWrapper(cst.parse_module(source)),
module_bytes=source,
) for rule in rules_in_lint_run if rule not in rules_without_report
]
tokens = _get_tokens(source)
ignore_info = IgnoreInfo.compute(
comment_info=CommentInfo.compute(tokens=tokens),
line_mapping_info=LineMappingInfo.compute(tokens=tokens),
)
cst_wrapper = MetadataWrapper(cst.parse_module(source),
unsafe_skip_copy=True)
config = LintConfig(
rule_config={
RemoveUnusedSuppressionsRule.__name__: {
"ignore_info": ignore_info,
"rules": rules_in_lint_run,
}
})
unused_suppressions_context = CstContext(cst_wrapper, source,
FILE_PATH, config)
for report in reports:
ignore_info.should_ignore_report(report)
_visit_cst_rules_with_context(cst_wrapper,
[RemoveUnusedSuppressionsRule],
unused_suppressions_context)
messages = []
patches = []
for report in unused_suppressions_context.reports:
messages.append(report.message)
patches.append(report.patch)
self.assertEqual(messages,
expected_unused_suppressions_report_messages)
if expected_replacements is None:
self.assertEqual(len(patches), 0)
else:
self.assertEqual(len(patches), len(expected_replacements))
for idx, patch in enumerate(patches):
replacement = patch.apply(source.decode())
self.assertEqual(replacement, expected_replacements[idx])
def test_ignored_lines(self, *, source: str, ignored_code: str,
ignored_lines: Container[int]) -> None:
tokens = tuple(
tokenize.tokenize(BytesIO(source.encode("utf-8")).readline))
ignore_info = IgnoreInfo.compute(
comment_info=CommentInfo.compute(tokens=tokens),
line_mapping_info=LineMappingInfo.compute(tokens=tokens),
)
lines = range(1, tokens[-1].end[0] + 1)
actual_ignored_lines = []
for line in lines:
ignored = ignore_info.should_ignore_report(
CstLintRuleReport(
file_path=Path("fake/path.py"),
node=cst.EmptyLine(),
code=ignored_code,
message="message",
line=line,
column=0,
module=cst.MetadataWrapper(cst.parse_module(source)),
module_bytes=source.encode("utf-8"),
))
if ignored:
actual_ignored_lines.append(line)
# pyre-fixme[6]: Expected `Iterable[Variable[_T]]` for 1st param but got
# `Container[int]`.
self.assertEqual(actual_ignored_lines, list(ignored_lines))
def __init__(self, context: CstContext) -> None:
super().__init__(context)
config = self.context.config.rule_config.get(self.__class__.__name__,
None)
if config is None:
self.rule_disabled: bool = True
else:
if not isinstance(config, dict) or "header" not in config:
raise ValueError(
"A ``header`` str config is required by AddMissingHeaderRule."
)
header_str = config["header"]
if not isinstance(header_str, str):
raise ValueError(
"A ``header`` str config is required by AddMissingHeaderRule."
)
lines = header_str.split("\n")
self.header_matcher: List[m.EmptyLine] = [
m.EmptyLine(comment=m.Comment(value=line)) for line in lines
]
self.header_replacement: List[cst.EmptyLine] = [
cst.EmptyLine(comment=cst.Comment(value=line))
for line in lines
]
if "path" in config:
path_pattern = config["path"]
if not isinstance(path_pattern, str):
raise ValueError(
"``path`` config should be a str in AddMissingHeaderRule."
)
else:
path_pattern = "*.py"
self.rule_disabled = not self.context.file_path.match(path_pattern)
def assign_properties(
p: typing.Dict[str, typing.Tuple[Metadata, Type]],
is_classvar=False) -> typing.Iterable[cst.SimpleStatementLine]:
for name, metadata_and_tp in sort_items(p):
if bad_name(name):
continue
metadata, tp = metadata_and_tp
ann = tp.annotation
yield cst.SimpleStatementLine(
[
cst.AnnAssign(
cst.Name(name),
cst.Annotation(
cst.Subscript(cst.Name("ClassVar"),
[cst.SubscriptElement(cst.Index(ann))]
) if is_classvar else ann),
)
],
leading_lines=[cst.EmptyLine()] + [
cst.EmptyLine(comment=cst.Comment("# " + l))
for l in metadata_lines(metadata)
],
)
def fixup_whitespace(initial_blank: Sequence[cst.EmptyLine],
imports: List[SortableImport]) -> List[SortableImport]:
cur_category = None
# TODO if they've already been reshuffled, there may have been a blank
# (separator) line between a non-block and the first import, that's now in
# the middle.
for i in imports:
old_blanks, old_comments = partition_leading_lines(
i.node.leading_lines)
if cur_category is None:
blanks = initial_blank
elif i.sort_key.category_index != cur_category:
blanks = (cst.EmptyLine(), )
elif old_comments:
# preserves black formatting
blanks = (cst.EmptyLine(), )
else:
blanks = ()
i.node = i.node.with_changes(leading_lines=(*blanks, *old_comments))
cur_category = i.sort_key.category_index
return imports
def test_deep_replace_identity(self) -> None:
old_code = """
pass
"""
new_code = """
break
"""
module = cst.parse_module(dedent(old_code))
new_module = module.deep_replace(
module,
cst.Module(
header=(cst.EmptyLine(), ),
body=(cst.SimpleStatementLine(body=(cst.Break(), )), ),
),
)
self.assertEqual(new_module.code, dedent(new_code))
def insert_header_comments(node: libcst.Module,
comments: List[str]) -> libcst.Module:
"""Insert comments after last non-empty line in header."""
# Split the lines up into a contiguous comment-containing section and
# the empty whitespace section that follows
last_comment_index = -1
for i, line in enumerate(node.header):
if line.comment is not None:
last_comment_index = i
comment_lines = islice(node.header, last_comment_index + 1)
empty_lines = islice(node.header, last_comment_index + 1, None)
inserted_lines = [
libcst.EmptyLine(comment=libcst.Comment(value=comment))
for comment in comments
]
return node.with_changes(header=(*comment_lines, *inserted_lines,
*empty_lines))
def test_unused_comments(
self,
*,
source: str,
reports_on_lines: Iterable[Tuple[int, str]],
unused_comments: Iterable[int],
) -> None:
"""
Verify that we can correctly track which lint comments were used and which were
unused.
TODO: We don't track usage of global ignore comments, so we can't know if
they're unused.
"""
tokens = tuple(
tokenize.tokenize(BytesIO(source.encode("utf-8")).readline))
ignore_info = IgnoreInfo.compute(
comment_info=CommentInfo.compute(tokens=tokens),
line_mapping_info=LineMappingInfo.compute(tokens=tokens),
use_noqa=True,
)
for line, code in reports_on_lines:
ignore_info.should_ignore_report(
CstLintRuleReport(
file_path=Path("fake/path.py"),
node=cst.EmptyLine(),
code=code,
message="message",
line=line,
column=0,
module=cst.MetadataWrapper(cst.parse_module(source)),
module_bytes=source.encode("utf-8"),
))
self.assertEqual(
sorted([
min(tok.start[0] for tok in c.tokens)
for c in ignore_info.suppression_comments if not c.used_by
]),
sorted(unused_comments),
)
def insert_header_comments(node: libcst.Module, comments: List[str]) -> libcst.Module:
"""
Insert comments after last non-empty line in header. Use this to insert one or more
comments after any copyright preamble in a :class:`~libcst.Module`. Each comment in
the list of ``comments`` must start with a ``#`` and will be placed on its own line
in the appropriate location.
"""
# Split the lines up into a contiguous comment-containing section and
# the empty whitespace section that follows
last_comment_index = -1
for i, line in enumerate(node.header):
if line.comment is not None:
last_comment_index = i
comment_lines = islice(node.header, last_comment_index + 1)
empty_lines = islice(node.header, last_comment_index + 1, None)
inserted_lines = [
libcst.EmptyLine(comment=libcst.Comment(value=comment)) for comment in comments
]
# pyre-fixme[60]: Concatenation not yet support for multiple variadic tuples:
# `*comment_lines, *inserted_lines, *empty_lines`.
return node.with_changes(header=(*comment_lines, *inserted_lines, *empty_lines))
class EmptyLineTest(CSTNodeTest):
@data_provider((
(cst.EmptyLine(), "\n"),
(cst.EmptyLine(whitespace=cst.SimpleWhitespace(" ")), " \n"),
(cst.EmptyLine(comment=cst.Comment("# comment")), "# comment\n"),
(cst.EmptyLine(newline=cst.Newline("\r\n")), "\r\n"),
(DummyIndentedBlock(" ", cst.EmptyLine()), " \n"),
(DummyIndentedBlock(" ", cst.EmptyLine(indent=False)), "\n"),
(
DummyIndentedBlock(
"\t",
cst.EmptyLine(
whitespace=cst.SimpleWhitespace(" "),
comment=cst.Comment("# comment"),
newline=cst.Newline("\r\n"),
),
),
"\t # comment\r\n",
),
))
def test_valid(self, node: cst.CSTNode, code: str) -> None:
self.validate_node(node, code)
class WhitespaceParserTest(UnitTest):
@data_provider(
{
"simple_whitespace_empty": {
"parser": parse_simple_whitespace,
"config": Config(
lines=["not whitespace\n", " another line\n"], default_newline="\n"
),
"start_state": State(
line=1, column=0, absolute_indent="", is_parenthesized=False
),
"end_state": State(
line=1, column=0, absolute_indent="", is_parenthesized=False
),
"expected_node": cst.SimpleWhitespace(""),
},
"simple_whitespace_start_of_line": {
"parser": parse_simple_whitespace,
"config": Config(
lines=["\t <-- There's some whitespace there\n"],
default_newline="\n",
),
"start_state": State(
line=1, column=0, absolute_indent="", is_parenthesized=False
),
"end_state": State(
line=1, column=3, absolute_indent="", is_parenthesized=False
),
"expected_node": cst.SimpleWhitespace("\t "),
},
"simple_whitespace_end_of_line": {
"parser": parse_simple_whitespace,
"config": Config(lines=["prefix "], default_newline="\n"),
"start_state": State(
line=1, column=6, absolute_indent="", is_parenthesized=False
),
"end_state": State(
line=1, column=9, absolute_indent="", is_parenthesized=False
),
"expected_node": cst.SimpleWhitespace(" "),
},
"simple_whitespace_line_continuation": {
"parser": parse_simple_whitespace,
"config": Config(
lines=["prefix \\\n", " \\\n", " # suffix\n"],
default_newline="\n",
),
"start_state": State(
line=1, column=6, absolute_indent="", is_parenthesized=False
),
"end_state": State(
line=3, column=4, absolute_indent="", is_parenthesized=False
),
"expected_node": cst.SimpleWhitespace(" \\\n \\\n "),
},
"empty_lines_empty_list": {
"parser": parse_empty_lines,
"config": Config(
lines=["this is not an empty line"], default_newline="\n"
),
"start_state": State(
line=1, column=0, absolute_indent="", is_parenthesized=False
),
"end_state": State(
line=1, column=0, absolute_indent="", is_parenthesized=False
),
"expected_node": [],
},
"empty_lines_single_line": {
"parser": parse_empty_lines,
"config": Config(
lines=[" # comment\n", "this is not an empty line\n"],
default_newline="\n",
),
"start_state": State(
line=1, column=0, absolute_indent=" ", is_parenthesized=False
),
"end_state": State(
line=2, column=0, absolute_indent=" ", is_parenthesized=False
),
"expected_node": [
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(""),
comment=cst.Comment("# comment"),
newline=cst.Newline(),
)
],
},
"empty_lines_multiple": {
"parser": parse_empty_lines,
"config": Config(
lines=[
"\n",
" \n",
" # comment with indent and whitespace\n",
"# comment without indent\n",
" # comment with no indent but some whitespace\n",
],
default_newline="\n",
),
"start_state": State(
line=1, column=0, absolute_indent=" ", is_parenthesized=False
),
"end_state": State(
line=5, column=47, absolute_indent=" ", is_parenthesized=False
),
"expected_node": [
cst.EmptyLine(
indent=False,
whitespace=cst.SimpleWhitespace(""),
comment=None,
newline=cst.Newline(),
),
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(""),
comment=None,
newline=cst.Newline(),
),
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(" "),
comment=cst.Comment("# comment with indent and whitespace"),
newline=cst.Newline(),
),
cst.EmptyLine(
indent=False,
whitespace=cst.SimpleWhitespace(""),
comment=cst.Comment("# comment without indent"),
newline=cst.Newline(),
),
cst.EmptyLine(
indent=False,
whitespace=cst.SimpleWhitespace(" "),
comment=cst.Comment(
"# comment with no indent but some whitespace"
),
newline=cst.Newline(),
),
],
},
"empty_lines_non_default_newline": {
"parser": parse_empty_lines,
"config": Config(lines=["\n", "\r\n", "\r"], default_newline="\n"),
"start_state": State(
line=1, column=0, absolute_indent="", is_parenthesized=False
),
"end_state": State(
line=3, column=1, absolute_indent="", is_parenthesized=False
),
"expected_node": [
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(""),
comment=None,
newline=cst.Newline(None), # default newline
),
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(""),
comment=None,
newline=cst.Newline("\r\n"), # non-default
),
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(""),
comment=None,
newline=cst.Newline("\r"), # non-default
),
],
},
"trailing_whitespace": {
"parser": parse_trailing_whitespace,
"config": Config(
lines=["some code # comment\n"], default_newline="\n"
),
"start_state": State(
line=1, column=9, absolute_indent="", is_parenthesized=False
),
"end_state": State(
line=1, column=21, absolute_indent="", is_parenthesized=False
),
"expected_node": cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(" "),
comment=cst.Comment("# comment"),
newline=cst.Newline(),
),
},
}
)
def test_parsers(
self,
parser: Callable[[Config, State], _T],
config: Config,
start_state: State,
end_state: State,
expected_node: _T,
) -> None:
# Uses internal `deep_equals` function instead of `CSTNode.deep_equals`, because
# we need to compare sequences of nodes, and this is the easiest way. :/
parsed_node = parser(config, start_state)
self.assertTrue(
deep_equals(parsed_node, expected_node),
msg=f"\n{parsed_node!r}\nis not deeply equal to \n{expected_node!r}",
)
self.assertEqual(start_state, end_state)
class ClassDefParserTest(CSTNodeTest):
@data_provider((
# Simple classdef
# pyre-fixme[6]: Incompatible parameter type
{
"node":
cst.ClassDef(cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), ))),
"code":
"class Foo: pass\n",
},
{
"node":
cst.ClassDef(
cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(),
rpar=cst.RightParen(),
),
"code":
"class Foo(): pass\n",
},
# Positional arguments render test
{
"node":
cst.ClassDef(
cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(),
bases=(cst.Arg(cst.Name("obj")), ),
rpar=cst.RightParen(),
),
"code":
"class Foo(obj): pass\n",
},
{
"node":
cst.ClassDef(
cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(),
bases=(
cst.Arg(
cst.Name("Bar"),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")),
),
cst.Arg(
cst.Name("Baz"),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")),
),
cst.Arg(cst.Name("object")),
),
rpar=cst.RightParen(),
),
"code":
"class Foo(Bar, Baz, object): pass\n",
},
# Keyword arguments render test
{
"node":
cst.ClassDef(
cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(),
keywords=(cst.Arg(
keyword=cst.Name("metaclass"),
equal=cst.AssignEqual(),
value=cst.Name("Bar"),
), ),
rpar=cst.RightParen(),
),
"code":
"class Foo(metaclass = Bar): pass\n",
},
# Iterator expansion render test
{
"node":
cst.ClassDef(
cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(),
bases=(cst.Arg(star="*", value=cst.Name("one")), ),
rpar=cst.RightParen(),
),
"code":
"class Foo(*one): pass\n",
},
{
"node":
cst.ClassDef(
cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(),
bases=(
cst.Arg(
star="*",
value=cst.Name("one"),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")),
),
cst.Arg(
star="*",
value=cst.Name("two"),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")),
),
cst.Arg(star="*", value=cst.Name("three")),
),
rpar=cst.RightParen(),
),
"code":
"class Foo(*one, *two, *three): pass\n",
},
# Dictionary expansion render test
{
"node":
cst.ClassDef(
cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(),
keywords=(cst.Arg(star="**", value=cst.Name("one")), ),
rpar=cst.RightParen(),
),
"code":
"class Foo(**one): pass\n",
},
{
"node":
cst.ClassDef(
cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(),
keywords=(
cst.Arg(
star="**",
value=cst.Name("one"),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")),
),
cst.Arg(
star="**",
value=cst.Name("two"),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")),
),
cst.Arg(star="**", value=cst.Name("three")),
),
rpar=cst.RightParen(),
),
"code":
"class Foo(**one, **two, **three): pass\n",
},
# Decorator render tests
{
"node":
cst.ClassDef(
cst.Name("Foo"),
cst.SimpleStatementSuite((cst.Pass(), )),
decorators=(cst.Decorator(cst.Name("foo")), ),
lpar=cst.LeftParen(),
rpar=cst.RightParen(),
),
"code":
"@foo\nclass Foo(): pass\n",
"expected_position":
CodeRange((2, 0), (2, 17)),
},
{
"node":
cst.ClassDef(
leading_lines=(
cst.EmptyLine(),
cst.EmptyLine(comment=cst.Comment("# leading comment 1")),
),
decorators=(
cst.Decorator(cst.Name("foo"), leading_lines=()),
cst.Decorator(
cst.Name("bar"),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment 2")), ),
),
cst.Decorator(
cst.Name("baz"),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment 3")), ),
),
),
lines_after_decorators=(cst.EmptyLine(
comment=cst.Comment("# class comment")), ),
name=cst.Name("Foo"),
body=cst.SimpleStatementSuite((cst.Pass(), )),
lpar=cst.LeftParen(),
rpar=cst.RightParen(),
),
"code":
"\n# leading comment 1\n@foo\n# leading comment 2\n@bar\n# leading comment 3\n@baz\n# class comment\nclass Foo(): pass\n",
"expected_position":
CodeRange((9, 0), (9, 17)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs, parser=parse_statement)
class ForTest(CSTNodeTest):
@data_provider((
# Simple for block
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"for target in iter(): pass\n",
"parser":
parse_statement,
},
# Simple async for block
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async for target in iter(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.7")),
},
# Python 3.6 async for block
{
"node":
cst.FunctionDef(
cst.Name("foo"),
cst.Parameters(),
cst.IndentedBlock((cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async def foo():\n async for target in iter(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.6")),
},
# For block with else
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
cst.Else(cst.SimpleStatementSuite((cst.Pass(), ))),
),
"code":
"for target in iter(): pass\nelse: pass\n",
"parser":
parse_statement,
},
# indentation
{
"node":
DummyIndentedBlock(
" ",
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
),
),
"code":
" for target in iter(): pass\n",
"parser":
None,
},
# for an indented body
{
"node":
DummyIndentedBlock(
" ",
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
),
),
"code":
" for target in iter():\n pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (2, 12)),
},
# leading_lines
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(), )), )),
cst.Else(
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# else comment")), ),
),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment")), ),
),
"code":
"# leading comment\nfor target in iter():\n pass\n# else comment\nelse:\n pass\n",
"parser":
None,
"expected_position":
CodeRange((2, 0), (6, 8)),
},
# Weird spacing rules
{
"node":
cst.For(
cst.Name("target",
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), )),
cst.Call(
cst.Name("iter"),
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), ),
),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_for=cst.SimpleWhitespace(""),
whitespace_before_in=cst.SimpleWhitespace(""),
whitespace_after_in=cst.SimpleWhitespace(""),
),
"code":
"for(target)in(iter()): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 27)),
},
# Whitespace
{
"node":
cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_for=cst.SimpleWhitespace(" "),
whitespace_before_in=cst.SimpleWhitespace(" "),
whitespace_after_in=cst.SimpleWhitespace(" "),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
"code":
"for target in iter() : pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 31)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
@data_provider((
{
"get_node":
lambda: cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_for=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space after 'for' keyword",
},
{
"get_node":
lambda: cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_before_in=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space before 'in' keyword",
},
{
"get_node":
lambda: cst.For(
cst.Name("target"),
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_in=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space after 'in' keyword",
},
))
def test_invalid(self, **kwargs: Any) -> None:
self.assert_invalid(**kwargs)
class WithTest(CSTNodeTest):
@data_provider((
# Simple with block
{
"node":
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with context_mgr(): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 24)),
},
# Simple async with block
{
"node":
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async with context_mgr(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.7")),
},
# Python 3.6 async with block
{
"node":
cst.FunctionDef(
cst.Name("foo"),
cst.Parameters(),
cst.IndentedBlock((cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
asynchronous=cst.Asynchronous(),
), )),
asynchronous=cst.Asynchronous(),
),
"code":
"async def foo():\n async with context_mgr(): pass\n",
"parser":
lambda code: parse_statement(
code, config=PartialParserConfig(python_version="3.6")),
},
# Multiple context managers
{
"node":
cst.With(
(
cst.WithItem(cst.Call(cst.Name("foo"))),
cst.WithItem(cst.Call(cst.Name("bar"))),
),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with foo(), bar(): pass\n",
"parser":
None,
},
{
"node":
cst.With(
(
cst.WithItem(
cst.Call(cst.Name("foo")),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")),
),
cst.WithItem(cst.Call(cst.Name("bar"))),
),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with foo(), bar(): pass\n",
"parser":
parse_statement,
},
# With block containing variable for context manager.
{
"node":
cst.With(
(cst.WithItem(
cst.Call(cst.Name("context_mgr")),
cst.AsName(cst.Name("ctx")),
), ),
cst.SimpleStatementSuite((cst.Pass(), )),
),
"code":
"with context_mgr() as ctx: pass\n",
"parser":
parse_statement,
},
# indentation
{
"node":
DummyIndentedBlock(
" ",
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
),
),
"code":
" with context_mgr(): pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (1, 28)),
},
# with an indented body
{
"node":
DummyIndentedBlock(
" ",
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
),
),
"code":
" with context_mgr():\n pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (2, 12)),
},
# leading_lines
{
"node":
cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment")), ),
),
"code":
"# leading comment\nwith context_mgr(): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((2, 0), (2, 24)),
},
# Weird spacing rules
{
"node":
cst.With(
(cst.WithItem(
cst.Call(
cst.Name("context_mgr"),
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), ),
)), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(""),
),
"code":
"with(context_mgr()): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 25)),
},
# Whitespace
{
"node":
cst.With(
(cst.WithItem(
cst.Call(cst.Name("context_mgr")),
cst.AsName(
cst.Name("ctx"),
whitespace_before_as=cst.SimpleWhitespace(" "),
whitespace_after_as=cst.SimpleWhitespace(" "),
),
), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(" "),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
"code":
"with context_mgr() as ctx : pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 36)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
@data_provider((
{
"get_node":
lambda: cst.With((),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), ))),
"expected_re":
"A With statement must have at least one WithItem",
},
{
"get_node":
lambda: cst.With(
(cst.WithItem(
cst.Call(cst.Name("foo")),
comma=cst.Comma(whitespace_after=cst.SimpleWhitespace(" ")
),
), ),
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(), )), )),
),
"expected_re":
"The last WithItem in a With cannot have a trailing comma",
},
{
"get_node":
lambda: cst.With(
(cst.WithItem(cst.Call(cst.Name("context_mgr"))), ),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_with=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space after with keyword",
},
))
def test_invalid(self, **kwargs: Any) -> None:
self.assert_invalid(**kwargs)
@data_provider((
{
"code": "with a, b: pass",
"parser": parse_statement_as(python_version="3.1"),
"expect_success": True,
},
{
"code": "with a, b: pass",
"parser": parse_statement_as(python_version="3.0"),
"expect_success": False,
},
))
def test_versions(self, **kwargs: Any) -> None:
self.assert_parses(**kwargs)
class ModuleTest(CSTNodeTest):
@data_provider((
# simplest possible program
(cst.Module((cst.SimpleStatementLine((cst.Pass(), )), )), "pass\n"),
# test default_newline
(
cst.Module((cst.SimpleStatementLine((cst.Pass(), )), ),
default_newline="\r"),
"pass\r",
),
# test header/footer
(
cst.Module(
(cst.SimpleStatementLine((cst.Pass(), )), ),
header=(cst.EmptyLine(comment=cst.Comment("# header")), ),
footer=(cst.EmptyLine(comment=cst.Comment("# footer")), ),
),
"# header\npass\n# footer\n",
),
# test has_trailing_newline
(
cst.Module(
(cst.SimpleStatementLine((cst.Pass(), )), ),
has_trailing_newline=False,
),
"pass",
),
# an empty file
(cst.Module((), has_trailing_newline=False), ""),
# a file with only comments
(
cst.Module(
(),
header=(cst.EmptyLine(
comment=cst.Comment("# nothing to see here")), ),
),
"# nothing to see here\n",
),
# TODO: test default_indent
))
def test_code_and_bytes_properties(self, module: cst.Module,
expected: str) -> None:
self.assertEqual(module.code, expected)
self.assertEqual(module.bytes, expected.encode("utf-8"))
@data_provider((
(cst.Module(()), cst.Newline(), "\n"),
(cst.Module((), default_newline="\r\n"), cst.Newline(), "\r\n"),
# has_trailing_newline has no effect on code_for_node
(cst.Module((), has_trailing_newline=False), cst.Newline(), "\n"),
# TODO: test default_indent
))
def test_code_for_node(self, module: cst.Module, node: cst.CSTNode,
expected: str) -> None:
self.assertEqual(module.code_for_node(node), expected)
@data_provider({
"empty_program": {
"code": "",
"expected": cst.Module([], has_trailing_newline=False),
},
"empty_program_with_newline": {
"code": "\n",
"expected": cst.Module([], has_trailing_newline=True),
"enabled_for_native": False,
},
"empty_program_with_comments": {
"code":
"# some comment\n",
"expected":
cst.Module(
[],
header=[cst.EmptyLine(comment=cst.Comment("# some comment"))]),
},
"simple_pass": {
"code": "pass\n",
"expected": cst.Module([cst.SimpleStatementLine([cst.Pass()])]),
},
"simple_pass_with_header_footer": {
"code":
"# header\npass # trailing\n# footer\n",
"expected":
cst.Module(
[
cst.SimpleStatementLine(
[cst.Pass()],
trailing_whitespace=cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(" "),
comment=cst.Comment("# trailing"),
),
)
],
header=[cst.EmptyLine(comment=cst.Comment("# header"))],
footer=[cst.EmptyLine(comment=cst.Comment("# footer"))],
),
},
})
def test_parser(self,
*,
code: str,
expected: cst.Module,
enabled_for_native: bool = True) -> None:
if is_native() and not enabled_for_native:
self.skipTest("Disabled for native parser")
self.assertEqual(parse_module(code), expected)
@data_provider({
"empty": {
"code": "",
"expected": CodeRange((1, 0), (1, 0))
},
"empty_with_newline": {
"code": "\n",
"expected": CodeRange((1, 0), (2, 0))
},
"empty_program_with_comments": {
"code": "# 2345",
"expected": CodeRange((1, 0), (2, 0)),
},
"simple_pass": {
"code": "pass\n",
"expected": CodeRange((1, 0), (2, 0))
},
"simple_pass_with_header_footer": {
"code": "# header\npass # trailing\n# footer\n",
"expected": CodeRange((1, 0), (4, 0)),
},
})
def test_module_position(self, *, code: str, expected: CodeRange) -> None:
wrapper = MetadataWrapper(parse_module(code))
positions = wrapper.resolve(PositionProvider)
self.assertEqual(positions[wrapper.module], expected)
def cmp_position(self, actual: CodeRange, start: Tuple[int, int],
end: Tuple[int, int]) -> None:
self.assertEqual(actual, CodeRange(start, end))
def test_function_position(self) -> None:
wrapper = MetadataWrapper(parse_module("def foo():\n pass"))
module = wrapper.module
positions = wrapper.resolve(PositionProvider)
fn = cast(cst.FunctionDef, module.body[0])
stmt = cast(cst.SimpleStatementLine, fn.body.body[0])
pass_stmt = cast(cst.Pass, stmt.body[0])
self.cmp_position(positions[stmt], (2, 4), (2, 8))
self.cmp_position(positions[pass_stmt], (2, 4), (2, 8))
def test_nested_indent_position(self) -> None:
wrapper = MetadataWrapper(
parse_module(
"if True:\n if False:\n x = 1\nelse:\n return"))
module = wrapper.module
positions = wrapper.resolve(PositionProvider)
outer_if = cast(cst.If, module.body[0])
inner_if = cast(cst.If, outer_if.body.body[0])
assign = cast(cst.SimpleStatementLine, inner_if.body.body[0]).body[0]
outer_else = cast(cst.Else, outer_if.orelse)
return_stmt = cast(cst.SimpleStatementLine,
outer_else.body.body[0]).body[0]
self.cmp_position(positions[outer_if], (1, 0), (5, 10))
self.cmp_position(positions[inner_if], (2, 4), (3, 13))
self.cmp_position(positions[assign], (3, 8), (3, 13))
self.cmp_position(positions[outer_else], (4, 0), (5, 10))
self.cmp_position(positions[return_stmt], (5, 4), (5, 10))
def test_multiline_string_position(self) -> None:
wrapper = MetadataWrapper(parse_module('"abc"\\\n"def"'))
module = wrapper.module
positions = wrapper.resolve(PositionProvider)
stmt = cast(cst.SimpleStatementLine, module.body[0])
expr = cast(cst.Expr, stmt.body[0])
string = expr.value
self.cmp_position(positions[stmt], (1, 0), (2, 5))
self.cmp_position(positions[expr], (1, 0), (2, 5))
self.cmp_position(positions[string], (1, 0), (2, 5))
def test_module_config_for_parsing(self) -> None:
module = parse_module("pass\r")
statement = parse_statement("if True:\r pass",
config=module.config_for_parsing)
self.assertEqual(
statement,
cst.If(
test=cst.Name(value="True"),
body=cst.IndentedBlock(
body=[cst.SimpleStatementLine(body=[cst.Pass()])],
header=cst.TrailingWhitespace(newline=cst.Newline(
# This would be "\r" if we didn't pass the module config forward.
value=None)),
),
),
)
class IfTest(CSTNodeTest):
@data_provider((
# Simple if without elif or else
{
"node":
cst.If(cst.Name("conditional"),
cst.SimpleStatementSuite((cst.Pass(), ))),
"code":
"if conditional: pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 20)),
},
# else clause
{
"node":
cst.If(
cst.Name("conditional"),
cst.SimpleStatementSuite((cst.Pass(), )),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(), ))),
),
"code":
"if conditional: pass\nelse: pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (2, 10)),
},
# elif clause
{
"node":
cst.If(
cst.Name("conditional"),
cst.SimpleStatementSuite((cst.Pass(), )),
orelse=cst.If(
cst.Name("other_conditional"),
cst.SimpleStatementSuite((cst.Pass(), )),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(), ))),
),
),
"code":
"if conditional: pass\nelif other_conditional: pass\nelse: pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (3, 10)),
},
# indentation
{
"node":
DummyIndentedBlock(
" ",
cst.If(
cst.Name("conditional"),
cst.SimpleStatementSuite((cst.Pass(), )),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(), ))),
),
),
"code":
" if conditional: pass\n else: pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (2, 14)),
},
# with an indented body
{
"node":
DummyIndentedBlock(
" ",
cst.If(
cst.Name("conditional"),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
),
),
"code":
" if conditional:\n pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (2, 12)),
},
# leading_lines
{
"node":
cst.If(
cst.Name("conditional"),
cst.SimpleStatementSuite((cst.Pass(), )),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment")), ),
),
"code":
"# leading comment\nif conditional: pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((2, 0), (2, 20)),
},
# whitespace before/after test and else
{
"node":
cst.If(
cst.Name("conditional"),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_before_test=cst.SimpleWhitespace(" "),
whitespace_after_test=cst.SimpleWhitespace(" "),
orelse=cst.Else(
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
),
"code":
"if conditional : pass\nelse : pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (2, 11)),
},
# empty lines between if/elif/else clauses, not captured by the suite.
{
"node":
cst.If(
cst.Name("test_a"),
cst.SimpleStatementSuite((cst.Pass(), )),
orelse=cst.If(
cst.Name("test_b"),
cst.SimpleStatementSuite((cst.Pass(), )),
leading_lines=(cst.EmptyLine(), ),
orelse=cst.Else(
cst.SimpleStatementSuite((cst.Pass(), )),
leading_lines=(cst.EmptyLine(), ),
),
),
),
"code":
"if test_a: pass\n\nelif test_b: pass\n\nelse: pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (5, 10)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
class SimpleStatementTest(CSTNodeTest):
@data_provider((
# a single-element SimpleStatementLine
# pyre-fixme[6]: Incompatible parameter type
{
"node": cst.SimpleStatementLine((cst.Pass(), )),
"code": "pass\n",
"parser": parse_statement,
},
# a multi-element SimpleStatementLine
{
"node":
cst.SimpleStatementLine(
(cst.Pass(semicolon=cst.Semicolon()), cst.Continue())),
"code":
"pass;continue\n",
"parser":
parse_statement,
},
# a multi-element SimpleStatementLine with whitespace
{
"node":
cst.SimpleStatementLine((
cst.Pass(semicolon=cst.Semicolon(
whitespace_before=cst.SimpleWhitespace(" "),
whitespace_after=cst.SimpleWhitespace(" "),
)),
cst.Continue(),
)),
"code":
"pass ; continue\n",
"parser":
parse_statement,
},
# A more complicated SimpleStatementLine
{
"node":
cst.SimpleStatementLine((
cst.Pass(semicolon=cst.Semicolon()),
cst.Continue(semicolon=cst.Semicolon()),
cst.Break(),
)),
"code":
"pass;continue;break\n",
"parser":
parse_statement,
"expected_position":
CodeRange.create((1, 0), (1, 19)),
},
# a multi-element SimpleStatementLine, inferred semicolons
{
"node":
cst.SimpleStatementLine((cst.Pass(), cst.Continue(), cst.Break())),
"code":
"pass; continue; break\n",
"parser":
None, # No test for parsing, since we are using sentinels.
},
# some expression statements
{
"node": cst.SimpleStatementLine((cst.Expr(cst.Name("None")), )),
"code": "None\n",
"parser": parse_statement,
},
{
"node": cst.SimpleStatementLine((cst.Expr(cst.Name("True")), )),
"code": "True\n",
"parser": parse_statement,
},
{
"node": cst.SimpleStatementLine((cst.Expr(cst.Name("False")), )),
"code": "False\n",
"parser": parse_statement,
},
{
"node": cst.SimpleStatementLine((cst.Expr(cst.Ellipsis()), )),
"code": "...\n",
"parser": parse_statement,
},
# Test some numbers
{
"node": cst.SimpleStatementLine((cst.Expr(cst.Integer("5")), )),
"code": "5\n",
"parser": parse_statement,
},
{
"node": cst.SimpleStatementLine((cst.Expr(cst.Float("5.5")), )),
"code": "5.5\n",
"parser": parse_statement,
},
{
"node": cst.SimpleStatementLine((cst.Expr(cst.Imaginary("5j")), )),
"code": "5j\n",
"parser": parse_statement,
},
# Test some numbers with parens
{
"node":
cst.SimpleStatementLine((cst.Expr(
cst.Integer("5",
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), ))), )),
"code":
"(5)\n",
"parser":
parse_statement,
"expected_position":
CodeRange.create((1, 0), (1, 3)),
},
{
"node":
cst.SimpleStatementLine((cst.Expr(
cst.Float("5.5",
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), ))), )),
"code":
"(5.5)\n",
"parser":
parse_statement,
},
{
"node":
cst.SimpleStatementLine((cst.Expr(
cst.Imaginary("5j",
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), ))), )),
"code":
"(5j)\n",
"parser":
parse_statement,
},
# Test some strings
{
"node":
cst.SimpleStatementLine((cst.Expr(cst.SimpleString('"abc"')), )),
"code":
'"abc"\n',
"parser":
parse_statement,
},
{
"node":
cst.SimpleStatementLine((cst.Expr(
cst.ConcatenatedString(cst.SimpleString('"abc"'),
cst.SimpleString('"def"'))), )),
"code":
'"abc""def"\n',
"parser":
parse_statement,
},
{
"node":
cst.SimpleStatementLine((cst.Expr(
cst.ConcatenatedString(
left=cst.SimpleString('"abc"'),
whitespace_between=cst.SimpleWhitespace(" "),
right=cst.ConcatenatedString(
left=cst.SimpleString('"def"'),
whitespace_between=cst.SimpleWhitespace(" "),
right=cst.SimpleString('"ghi"'),
),
)), )),
"code":
'"abc" "def" "ghi"\n',
"parser":
parse_statement,
"expected_position":
CodeRange.create((1, 0), (1, 17)),
},
# Test parenthesis rules
{
"node":
cst.SimpleStatementLine((cst.Expr(
cst.Ellipsis(lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), ))), )),
"code":
"(...)\n",
"parser":
parse_statement,
},
# Test parenthesis with whitespace ownership
{
"node":
cst.SimpleStatementLine((cst.Expr(
cst.Ellipsis(
lpar=(cst.LeftParen(
whitespace_after=cst.SimpleWhitespace(" ")), ),
rpar=(cst.RightParen(
whitespace_before=cst.SimpleWhitespace(" ")), ),
)), )),
"code":
"( ... )\n",
"parser":
parse_statement,
},
{
"node":
cst.SimpleStatementLine((cst.Expr(
cst.Ellipsis(
lpar=(
cst.LeftParen(
whitespace_after=cst.SimpleWhitespace(" ")),
cst.LeftParen(
whitespace_after=cst.SimpleWhitespace(" ")),
cst.LeftParen(
whitespace_after=cst.SimpleWhitespace(" ")),
),
rpar=(
cst.RightParen(
whitespace_before=cst.SimpleWhitespace(" ")),
cst.RightParen(
whitespace_before=cst.SimpleWhitespace(" ")),
cst.RightParen(
whitespace_before=cst.SimpleWhitespace(" ")),
),
)), )),
"code":
"( ( ( ... ) ) )\n",
"parser":
parse_statement,
"expected_position":
CodeRange.create((1, 0), (1, 21)),
},
# Test parenthesis rules with expressions
{
"node":
cst.SimpleStatementLine((cst.Expr(
cst.Ellipsis(
lpar=(cst.LeftParen(
whitespace_after=cst.ParenthesizedWhitespace(
first_line=cst.TrailingWhitespace(),
empty_lines=(cst.EmptyLine(
comment=cst.Comment("# Wow, a comment!")), ),
indent=True,
last_line=cst.SimpleWhitespace(" "),
)), ),
rpar=(cst.RightParen(
whitespace_before=cst.ParenthesizedWhitespace(
first_line=cst.TrailingWhitespace(),
empty_lines=(),
indent=True,
last_line=cst.SimpleWhitespace(""),
)), ),
)), )),
"code":
"(\n# Wow, a comment!\n ...\n)\n",
"parser":
parse_statement,
"expected_position":
CodeRange.create((1, 0), (4, 1)),
},
# test trailing whitespace
{
"node":
cst.SimpleStatementLine(
(cst.Pass(), ),
trailing_whitespace=cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(" "),
comment=cst.Comment("# trailing comment"),
),
),
"code":
"pass # trailing comment\n",
"parser":
parse_statement,
"expected_position":
CodeRange.create((1, 0), (1, 4)),
},
# test leading comment
{
"node":
cst.SimpleStatementLine(
(cst.Pass(), ),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# comment")), ),
),
"code":
"# comment\npass\n",
"parser":
parse_statement,
"expected_position":
CodeRange.create((2, 0), (2, 4)),
},
# test indentation
{
"node":
DummyIndentedBlock(
" ",
cst.SimpleStatementLine(
(cst.Pass(), ),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# comment")), ),
),
),
"code":
" # comment\n pass\n",
"expected_position":
CodeRange.create((2, 4), (2, 8)),
},
# test suite variant
{
"node": cst.SimpleStatementSuite((cst.Pass(), )),
"code": " pass\n",
"expected_position": CodeRange.create((1, 1), (1, 5)),
},
{
"node":
cst.SimpleStatementSuite(
(cst.Pass(), ), leading_whitespace=cst.SimpleWhitespace("")),
"code":
"pass\n",
"expected_position":
CodeRange.create((1, 0), (1, 4)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
def leave_Module(self, original_node: libcst.Module,
updated_node: libcst.Module) -> libcst.Module:
# Don't try to modify if we have nothing to do
if (not self.module_imports and not self.module_mapping
and not self.module_aliases and not self.alias_mapping):
return updated_node
# First, find the insertion point for imports
(
statements_before_imports,
statements_until_add_imports,
statements_after_imports,
) = self._split_module(original_node, updated_node)
# Make sure there's at least one empty line before the first non-import
statements_after_imports = self._insert_empty_line(
statements_after_imports)
# Mapping of modules we're adding to the object with and without alias they should import
module_and_alias_mapping = defaultdict(list)
for module, aliases in self.alias_mapping.items():
module_and_alias_mapping[module].extend(aliases)
for module, imports in self.module_mapping.items():
module_and_alias_mapping[module].extend([(object, None)
for object in imports])
module_and_alias_mapping = {
module: sorted(aliases)
for module, aliases in module_and_alias_mapping.items()
}
import_cycle_safe_module_names = [
'mypy_extensions',
'typing',
'typing_extensions',
]
type_checking_cond_import = parse_statement(
f"from typing import TYPE_CHECKING",
config=updated_node.config_for_parsing,
)
type_checking_cond_statement = libcst.If(
test=libcst.Name("TYPE_CHECKING"),
body=libcst.IndentedBlock(body=[
parse_statement(
f"from {module} import " + ", ".join([
obj if alias is None else f"{obj} as {alias}"
for (obj, alias) in aliases
]),
config=updated_node.config_for_parsing,
) for module, aliases in module_and_alias_mapping.items()
if module != "__future__"
and module not in import_cycle_safe_module_names
], ),
)
if not type_checking_cond_statement.body.body:
type_checking_cond_statement = libcst.EmptyLine()
type_checking_cond_import = libcst.EmptyLine()
# import ptvsd; ptvsd.set_trace()
# Now, add all of the imports we need!
return updated_node.with_changes(body=(
*statements_before_imports,
*[
parse_statement(
f"from {module} import " + ", ".join([
obj if alias is None else f"{obj} as {alias}"
for (obj, alias) in aliases
]),
config=updated_node.config_for_parsing,
) for module, aliases in module_and_alias_mapping.items()
if module == "__future__"
],
*statements_until_add_imports,
*[
parse_statement(f"import {module}",
config=updated_node.config_for_parsing)
for module in sorted(self.module_imports)
],
*[
parse_statement(
f"import {module} as {asname}",
config=updated_node.config_for_parsing,
) for (module, asname) in self.module_aliases.items()
],
# TODO: 可以进一步用 `from __future__ import annotations` 解决forward ref, 这里加也可以，用其他工具也可以
type_checking_cond_import,
type_checking_cond_statement,
*[
parse_statement(
f"from {module} import " + ", ".join([
obj if alias is None else f"{obj} as {alias}"
for (obj, alias) in aliases
]),
config=updated_node.config_for_parsing,
) for module, aliases in module_and_alias_mapping.items()
if module != "__future__"
and module in import_cycle_safe_module_names
and not module.startswith("monkeytype")
],
*statements_after_imports,
))
def test_repr(self) -> None:
self.assertEqual(
repr(
cst.SimpleStatementLine(
body=[cst.Pass()],
# tuple with multiple items
leading_lines=(
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(""),
comment=None,
newline=cst.Newline(),
),
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(""),
comment=None,
newline=cst.Newline(),
),
),
trailing_whitespace=cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(" "),
comment=cst.Comment("# comment"),
newline=cst.Newline(),
),
)
),
dedent(
"""
SimpleStatementLine(
body=[
Pass(
semicolon=MaybeSentinel.DEFAULT,
),
],
leading_lines=[
EmptyLine(
indent=True,
whitespace=SimpleWhitespace(
value='',
),
comment=None,
newline=Newline(
value=None,
),
),
EmptyLine(
indent=True,
whitespace=SimpleWhitespace(
value='',
),
comment=None,
newline=Newline(
value=None,
),
),
],
trailing_whitespace=TrailingWhitespace(
whitespace=SimpleWhitespace(
value=' ',
),
comment=Comment(
value='# comment',
),
newline=Newline(
value=None,
),
),
)
"""
).strip(),
)
class TryTest(CSTNodeTest):
@data_provider(
(
# Simple try/except block
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_after_except=cst.SimpleWhitespace(""),
),
),
),
"code": "try: pass\nexcept: pass\n",
"parser": parse_statement,
"expected_position": CodeRange((1, 0), (2, 12)),
},
# Try/except with a class
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
type=cst.Name("Exception"),
),
),
),
"code": "try: pass\nexcept Exception: pass\n",
"parser": parse_statement,
},
# Try/except with a named class
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
type=cst.Name("Exception"),
name=cst.AsName(cst.Name("exc")),
),
),
),
"code": "try: pass\nexcept Exception as exc: pass\n",
"parser": parse_statement,
"expected_position": CodeRange((1, 0), (2, 29)),
},
# Try/except with multiple clauses
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
type=cst.Name("TypeError"),
name=cst.AsName(cst.Name("e")),
),
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
type=cst.Name("KeyError"),
name=cst.AsName(cst.Name("e")),
),
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_after_except=cst.SimpleWhitespace(""),
),
),
),
"code": "try: pass\n"
+ "except TypeError as e: pass\n"
+ "except KeyError as e: pass\n"
+ "except: pass\n",
"parser": parse_statement,
"expected_position": CodeRange((1, 0), (4, 12)),
},
# Simple try/finally block
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
finalbody=cst.Finally(cst.SimpleStatementSuite((cst.Pass(),))),
),
"code": "try: pass\nfinally: pass\n",
"parser": parse_statement,
"expected_position": CodeRange((1, 0), (2, 13)),
},
# Simple try/except/finally block
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_after_except=cst.SimpleWhitespace(""),
),
),
finalbody=cst.Finally(cst.SimpleStatementSuite((cst.Pass(),))),
),
"code": "try: pass\nexcept: pass\nfinally: pass\n",
"parser": parse_statement,
"expected_position": CodeRange((1, 0), (3, 13)),
},
# Simple try/except/else block
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_after_except=cst.SimpleWhitespace(""),
),
),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(),))),
),
"code": "try: pass\nexcept: pass\nelse: pass\n",
"parser": parse_statement,
"expected_position": CodeRange((1, 0), (3, 10)),
},
# Simple try/except/else block/finally
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_after_except=cst.SimpleWhitespace(""),
),
),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(),))),
finalbody=cst.Finally(cst.SimpleStatementSuite((cst.Pass(),))),
),
"code": "try: pass\nexcept: pass\nelse: pass\nfinally: pass\n",
"parser": parse_statement,
"expected_position": CodeRange((1, 0), (4, 13)),
},
# Verify whitespace in various locations
{
"node": cst.Try(
leading_lines=(cst.EmptyLine(comment=cst.Comment("# 1")),),
body=cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
leading_lines=(cst.EmptyLine(comment=cst.Comment("# 2")),),
type=cst.Name("TypeError"),
name=cst.AsName(
cst.Name("e"),
whitespace_before_as=cst.SimpleWhitespace(" "),
whitespace_after_as=cst.SimpleWhitespace(" "),
),
whitespace_after_except=cst.SimpleWhitespace(" "),
whitespace_before_colon=cst.SimpleWhitespace(" "),
body=cst.SimpleStatementSuite((cst.Pass(),)),
),
),
orelse=cst.Else(
leading_lines=(cst.EmptyLine(comment=cst.Comment("# 3")),),
body=cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
finalbody=cst.Finally(
leading_lines=(cst.EmptyLine(comment=cst.Comment("# 4")),),
body=cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
"code": "# 1\ntry : pass\n# 2\nexcept TypeError as e : pass\n# 3\nelse : pass\n# 4\nfinally : pass\n",
"parser": parse_statement,
"expected_position": CodeRange((2, 0), (8, 14)),
},
# Please don't write code like this
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
type=cst.Name("TypeError"),
name=cst.AsName(cst.Name("e")),
),
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
type=cst.Name("KeyError"),
name=cst.AsName(cst.Name("e")),
),
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_after_except=cst.SimpleWhitespace(""),
),
),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(),))),
finalbody=cst.Finally(cst.SimpleStatementSuite((cst.Pass(),))),
),
"code": "try: pass\n"
+ "except TypeError as e: pass\n"
+ "except KeyError as e: pass\n"
+ "except: pass\n"
+ "else: pass\n"
+ "finally: pass\n",
"parser": parse_statement,
"expected_position": CodeRange((1, 0), (6, 13)),
},
# Verify indentation
{
"node": DummyIndentedBlock(
" ",
cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
type=cst.Name("TypeError"),
name=cst.AsName(cst.Name("e")),
),
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
type=cst.Name("KeyError"),
name=cst.AsName(cst.Name("e")),
),
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_after_except=cst.SimpleWhitespace(""),
),
),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(),))),
finalbody=cst.Finally(cst.SimpleStatementSuite((cst.Pass(),))),
),
),
"code": " try: pass\n"
+ " except TypeError as e: pass\n"
+ " except KeyError as e: pass\n"
+ " except: pass\n"
+ " else: pass\n"
+ " finally: pass\n",
"parser": None,
},
# Verify indentation in bodies
{
"node": DummyIndentedBlock(
" ",
cst.Try(
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(),)),)),
handlers=(
cst.ExceptHandler(
cst.IndentedBlock(
(cst.SimpleStatementLine((cst.Pass(),)),)
),
whitespace_after_except=cst.SimpleWhitespace(""),
),
),
orelse=cst.Else(
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(),)),))
),
finalbody=cst.Finally(
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(),)),))
),
),
),
"code": " try:\n"
+ " pass\n"
+ " except:\n"
+ " pass\n"
+ " else:\n"
+ " pass\n"
+ " finally:\n"
+ " pass\n",
"parser": None,
},
# No space when using grouping parens
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_after_except=cst.SimpleWhitespace(""),
type=cst.Name(
"Exception",
lpar=(cst.LeftParen(),),
rpar=(cst.RightParen(),),
),
),
),
),
"code": "try: pass\nexcept(Exception): pass\n",
"parser": parse_statement,
},
# No space when using tuple
{
"node": cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
handlers=(
cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
whitespace_after_except=cst.SimpleWhitespace(""),
type=cst.Tuple(
[
cst.Element(
cst.Name("IOError"),
comma=cst.Comma(
whitespace_after=cst.SimpleWhitespace(" ")
),
),
cst.Element(cst.Name("ImportError")),
]
),
),
),
),
"code": "try: pass\nexcept(IOError, ImportError): pass\n",
"parser": parse_statement,
},
)
)
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
@data_provider(
(
{
"get_node": lambda: cst.AsName(cst.Name("")),
"expected_re": "empty name identifier",
},
{
"get_node": lambda: cst.AsName(
cst.Name("bla"), whitespace_after_as=cst.SimpleWhitespace("")
),
"expected_re": "between 'as'",
},
{
"get_node": lambda: cst.AsName(
cst.Name("bla"), whitespace_before_as=cst.SimpleWhitespace("")
),
"expected_re": "before 'as'",
},
{
"get_node": lambda: cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
name=cst.AsName(cst.Name("bla")),
),
"expected_re": "name for an empty type",
},
{
"get_node": lambda: cst.ExceptHandler(
cst.SimpleStatementSuite((cst.Pass(),)),
type=cst.Name("TypeError"),
whitespace_after_except=cst.SimpleWhitespace(""),
),
"expected_re": "at least one space after except",
},
{
"get_node": lambda: cst.Try(cst.SimpleStatementSuite((cst.Pass(),))),
"expected_re": "at least one ExceptHandler or Finally",
},
{
"get_node": lambda: cst.Try(
cst.SimpleStatementSuite((cst.Pass(),)),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(),))),
finalbody=cst.Finally(cst.SimpleStatementSuite((cst.Pass(),))),
),
"expected_re": "at least one ExceptHandler in order to have an Else",
},
)
)
def test_invalid(self, **kwargs: Any) -> None:
self.assert_invalid(**kwargs)
class CSTNodeTest(UnitTest):
def test_with_changes(self) -> None:
initial = cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(" \\\n "),
comment=cst.Comment("# initial"),
newline=cst.Newline("\r\n"),
)
changed = initial.with_changes(comment=cst.Comment("# new comment"))
# see that we have the updated fields
self.assertEqual(none_throws(changed.comment).value, "# new comment")
# and that the old fields are still there
self.assertEqual(changed.whitespace.value, " \\\n ")
self.assertEqual(changed.newline.value, "\r\n")
# ensure no mutation actually happened
self.assertEqual(none_throws(initial.comment).value, "# initial")
def test_default_eq(self) -> None:
sw1 = cst.SimpleWhitespace("")
sw2 = cst.SimpleWhitespace("")
self.assertNotEqual(sw1, sw2)
self.assertEqual(sw1, sw1)
self.assertEqual(sw2, sw2)
self.assertTrue(sw1.deep_equals(sw2))
self.assertTrue(sw2.deep_equals(sw1))
def test_hash(self) -> None:
sw1 = cst.SimpleWhitespace("")
sw2 = cst.SimpleWhitespace("")
self.assertNotEqual(hash(sw1), hash(sw2))
self.assertEqual(hash(sw1), hash(sw1))
self.assertEqual(hash(sw2), hash(sw2))
@data_provider(
{
"simple": (cst.SimpleWhitespace(""), cst.SimpleWhitespace("")),
"identity": (_EMPTY_SIMPLE_WHITESPACE, _EMPTY_SIMPLE_WHITESPACE),
"nested": (
cst.EmptyLine(whitespace=cst.SimpleWhitespace("")),
cst.EmptyLine(whitespace=cst.SimpleWhitespace("")),
),
"tuple_versus_list": (
cst.SimpleStatementLine(body=[cst.Pass()]),
cst.SimpleStatementLine(body=(cst.Pass(),)),
),
}
)
def test_deep_equals_success(self, a: cst.CSTNode, b: cst.CSTNode) -> None:
self.assertTrue(a.deep_equals(b))
@data_provider(
{
"simple": (cst.SimpleWhitespace(" "), cst.SimpleWhitespace(" ")),
"nested": (
cst.EmptyLine(whitespace=cst.SimpleWhitespace(" ")),
cst.EmptyLine(whitespace=cst.SimpleWhitespace(" ")),
),
"list": (
cst.SimpleStatementLine(body=[cst.Pass(semicolon=cst.Semicolon())]),
cst.SimpleStatementLine(body=[cst.Pass(semicolon=cst.Semicolon())] * 2),
),
}
)
def test_deep_equals_fails(self, a: cst.CSTNode, b: cst.CSTNode) -> None:
self.assertFalse(a.deep_equals(b))
def test_repr(self) -> None:
self.assertEqual(
repr(
cst.SimpleStatementLine(
body=[cst.Pass()],
# tuple with multiple items
leading_lines=(
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(""),
comment=None,
newline=cst.Newline(),
),
cst.EmptyLine(
indent=True,
whitespace=cst.SimpleWhitespace(""),
comment=None,
newline=cst.Newline(),
),
),
trailing_whitespace=cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(" "),
comment=cst.Comment("# comment"),
newline=cst.Newline(),
),
)
),
dedent(
"""
SimpleStatementLine(
body=[
Pass(
semicolon=MaybeSentinel.DEFAULT,
),
],
leading_lines=[
EmptyLine(
indent=True,
whitespace=SimpleWhitespace(
value='',
),
comment=None,
newline=Newline(
value=None,
),
),
EmptyLine(
indent=True,
whitespace=SimpleWhitespace(
value='',
),
comment=None,
newline=Newline(
value=None,
),
),
],
trailing_whitespace=TrailingWhitespace(
whitespace=SimpleWhitespace(
value=' ',
),
comment=Comment(
value='# comment',
),
newline=Newline(
value=None,
),
),
)
"""
).strip(),
)
def test_visit(self) -> None:
tree = cst.Module((cst.SimpleStatementLine((cst.Pass(),)),))
tree.visit(_TestVisitor(self))
class FooterBehaviorTest(UnitTest):
@data_provider({
# Literally the most basic example
"simple_module": {
"code": "\n",
"expected_module": cst.Module(body=())
},
# A module with a header comment
"header_only_module": {
"code":
"# This is a header comment\n",
"expected_module":
cst.Module(
header=[
cst.EmptyLine(comment=cst.Comment(
value="# This is a header comment"))
],
body=[],
),
},
# A module with a header and footer
"simple_header_footer_module": {
"code":
"# This is a header comment\npass\n# This is a footer comment\n",
"expected_module":
cst.Module(
header=[
cst.EmptyLine(comment=cst.Comment(
value="# This is a header comment"))
],
body=[cst.SimpleStatementLine([cst.Pass()])],
footer=[
cst.EmptyLine(comment=cst.Comment(
value="# This is a footer comment"))
],
),
},
# A module which should have a footer comment taken from the
# if statement's indented block.
"simple_reparented_footer_module": {
"code":
"# This is a header comment\nif True:\n pass\n# This is a footer comment\n",
"expected_module":
cst.Module(
header=[
cst.EmptyLine(comment=cst.Comment(
value="# This is a header comment"))
],
body=[
cst.If(
test=cst.Name(value="True"),
body=cst.IndentedBlock(
header=cst.TrailingWhitespace(),
body=[
cst.SimpleStatementLine(
body=[cst.Pass()],
trailing_whitespace=cst.TrailingWhitespace(
),
)
],
),
)
],
footer=[
cst.EmptyLine(comment=cst.Comment(
value="# This is a footer comment"))
],
),
},
# Verifying that we properly parse and spread out footer comments to the
# relative indents they go with.
"complex_reparented_footer_module": {
"code":
("# This is a header comment\nif True:\n if True:\n pass"
+
"\n # This is an inner indented block comment\n # This "
+
"is an outer indented block comment\n# This is a footer comment\n"
),
"expected_module":
cst.Module(
body=[
cst.If(
test=cst.Name(value="True"),
body=cst.IndentedBlock(
body=[
cst.If(
test=cst.Name(value="True"),
body=cst.IndentedBlock(
body=[
cst.SimpleStatementLine(
body=[cst.Pass()])
],
footer=[
cst.EmptyLine(comment=cst.Comment(
value=
"# This is an inner indented block comment"
))
],
),
)
],
footer=[
cst.EmptyLine(comment=cst.Comment(
value=
"# This is an outer indented block comment"
))
],
),
)
],
header=[
cst.EmptyLine(comment=cst.Comment(
value="# This is a header comment"))
],
footer=[
cst.EmptyLine(comment=cst.Comment(
value="# This is a footer comment"))
],
),
},
# Verify that comments belonging to statements are still owned even
# after an indented block.
"statement_comment_reparent": {
"code":
"if foo:\n return\n# comment\nx = 7\n",
"expected_module":
cst.Module(body=[
cst.If(
test=cst.Name(value="foo"),
body=cst.IndentedBlock(body=[
cst.SimpleStatementLine(body=[
cst.Return(
whitespace_after_return=cst.SimpleWhitespace(
value=""))
])
]),
),
cst.SimpleStatementLine(
body=[
cst.Assign(
targets=[
cst.AssignTarget(target=cst.Name(value="x"))
],
value=cst.Integer(value="7"),
)
],
leading_lines=[
cst.EmptyLine(comment=cst.Comment(value="# comment"))
],
),
]),
},
# Verify that even if there are completely empty lines, we give all lines
# up to and including the last line that's indented correctly. That way
# comments that line up with indented block's indentation level aren't
# parented to the next line just because there's a blank line or two
# between them.
"statement_comment_with_empty_lines": {
"code":
("def foo():\n if True:\n pass\n\n # Empty " +
"line before me\n\n else:\n pass\n"),
"expected_module":
cst.Module(body=[
cst.FunctionDef(
name=cst.Name(value="foo"),
params=cst.Parameters(),
body=cst.IndentedBlock(body=[
cst.If(
test=cst.Name(value="True"),
body=cst.IndentedBlock(
body=[
cst.SimpleStatementLine(body=[cst.Pass()])
],
footer=[
cst.EmptyLine(indent=False),
cst.EmptyLine(comment=cst.Comment(
value="# Empty line before me")),
],
),
orelse=cst.Else(
body=cst.IndentedBlock(body=[
cst.SimpleStatementLine(body=[cst.Pass()])
]),
leading_lines=[cst.EmptyLine(indent=False)],
),
)
]),
)
]),
},
})
def test_parsers(self, code: str, expected_module: cst.CSTNode) -> None:
parsed_module = parse_module(dedent(code))
self.assertTrue(
deep_equals(parsed_module, expected_module),
msg=
f"\n{parsed_module!r}\nis not deeply equal to \n{expected_module!r}",
)
class ParenthesizedWhitespaceTest(CSTNodeTest):
@data_provider(
(
(cst.ParenthesizedWhitespace(), "\n"),
(
cst.ParenthesizedWhitespace(first_line=cst.TrailingWhitespace(
cst.SimpleWhitespace(" "),
cst.Comment("# This is a comment"))),
" # This is a comment\n",
),
(
cst.ParenthesizedWhitespace(
first_line=cst.TrailingWhitespace(
cst.SimpleWhitespace(" "),
cst.Comment("# This is a comment")),
empty_lines=(cst.EmptyLine(), cst.EmptyLine(),
cst.EmptyLine()),
),
" # This is a comment\n\n\n\n",
),
(
cst.ParenthesizedWhitespace(
first_line=cst.TrailingWhitespace(
cst.SimpleWhitespace(" "),
cst.Comment("# This is a comment")),
empty_lines=(cst.EmptyLine(), cst.EmptyLine(),
cst.EmptyLine()),
indent=False,
last_line=cst.SimpleWhitespace(" "),
),
" # This is a comment\n\n\n\n ",
),
(
DummyIndentedBlock(
" ",
cst.ParenthesizedWhitespace(
first_line=cst.TrailingWhitespace(
cst.SimpleWhitespace(" "),
cst.Comment("# This is a comment"),
),
empty_lines=(cst.EmptyLine(), cst.EmptyLine(),
cst.EmptyLine()),
indent=True,
last_line=cst.SimpleWhitespace(" "),
),
),
" # This is a comment\n \n \n \n ",
),
(
DummyIndentedBlock(
" ",
cst.ParenthesizedWhitespace(
first_line=cst.TrailingWhitespace(
cst.SimpleWhitespace(" "),
cst.Comment("# This is a comment"),
),
indent=True,
last_line=cst.SimpleWhitespace(""),
),
),
" # This is a comment\n ",
),
))
def test_valid(self, node: cst.CSTNode, code: str) -> None:
self.validate_node(node, code)
class WhileTest(CSTNodeTest):
@data_provider((
# Simple while block
# pyre-fixme[6]: Incompatible parameter type
{
"node":
cst.While(cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), ))),
"code":
"while iter(): pass\n",
"parser":
parse_statement,
},
# While block with else
{
"node":
cst.While(
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
cst.Else(cst.SimpleStatementSuite((cst.Pass(), ))),
),
"code":
"while iter(): pass\nelse: pass\n",
"parser":
parse_statement,
},
# indentation
{
"node":
DummyIndentedBlock(
" ",
cst.While(
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
),
),
"code":
" while iter(): pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (1, 22)),
},
# while an indented body
{
"node":
DummyIndentedBlock(
" ",
cst.While(
cst.Call(cst.Name("iter")),
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
),
),
"code":
" while iter():\n pass\n",
"parser":
None,
"expected_position":
CodeRange((1, 4), (2, 12)),
},
# leading_lines
{
"node":
cst.While(
cst.Call(cst.Name("iter")),
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(), )), )),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment")), ),
),
"code":
"# leading comment\nwhile iter():\n pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((2, 0), (3, 8)),
},
{
"node":
cst.While(
cst.Call(cst.Name("iter")),
cst.IndentedBlock((cst.SimpleStatementLine((cst.Pass(), )), )),
cst.Else(
cst.IndentedBlock((cst.SimpleStatementLine(
(cst.Pass(), )), )),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# else comment")), ),
),
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# leading comment")), ),
),
"code":
"# leading comment\nwhile iter():\n pass\n# else comment\nelse:\n pass\n",
"parser":
None,
"expected_position":
CodeRange((2, 0), (6, 8)),
},
# Weird spacing rules
{
"node":
cst.While(
cst.Call(
cst.Name("iter"),
lpar=(cst.LeftParen(), ),
rpar=(cst.RightParen(), ),
),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_while=cst.SimpleWhitespace(""),
),
"code":
"while(iter()): pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 19)),
},
# Whitespace
{
"node":
cst.While(
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_while=cst.SimpleWhitespace(" "),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
"code":
"while iter() : pass\n",
"parser":
parse_statement,
"expected_position":
CodeRange((1, 0), (1, 21)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
@data_provider(({
"get_node":
lambda: cst.While(
cst.Call(cst.Name("iter")),
cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_after_while=cst.SimpleWhitespace(""),
),
"expected_re":
"Must have at least one space after 'while' keyword",
}, ))
def test_invalid(self, **kwargs: Any) -> None:
self.assert_invalid(**kwargs)
class TryStarTest(CSTNodeTest):
@data_provider((
# Try/except with a class
{
"node":
cst.TryStar(
cst.SimpleStatementSuite((cst.Pass(), )),
handlers=(cst.ExceptStarHandler(
cst.SimpleStatementSuite((cst.Pass(), )),
type=cst.Name("Exception"),
), ),
),
"code":
"try: pass\nexcept* Exception: pass\n",
"parser":
native_parse_statement,
},
# Try/except with a named class
{
"node":
cst.TryStar(
cst.SimpleStatementSuite((cst.Pass(), )),
handlers=(cst.ExceptStarHandler(
cst.SimpleStatementSuite((cst.Pass(), )),
type=cst.Name("Exception"),
name=cst.AsName(cst.Name("exc")),
), ),
),
"code":
"try: pass\nexcept* Exception as exc: pass\n",
"parser":
native_parse_statement,
"expected_position":
CodeRange((1, 0), (2, 30)),
},
# Try/except with multiple clauses
{
"node":
cst.TryStar(
cst.SimpleStatementSuite((cst.Pass(), )),
handlers=(
cst.ExceptStarHandler(
cst.SimpleStatementSuite((cst.Pass(), )),
type=cst.Name("TypeError"),
name=cst.AsName(cst.Name("e")),
),
cst.ExceptStarHandler(
cst.SimpleStatementSuite((cst.Pass(), )),
type=cst.Name("KeyError"),
name=cst.AsName(cst.Name("e")),
),
),
),
"code":
"try: pass\n" + "except* TypeError as e: pass\n" +
"except* KeyError as e: pass\n",
"parser":
native_parse_statement,
"expected_position":
CodeRange((1, 0), (3, 27)),
},
# Simple try/except/finally block
{
"node":
cst.TryStar(
cst.SimpleStatementSuite((cst.Pass(), )),
handlers=(cst.ExceptStarHandler(
cst.SimpleStatementSuite((cst.Pass(), )),
type=cst.Name("KeyError"),
whitespace_after_except=cst.SimpleWhitespace(""),
), ),
finalbody=cst.Finally(cst.SimpleStatementSuite(
(cst.Pass(), ))),
),
"code":
"try: pass\nexcept* KeyError: pass\nfinally: pass\n",
"parser":
native_parse_statement,
"expected_position":
CodeRange((1, 0), (3, 13)),
},
# Simple try/except/else block
{
"node":
cst.TryStar(
cst.SimpleStatementSuite((cst.Pass(), )),
handlers=(cst.ExceptStarHandler(
cst.SimpleStatementSuite((cst.Pass(), )),
type=cst.Name("KeyError"),
whitespace_after_except=cst.SimpleWhitespace(""),
), ),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(), ))),
),
"code":
"try: pass\nexcept* KeyError: pass\nelse: pass\n",
"parser":
native_parse_statement,
"expected_position":
CodeRange((1, 0), (3, 10)),
},
# Verify whitespace in various locations
{
"node":
cst.TryStar(
leading_lines=(cst.EmptyLine(comment=cst.Comment("# 1")), ),
body=cst.SimpleStatementSuite((cst.Pass(), )),
handlers=(cst.ExceptStarHandler(
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# 2")), ),
type=cst.Name("TypeError"),
name=cst.AsName(
cst.Name("e"),
whitespace_before_as=cst.SimpleWhitespace(" "),
whitespace_after_as=cst.SimpleWhitespace(" "),
),
whitespace_after_except=cst.SimpleWhitespace(" "),
whitespace_after_star=cst.SimpleWhitespace(""),
whitespace_before_colon=cst.SimpleWhitespace(" "),
body=cst.SimpleStatementSuite((cst.Pass(), )),
), ),
orelse=cst.Else(
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# 3")), ),
body=cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
finalbody=cst.Finally(
leading_lines=(cst.EmptyLine(
comment=cst.Comment("# 4")), ),
body=cst.SimpleStatementSuite((cst.Pass(), )),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
whitespace_before_colon=cst.SimpleWhitespace(" "),
),
"code":
"# 1\ntry : pass\n# 2\nexcept *TypeError as e : pass\n# 3\nelse : pass\n# 4\nfinally : pass\n",
"parser":
native_parse_statement,
"expected_position":
CodeRange((2, 0), (8, 14)),
},
# Now all together
{
"node":
cst.TryStar(
cst.SimpleStatementSuite((cst.Pass(), )),
handlers=(
cst.ExceptStarHandler(
cst.SimpleStatementSuite((cst.Pass(), )),
type=cst.Name("TypeError"),
name=cst.AsName(cst.Name("e")),
),
cst.ExceptStarHandler(
cst.SimpleStatementSuite((cst.Pass(), )),
type=cst.Name("KeyError"),
name=cst.AsName(cst.Name("e")),
),
),
orelse=cst.Else(cst.SimpleStatementSuite((cst.Pass(), ))),
finalbody=cst.Finally(cst.SimpleStatementSuite(
(cst.Pass(), ))),
),
"code":
"try: pass\n" + "except* TypeError as e: pass\n" +
"except* KeyError as e: pass\n" + "else: pass\n" +
"finally: pass\n",
"parser":
native_parse_statement,
"expected_position":
CodeRange((1, 0), (5, 13)),
},
))
def test_valid(self, **kwargs: Any) -> None:
self.validate_node(**kwargs)
def import_to_node_multi(imp: SortableImport,
module: cst.Module) -> cst.BaseStatement:
body: List[cst.BaseSmallStatement] = []
names: List[cst.ImportAlias] = []
prev: Optional[cst.ImportAlias] = None
following: List[str] = []
lpar_lines: List[cst.EmptyLine] = []
lpar_inline: cst.TrailingWhitespace = cst.TrailingWhitespace()
item_count = len(imp.items)
for idx, item in enumerate(imp.items):
name = name_to_node(item.name)
asname = cst.AsName(
name=cst.Name(item.asname)) if item.asname else None
# Leading comments actually have to be trailing comments on the previous node.
# That means putting them on the lpar node for the first item
if item.comments.before:
lines = [
cst.EmptyLine(
indent=True,
comment=cst.Comment(c),
whitespace=cst.SimpleWhitespace(module.default_indent),
) for c in item.comments.before
]
if prev is None:
lpar_lines.extend(lines)
else:
prev.comma.whitespace_after.empty_lines.extend(
lines) # type: ignore
# all items except the last needs whitespace to indent the *next* line/item
indent = idx != (len(imp.items) - 1)
first_line = cst.TrailingWhitespace()
inline = COMMENT_INDENT.join(item.comments.inline)
if inline:
first_line = cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(COMMENT_INDENT),
comment=cst.Comment(inline),
)
if idx == item_count - 1:
following = item.comments.following + imp.comments.final
else:
following = item.comments.following
after = cst.ParenthesizedWhitespace(
indent=True,
first_line=first_line,
empty_lines=[
cst.EmptyLine(
indent=True,
comment=cst.Comment(c),
whitespace=cst.SimpleWhitespace(module.default_indent),
) for c in following
],
last_line=cst.SimpleWhitespace(
module.default_indent if indent else ""),
)
node = cst.ImportAlias(
name=name,
asname=asname,
comma=cst.Comma(whitespace_after=after),
)
names.append(node)
prev = node
# from foo import (
# bar
# )
if imp.stem:
stem, ndots = split_relative(imp.stem)
if not stem:
module_name = None
else:
module_name = name_to_node(stem)
relative = (cst.Dot(), ) * ndots
# inline comment following lparen
if imp.comments.first_inline:
inline = COMMENT_INDENT.join(imp.comments.first_inline)
lpar_inline = cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(COMMENT_INDENT),
comment=cst.Comment(inline),
)
body = [
cst.ImportFrom(
module=module_name,
names=names,
relative=relative,
lpar=cst.LeftParen(
whitespace_after=cst.ParenthesizedWhitespace(
indent=True,
first_line=lpar_inline,
empty_lines=lpar_lines,
last_line=cst.SimpleWhitespace(module.default_indent),
), ),
rpar=cst.RightParen(),
)
]
# import foo
else:
raise ValueError("can't render basic imports on multiple lines")
# comment lines above import
leading_lines = [
cst.EmptyLine(indent=True, comment=cst.Comment(line))
if line.startswith("#") else cst.EmptyLine(indent=False)
for line in imp.comments.before
]
# inline comments following import/rparen
if imp.comments.last_inline:
inline = COMMENT_INDENT.join(imp.comments.last_inline)
trailing = cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(COMMENT_INDENT),
comment=cst.Comment(inline))
else:
trailing = cst.TrailingWhitespace()
return cst.SimpleStatementLine(
body=body,
leading_lines=leading_lines,
trailing_whitespace=trailing,
)
class ModuleTest(CSTNodeTest):
@data_provider((
# simplest possible program
(cst.Module((cst.SimpleStatementLine((cst.Pass(), )), )), "pass\n"),
# test default_newline
(
cst.Module((cst.SimpleStatementLine((cst.Pass(), )), ),
default_newline="\r"),
"pass\r",
),
# test header/footer
(
cst.Module(
(cst.SimpleStatementLine((cst.Pass(), )), ),
header=(cst.EmptyLine(comment=cst.Comment("# header")), ),
footer=(cst.EmptyLine(comment=cst.Comment("# footer")), ),
),
"# header\npass\n# footer\n",
),
# test has_trailing_newline
(
cst.Module(
(cst.SimpleStatementLine((cst.Pass(), )), ),
has_trailing_newline=False,
),
"pass",
),
# an empty file
(cst.Module((), has_trailing_newline=False), ""),
# a file with only comments
(
cst.Module(
(),
header=(cst.EmptyLine(
comment=cst.Comment("# nothing to see here")), ),
),
"# nothing to see here\n",
),
# TODO: test default_indent
))
def test_code_and_bytes_properties(self, module: cst.Module,
expected: str) -> None:
self.assertEqual(module.code, expected)
self.assertEqual(module.bytes, expected.encode("utf-8"))
@data_provider((
(cst.Module(()), cst.Newline(), "\n"),
(cst.Module((), default_newline="\r\n"), cst.Newline(), "\r\n"),
# has_trailing_newline has no effect on code_for_node
(cst.Module((), has_trailing_newline=False), cst.Newline(), "\n"),
# TODO: test default_indent
))
def test_code_for_node(self, module: cst.Module, node: cst.CSTNode,
expected: str) -> None:
self.assertEqual(module.code_for_node(node), expected)
@data_provider({
"empty_program": {
"code": "",
"expected": cst.Module([], has_trailing_newline=False),
},
"empty_program_with_newline": {
"code": "\n",
"expected": cst.Module([], has_trailing_newline=True),
},
"empty_program_with_comments": {
"code":
"# some comment\n",
"expected":
cst.Module(
[],
header=[cst.EmptyLine(comment=cst.Comment("# some comment"))]),
},
"simple_pass": {
"code": "pass\n",
"expected": cst.Module([cst.SimpleStatementLine([cst.Pass()])]),
},
"simple_pass_with_header_footer": {
"code":
"# header\npass # trailing\n# footer\n",
"expected":
cst.Module(
[
cst.SimpleStatementLine(
[cst.Pass()],
trailing_whitespace=cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(" "),
comment=cst.Comment("# trailing"),
),
)
],
header=[cst.EmptyLine(comment=cst.Comment("# header"))],
footer=[cst.EmptyLine(comment=cst.Comment("# footer"))],
),
},
})
def test_parser(self, *, code: str, expected: cst.Module) -> None:
self.assertEqual(parse_module(code), expected)
@data_provider({
"empty": {
"code": "",
"expected": CodeRange.create((1, 0), (1, 0))
},
"empty_with_newline": {
"code": "\n",
"expected": CodeRange.create((1, 0), (2, 0)),
},
"empty_program_with_comments": {
"code": "# 2345",
"expected": CodeRange.create((1, 0), (2, 0)),
},
"simple_pass": {
"code": "pass\n",
"expected": CodeRange.create((1, 0), (2, 0)),
},
"simple_pass_with_header_footer": {
"code": "# header\npass # trailing\n# footer\n",
"expected": CodeRange.create((1, 0), (4, 0)),
},
})
def test_module_position(self, *, code: str, expected: CodeRange) -> None:
module = parse_module(code)
provider = SyntacticPositionProvider()
module.code_for_node(module, provider)
self.assertEqual(provider._computed[module], expected)
# TODO: remove this
self.assertEqual(module._metadata[SyntacticPositionProvider], expected)
def cmp_position(self, actual: CodeRange, start: Tuple[int, int],
end: Tuple[int, int]) -> None:
self.assertEqual(actual, CodeRange.create(start, end))
def test_function_position(self) -> None:
module = parse_module("def foo():\n pass")
provider = SyntacticPositionProvider()
module.code_for_node(module, provider)
fn = cast(cst.FunctionDef, module.body[0])
stmt = cast(cst.SimpleStatementLine, fn.body.body[0])
pass_stmt = cast(cst.Pass, stmt.body[0])
self.cmp_position(provider._computed[stmt], (2, 4), (2, 8))
self.cmp_position(provider._computed[pass_stmt], (2, 4), (2, 8))
def test_nested_indent_position(self) -> None:
module = parse_module(
"if True:\n if False:\n x = 1\nelse:\n return")
provider = SyntacticPositionProvider()
module.code_for_node(module, provider)
outer_if = cast(cst.If, module.body[0])
inner_if = cast(cst.If, outer_if.body.body[0])
assign = cast(cst.SimpleStatementLine, inner_if.body.body[0]).body[0]
outer_else = cast(cst.Else, outer_if.orelse)
return_stmt = cast(cst.SimpleStatementLine,
outer_else.body.body[0]).body[0]
self.cmp_position(provider._computed[outer_if], (1, 0), (5, 10))
self.cmp_position(provider._computed[inner_if], (2, 4), (3, 13))
self.cmp_position(provider._computed[assign], (3, 8), (3, 13))
self.cmp_position(provider._computed[outer_else], (4, 0), (5, 10))
self.cmp_position(provider._computed[return_stmt], (5, 4), (5, 10))
def test_multiline_string_position(self) -> None:
module = parse_module('"abc"\\\n"def"')
provider = SyntacticPositionProvider()
module.code_for_node(module, provider)
stmt = cast(cst.SimpleStatementLine, module.body[0])
expr = cast(cst.Expr, stmt.body[0])
string = expr.value
self.cmp_position(provider._computed[stmt], (1, 0), (2, 5))
self.cmp_position(provider._computed[expr], (1, 0), (2, 5))
self.cmp_position(provider._computed[string], (1, 0), (2, 5))