class TestConfig(UnitTest):
@data_provider({
"empty": (PartialParserConfig, ),
"python_version_a":
(lambda: PartialParserConfig(python_version="3.7"), ),
"python_version_b":
(lambda: PartialParserConfig(python_version="3.7.1"), ),
"encoding": (lambda: PartialParserConfig(encoding="latin-1"), ),
"default_indent":
(lambda: PartialParserConfig(default_indent="\t "), ),
"default_newline":
(lambda: PartialParserConfig(default_newline="\r\n"), ),
})
def test_valid_partial_parser_config(
self, factory: Callable[[], PartialParserConfig]) -> None:
self.assertIsInstance(factory(), PartialParserConfig)
@data_provider({
"python_version": (
lambda: PartialParserConfig(python_version="3.7.1.0"),
"The given version is not in the right format",
),
"python_version_unsupported": (
lambda: PartialParserConfig(python_version="3.4"),
"LibCST can only parse code using one of the following versions of Python's grammar",
),
"encoding": (
lambda: PartialParserConfig(encoding="utf-42"),
"not a supported encoding",
),
"default_indent": (
lambda: PartialParserConfig(default_indent="badinput"),
"invalid value for default_indent",
),
"default_newline": (
lambda: PartialParserConfig(default_newline="\n\r"),
"invalid value for default_newline",
),
})
def test_invalid_partial_parser_config(self, factory: Callable[
[], PartialParserConfig], expected_re: str) -> None:
with self.assertRaisesRegex(ValueError, expected_re):
factory()
def lenient_parse_module(
source: Union[str, bytes],
config: PartialParserConfig = PartialParserConfig(),
) -> Module:
try:
result = _lenient_parse(
"file_input",
source,
config,
detect_trailing_newline=True,
detect_default_newline=True,
)
assert isinstance(result, Module)
return result
except ParseTokenError as pse:
token, idx = pse.get_erroneous_token()
return lenient_parse_module(source[:idx] + "" + source[idx + 1:],
config)
def config_for_parsing(self) -> "PartialParserConfig":
"""
Generates a parser config appropriate for passing to a :func:`parse_expression`
or :func:`parse_statement` call. This is useful when using either parser
function to generate code from a string template. By using a generated parser
config instead of the default, you can guarantee that trees generated from
both statement and expression strings have the same inferred defaults for things
like newlines, indents and similar::
module = cst.parse_module("pass\\n")
expression = cst.parse_expression("1 + 2", config=module.config_for_parsing)
"""
from libcst._parser.types.config import PartialParserConfig
return PartialParserConfig(
encoding=self.encoding,
default_indent=self.default_indent,
default_newline=self.default_newline,
)
parser. A parser entrypoint should take the source code and some configuration
information
"""
from typing import Union
from libcst._nodes.base import CSTNode
from libcst._nodes.expression import BaseExpression
from libcst._nodes.module import Module
from libcst._nodes.statement import BaseCompoundStatement, SimpleStatementLine
from libcst._parser.detect_config import detect_config
from libcst._parser.grammar import get_grammar, validate_grammar
from libcst._parser.python_parser import PythonCSTParser
from libcst._parser.types.config import PartialParserConfig
_DEFAULT_PARTIAL_PARSER_CONFIG: PartialParserConfig = PartialParserConfig()
def _parse(
entrypoint: str,
source: Union[str, bytes],
config: PartialParserConfig,
*,
detect_trailing_newline: bool,
detect_default_newline: bool,
) -> CSTNode:
detection_result = detect_config(
source,
partial=config,
detect_trailing_newline=detect_trailing_newline,
detect_default_newline=detect_default_newline,
class TestDetectConfig(UnitTest):
@data_provider({
"empty_input": {
"source":
b"",
"partial":
PartialParserConfig(python_version="3.7"),
"detect_trailing_newline":
True,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=["\n", ""],
encoding="utf-8",
default_indent=" ",
default_newline="\n",
has_trailing_newline=False,
version=PythonVersionInfo(3, 7),
),
},
"detect_trailing_newline_disabled": {
"source":
b"",
"partial":
PartialParserConfig(python_version="3.7"),
"detect_trailing_newline":
False,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=[""], # the trailing newline isn't inserted
encoding="utf-8",
default_indent=" ",
default_newline="\n",
has_trailing_newline=False,
version=PythonVersionInfo(3, 7),
),
},
"detect_default_newline_disabled": {
"source":
b"pass\r",
"partial":
PartialParserConfig(python_version="3.7"),
"detect_trailing_newline":
False,
"detect_default_newline":
False,
"expected_config":
ParserConfig(
lines=["pass\r", ""], # the trailing newline isn't inserted
encoding="utf-8",
default_indent=" ",
default_newline="\n",
has_trailing_newline=False,
version=PythonVersionInfo(3, 7),
),
},
"newline_inferred": {
"source":
b"first_line\r\n\nsomething\n",
"partial":
PartialParserConfig(python_version="3.7"),
"detect_trailing_newline":
True,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=["first_line\r\n", "\n", "something\n", ""],
encoding="utf-8",
default_indent=" ",
default_newline="\r\n",
has_trailing_newline=True,
version=PythonVersionInfo(3, 7),
),
},
"newline_partial_given": {
"source":
b"first_line\r\nsecond_line\r\n",
"partial":
PartialParserConfig(default_newline="\n", python_version="3.7"),
"detect_trailing_newline":
True,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=["first_line\r\n", "second_line\r\n", ""],
encoding="utf-8",
default_indent=" ",
default_newline="\n", # The given partial disables inference
has_trailing_newline=True,
version=PythonVersionInfo(3, 7),
),
},
"indent_inferred": {
"source":
b"if test:\n\t something\n",
"partial":
PartialParserConfig(python_version="3.7"),
"detect_trailing_newline":
True,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=["if test:\n", "\t something\n", ""],
encoding="utf-8",
default_indent="\t ",
default_newline="\n",
has_trailing_newline=True,
version=PythonVersionInfo(3, 7),
),
},
"indent_partial_given": {
"source":
b"if test:\n\t something\n",
"partial":
PartialParserConfig(default_indent=" ", python_version="3.7"),
"detect_trailing_newline":
True,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=["if test:\n", "\t something\n", ""],
encoding="utf-8",
default_indent=" ",
default_newline="\n",
has_trailing_newline=True,
version=PythonVersionInfo(3, 7),
),
},
"encoding_inferred": {
"source":
b"#!/usr/bin/python3\n# -*- coding: latin-1 -*-\npass\n",
"partial":
PartialParserConfig(python_version="3.7"),
"detect_trailing_newline":
True,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=[
"#!/usr/bin/python3\n",
"# -*- coding: latin-1 -*-\n",
"pass\n",
"",
],
encoding="iso-8859-1", # this is an alias for latin-1
default_indent=" ",
default_newline="\n",
has_trailing_newline=True,
version=PythonVersionInfo(3, 7),
),
},
"encoding_partial_given": {
"source":
b"#!/usr/bin/python3\n# -*- coding: latin-1 -*-\npass\n",
"partial":
PartialParserConfig(encoding="us-ascii", python_version="3.7"),
"detect_trailing_newline":
True,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=[
"#!/usr/bin/python3\n",
"# -*- coding: latin-1 -*-\n",
"pass\n",
"",
],
encoding="us-ascii",
default_indent=" ",
default_newline="\n",
has_trailing_newline=True,
version=PythonVersionInfo(3, 7),
),
},
"encoding_str_not_bytes_disables_inference": {
"source":
"#!/usr/bin/python3\n# -*- coding: latin-1 -*-\npass\n",
"partial":
PartialParserConfig(python_version="3.7"),
"detect_trailing_newline":
True,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=[
"#!/usr/bin/python3\n",
"# -*- coding: latin-1 -*-\n",
"pass\n",
"",
],
encoding="utf-8", # because source is a str, don't infer latin-1
default_indent=" ",
default_newline="\n",
has_trailing_newline=True,
version=PythonVersionInfo(3, 7),
),
},
"encoding_non_ascii_compatible_utf_16_with_bom": {
"source":
b"\xff\xfet\x00e\x00s\x00t\x00",
"partial":
PartialParserConfig(encoding="utf-16", python_version="3.7"),
"detect_trailing_newline":
True,
"detect_default_newline":
True,
"expected_config":
ParserConfig(
lines=["test\n", ""],
encoding="utf-16",
default_indent=" ",
default_newline="\n",
has_trailing_newline=False,
version=PythonVersionInfo(3, 7),
),
},
})
def test_detect_module_config(
self,
*,
source: Union[str, bytes],
partial: PartialParserConfig,
detect_trailing_newline: bool,
detect_default_newline: bool,
expected_config: ParserConfig,
) -> None:
self.assertEqual(
detect_config(
source,
partial=partial,
detect_trailing_newline=detect_trailing_newline,
detect_default_newline=detect_default_newline,
).config,
expected_config,
)