def _parse(
entrypoint: str,
source: Union[str, bytes],
config: PartialParserConfig,
*,
detect_trailing_newline: bool,
detect_default_newline: bool,
) -> CSTNode:
detection_result = detect_config(
source,
partial=config,
detect_trailing_newline=detect_trailing_newline,
detect_default_newline=detect_default_newline,
)
validate_grammar()
grammar = get_grammar(config.parsed_python_version)
parser = PythonCSTParser(
tokens=detection_result.tokens,
config=detection_result.config,
pgen_grammar=grammar,
start_nonterminal=entrypoint,
)
# The parser has an Any return type, we can at least refine it to CSTNode here.
result = parser.parse()
assert isinstance(result, CSTNode)
return result
def _lenient_parse(
entrypoint: str,
source: Union[str, bytes],
config: PartialParserConfig,
*,
detect_trailing_newline: bool,
detect_default_newline: bool,
) -> CSTNode:
detection_result = detect_config(
source,
partial=config,
detect_trailing_newline=detect_trailing_newline,
detect_default_newline=detect_default_newline,
)
validate_grammar()
grammar = get_grammar(config.parsed_python_version, config.future_imports)
parser = LenientPythonParser(
tokens=detection_result.tokens,
config=detection_result.config,
pgen_grammar=grammar,
start_nonterminal=entrypoint,
)
result = parser.parse()
assert isinstance(result, CSTNode)
return result
def EntryParserHolder():
"""stmt_input this is False stmt_input
_make_transition(token_namespace,
reserved_syntax_strings: ReservedString,
label)
"stmt_input":
detect_trailing_newline=True,
detect_default_newline=False,
ret = Union[SimpleStatementLine, BaseCompoundStatement]
"expression_input":
detect_trailing_newline=False,
detect_default_newline=False,
ret = BaseExpression
"""
pass
entrypoint: ENTRYPOINTS = "file_input"
config: PartialParserConfig
detect_trailing_newline: bool = field(default=True)
detect_default_newlinee: bool = field(
default=True) # detect_default_newline
retvar: libcst.CSTNode = field(default=Module)
detection_result: libcst._parser.detect_config.ConfigDetectionResult = (
None # detect_config( source,
)
from libcst._parser.detect_config import detect_config
# _DEFAULT_PARTIAL_PARSER_CONFIG
version: Any = ""
source: str = "import os"
_config: _DEFAULT_PARTIAL_PARSER_CONFIG
grammar: generator.Grammar
detect_trailing_newline = True
detect_default_newline = True
detection_result = detect_config(
source,
partial=_config,
detect_trailing_newline=detect_trailing_newline,
detect_default_newline=detect_default_newline,
)
grammar = None # _pgen_grammar; validate_grammar()
# grammar = get_grammar(config.parsed_python_version, config.future_imports)
grammar = get_grammar(config.parsed_python_version,
config.future_imports) # Grammar[TokenType]
parser: PythonCSTParser = field(default=None, init=False)
result = None # retvar
extra = dict() # _pgen_grammar
to_dfa = None
def test_detect_module_config(
self,
*,
source: Union[str, bytes],
partial: PartialParserConfig,
detect_trailing_newline: bool,
detect_default_newline: bool,
expected_config: ParserConfig,
) -> None:
self.assertEqual(
detect_config(
source,
partial=partial,
detect_trailing_newline=detect_trailing_newline,
detect_default_newline=detect_default_newline,
).config,
expected_config,
)
def _parse(
entrypoint: str,
source: Union[str, bytes],
config: PartialParserConfig,
*,
detect_trailing_newline: bool,
detect_default_newline: bool,
) -> CSTNode:
detection_result = detect_config(
source,
partial=config,
detect_trailing_newline=detect_trailing_newline,
detect_default_newline=detect_default_newline,
)
# from libcst._parser.grammar import get_grammar, validate_grammar, PythonVersionInfo, parse_version_string, generate_grammar, Grammar
validate_grammar()
# libcst._parser.parso.pgen2.generator.generate_grammar(bnf_string, token_namespac: PythonTokenTypes)
grammar = get_grammar(config.parsed_python_version,
config.future_imports) # Grammar[TokenType]
"""
Grammar[TokenType]
grammar.nonterminal_to_dfas, reserved_syntax_strings, start_nonterminal
parser:
for token in self.tokens:
self._add_token(token)
while True:
tos = self.stack[-1] #: libcst._parser.base_parser.StackNode
if not tos.dfa.is_final:
pass
if len(self.stack) > 1:
self._pop()
else: tos.dfa.from_rule
return self.convert_nonterminal(tos.nonterminal = 'file_input', tos.nodes)
parser.nonterminal_conversions['file_input'](self.config, children = tos.nodes) -> Module
_add_token(token)
grammar = self._pgen_grammar
stack = self.stack
transition = libcst._parser.base_parser._token_to_transition(grammar, token.type, token.string)
while True:
try:
plan = stack[-1].dfa.transitions[transition]
break
except KeyError:
if stack[-1].dfa.is_final:
try:
self._pop()
else:
EOF Error
stack[-1].dfa = plan.next_dfa
for push in plan.dfa_pushes:
stack.append(StackNode(push))
leaf = self.convert_terminal(token)
stack[-1].nodes.append(leaf)
"""
parser = PythonCSTParser(
tokens=detection_result.tokens,
config=detection_result.config,
pgen_grammar=grammar,
start_nonterminal=entrypoint,
)
# The parser has an Any return type, we can at least refine it to CSTNode here.
result = parser.parse()