def load_grammar(*, version: str = None, path: str = None):
"""
Loads a :py:class:`parso.Grammar`. The default version is the current Python
version.
:param str version: A python version string, e.g. ``version='3.8'``.
:param str path: A path to a grammar file
"""
version_info = parse_version_string(version)
file = path or os.path.join(
'python', 'grammar%s%s.txt' % (version_info.major, version_info.minor))
global _loaded_grammars
path = os.path.join(os.path.dirname(__file__), file)
try:
return _loaded_grammars[path]
except KeyError:
try:
bnf_text = pkgutil.get_data("parso", file).decode("utf-8")
if bnf_text is None:
raise FileNotFoundError
grammar = PythonGrammar(version_info, bnf_text)
return _loaded_grammars.setdefault(path, grammar)
except (FileNotFoundError, IOError):
message = "Python version %s.%s is currently not supported." % (
version_info.major, version_info.minor)
raise NotImplementedError(message)
def load_grammar(language='python', version=None):
if language == 'python':
version_info = parse_version_string(version)
file = 'python/grammar%s%s.txt' % (version_info.major,
version_info.minor)
global _loaded_grammars
path = os.path.join(os.path.dirname(__file__), file)
try:
return _loaded_grammars[path]
except KeyError:
try:
with open(path) as f:
bnf_text = f.read()
grammar = PythonGrammar(version_info, bnf_text)
return _loaded_grammars.setdefault(path, grammar)
except FileNotFoundError:
message = "Python version %s is currently not supported." % version
raise NotImplementedError(message)
elif language == 'python-f-string':
if version is not None:
raise NotImplementedError(
"Currently different versions are not supported.")
return PythonFStringGrammar()
else:
raise NotImplementedError("No support for language %s." % language)
def __init__(self, bnf_grammar):
self._bnf_grammar = bnf_grammar
self.generator = tokenize(
bnf_grammar,
version_info=parse_version_string('3.6')
)
self._gettoken() # Initialize lookahead
def load_grammar(language='python', version=None, path=None):
if language == 'python':
version_info = parse_version_string(version)
file = path or os.path.join(
'python', 'grammar%s%s.txt' %
(version_info.major, version_info.minor))
global _loaded_grammars
path = os.path.join(os.path.dirname(__file__), file)
try:
return _loaded_grammars[path]
except KeyError:
try:
bnf_text = pkgutil.get_data("parso", file)
if bnf_text is None:
raise FileNotFoundError
if sys.version_info[0] == 3:
bnf_text = bnf_text.decode("ascii")
grammar = PythonGrammar(version_info, bnf_text)
return _loaded_grammars.setdefault(path, grammar)
except (FileNotFoundError, IOError):
message = "Python version %s.%s is currently not supported." % (
version_info.major, version_info.minor)
raise NotImplementedError(message)
else:
raise NotImplementedError("No support for language %s." % language)
def __init__(self, bnf_grammar):
self._bnf_grammar = bnf_grammar
self.generator = tokenize(
bnf_grammar,
version_info=parse_version_string('3.6')
)
self._gettoken() # Initialize lookahead
def load_grammar(language='python', version=None, path=None):
if language == 'python':
version_info = parse_version_string(version)
file = path or os.path.join(
'python',
'grammar%s%s.txt' % (version_info.major, version_info.minor)
)
global _loaded_grammars
path = os.path.join(os.path.dirname(__file__), file)
try:
return _loaded_grammars[path]
except KeyError:
try:
with open(path) as f:
bnf_text = f.read()
grammar = PythonGrammar(version_info, bnf_text)
return _loaded_grammars.setdefault(path, grammar)
except FileNotFoundError:
message = "Python version %s is currently not supported." % version
raise NotImplementedError(message)
else:
raise NotImplementedError("No support for language %s." % language)
def __init__(self, bnf_text, token_namespace):
self._bnf_text = bnf_text
self.generator = tokenize.tokenize(
bnf_text, version_info=parse_version_string('3.6'))
self._gettoken() # Initialize lookahead
self.dfas, self.startsymbol = self._parse()
self.first = {} # map from symbol name to set of tokens
self._addfirstsets()
self._token_namespace = token_namespace
def __init__(self, bnf_text, token_namespace):
self._bnf_text = bnf_text
self.generator = tokenize.tokenize(
bnf_text,
version_info=parse_version_string('3.6')
)
self._gettoken() # Initialize lookahead
self.dfas, self.startsymbol = self._parse()
self.first = {} # map from symbol name to set of tokens
self._addfirstsets()
self._token_namespace = token_namespace
def __post_init__(self) -> None:
raw_python_version = self.python_version
# `parse_version_string` will raise a ValueError if the version is invalid.
#
# We use object.__setattr__ because the dataclass is frozen. See:
# https://docs.python.org/3/library/dataclasses.html#frozen-instances
# This should be safe behavior inside of `__post_init__`.
parsed_python_version = parse_version_string(
# TODO: We currently hardcode support for Py3.7 both in our grammar productions and in
# our tokenizer config. If we want to support multiple versions, we will need to switch
# on version-pinned grammar productions and also fix Parso's tokenizer to allow for
# 3.6 and below handling of ASYNC/AWAIT. This should be changed back to None once we
# support multiple versions, so that parso can derive the version from `sys.version_info`
"3.7" if isinstance(raw_python_version, AutoConfig
) else raw_python_version)
# Once we add support for more versions of Python, we can change this to detect
# the supported version range.
if parsed_python_version != PythonVersionInfo(3, 7):
raise ValueError(
"LibCST can only parse code using Python 3.7's grammar. More versions "
+ "may be supported by future releases.")
object.__setattr__(self, "parsed_python_version",
parsed_python_version)
encoding = self.encoding
if not isinstance(encoding, AutoConfig):
try:
codecs.lookup(encoding)
except LookupError:
raise ValueError(
f"{repr(encoding)} is not a supported encoding")
newline = self.default_newline
if (not isinstance(newline, AutoConfig)
and NEWLINE_RE.fullmatch(newline) is None):
raise ValueError(
f"Got an invalid value for default_newline: {repr(newline)}")
indent = self.default_indent
if not isinstance(indent,
AutoConfig) and _INDENT_RE.fullmatch(indent) is None:
raise ValueError(
f"Got an invalid value for default_indent: {repr(indent)}")
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
from typing import Sequence
from parso.python.token import PythonTokenTypes
from parso.utils import parse_version_string
from libcst._exceptions import ParserSyntaxError
from libcst._parser._types.whitespace_state import WhitespaceState
from libcst._parser._wrapped_tokenize import Token, tokenize
from libcst.testing.utils import UnitTest, data_provider
_PY38 = parse_version_string("3.8.0")
class WrappedTokenizeTest(UnitTest):
maxDiff = 10000
@data_provider({
"simple": (
"pass;\n",
(
Token(
type=PythonTokenTypes.NAME,
string="pass",
start_pos=(1, 0),
end_pos=(1, 4),
whitespace_before=WhitespaceState(line=1,
yield PythonToken(OP, token, spos, prefix)
if contstr:
yield PythonToken(ERRORTOKEN, contstr, contstr_start, prefix)
if contstr.endswith('\n') or contstr.endswith('\r'):
new_line = True
end_pos = lnum, max
# As the last position we just take the maximally possible position. We
# remove -1 for the last new line.
for indent in indents[1:]:
yield PythonToken(DEDENT, '', end_pos, '')
yield PythonToken(ENDMARKER, '', end_pos, additional_prefix)
if __name__ == "__main__":
if len(sys.argv) >= 2:
path = sys.argv[1]
with open(path) as f:
code = f.read()
else:
code = sys.stdin.read()
from parso.utils import python_bytes_to_unicode, parse_version_string
if isinstance(code, bytes):
code = python_bytes_to_unicode(code)
for token in tokenize(code, parse_version_string()):
print(token)
def _get_token_list(string):
# Load the current version.
version_info = parse_version_string()
return list(tokenize.tokenize(string, version_info))
def works_ge_py35(each_version):
"""
Works only greater equal Python 3.3.
"""
version_info = parse_version_string(each_version)
return Checker(each_version, version_info >= (3, 5))
def works_ge_py3(each_version):
version_info = parse_version_string(each_version)
return Checker(each_version, version_info >= (3, 0))
yield PythonToken(typ, token, spos, prefix)
if contstr:
yield PythonToken(ERRORTOKEN, contstr, contstr_start, prefix)
if contstr.endswith('\n'):
new_line = True
end_pos = lnum, max
# As the last position we just take the maximally possible position. We
# remove -1 for the last new line.
for indent in indents[1:]:
yield PythonToken(DEDENT, '', end_pos, '')
yield PythonToken(ENDMARKER, '', end_pos, additional_prefix)
if __name__ == "__main__":
if len(sys.argv) >= 2:
path = sys.argv[1]
with open(path) as f:
code = f.read()
else:
code = sys.stdin.read()
from parso.utils import python_bytes_to_unicode, parse_version_string
if isinstance(code, bytes):
code = python_bytes_to_unicode(code)
for token in tokenize(code, parse_version_string()):
print(token)
def works_ge_py35(each_version):
"""
Works only greater equal Python 3.3.
"""
version_info = parse_version_string(each_version)
return Checker(each_version, version_info >= (3, 5))
def works_ge_py3(each_version):
version_info = parse_version_string(each_version)
return Checker(each_version, version_info >= (3, 0))
def test_parse_version_string(version_str, version):
parsed_version = parse_version_string(version_str)
if len(version) == 1:
assert parsed_version[0] == version[0]
else:
assert parsed_version == version
yield create_token()
found = char
is_illegal = False
prefix = ''
pos = start_pos[0], start_pos[1] + i
else:
found += char
else:
new_found = found + char
if new_found.isidentifier():
found = new_found
else:
if found:
yield create_token()
prefix = ''
pos = start_pos[0], start_pos[1] + i
found = char
is_illegal = True
if found:
yield create_token()
if __name__ == "__main__":
path = sys.argv[1]
with open(path) as f:
code = f.read()
for token in tokenize(code, version_info=parse_version_string('3.10')):
print(token)