def __init__(self, bnf_grammar):
self._bnf_grammar = bnf_grammar
self.generator = tokenize(
bnf_grammar,
version_info=parse_version_string('3.6')
)
self._gettoken() # Initialize lookahead
def __init__(self, bnf_grammar):
self._bnf_grammar = bnf_grammar
self.generator = tokenize(
bnf_grammar,
version_info=parse_version_string('3.6')
)
self._gettoken() # Initialize lookahead
def _(source, source_file):
source_string = python_bytes_to_unicode(source)
for token in tokenize(
source_string,
version_info=parse_version_string('3.10')
):
pass
def __init__(self, bnf_text, token_namespace):
self._bnf_text = bnf_text
self.generator = tokenize.tokenize(
bnf_text, version_info=parse_version_string('3.6'))
self._gettoken() # Initialize lookahead
self.dfas, self.startsymbol = self._parse()
self.first = {} # map from symbol name to set of tokens
self._addfirstsets()
self._token_namespace = token_namespace
def tokenize_code(code, inference=False):
if inference:
return [
token_filter(token.type, token.string)
for token in tokenize_without_endmarker(code)
]
return [
token_filter(token.type, token.string)
for token in tokenize(code, (2, 7))
]
def tokenize_without_endmarker(code):
safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE'
for token in tokenize(code + safeword, (2, 7)):
if token.string == safeword:
return
elif token.string.endswith(safeword):
yield PythonToken(token.type, token.string[:-len(safeword)],
token.start_pos, token.prefix)
return
else:
yield token
def __init__(self, bnf_text, token_namespace):
self._bnf_text = bnf_text
self.generator = tokenize.tokenize(
bnf_text,
version_info=parse_version_string('3.6')
)
self._gettoken() # Initialize lookahead
self.dfas, self.startsymbol = self._parse()
self.first = {} # map from symbol name to set of tokens
self._addfirstsets()
self._token_namespace = token_namespace
def _tokenize(self, code):
# Used by Jedi.
return tokenize(code, self.version_info)
def _tokenize(self, code):
# Used by Jedi.
return tokenize(code, self.version_info)
def _get_token_list(string):
# Load the current version.
version_info = parse_version_string()
return list(tokenize.tokenize(string, version_info))
def test_tokenize_start_pos(code, positions):
tokens = list(tokenize(code, version_info=(3, 6)))
assert positions == [p.start_pos for p in tokens]
def tokenize_code(code):
return [(token.type, token.string if token.type != 3 else '')
for token in tokenize(code, (2, 7))]
from parso.python.tokenize import tokenize
import json
import sys
file = open(sys.argv[1], 'r')
file_content = file.read()
tokens = []
for token in tokenize(file_content, version_info=(3, 6)):
if (str(token.string) != ""):
tmp = dict()
tmp["line"] = (token.start_pos)[0]
tmp["char"] = ((token.start_pos)[1]) + 1
tmp["type"] = ((str(token.type))[17:]).strip(")")
if tmp["type"] == "OP":
tmp["type"] += "-" + str(token.string)
tmp["value"] = str(token.string)
tokens.append(tmp)
print(json.dumps(tokens, indent=4, sort_keys=True))
def test_tokenize_start_pos(code, positions):
tokens = list(tokenize(code, version_info=(3, 6)))
assert positions == [p.start_pos for p in tokens]
