def next(self):
try:
tok = self._queue.pop()
except IndexError:
tok = next(self.tokens)
# foo.1 will be split into NAME(foo), NUMBER(.1)
# We need to re-work this to yield NAME(foo), OP(.), NUMBER(1)
if tok.exact_type == tokenize.NUMBER and tok.string.startswith(
'.'):
# Split into two tokens
self._queue.append(
tokenize.TokenInfo(
type=tokenize.NUMBER,
string=tok.string,
start=tok.start,
end=tok.end,
line=tok.line,
))
tok = tokenize.TokenInfo(
type=tokenize.DOT,
string='.',
start=tok.start,
end=tok.end,
line=tok.line,
)
return tok
return tok
def fix_line(line):
if len(line) < 2:
return line
if line[0].type != token.NAME:
return line
if line[0].string != "print":
return line
if line[1].exact_type == token.LPAR:
return line
if line[1].exact_type == token.RIGHTSHIFT:
newline = line[2:]
else:
newline = line[1:]
# Replace the print statement 0, arguments.
old = line[0]
newline.insert(0, tokenize.TokenInfo(token.NUMBER, "0", old.start, old.start, old.line))
newline.insert(1, tokenize.TokenInfo(token.OP, ",", old.end, old.end, old.line))
return newline
def translate_ptl(tokens):
i = 0
while i < len(tokens):
tok = tokens[i]
if tok.type == tokenize.NAME and tok.string == 'def':
if (tokens[i + 2][:2] == (tokenize.OP, '[')
and tokens[i + 3][1] in {'html', 'plain'}
and tokens[i + 4][:2] == (tokenize.OP, ']')):
template_type = tokens[i + 3][1]
template_func = 'ptl_' + template_type
t = tokens[i + 1]
line, col = tok.start
# delete extra tokens from [html] or [plain]
tokens[i + 1:i + 4] = []
end = (t.end[0], t.end[1] + len(template_type) + 4)
start = (t.start[0], tok.end[1] + 1)
indent = ' ' * tok.start[1]
# putting newline in tok.string is ugly but it works
s = '@' + template_func + '\n' + indent + tok.string
tokens[i] = tokenize.TokenInfo(tok.type, s, tok.start, tok.end,
tok.line)
tokens[i + 1] = tokenize.TokenInfo(t.type, t.string, start,
end, t.line)
#print('tokens', tokens[i-1:i+3])
elif tok.type == tokenize.NAME and tok.string == 'h':
try:
str_tok = tokens[i + 1]
if str_tok.type == tokenize.STRING:
t = tokens[i]
tokens[i] = tokenize.TokenInfo(t.type, 'F', t.start, t.end,
t.line)
except IndexError:
pass
i += 1
def translate_ptl(tokens):
i = 0
while i < len(tokens):
tok = tokens[i]
if tok.type == tokenize.NAME and tok.string == 'def':
if (tokens[i + 2][:2] == (tokenize.OP, '[')
and tokens[i + 3][1] in {'html', 'plain'}
and tokens[i + 4][:2] == (tokenize.OP, ']')):
template_type = tokens[i + 3][1]
t = tokens[i + 1]
tokens[i + 1:i + 4] = []
end = t.end
if str(template_type) == 'plain':
prefix = PLAIN_PREFIX
else:
prefix = HTML_PREFIX
tokens[i + 1] = tokenize.TokenInfo(
t.type,
prefix + t.string,
t.start,
(end[0], end[1] + len(prefix)),
t.line,
)
elif tok.type == tokenize.NAME and tok.string == 'h':
try:
str_tok = tokens[i + 1]
if str_tok.type == tokenize.STRING:
t = tokens[i]
tokens[i] = tokenize.TokenInfo(t.type, 'F', t.start, t.end,
t.line)
except IndexError:
pass
i += 1
def googleTranslate(tokenL):
gTranslator = ryGoogleTranslate.Translator(from_lang='auto',
to_lang='zh-tw')
for n, t in enumerate(tokenL):
if ((t.type == tn.STRING)
and (('"""' in t.string) or ("'''" in t.string))):
triQuot = '"""' if ('"""' in t.string) else "'''"
text = t.string.strip(triQuot)
#text= text.strip("'''")
try:
gTranslation = gTranslator.translate(text)
gTranslation = google翻譯修正(gTranslation)
except:
gTranslation = 'Google翻譯失敗，網路可能有問題！保留原文 ... ' + text
newString = "'''" # 我愛用 三單引號 '''
#newString += '== begin Google 翻譯 =='
newString += gTranslation
newString += '=== 以上由 Google 翻譯，請協助改善 ===\n'
#newString += text # 必要時，印出原文。the original english comment
newString += "'''"
t0 = t.type
t1 = newString #D[t.string]
t2 = t.start # 可能也要改
t3 = t.end # 可能也要改
t4 = t.line # 可能也要改
tokenL[n] = tn.TokenInfo(t0, t1, t2, t3, t4)
if ((t.type == tn.COMMENT) and ('#' in t.string)
and ('#!' not in t.string)):
text = t.string.strip('#')
gTranslation = gTranslator.translate(text)
gTranslation = google翻譯修正(gTranslation)
newString = "# "
newString += gTranslation
newString += '' #' ..by Google'
t0 = t.type
t1 = newString #D[t.string]
t2 = t.start # 可能也要改
t3 = t.end # 可能也要改
t4 = t.line # 可能也要改
tokenL[n] = tn.TokenInfo(t0, t1, t2, t3, t4)
return tokenL
def fix_token_error(self, err: Exception) -> tokenize.TokenInfo:
msg = err.args[0]
if msg == "EOF in multi-line statement":
return tokenize.TokenInfo(token.ENDMARKER, "", (0, 0), (0, 0), "")
elif msg == "EOF in multi-line string":
return tokenize.TokenInfo(token.ERRORTOKEN, "", (0, 0), (0, 0), "")
elif msg == "unindent does not match any outer indentation level":
return tokenize.TokenInfo(token.ERRORTOKEN, "", (0, 0), (0, 0), "")
else:
raise err
def 外部連結翻譯(標記表, from_l='auto', to_l='zh-tw'):
翻譯器 = ryOuterTranslate.Translator(from_lang=from_l, to_lang=to_l)
for n, t in enumerate(標記表):
# 檢測字串是否為多行字串，是才進行翻譯
if ((t.type == tn.STRING)
and (('"""' in t.string) or ("'''" in t.string))):
# 檢測是用哪種三引號'''或"""
三引號 = '"""' if ('"""' in t.string) else "'''"
# 將字串的前後三引號去掉
待翻譯文字 = t.string.strip(三引號)
try:
翻譯後文字 = 翻譯器.translate(待翻譯文字)
翻譯後文字 = 翻譯文字修正(翻譯後文字)
except:
翻譯後文字 = 'Google翻譯失敗，網路可能有問題！保留原文 ... ' + 待翻譯文字
新字串 = "'''" # 我愛用 三單引號 '''
新字串 += 翻譯後文字
新字串 += "'''"
標記型態 = t.type
標記字串 = 新字串 #D[t.string]
開始位置 = t.start # 可能也要改
結束位置 = t.end # 可能也要改
整行文字 = t.line # 可能也要改
標記表[n] = tn.TokenInfo(標記型態, 標記字串, 開始位置, 結束位置, 整行文字)
# 檢測是否為註解，如為#!則不進行
if ((t.type == tn.COMMENT) and ('#' in t.string)
and ('#!' not in t.string)):
# 將字串的#去掉
待翻譯文字 = t.string.strip('#')
翻譯後文字 = 翻譯器.translate(待翻譯文字)
翻譯後文字 = 翻譯文字修正(翻譯後文字)
新字串 = "# "
新字串 += 翻譯後文字
新字串 += '' #' ..by Google'
標記型態 = t.type
標記字串 = 新字串 #D[t.string]
開始位置 = t.start # 可能也要改
結束位置 = t.end # 可能也要改
整行文字 = t.line # 可能也要改
標記表[n] = tn.TokenInfo(標記型態, 標記字串, 開始位置, 結束位置, 整行文字)
return 標記表
async def post(self):
path = configuration.PROJECT_PATH / self.request.match_info.get(
'file_name', '')
body = await self.request.json()
print('INPUT BODY:', body)
token_info = None
for i in tokenize.tokenize(
BytesIO(body['code_string'].encode('utf8')).readline):
if i.end[1] >= body['cursor_position']:
token_info = tokenize.TokenInfo(
type=i.type,
string=i.string,
start=(body['code_line_number'], i.start[1]),
end=(body['code_line_number'], i.end[1]),
line=i.line)
break
node_info, node_scope_id, scope_tree = AstParser.search_token(
token_info, AST_PARSER[path]['ast_tree'])
node_info = AstParser.get_token_definition(token_info, node_scope_id,
scope_tree)
print('node_info in gotoddefinition: ', node_info, node_info.lineno)
if node_info:
# TODO add file name in case cross file definition
return aiohttp.web.json_response({
'code_line_number':
node_info.lineno - 1,
'cursor_position':
node_info.col_offset
})
else:
raise aiohttp.web.HTTPNotFound(
text=f'Go to definition not found: {body}')
def parse(self, token: tokenize.TokenInfo,
line_tokens: List[tokenize.TokenInfo], args: SimpleNamespace,
token_dict: Dict[str, Any], *_args: Any,
**kwargs: Any) -> tokenize.TokenInfo:
"""Make furbFURB prefixes lowercase.
Original from https://github.com/psf/black
"""
if args.lowercase_string_prefix:
if token.type != tokenize.STRING:
return token
leaf = token.string
match = re.match(r"^([furbFURB]*)(.*)$", leaf, re.DOTALL)
assert match is not None, f"failed to match string {leaf!r}"
orig_prefix = match.group(1)
new_prefix = orig_prefix
new_prefix = new_prefix.replace("F", "f")
new_prefix = new_prefix.replace("U", "u")
new_prefix = new_prefix.replace("R", "r")
new_prefix = new_prefix.replace("B", "b")
leaf = f"{new_prefix}{match.group(2)}"
token = tokenize.TokenInfo(type=token.type,
string=leaf,
start=token.start,
end=token.end,
line=token.line)
return token
def getnext(self) -> tokenize.TokenInfo:
"""Return the next token and updates the index."""
cached = True
while self._index == len(self._tokens):
tok = next(self._tokengen)
if tok.type in (
tokenize.COMMENT,
tokenize.INDENT,
tokenize.DEDENT,
):
continue
# Transform NL to NEWLINE
if tok.type == token.NL:
tok = tokenize.TokenInfo(
token.NEWLINE,
tok.string,
start=tok.start,
end=tok.end,
line=tok.line,
)
if tok.type == token.ERRORTOKEN and tok.string.isspace():
continue
self._tokens.append(tok)
cached = False
tok = self._tokens[self._index]
self._index += 1
if self._verbose:
self.report(cached, False)
return tok
def _gen_eval_tokens(self, tokens):
"""Wrap tokens in parens so multi-line input works as expected"""
# untokenize() stops at ENDMARKER, causing rparen to be discarded.
tokens = [t for t in tokens if t.type != tokenize.ENDMARKER]
if not tokens:
tokens = [tokenize.TokenInfo(
tokenize.NAME, 'None', (1, 0), (1, 4), 'None')]
last = tokens[-1]
lparen = tokenize.TokenInfo(tokenize.OP, '(', (1, 0), (1, 0), '(')
rparen = tokenize.TokenInfo(tokenize.OP, ')', last.end,
(last.end[0], last.end[1] + 1), ')')
return [lparen] + tokens + [rparen]
def test_simple_strings_are_not_passwords(self):
"""
Tests whether simple strings with 2 character classes are passwords
"""
tokens = [
tokenize.TokenInfo(tokenize.STRING, "this-is-not-my-password",
(0, 0), (0, 0), ""),
tokenize.TokenInfo(tokenize.STRING, "123456789", (0, 0), (0, 0),
""),
tokenize.TokenInfo(tokenize.STRING, "!@#$%^&**", (0, 0), (0, 0),
"")
]
with self.assertNoMessages():
self.checker.process_tokens(tokens)
def substitueFormulaReference(token):
"""Given a token, inspect for cell references and return either
the given token or tokens that represent formula.
If a NAME token corresponds to a formula rather than a value,
replace the NAME with the respective formula taking care the context
of resulting NAMEs is preserved e.g.
$Sheet1.A1 = A3 * A2 --> $Sheet1.A3 * $Sheet1.A2
Parameters
token : TokenInfo
Returns
[TokenInfo]
"""
if token == None or token.type != tokenize.NAME: return [token]
formula = getFormula(token.string)
if formula == None: return [token]
result = []
formula = tokenize.untokenize(getTokens(formula)) # expunge leading =
if '.' in token.string: # qualified name
sheet, cell = token.string.split('.')
for tok in getTokens('(' + formula + ')'):
if tok.type == tokenize.NAME and '.' not in tok.string:
result.append(
tokenize.TokenInfo(tok.type, sheet + '.' + tok.string,
tok.start, tok.end, tok.line))
else:
result.append(tok)
else:
result.extend(getTokens('(' + formula + ')'))
return result[:-1] # drop end marker
def __init__(self, error_info):
super().__init__(error_info)
self.tokens = []
self.token_error = None
if self.error_info["message"] == "EOL while scanning string literal":
self.intro_text = (
"You haven't properly closed the string on line %s."
% self.error_info["lineno"]
+ "\n(If you want a multi-line string, then surround it with"
+ " `'''` or `\"\"\"` at both ends.)"
)
elif (
self.error_info["message"]
== "EOF while scanning triple-quoted string literal"
):
# lineno is not useful, as it is at the end of the file and user probably
# didn't want the string to end there
self.intro_text = "You haven't properly closed a triple-quoted string"
else:
if self.error_info["filename"] and os.path.isfile(
self.error_info["filename"]
):
with open(self.error_info["filename"], mode="rb") as fp:
try:
for t in tokenize.tokenize(fp.readline):
self.tokens.append(t)
except tokenize.TokenError as e:
self.token_error = e
if not self.tokens or self.tokens[-1].type not in [
token.ERRORTOKEN,
token.ENDMARKER,
]:
self.tokens.append(
tokenize.TokenInfo(token.ERRORTOKEN, "", None, None, "")
)
else:
self.tokens = []
unbalanced = self._sug_unbalanced_parens()
if unbalanced:
self.intro_text = (
"Unbalanced parentheses, brackets or braces:\n\n" + unbalanced.body
)
self.intro_confidence = 5
else:
self.intro_text = "Python doesn't know how to read your program."
if "^" in str(self.error_info):
self.intro_text += (
"\n\nSmall `^` in the original error message shows where it gave up,"
+ " but the actual mistake can be before this."
)
self.suggestions = [self._sug_missing_or_misplaced_colon()]
def tokenize(src):
byte_src = io.BytesIO(src.encode('utf-8'))
for token in py_tokenize.tokenize(byte_src.readline):
if token.type == py_tokenize.ENCODING: # discard encoding token
continue
if token.type == py_token.OP:
token = py_tokenize.TokenInfo(token.exact_type, *token[1:])
yield token
def fixNames(tokenList):
"""Tweak a token list taking spreadsheet references into account.
I'm co-opting the python tokenizer and have to tweak the NAME tokens
to allow for spreadsheet references.
A custom tokenizer or, even better, one for calc sheets would obviate
this completely.
Parameters
tokenList : list of TokenInfo from tokenizer
Returns
list of TokenInfo
"""
result = []
while len(tokenList) > 0:
token = tokenList.pop(0)
if token.type in (tokenize.ERRORTOKEN, tokenize.COMMENT) \
or (token.type == tokenize.OP and token.string in ('.',':')):
# if previous isn't an OP, have to bind to that
# ditto when following.
if len(result) > 0:
prv = result.pop()
if prv.type == tokenize.OP:
result.append(prv) # put it back
else:
token = tokenize.TokenInfo(tokenize.NAME,
prv.string + token.string,
prv.start, token.end,
token.line)
nxt = tokenList.pop(0) if len(tokenList) > 0 else None
while nxt != None and (nxt.type
not in (tokenize.OP, tokenize.NEWLINE)
or nxt.string == '.'):
token = tokenize.TokenInfo(tokenize.NAME,
token.string + nxt.string,
token.start, nxt.end, token.line)
nxt = tokenList.pop(0) if len(tokenList) > 0 else None
if nxt != None: tokenList.insert(0, nxt)
result.append(token)
return result
def replacements(self) -> typing.Iterator[Replacement]:
token = tokenize.TokenInfo(
type=tokenize.STRING,
string=self.content,
start=(1, 0),
end=(2, 0),
line=self.content,
)
yield from Replacement.from_token(token, self.words)
def test_encoding_token(self):
"""Make sure the encoding token doesn't change the checker's behavior
_tokenize_str doesn't produce an encoding token, but
reading a file does
"""
with self.assertNoMessages():
encoding_token = tokenize.TokenInfo(tokenize.ENCODING, "utf-8", (0, 0), (0, 0), '')
tokens = [encoding_token] + _tokenize_str('if (\n None):\n pass\n')
self.checker.process_tokens(tokens)
def _replace_with_newline(tokens: List[tokenize.TokenInfo], pos: int) -> None:
tok = tokens[pos]
tokens[pos] = tokenize.TokenInfo(
token.NEWLINE,
'\n',
tok.start,
tok.end,
tok.line)
line_offset = 0
for i, tok2 in enumerate(tokens[pos + 1:]):
if i == 0:
line_offset = tok2.start[1]
if tok2.start[0] != tok.start[0]:
line_offset = 0
tokens[i + pos + 1] = tokenize.TokenInfo(
tok2.type, tok2.string,
start=(tok2.start[0] + 1, tok2.start[1] - line_offset),
end=(tok2.end[0] + 1, tok2.end[1] - line_offset),
line=tok2.line)
def translate_hstrings(tokens):
i = 0
context = [False]
html_def = False # True if upcoming indent enters html template
# test if string contains markup characters and needs escaping. We
# also include strings that contain % format characters (otherwise str
# formatting would convert htmltext to str). If the string doesn't
# contain any of these characters, we will leave it as an ordinary
# string.
need_escape = re.compile(r'[&<>"%]').search
# test if a string appears on the same line as a .join call, if so,
# change it to an h-string since str.join doesn't handle htmltext
# strings.
is_join = re.compile(r'["\']\.join').search
while i < len(tokens):
tok = tokens[i]
if tok.type == tokenize.NAME and tok.string == 'def':
if (
tokens[i + 2][:2] == (tokenize.OP, '[')
and tokens[i + 3][1] == 'html'
and tokens[i + 4][:2] == (tokenize.OP, ']')
):
html_def = True
else:
html_def = False
elif tok.type == tokenize.INDENT:
if html_def:
context.append(True)
html_def = False
else:
context.append(context[-1])
elif tok.type == tokenize.DEDENT:
context.pop()
elif tok.type == tokenize.STRING and context[-1]:
# found literal string inside html template
str_tok = list(tokens[i])
# prefix string with marker
s = str_tok[1]
if s[:1] == 'f':
# we could support this, just gets more complicated
raise RuntimeError(
'f-strings in PTL source not supported '
'by conversion. %r' % tok.line
)
if s[:1] not in {'"', "'", 'u'}:
raise RuntimeError('bad string in html template %r' % s)
if need_escape(ast.literal_eval(s)) or is_join(tok.line):
# we are creating h-string
if '{' in s:
# escape f-string chars
s = s.replace('{', '{{')
s = s.replace('}', '}}')
str_tok[1] = 'h' + s
tokens[i] = tokenize.TokenInfo(*str_tok)
i += 1
def replacements(self) -> typing.Iterator[Replacement]:
for row_offset, line in enumerate(self.content.split('\n')):
for match in REX_STRING.finditer(line):
token = tokenize.TokenInfo(
type=tokenize.STRING,
string=match.group(0),
start=(1+row_offset, match.start()),
end=(2+row_offset, 0),
line=self.content,
)
yield from Replacement.from_token(token, self.words)
def pysh_tokenize(readline):
tokens = tokenize.generate_tokens(readline)
parse = parser.statement(tokens)
# print(list(parse))
return parse
indent = 0
last_obj = None
gen_name = get_temp()
try:
while True:
token = next(tokens)
print(token)
ttype, tvalue, tstart, tend, tline = token
if ttype == tokenize.NAME and tvalue == 'e':
tnext = next(tokens)
print(tnext)
if tnext[0] == tokenize.STRING and tnext[2][1] == tend[1]:
# 解析e字符串
last_obj = next(gen_name)
# yield tokenize.TokenInfo(tokenize.NAME, last_obj, token[2], (0, 0), '')
# yield tokenize.TokenInfo(tokenize.OP, '=', (0, 0), (0, 0), '')
yield tokenize.TokenInfo(tokenize.NAME, 'Exec', token[2], (0, 0), '')
yield tokenize.TokenInfo(tokenize.OP, '(', (0, 0), (0, 0), '')
yield tokenize.TokenInfo(tokenize.STRING, tnext[1], (0, 0), (0, 0), '')
yield tokenize.TokenInfo(tokenize.OP, ')', (0, 0), tnext[3], '')
# yield tokenize.TokenInfo(tokenize.NEWLINE, '\n', (0, 0), tnext[3], '')
# for x in indents(indent):
# yield x
continue
else:
# e后面不是紧跟字符串，原样返回
yield token
yield tnext
continue
elif ttype == tokenize.INDENT:
indent += 1
elif ttype == tokenize.NEWLINE:
indent = 0
yield token
except (StopIteration, tokenize.TokenError):
pass
def test_simple_strings_not_credentials(self):
"""
Checks that the checker doesn't raise a message for regular strings
"""
tokens = [
tokenize.TokenInfo(tokenize.STRING, "This is my string!", (0, 0),
(0, 0), "")
]
with self.assertNoMessages():
self.checker.process_tokens(tokens)
def replace_keywords(token_list):
for t in token_list:
if (t.type, t.string) in REPLACEMENTS:
yield tokenize.TokenInfo(
type=t.type,
string=REPLACEMENTS[(t.type, t.string)],
start=t.start,
end=t.end,
line=t.line
)
else:
yield t
def next_atom(self):
tok = self.next()
if tok.string == '-':
tok = self.next()
if tok.type != tokenize.NUMBER:
raise ParseError('%s unexpected `%s`' %
(tok.start, tok.string))
return tokenize.TokenInfo(type=tok.type,
string='-' + tok.string,
start=tok.start,
end=tok.end,
line=tok.line)
return tok
def 外部連結翻譯(Token表,from_l='auto',to_l='zh-tw'):
翻譯器= ryOuterTranslate.Translator(from_lang=from_l, to_lang=to_l)
for n,t in enumerate(Token表):
連結翻譯= False
註解類型=0
if ((t.type==tn.STRING)
and ( ('"""' in t.string)
or ("'''" in t.string) )):
三引號= '"""' if ('"""' in t.string) else "'''"
待翻譯文字= t.string.strip(三引號)
註解類型=1
連結翻譯= True
# 檢測是否為註解，如為#!則不進行
if ( (t.type==tn.COMMENT)
and('#' in t.string)
and('#!' not in t.string)
):
# 將字串的#去掉
待翻譯文字= t.string.strip('#')
註解類型=2
連結翻譯= True
if 連結翻譯== True:
try:
翻譯後文字= 翻譯器.translate(待翻譯文字)
翻譯後文字= 翻譯文字修正(翻譯後文字)
except:
翻譯後文字= '翻譯失敗，網路可能有問題！保留原文 ... ' + 待翻譯文字
if 註解類型==1:
新字串 = "'''"
新字串 += 翻譯後文字
新字串 += "'''"
elif 註解類型==2:
新字串 = "# "
新字串 += 翻譯後文字
新字串 += ''
標記型態= t.type
標記字串= 新字串 #D[t.string]
開始位置= t.start # 可能也要改
結束位置= t.end # 可能也要改
整行文字= t.line # 可能也要改
Token表[n]= tn.TokenInfo(標記型態,標記字串,開始位置,結束位置,整行文字)
return Token表
def translate_py(tokens):
i = 0
while i < len(tokens):
tok = tokens[i]
if tok.type == tokenize.NAME and tok.string == 'def':
t = tokens[i + 1]
if t.type == tokenize.NAME:
if t.string.startswith(HTML_PREFIX):
template_type = 'html'
prefix = HTML_PREFIX
elif t.string.startswith(PLAIN_PREFIX):
template_type = 'plain'
prefix = PLAIN_PREFIX
else:
template_type = ''
if template_type:
func_name = t.string[len(prefix):]
func_name += ' [' + template_type + '] '
tokens[i + 1] = tokenize.TokenInfo(t.type, func_name,
t.start, t.end, t.line)
elif tok.type == tokenize.STRING and tok.string[0] == 'F':
tokens[i] = tokenize.TokenInfo(tok.type, 'h' + tok.string[1:],
tok.start, tok.end, tok.line)
i += 1
def check_for_dummy(self, type: str) -> None:
"""Check whether this is the position where we should insert a dummy token."""
if self._dummy_pos is None or self._dummy_pos != self.mark():
return
assert self._dummy_count is not None
if self._dummy_count > 0:
self._dummy_count -= 1
return
pos = self._dummy_pos
self._dummy_pos = None
self._dummy_count = None
tok = tokenize.TokenInfo(make_dummy_token_type(type), type, (0, 0),
(0, 0), "")
self._tokenizer._tokens.insert(pos, tok) # TODO: Make an API for this
self._dummy_inserted = pos
def demojize(lines, delimiters=('_', '_')):
str = ''.join(lines or [])
import tokenize
import emoji
tokens = []
try:
for token in list(tokenize.tokenize(
__import__('io').BytesIO(str.encode()).readline)):
if token.type == tokenize.ERRORTOKEN:
string = emoji.demojize(token.string, delimiters=delimiters
).replace('-', '_').replace("’", "_")
if tokens and tokens[-1].type == tokenize.NAME:
tokens[-1] = tokenize.TokenInfo(tokens[-1].type, tokens[-1].string +
string, tokens[-1].start, tokens[-1].end, tokens[-1].line)
else:
tokens.append(
tokenize.TokenInfo(
tokenize.NAME, string, token.start, token.end, token.line))
else:
tokens.append(token)
return tokenize.untokenize(tokens).decode().splitlines(True)
except BaseException:
...
return ''.join(lines)
def group_operand(tokens: TokenTuple) -> typing.Optional[tokenize.TokenInfo]:
token_length = len(tokens)
if token_length == 0:
return None
if token_length == 1:
return tokens[0]
first_char_position = tokens[0].start[1]
last_char_position = tokens[token_length - 1].end[1]
column_name = tokens[0].line[first_char_position:last_char_position]
return tokenize.TokenInfo(type=1,
string=column_name,
start=(1, first_char_position),
end=(1, last_char_position),
line=tokens[0].line)
