def parsekeywordpairs(signature):
tokens = PythonLexer().get_tokens(signature)
preamble = True
stack = []
substack = []
parendepth = 0
for token, value in tokens:
if preamble:
if token is Token.Punctuation and value == u"(":
preamble = False
continue
if token is Token.Punctuation:
if value in [u'(', u'{', u'[']:
parendepth += 1
elif value in [u')', u'}', u']']:
parendepth -= 1
elif value == ':' and parendepth == -1:
# End of signature reached
break
if ((value == ',' and parendepth == 0) or
(value == ')' and parendepth == -1)):
stack.append(substack)
substack = []
continue
if value and (parendepth > 0 or value.strip()):
substack.append(value)
d = {}
for item in stack:
if len(item) >= 3:
d[item[0]] = ''.join(item[2:])
return d
def string_to_fmtstr(x):
from pygments import format
from bpython.formatter import BPythonFormatter
from bpython._py3compat import PythonLexer
from bpython.config import Struct, loadini, default_config_path
config = Struct()
loadini(config, default_config_path())
return parse(format(PythonLexer().get_tokens(x), BPythonFormatter(config.color_scheme)))
def _funcname_and_argnum(cls, line):
"""Parse out the current function name and arg from a line of code."""
# each list in stack:
# [full_expr, function_expr, arg_number, opening]
# arg_number may be a string if we've encountered a keyword
# argument so we're done counting
stack = [['', '', 0, '']]
try:
for (token, value) in PythonLexer().get_tokens(line):
if token is Token.Punctuation:
if value in '([{':
stack.append(['', '', 0, value])
elif value in ')]}':
full, _, _, start = stack.pop()
expr = start + full + value
stack[-1][1] += expr
stack[-1][0] += expr
elif value == ',':
try:
stack[-1][2] += 1
except TypeError:
stack[-1][2] = ''
stack[-1][1] = ''
stack[-1][0] += value
elif value == ':' and stack[-1][3] == 'lambda':
expr = stack.pop()[0] + ':'
stack[-1][1] += expr
stack[-1][0] += expr
else:
stack[-1][1] = ''
stack[-1][0] += value
elif (token is Token.Number or
token in Token.Number.subtypes or
token is Token.Name or token in Token.Name.subtypes or
token is Token.Operator and value == '.'):
stack[-1][1] += value
stack[-1][0] += value
elif token is Token.Operator and value == '=':
stack[-1][2] = stack[-1][1]
stack[-1][1] = ''
stack[-1][0] += value
elif token is Token.Number or token in Token.Number.subtypes:
stack[-1][1] = value
stack[-1][0] += value
elif token is Token.Keyword and value == 'lambda':
stack.append([value, '', 0, value])
else:
stack[-1][1] = ''
stack[-1][0] += value
while stack[-1][3] in '[{':
stack.pop()
_, _, arg_number, _ = stack.pop()
_, func, _, _ = stack.pop()
return func, arg_number
except IndexError:
return None, None
def next_token_inside_string(s, inside_string):
"""Given a code string s and an initial state inside_string, return
whether the next token will be inside a string or not."""
for token, value in PythonLexer().get_tokens(s):
if token is Token.String:
value = value.lstrip('bBrRuU')
if value in ['"""', "'''", '"', "'"]:
if not inside_string:
inside_string = value
elif value == inside_string:
inside_string = False
return inside_string
def test():
from pygments import format
from bpython.formatter import BPythonFormatter
from bpython._py3compat import PythonLexer
from bpython.config import Struct, loadini, default_config_path
config = Struct()
loadini(config, default_config_path())
all_tokens = list(PythonLexer().get_tokens('print(1 + 2)'))
formatted_line = format(all_tokens, BPythonFormatter(config.color_scheme))
print((repr(formatted_line)))
fs = parse(formatted_line)
print((repr(fs)))
print(fs)
string_to_fmtstr('asdf')
def show_source_in_new_window(source, color_scheme=None, highlight=True):
win = gtk.Window()
sw = gtk.ScrolledWindow()
view = gtk.TextView()
buffer = view.get_buffer()
if highlight:
add_tags_to_buffer(color_scheme, buffer)
for (token, value) in PythonLexer().get_tokens(source):
while token not in theme_map:
token = token.parent
iter_ = buffer.get_end_iter()
buffer.insert_with_tags_by_name(iter_, value, theme_map[token])
else:
buffer.insert(buffer.get_end_iter(), source)
sw.add(view)
win.add(sw)
win.show_all()
def tokenize(self, s, newline=False):
"""Tokenize a line of code."""
source = '\n'.join(self.buffer + [s])
cursor = len(source) - self.cpos
if self.cpos:
cursor += 1
stack = list()
all_tokens = list(PythonLexer().get_tokens(source))
# Unfortunately, Pygments adds a trailing newline and strings with
# no size, so strip them
while not all_tokens[-1][1]:
all_tokens.pop()
all_tokens[-1] = (all_tokens[-1][0], all_tokens[-1][1].rstrip('\n'))
line = pos = 0
parens = dict(list(zip('{([', '})]')))
line_tokens = list()
saved_tokens = list()
search_for_paren = True
for (token, value) in split_lines(all_tokens):
pos += len(value)
if token is Token.Text and value == '\n':
line += 1
# Remove trailing newline
line_tokens = list()
saved_tokens = list()
continue
line_tokens.append((token, value))
saved_tokens.append((token, value))
if not search_for_paren:
continue
under_cursor = (pos == cursor)
if token is Token.Punctuation:
if value in parens:
if under_cursor:
line_tokens[-1] = (Parenthesis.UnderCursor, value)
# Push marker on the stack
stack.append((Parenthesis, value))
else:
stack.append(
(line, len(line_tokens) - 1, line_tokens, value))
elif value in iter(parens.values()):
saved_stack = list(stack)
try:
while True:
opening = stack.pop()
if parens[opening[-1]] == value:
break
except IndexError:
# SyntaxError.. more closed parentheses than
# opened or a wrong closing paren
opening = None
if not saved_stack:
search_for_paren = False
else:
stack = saved_stack
if opening and opening[0] is Parenthesis:
# Marker found
line_tokens[-1] = (Parenthesis, value)
search_for_paren = False
elif opening and under_cursor and not newline:
if self.cpos:
line_tokens[-1] = (Parenthesis.UnderCursor, value)
else:
# The cursor is at the end of line and next to
# the paren, so it doesn't reverse the paren.
# Therefore, we insert the Parenthesis token
# here instead of the Parenthesis.UnderCursor
# token.
line_tokens[-1] = (Parenthesis, value)
(lineno, i, tokens, opening) = opening
if lineno == len(self.buffer):
self.highlighted_paren = (lineno, saved_tokens)
line_tokens[i] = (Parenthesis, opening)
else:
self.highlighted_paren = (lineno, list(tokens))
# We need to redraw a line
tokens[i] = (Parenthesis, opening)
self.reprint_line(lineno, tokens)
search_for_paren = False
elif under_cursor:
search_for_paren = False
if line != len(self.buffer):
return list()
return line_tokens
def get_args(self):
"""Check if an unclosed parenthesis exists, then attempt to get the
argspec() for it. On success, update self.argspec and return True,
otherwise set self.argspec to None and return False"""
self.current_func = None
if not self.config.arg_spec:
return False
# Get the name of the current function and where we are in
# the arguments
stack = [['', 0, '']]
try:
for (token,
value) in PythonLexer().get_tokens(self.current_line()):
if token is Token.Punctuation:
if value in '([{':
stack.append(['', 0, value])
elif value in ')]}':
stack.pop()
elif value == ',':
try:
stack[-1][1] += 1
except TypeError:
stack[-1][1] = ''
stack[-1][0] = ''
elif value == ':' and stack[-1][2] == 'lambda':
stack.pop()
else:
stack[-1][0] = ''
elif (token is Token.Name or token in Token.Name.subtypes
or token is Token.Operator and value == '.'):
stack[-1][0] += value
elif token is Token.Operator and value == '=':
stack[-1][1] = stack[-1][0]
stack[-1][0] = ''
elif token is Token.Keyword and value == 'lambda':
stack.append(['', 0, value])
else:
stack[-1][0] = ''
while stack[-1][2] in '[{':
stack.pop()
_, arg_number, _ = stack.pop()
func, _, _ = stack.pop()
except IndexError:
return False
if not func:
return False
try:
f = self.get_object(func)
except (AttributeError, NameError, SyntaxError):
return False
if inspect.isclass(f):
try:
if f.__init__ is not object.__init__:
f = f.__init__
except AttributeError:
return None
self.current_func = f
self.argspec = inspection.getargspec(func, f)
if self.argspec:
self.argspec.append(arg_number)
return True
return False
def get_args(self):
"""Check if an unclosed parenthesis exists, then attempt to get the
argspec() for it. On success, update self.funcprops,self.arg_pos and
return True, otherwise set self.funcprops to None and return False"""
self.current_func = None
if not self.config.arg_spec:
return False
# Get the name of the current function and where we are in
# the arguments
stack = [['', 0, '']]
try:
for (token, value) in PythonLexer().get_tokens(self.current_line):
if token is Token.Punctuation:
if value in '([{':
stack.append(['', 0, value])
elif value in ')]}':
stack.pop()
elif value == ',':
try:
stack[-1][1] += 1
except TypeError:
stack[-1][1] = ''
stack[-1][0] = ''
elif value == ':' and stack[-1][2] == 'lambda':
stack.pop()
else:
stack[-1][0] = ''
elif (token is Token.Name or token in Token.Name.subtypes
or token is Token.Operator and value == '.'):
stack[-1][0] += value
elif token is Token.Operator and value == '=':
stack[-1][1] = stack[-1][0]
stack[-1][0] = ''
elif token is Token.Keyword and value == 'lambda':
stack.append(['', 0, value])
else:
stack[-1][0] = ''
while stack[-1][2] in '[{':
stack.pop()
_, arg_number, _ = stack.pop()
func, _, _ = stack.pop()
except IndexError:
return False
if not func:
return False
try:
f = self.get_object(func)
except Exception:
# another case of needing to catch every kind of error
# since user code is run in the case of descriptors
# XXX: Make sure you raise here if you're debugging the completion
# stuff !
return False
if inspect.isclass(f):
class_f = None
if (hasattr(f, '__init__') and f.__init__ is not object.__init__):
class_f = f.__init__
if ((not class_f or not inspection.getfuncprops(func, class_f))
and hasattr(f, '__new__')
and f.__new__ is not object.__new__ and f.__new__.__class__
is not object.__new__.__class__): # py3
class_f = f.__new__
if class_f:
f = class_f
self.current_func = f
self.funcprops = inspection.getfuncprops(func, f)
if self.funcprops:
self.arg_pos = arg_number
return True
self.arg_pos = None
return False
def string_to_fmtstr(x):
config = Struct()
loadini(config, default_config_path())
return parse(format(PythonLexer().get_tokens(x), BPythonFormatter(config.color_scheme)))
