def parsekeywordpairs(signature): tokens = PythonLexer().get_tokens(signature) preamble = True stack = [] substack = [] parendepth = 0 for token, value in tokens: if preamble: if token is Token.Punctuation and value == u"(": preamble = False continue if token is Token.Punctuation: if value in [u'(', u'{', u'[']: parendepth += 1 elif value in [u')', u'}', u']']: parendepth -= 1 elif value == ':' and parendepth == -1: # End of signature reached break if ((value == ',' and parendepth == 0) or (value == ')' and parendepth == -1)): stack.append(substack) substack = [] continue if value and (parendepth > 0 or value.strip()): substack.append(value) d = {} for item in stack: if len(item) >= 3: d[item[0]] = ''.join(item[2:]) return d
def string_to_fmtstr(x): from pygments import format from bpython.formatter import BPythonFormatter from bpython._py3compat import PythonLexer from bpython.config import Struct, loadini, default_config_path config = Struct() loadini(config, default_config_path()) return parse(format(PythonLexer().get_tokens(x), BPythonFormatter(config.color_scheme)))
def _funcname_and_argnum(cls, line): """Parse out the current function name and arg from a line of code.""" # each list in stack: # [full_expr, function_expr, arg_number, opening] # arg_number may be a string if we've encountered a keyword # argument so we're done counting stack = [['', '', 0, '']] try: for (token, value) in PythonLexer().get_tokens(line): if token is Token.Punctuation: if value in '([{': stack.append(['', '', 0, value]) elif value in ')]}': full, _, _, start = stack.pop() expr = start + full + value stack[-1][1] += expr stack[-1][0] += expr elif value == ',': try: stack[-1][2] += 1 except TypeError: stack[-1][2] = '' stack[-1][1] = '' stack[-1][0] += value elif value == ':' and stack[-1][3] == 'lambda': expr = stack.pop()[0] + ':' stack[-1][1] += expr stack[-1][0] += expr else: stack[-1][1] = '' stack[-1][0] += value elif (token is Token.Number or token in Token.Number.subtypes or token is Token.Name or token in Token.Name.subtypes or token is Token.Operator and value == '.'): stack[-1][1] += value stack[-1][0] += value elif token is Token.Operator and value == '=': stack[-1][2] = stack[-1][1] stack[-1][1] = '' stack[-1][0] += value elif token is Token.Number or token in Token.Number.subtypes: stack[-1][1] = value stack[-1][0] += value elif token is Token.Keyword and value == 'lambda': stack.append([value, '', 0, value]) else: stack[-1][1] = '' stack[-1][0] += value while stack[-1][3] in '[{': stack.pop() _, _, arg_number, _ = stack.pop() _, func, _, _ = stack.pop() return func, arg_number except IndexError: return None, None
def next_token_inside_string(s, inside_string): """Given a code string s and an initial state inside_string, return whether the next token will be inside a string or not.""" for token, value in PythonLexer().get_tokens(s): if token is Token.String: value = value.lstrip('bBrRuU') if value in ['"""', "'''", '"', "'"]: if not inside_string: inside_string = value elif value == inside_string: inside_string = False return inside_string
def test(): from pygments import format from bpython.formatter import BPythonFormatter from bpython._py3compat import PythonLexer from bpython.config import Struct, loadini, default_config_path config = Struct() loadini(config, default_config_path()) all_tokens = list(PythonLexer().get_tokens('print(1 + 2)')) formatted_line = format(all_tokens, BPythonFormatter(config.color_scheme)) print((repr(formatted_line))) fs = parse(formatted_line) print((repr(fs))) print(fs) string_to_fmtstr('asdf')
def show_source_in_new_window(source, color_scheme=None, highlight=True): win = gtk.Window() sw = gtk.ScrolledWindow() view = gtk.TextView() buffer = view.get_buffer() if highlight: add_tags_to_buffer(color_scheme, buffer) for (token, value) in PythonLexer().get_tokens(source): while token not in theme_map: token = token.parent iter_ = buffer.get_end_iter() buffer.insert_with_tags_by_name(iter_, value, theme_map[token]) else: buffer.insert(buffer.get_end_iter(), source) sw.add(view) win.add(sw) win.show_all()
def tokenize(self, s, newline=False): """Tokenize a line of code.""" source = '\n'.join(self.buffer + [s]) cursor = len(source) - self.cpos if self.cpos: cursor += 1 stack = list() all_tokens = list(PythonLexer().get_tokens(source)) # Unfortunately, Pygments adds a trailing newline and strings with # no size, so strip them while not all_tokens[-1][1]: all_tokens.pop() all_tokens[-1] = (all_tokens[-1][0], all_tokens[-1][1].rstrip('\n')) line = pos = 0 parens = dict(list(zip('{([', '})]'))) line_tokens = list() saved_tokens = list() search_for_paren = True for (token, value) in split_lines(all_tokens): pos += len(value) if token is Token.Text and value == '\n': line += 1 # Remove trailing newline line_tokens = list() saved_tokens = list() continue line_tokens.append((token, value)) saved_tokens.append((token, value)) if not search_for_paren: continue under_cursor = (pos == cursor) if token is Token.Punctuation: if value in parens: if under_cursor: line_tokens[-1] = (Parenthesis.UnderCursor, value) # Push marker on the stack stack.append((Parenthesis, value)) else: stack.append( (line, len(line_tokens) - 1, line_tokens, value)) elif value in iter(parens.values()): saved_stack = list(stack) try: while True: opening = stack.pop() if parens[opening[-1]] == value: break except IndexError: # SyntaxError.. more closed parentheses than # opened or a wrong closing paren opening = None if not saved_stack: search_for_paren = False else: stack = saved_stack if opening and opening[0] is Parenthesis: # Marker found line_tokens[-1] = (Parenthesis, value) search_for_paren = False elif opening and under_cursor and not newline: if self.cpos: line_tokens[-1] = (Parenthesis.UnderCursor, value) else: # The cursor is at the end of line and next to # the paren, so it doesn't reverse the paren. # Therefore, we insert the Parenthesis token # here instead of the Parenthesis.UnderCursor # token. line_tokens[-1] = (Parenthesis, value) (lineno, i, tokens, opening) = opening if lineno == len(self.buffer): self.highlighted_paren = (lineno, saved_tokens) line_tokens[i] = (Parenthesis, opening) else: self.highlighted_paren = (lineno, list(tokens)) # We need to redraw a line tokens[i] = (Parenthesis, opening) self.reprint_line(lineno, tokens) search_for_paren = False elif under_cursor: search_for_paren = False if line != len(self.buffer): return list() return line_tokens
def get_args(self): """Check if an unclosed parenthesis exists, then attempt to get the argspec() for it. On success, update self.argspec and return True, otherwise set self.argspec to None and return False""" self.current_func = None if not self.config.arg_spec: return False # Get the name of the current function and where we are in # the arguments stack = [['', 0, '']] try: for (token, value) in PythonLexer().get_tokens(self.current_line()): if token is Token.Punctuation: if value in '([{': stack.append(['', 0, value]) elif value in ')]}': stack.pop() elif value == ',': try: stack[-1][1] += 1 except TypeError: stack[-1][1] = '' stack[-1][0] = '' elif value == ':' and stack[-1][2] == 'lambda': stack.pop() else: stack[-1][0] = '' elif (token is Token.Name or token in Token.Name.subtypes or token is Token.Operator and value == '.'): stack[-1][0] += value elif token is Token.Operator and value == '=': stack[-1][1] = stack[-1][0] stack[-1][0] = '' elif token is Token.Keyword and value == 'lambda': stack.append(['', 0, value]) else: stack[-1][0] = '' while stack[-1][2] in '[{': stack.pop() _, arg_number, _ = stack.pop() func, _, _ = stack.pop() except IndexError: return False if not func: return False try: f = self.get_object(func) except (AttributeError, NameError, SyntaxError): return False if inspect.isclass(f): try: if f.__init__ is not object.__init__: f = f.__init__ except AttributeError: return None self.current_func = f self.argspec = inspection.getargspec(func, f) if self.argspec: self.argspec.append(arg_number) return True return False
def get_args(self): """Check if an unclosed parenthesis exists, then attempt to get the argspec() for it. On success, update self.funcprops,self.arg_pos and return True, otherwise set self.funcprops to None and return False""" self.current_func = None if not self.config.arg_spec: return False # Get the name of the current function and where we are in # the arguments stack = [['', 0, '']] try: for (token, value) in PythonLexer().get_tokens(self.current_line): if token is Token.Punctuation: if value in '([{': stack.append(['', 0, value]) elif value in ')]}': stack.pop() elif value == ',': try: stack[-1][1] += 1 except TypeError: stack[-1][1] = '' stack[-1][0] = '' elif value == ':' and stack[-1][2] == 'lambda': stack.pop() else: stack[-1][0] = '' elif (token is Token.Name or token in Token.Name.subtypes or token is Token.Operator and value == '.'): stack[-1][0] += value elif token is Token.Operator and value == '=': stack[-1][1] = stack[-1][0] stack[-1][0] = '' elif token is Token.Keyword and value == 'lambda': stack.append(['', 0, value]) else: stack[-1][0] = '' while stack[-1][2] in '[{': stack.pop() _, arg_number, _ = stack.pop() func, _, _ = stack.pop() except IndexError: return False if not func: return False try: f = self.get_object(func) except Exception: # another case of needing to catch every kind of error # since user code is run in the case of descriptors # XXX: Make sure you raise here if you're debugging the completion # stuff ! return False if inspect.isclass(f): class_f = None if (hasattr(f, '__init__') and f.__init__ is not object.__init__): class_f = f.__init__ if ((not class_f or not inspection.getfuncprops(func, class_f)) and hasattr(f, '__new__') and f.__new__ is not object.__new__ and f.__new__.__class__ is not object.__new__.__class__): # py3 class_f = f.__new__ if class_f: f = class_f self.current_func = f self.funcprops = inspection.getfuncprops(func, f) if self.funcprops: self.arg_pos = arg_number return True self.arg_pos = None return False
def string_to_fmtstr(x): config = Struct() loadini(config, default_config_path()) return parse(format(PythonLexer().get_tokens(x), BPythonFormatter(config.color_scheme)))