def error_recovery(self, grammar, stack, typ, value, start_pos, prefix, add_token_callback): """ This parser is written in a dynamic way, meaning that this parser allows using different grammars (even non-Python). However, error recovery is purely written for Python. """ def current_suite(stack): # For now just discard everything that is not a suite or # file_input, if we detect an error. for index, (dfa, state, (typ, nodes)) in reversed(list(enumerate(stack))): # `suite` can sometimes be only simple_stmt, not stmt. symbol = grammar.number2symbol[typ] if symbol == 'file_input': break elif symbol == 'suite' and len(nodes) > 1: # suites without an indent in them get discarded. break elif symbol == 'simple_stmt' and len(nodes) > 1: # simple_stmt can just be turned into a Node, if there are # enough statements. Ignore the rest after that. break return index, symbol, nodes index, symbol, nodes = current_suite(stack) if symbol == 'simple_stmt': index -= 2 (_, _, (typ, suite_nodes)) = stack[index] symbol = grammar.number2symbol[typ] suite_nodes.append(pt.Node(symbol, list(nodes))) # Remove nodes[:] = [] nodes = suite_nodes stack[index] #print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index) self._stack_removal(grammar, stack, index + 1, value, start_pos) if typ == INDENT: # For every deleted INDENT we have to delete a DEDENT as well. # Otherwise the parser will get into trouble and DEDENT too early. self._omit_dedent_list.append(self._indent_counter) if value in ('import', 'from', 'class', 'def', 'try', 'while', 'return'): # Those can always be new statements. add_token_callback(typ, value, prefix, start_pos) elif typ == DEDENT and symbol == 'suite': # Close the current suite, with DEDENT. # Note that this may cause some suites to not contain any # statements at all. This is contrary to valid Python syntax. We # keep incomplete suites in Jedi to be able to complete param names # or `with ... as foo` names. If we want to use this parser for # syntax checks, we have to check in a separate turn if suites # contain statements or not. However, a second check is necessary # anyway (compile.c does that for Python), because Python's grammar # doesn't stop you from defining `continue` in a module, etc. add_token_callback(typ, value, prefix, start_pos)
def get_decorated_func(self): """ Returns the function, that should to be executed in the end. This is also the places where the decorators are processed. """ f = self.base_func decorators = self.base_func.get_decorators() if not decorators or self.is_decorated: return self # Only enter it, if has not already been processed. if not self.is_decorated: for dec in reversed(decorators): debug.dbg('decorator: %s %s', dec, f) dec_results = self._evaluator.eval_element(dec.children[1]) trailer = dec.children[2:-1] if trailer: # Create a trailer and evaluate it. trailer = tree.Node('trailer', trailer) trailer.parent = dec dec_results = self._evaluator.eval_trailer( dec_results, trailer) if not len(dec_results): debug.warning('decorator not found: %s on %s', dec, self.base_func) return self decorator = dec_results.pop() if dec_results: debug.warning('multiple decorators found %s %s', self.base_func, dec_results) # Create param array. if isinstance(f, Function): old_func = f # TODO this is just hacky. change. id:133 gh:134 elif f.type == 'funcdef': old_func = Function(self._evaluator, f, is_decorated=True) else: old_func = f wrappers = self._evaluator.execute_evaluated( decorator, old_func) if not len(wrappers): debug.warning('no wrappers found %s', self.base_func) return self if len(wrappers) > 1: # TODO resolve issue with multiple wrappers -> multiple types id:88 gh:89 debug.warning('multiple wrappers found %s %s', self.base_func, wrappers) f = list(wrappers)[0] if isinstance(f, (Instance, Function)): f.decorates = self debug.dbg('decorator end %s', f) return f
def get_faked(module, obj, name=None): obj = obj.__class__ if is_class_instance(obj) else obj result = _faked(module, obj, name) # TODO may this ever happen? result None? if so, document! if not isinstance(result, pt.Class) and result is not None: # Set the docstr which was previously not set (faked modules don't # contain it). doc = '"""%s"""' % obj.__doc__ # TODO need escapes. suite = result.children[-1] string = pt.String(pt.zero_position_modifier, doc, (0, 0), '') new_line = pt.Whitespace('\n', (0, 0), '') docstr_node = pt.Node('simple_stmt', [string, new_line]) suite.children.insert(2, docstr_node) return result
def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix, add_token_callback): """ This parser is written in a dynamic way, meaning that this parser allows using different grammars (even non-Python). However, error recovery is purely written for Python. """ def current_suite(stack): # For now just discard everything that is not a suite or # file_input, if we detect an error. for index, (dfa, state, (type_, nodes)) in reversed(list(enumerate(stack))): # `suite` can sometimes be only simple_stmt, not stmt. symbol = grammar.number2symbol[type_] if symbol == 'file_input': break elif symbol == 'suite' and len(nodes) > 1: # suites without an indent in them get discarded. break elif symbol == 'simple_stmt' and len(nodes) > 1: # simple_stmt can just be turned into a Node, if there are # enough statements. Ignore the rest after that. break return index, symbol, nodes index, symbol, nodes = current_suite(stack) if symbol == 'simple_stmt': index -= 2 (_, _, (type_, suite_nodes)) = stack[index] symbol = grammar.number2symbol[type_] suite_nodes.append(pt.Node(symbol, list(nodes))) # Remove nodes[:] = [] nodes = suite_nodes stack[index] # print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index) if self._stack_removal(grammar, stack, arcs, index + 1, value, start_pos): add_token_callback(typ, value, start_pos, prefix) else: if typ == INDENT: # For every deleted INDENT we have to delete a DEDENT as well. # Otherwise the parser will get into trouble and DEDENT too early. self._omit_dedent_list.append(self._indent_counter) else: error_leaf = pt.ErrorLeaf(self.position_modifier, typ, value, start_pos, prefix) stack[-1][2][1].append(error_leaf)
def get_faked(module, obj, name=None): obj = obj.__class__ if is_class_instance(obj) else obj result = _faked(module, obj, name) if result is None or isinstance(result, pt.Class): # We're not interested in classes. What we want is functions. return None else: # Set the docstr which was previously not set (faked modules don't # contain it). doc = '"""%s"""' % obj.__doc__ # TODO need escapes. suite = result.children[-1] string = pt.String(pt.zero_position_modifier, doc, (0, 0), '') new_line = pt.Whitespace('\n', (0, 0), '') docstr_node = pt.Node('simple_stmt', [string, new_line]) suite.children.insert(2, docstr_node) return result
def call_of_leaf(leaf, cut_own_trailer=False): """ Creates a "call" node that consist of all ``trailer`` and ``power`` objects. E.g. if you call it with ``append``:: list([]).append(3) or None You would get a node with the content ``list([]).append`` back. This generates a copy of the original ast node. If you're using the leaf, e.g. the bracket `)` it will return ``list([])``. # TODO remove cut_own_trailer option, since its always used with it. Just id:80 gh:81 # ignore it, It's not what we want anyway. Or document it better? """ trailer = leaf.parent # The leaf may not be the last or first child, because there exist three # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples # we should not match anything more than x. if trailer.type != 'trailer' or leaf not in (trailer.children[0], trailer.children[-1]): if trailer.type == 'atom': return trailer return leaf power = trailer.parent index = power.children.index(trailer) power = deep_ast_copy(power) if cut_own_trailer: cut = index else: cut = index + 1 power.children[cut:] = [] if power.type == 'error_node': start = index while True: start -= 1 if power.children[start].type != 'trailer': break transformed = tree.Node('power', power.children[start:]) transformed.parent = power.parent return transformed return power
def _get_faked(module, obj, name=None): result, fake_module = _faked(module, obj, name) if result is None: # We're not interested in classes. What we want is functions. raise FakeDoesNotExist elif result.type == 'classdef': return result, fake_module else: # Set the docstr which was previously not set (faked modules don't # contain it). assert result.type == 'funcdef' doc = '"""%s"""' % obj.__doc__ # TODO need escapes. suite = result.children[-1] string = pt.String(doc, (0, 0), '') new_line = pt.Newline('\n', (0, 0)) docstr_node = pt.Node('simple_stmt', [string, new_line]) suite.children.insert(1, docstr_node) return result, fake_module
def convert_node(self, grammar, type, children): """ Convert raw node information to a Node instance. This is passed to the parser driver which calls it whenever a reduction of a grammar rule produces a new complete node, so that the tree is build strictly bottom-up. """ symbol = grammar.number2symbol[type] try: return Parser.AST_MAPPING[symbol](children) except KeyError: if symbol == 'suite': # We don't want the INDENT/DEDENT in our parser tree. Those # leaves are just cancer. They are virtual leaves and not real # ones and therefore have pseudo start/end positions and no # prefixes. Just ignore them. children = [children[0]] + children[2:-1] return pt.Node(symbol, children)
def call_of_leaf(leaf): """ Creates a "call" node that consist of all ``trailer`` and ``power`` objects. E.g. if you call it with ``append``:: list([]).append(3) or None You would get a node with the content ``list([]).append`` back. This generates a copy of the original ast node. If you're using the leaf, e.g. the bracket `)` it will return ``list([])``. """ # TODO this is the old version of this call. Try to remove it. trailer = leaf.parent # The leaf may not be the last or first child, because there exist three # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples # we should not match anything more than x. if trailer.type != 'trailer' or leaf not in (trailer.children[0], trailer.children[-1]): if trailer.type == 'atom': return trailer return leaf power = trailer.parent index = power.children.index(trailer) new_power = copy.copy(power) new_power.children = list(new_power.children) new_power.children[index + 1:] = [] if power.type == 'error_node': start = index while True: start -= 1 if power.children[start].type != 'trailer': break transformed = tree.Node('power', power.children[start:]) transformed.parent = power.parent return transformed return power
def convert_node(self, grammar, type, children): """ Convert raw node information to a Node instance. This is passed to the parser driver which calls it whenever a reduction of a grammar rule produces a new complete node, so that the tree is build strictly bottom-up. """ symbol = grammar.number2symbol[type] try: new_node = self._ast_mapping[symbol](children) except KeyError: new_node = pt.Node(symbol, children) # We need to check raw_node always, because the same node can be # returned by convert multiple times. if symbol == 'global_stmt': self._global_names += new_node.get_global_names() elif isinstance(new_node, pt.Lambda): new_node.names_dict = self._scope_names_stack.pop() elif isinstance(new_node, (pt.ClassOrFunc, pt.Module)) \ and symbol in ('funcdef', 'classdef', 'file_input'): # scope_name_stack handling scope_names = self._scope_names_stack.pop() if isinstance(new_node, pt.ClassOrFunc): n = new_node.name scope_names[n.value].remove(n) # Set the func name of the current node arr = self._scope_names_stack[-1].setdefault(n.value, []) arr.append(n) new_node.names_dict = scope_names elif isinstance(new_node, pt.CompFor): # The name definitions of comprehenions shouldn't be part of the # current scope. They are part of the comprehension scope. for n in new_node.get_defined_names(): self._scope_names_stack[-1][n.value].remove(n) return new_node
def _apply_decorators(evaluator, context, node): """ Returns the function, that should to be executed in the end. This is also the places where the decorators are processed. """ if node.type == 'classdef': decoratee_context = er.ClassContext(evaluator, parent_context=context, classdef=node) else: decoratee_context = er.FunctionContext(evaluator, parent_context=context, funcdef=node) initial = values = set([decoratee_context]) for dec in reversed(node.get_decorators()): debug.dbg('decorator: %s %s', dec, values) dec_values = context.eval_node(dec.children[1]) trailer_nodes = dec.children[2:-1] if trailer_nodes: # Create a trailer and evaluate it. trailer = tree.Node('trailer', trailer_nodes) trailer.parent = dec dec_values = evaluator.eval_trailer(context, dec_values, trailer) if not len(dec_values): debug.warning('decorator not found: %s on %s', dec, node) return initial values = unite( dec_value.execute(param.ValuesArguments([values])) for dec_value in dec_values) if not len(values): debug.warning('not possible to resolve wrappers found %s', node) return initial debug.dbg('decorator end %s', values) return values