def _evaluate_for_statement_string(evaluator, string, module): code = dedent(""" def pseudo_docstring_stuff(): # Create a pseudo function for docstring statements. %s """) if string is None: return [] for element in re.findall('((?:\w+\.)*\w+)\.', string): # Try to import module part in dotted name. # (e.g., 'threading' in 'threading.Thread'). string = 'import %s\n' % element + string # Take the default grammar here, if we load the Python 2.7 grammar here, it # will be impossible to use `...` (Ellipsis) as a token. Docstring types # don't need to conform with the current grammar. p = Parser(load_grammar(), code % indent_block(string)) try: pseudo_cls = p.module.subscopes[0] # First pick suite, then simple_stmt (-2 for DEDENT) and then the node, # which is also not the last item, because there's a newline. stmt = pseudo_cls.children[-1].children[-2].children[-2] except (AttributeError, IndexError): return [] # Use the module of the param. # TODO this module is not the module of the param in case of a function # call. In that case it's the module of the function call. # stuffed with content from a function call. pseudo_cls.parent = module return list(_execute_types_in_stmt(evaluator, stmt))
def _replace(nodes, expression_replacement, extracted, pos, insert_before_leaf=None, remaining_prefix=None): # Now try to replace the nodes found with a variable and move the code # before the current statement. definition = _get_parent_definition(nodes[0]) if insert_before_leaf is None: insert_before_leaf = definition.get_first_leaf() first_node_leaf = nodes[0].get_first_leaf() lines = split_lines(insert_before_leaf.prefix, keepends=True) if first_node_leaf is insert_before_leaf: if remaining_prefix is not None: # The remaining prefix has already been calculated. lines[:-1] = remaining_prefix lines[-1:-1] = [indent_block(extracted, lines[-1]) + '\n'] extracted_prefix = ''.join(lines) replacement_dct = {} if first_node_leaf is insert_before_leaf: replacement_dct[nodes[0]] = extracted_prefix + expression_replacement else: if remaining_prefix is None: p = first_node_leaf.prefix else: p = remaining_prefix + _get_indentation(nodes[0]) replacement_dct[nodes[0]] = p + expression_replacement replacement_dct[insert_before_leaf] = extracted_prefix + insert_before_leaf.value for node in nodes[1:]: replacement_dct[node] = '' return replacement_dct
def _evaluate_for_statement_string(evaluator, string, module): code = dedent(""" def pseudo_docstring_stuff(): '''Create a pseudo function for docstring statements.''' %s """) if string is None: return [] for element in re.findall('((?:\w+\.)*\w+)\.', string): # Try to import module part in dotted name. # (e.g., 'threading' in 'threading.Thread'). string = 'import %s\n' % element + string p = Parser(code % indent_block(string), no_docstr=True) pseudo_cls = p.module.subscopes[0] try: stmt = pseudo_cls.statements[-1] except IndexError: return [] # Use the module of the param. # TODO this module is not the module of the param in case of a function # call. In that case it's the module of the function call. # stuffed with content from a function call. pseudo_cls.parent = module definitions = evaluator.eval_statement(stmt) it = (evaluator.execute(d) for d in definitions) # TODO Executing tuples does not make sense, people tend to say # `(str, int)` in a type annotation, which means that it returns a tuple # with both types. # At this point we just return the classes if executing wasn't possible, # i.e. is a tuple. return list(chain.from_iterable(it)) or definitions
def _evaluate_for_statement_string(evaluator, string, module): code = dedent(""" def pseudo_docstring_stuff(): '''Create a pseudo function for docstring statements.''' %s """) if string is None: return [] for element in re.findall('((?:\w+\.)*\w+)\.', string): # Try to import module part in dotted name. # (e.g., 'threading' in 'threading.Thread'). string = 'import %s\n' % element + string p = Parser(code % indent_block(string), no_docstr=True) pseudo_cls = p.module.subscopes[0] try: stmt = pseudo_cls.statements[-1] except IndexError: return [] # Use the module of the param. # TODO this module is not the module of the param in case of a function # call. In that case it's the module of the function call. # stuffed with content from a function call. pseudo_cls.parent = module return list(_execute_types_in_stmt(evaluator, stmt))
def get_doc(obj, indent=False): doc = inspect.getdoc(obj) if doc: doc = ('r"""\n%s\n"""\n' % doc) if indent: doc = common.indent_block(doc) return doc return ''
def _infer_for_statement_string(module_context, string): code = dedent( u(""" def pseudo_docstring_stuff(): ''' Create a pseudo function for docstring statements. Need this docstring so that if the below part is not valid Python this is still a function. ''' {} """)) if string is None: return [] for element in re.findall(r'((?:\w+\.)*\w+)\.', string): # Try to import module part in dotted name. # (e.g., 'threading' in 'threading.Thread'). string = 'import %s\n' % element + string # Take the default grammar here, if we load the Python 2.7 grammar here, it # will be impossible to use `...` (Ellipsis) as a token. Docstring types # don't need to conform with the current grammar. debug.dbg('Parse docstring code %s', string, color='BLUE') grammar = module_context.inference_state.latest_grammar try: module = grammar.parse(code.format(indent_block(string)), error_recovery=False) except ParserSyntaxError: return [] try: funcdef = next(module.iter_funcdefs()) # First pick suite, then simple_stmt and then the node, # which is also not the last item, because there's a newline. stmt = funcdef.children[-1].children[-1].children[-2] except (AttributeError, IndexError): return [] if stmt.type not in ('name', 'atom', 'atom_expr'): return [] from jedi.inference.value import FunctionValue function_value = FunctionValue(module_context.inference_state, module_context, funcdef) func_execution_context = function_value.as_context() # Use the module of the param. # TODO this module is not the module of the param in case of a function # call. In that case it's the module of the function call. # stuffed with content from a function call. return list(_execute_types_in_stmt(func_execution_context, stmt))
def _evaluate_for_statement_string(module_context, string): code = dedent(u(""" def pseudo_docstring_stuff(): ''' Create a pseudo function for docstring statements. Need this docstring so that if the below part is not valid Python this is still a function. ''' {0} """)) if string is None: return [] for element in re.findall('((?:\w+\.)*\w+)\.', string): # Try to import module part in dotted name. # (e.g., 'threading' in 'threading.Thread'). string = 'import %s\n' % element + string # Take the default grammar here, if we load the Python 2.7 grammar here, it # will be impossible to use `...` (Ellipsis) as a token. Docstring types # don't need to conform with the current grammar. grammar = module_context.evaluator.latest_grammar module = grammar.parse(code.format(indent_block(string))) try: funcdef = next(module.iter_funcdefs()) # First pick suite, then simple_stmt and then the node, # which is also not the last item, because there's a newline. stmt = funcdef.children[-1].children[-1].children[-2] except (AttributeError, IndexError): return [] from jedi.evaluate.representation import FunctionContext function_context = FunctionContext( module_context.evaluator, module_context, funcdef ) func_execution_context = function_context.get_function_execution() # Use the module of the param. # TODO this module is not the module of the param in case of a function # call. In that case it's the module of the function call. # stuffed with content from a function call. return list(_execute_types_in_stmt(func_execution_context, stmt))
def get_code(self, first_indent=False, indention=' '): """ :return: Returns the code of the current scope. :rtype: str """ string = "" if len(self.docstr) > 0: string += '"""' + self.docstr + '"""\n' objs = self.subscopes + self.imports + self.statements + self.returns for obj in sorted(objs, key=lambda x: x.start_pos): if isinstance(obj, Scope): string += obj.get_code(first_indent=True, indention=indention) else: if obj in self.returns and not isinstance(self, Lambda): string += 'yield ' if self.is_generator else 'return ' string += obj.get_code() if first_indent: string = common.indent_block(string, indention=indention) return string
def get_code(self, first_indent=False, indention=' '): """ :return: Returns the code of the current scope. :rtype: str """ string = "" if len(self.docstr) > 0: string += '"""' + self.docstr + '"""\n' for i in self.imports: string += i.get_code() for sub in self.subscopes: string += sub.get_code(first_indent=True, indention=indention) returns = self.returns if hasattr(self, 'returns') else [] ret_str = '' if isinstance(self, Lambda) else 'return ' for stmt in self.statements + returns: string += (ret_str if stmt in returns else '') + stmt.get_code() if first_indent: string = common.indent_block(string, indention=indention) return string
def _generate_code(scope, mixin_funcs={}, depth=0): """ Generate a string, which uses python syntax as an input to the Parser. """ def get_doc(obj, indent=False): doc = inspect.getdoc(obj) if doc: doc = ('r"""\n%s\n"""\n' % doc) if indent: doc = common.indent_block(doc) return doc return '' def is_in_base_classes(cls, name, comparison): """ Base classes may contain the exact same object """ if name in mixin_funcs: return False try: mro = cls.mro() except TypeError: # this happens, if cls == type return False for base in mro[1:]: try: attr = getattr(base, name) except AttributeError: continue if attr == comparison: return True return False def get_scope_objects(names): """ Looks for the names defined with dir() in an objects and divides them into different object types. """ classes = {} funcs = {} stmts = {} members = {} for n in names: try: # this has a builtin_function_or_method exe = getattr(scope, n) except AttributeError: # happens e.g. in properties of # PyQt4.QtGui.QStyleOptionComboBox.currentText # -> just set it to None members[n] = None else: if inspect.isclass(scope): if is_in_base_classes(scope, n, exe): continue if inspect.isbuiltin(exe) or inspect.ismethod(exe) \ or inspect.ismethoddescriptor(exe): funcs[n] = exe elif inspect.isclass(exe) or inspect.ismodule(exe): classes[n] = exe elif inspect.ismemberdescriptor(exe): members[n] = exe else: stmts[n] = exe return classes, funcs, stmts, members code = '' if inspect.ismodule(scope): # generate comment where the code's from. try: path = scope.__file__ except AttributeError: path = '?' code += '# Generated module %s from %s\n' % (scope.__name__, path) code += get_doc(scope) names = set(dir(scope)) - set(['__file__', '__name__', '__doc__', '__path__', '__package__']) \ | set(['mro']) classes, funcs, stmts, members = get_scope_objects(names) # classes for name, cl in classes.items(): bases = (c.__name__ for c in cl.__bases__) if inspect.isclass(cl) \ else [] code += 'class %s(%s):\n' % (name, ','.join(bases)) if depth == 0: try: mixin = mixin_funcs[name] except KeyError: mixin = {} cl_code = _generate_code(cl, mixin, depth + 1) code += common.indent_block(cl_code) code += '\n' # functions for name, func in funcs.items(): params, ret = _parse_function_doc(func) if depth > 0: params = 'self, ' + params doc_str = get_doc(func, indent=True) try: mixin = mixin_funcs[name] except KeyError: # normal code generation code += 'def %s(%s):\n' % (name, params) code += doc_str code += common.indent_block('%s\n\n' % ret) else: # generation of code with mixins # the parser only supports basic functions with a newline after # the double dots # find doc_str place try: pos = re.search(r'\):\s*\n', mixin).end() except TypeError: # pypy uses a different reversed builtin if name == 'reversed': mixin = 'def reversed(sequence):\n' \ ' for i in self.__sequence: yield i' pos = 24 else: debug.warning('mixin trouble in pypy: %s', name) raise if pos is None: raise Exception("Builtin function not parsed correctly") code += mixin[:pos] + doc_str + mixin[pos:] # class members (functions) properties? for name, func in members.items(): # recursion problem in properties TODO remove if name in ['fget', 'fset', 'fdel']: continue ret = 'pass' code += '@property\ndef %s(self):\n' % (name) code += common.indent_block(get_doc(func) + '%s\n\n' % ret) # variables for name, value in stmts.items(): if is_py3k: file_type = io.TextIOWrapper else: file_type = types.FileType if isinstance(value, file_type): value = 'open()' elif name == 'None': value = '' elif type(value).__name__ in [ 'int', 'bool', 'float', 'dict', 'list', 'tuple' ]: value = repr(value) else: # get the type, if the type is not simple. mod = type(value).__module__ value = type(value).__name__ + '()' if mod != '__builtin__': value = '%s.%s' % (mod, value) code += '%s = %s\n' % (name, value) return code
def extract_function(inference_state, path, module_context, name, pos, until_pos): nodes = _find_nodes(module_context.tree_node, pos, until_pos) assert len(nodes) is_expression, _ = _is_expression_with_error(nodes) context = module_context.create_context(nodes[0]) is_bound_method = context.is_bound_method() params, return_variables = list( _find_inputs_and_outputs(module_context, context, nodes)) # Find variables # Is a class method / method if context.is_module(): insert_before_leaf = None # Leaf will be determined later else: node = _get_code_insertion_node(context.tree_node, is_bound_method) insert_before_leaf = node.get_first_leaf() if is_expression: code_block = 'return ' + _expression_nodes_to_string(nodes) + '\n' remaining_prefix = None has_ending_return_stmt = False else: has_ending_return_stmt = _is_node_ending_return_stmt(nodes[-1]) if not has_ending_return_stmt: # Find the actually used variables (of the defined ones). If none are # used (e.g. if the range covers the whole function), return the last # defined variable. return_variables = list( _find_needed_output_variables( context, nodes[0].parent, nodes[-1].end_pos, return_variables)) or [return_variables[-1] ] if return_variables else [] remaining_prefix, code_block = _suite_nodes_to_string(nodes, pos) after_leaf = nodes[-1].get_next_leaf() first, second = _split_prefix_at(after_leaf, until_pos[0]) code_block += first code_block = dedent(code_block) if not has_ending_return_stmt: output_var_str = ', '.join(return_variables) code_block += 'return ' + output_var_str + '\n' # Check if we have to raise RefactoringError _check_for_non_extractables( nodes[:-1] if has_ending_return_stmt else nodes) decorator = '' self_param = None if is_bound_method: if not function_is_staticmethod(context.tree_node): function_param_names = context.get_value().get_param_names() if len(function_param_names): self_param = function_param_names[0].string_name params = [p for p in params if p != self_param] if function_is_classmethod(context.tree_node): decorator = '@classmethod\n' else: code_block += '\n' function_code = '%sdef %s(%s):\n%s' % (decorator, name, ', '.join( params if self_param is None else [self_param] + params), indent_block(code_block)) function_call = '%s(%s)' % ( ('' if self_param is None else self_param + '.') + name, ', '.join(params)) if is_expression: replacement = function_call else: if has_ending_return_stmt: replacement = 'return ' + function_call + '\n' else: replacement = output_var_str + ' = ' + function_call + '\n' replacement_dct = _replace(nodes, replacement, function_code, pos, insert_before_leaf, remaining_prefix) if not is_expression: replacement_dct[after_leaf] = second + after_leaf.value file_to_node_changes = {path: replacement_dct} return Refactoring(inference_state, file_to_node_changes)
def assert_case_equal(case, actual, desired): """ Assert ``actual == desired`` with formatted message. This is not needed for typical pytest use case, but as we need ``--assert=plain`` (see ../pytest.ini) to workaround some issue due to pytest magic, let's format the message by hand. """ assert actual == desired, """ Test %r failed. actual = %s desired = %s """ % (case, indent_block(str(actual)), indent_block(str(desired))) def assert_static_analysis(case, actual, desired): """A nicer formatting for static analysis tests.""" a = set(actual) d = set(desired) assert actual == desired, """ Test %r failed. not raised = %s unspecified = %s """ % (case, sorted(d - a), sorted(a - d)) def test_completion(case, monkeypatch, environment, has_typing, has_django): skip_reason = case.get_skip_reason(environment)
def _generate_code(scope, mixin_funcs={}, depth=0): """ Generate a string, which uses python syntax as an input to the Parser. """ def get_doc(obj, indent=False): doc = inspect.getdoc(obj) if doc: doc = ('r"""\n%s\n"""\n' % doc) if indent: doc = common.indent_block(doc) return doc return '' def is_in_base_classes(cls, name, comparison): """ Base classes may contain the exact same object """ if name in mixin_funcs: return False try: mro = cls.mro() except TypeError: # this happens, if cls == type return False for base in mro[1:]: try: attr = getattr(base, name) except AttributeError: continue if attr == comparison: return True return False def get_scope_objects(names): """ Looks for the names defined with dir() in an objects and divides them into different object types. """ classes = {} funcs = {} stmts = {} members = {} for n in names: try: # this has a builtin_function_or_method exe = getattr(scope, n) except AttributeError: # happens e.g. in properties of # PyQt4.QtGui.QStyleOptionComboBox.currentText # -> just set it to None members[n] = None else: if inspect.isclass(scope): if is_in_base_classes(scope, n, exe): continue if inspect.isbuiltin(exe) or inspect.ismethod(exe) \ or inspect.ismethoddescriptor(exe): funcs[n] = exe elif inspect.isclass(exe) or inspect.ismodule(exe): classes[n] = exe elif inspect.ismemberdescriptor(exe): members[n] = exe else: stmts[n] = exe return classes, funcs, stmts, members code = '' if inspect.ismodule(scope): # generate comment where the code's from. try: path = scope.__file__ except AttributeError: path = '?' code += '# Generated module %s from %s\n' % (scope.__name__, path) code += get_doc(scope) names = set(dir(scope)) - set(['__file__', '__name__', '__doc__', '__path__', '__package__']) \ | set(['mro']) classes, funcs, stmts, members = get_scope_objects(names) # classes for name, cl in classes.items(): bases = (c.__name__ for c in cl.__bases__) if inspect.isclass(cl) \ else [] code += 'class %s(%s):\n' % (name, ','.join(bases)) if depth == 0: try: mixin = mixin_funcs[name] except KeyError: mixin = {} cl_code = _generate_code(cl, mixin, depth + 1) code += common.indent_block(cl_code) code += '\n' # functions for name, func in funcs.items(): params, ret = _parse_function_doc(func) if depth > 0: params = 'self, ' + params doc_str = get_doc(func, indent=True) try: mixin = mixin_funcs[name] except KeyError: # normal code generation code += 'def %s(%s):\n' % (name, params) code += doc_str code += common.indent_block('%s\n\n' % ret) else: # generation of code with mixins # the parser only supports basic functions with a newline after # the double dots # find doc_str place try: pos = re.search(r'\):\s*\n', mixin).end() except TypeError: # pypy uses a different reversed builtin if name == 'reversed': mixin = 'def reversed(sequence):\n' \ ' for i in self.__sequence: yield i' pos = 24 else: debug.warning('mixin trouble in pypy: %s', name) raise if pos is None: raise Exception("Builtin function not parsed correctly") code += mixin[:pos] + doc_str + mixin[pos:] # class members (functions) properties? for name, func in members.items(): # recursion problem in properties TODO remove if name in ['fget', 'fset', 'fdel']: continue ret = 'pass' code += '@property\ndef %s(self):\n' % (name) code += common.indent_block(get_doc(func) + '%s\n\n' % ret) # variables for name, value in stmts.items(): if is_py3k: file_type = io.TextIOWrapper else: file_type = types.FileType if isinstance(value, file_type): value = 'open()' elif name == 'None': value = '' elif type(value).__name__ in ['int', 'bool', 'float', 'dict', 'list', 'tuple']: value = repr(value) else: # get the type, if the type is not simple. mod = type(value).__module__ value = type(value).__name__ + '()' if mod != '__builtin__': value = '%s.%s' % (mod, value) code += '%s = %s\n' % (name, value) return code
def _generate_code(scope, mixin_funcs={}, depth=0): """ Generate a string, which uses python syntax as an input to the Parser. """ def get_doc(obj, indent=False): doc = inspect.getdoc(obj) if doc: doc = 'r"""\n%s\n"""\n' % doc if indent: doc = common.indent_block(doc) return doc return "" def is_in_base_classes(cls, name, comparison): """ Base classes may contain the exact same object """ if name in mixin_funcs: return False try: mro = cls.mro() except TypeError: # this happens, if cls == type return False for base in mro[1:]: try: attr = getattr(base, name) except AttributeError: continue if attr == comparison: return True return False def get_scope_objects(names): """ Looks for the names defined with dir() in an objects and divides them into different object types. """ classes = {} funcs = {} stmts = {} members = {} for n in names: try: # this has a builtin_function_or_method exe = getattr(scope, n) except AttributeError: # happens e.g. in properties of # PyQt4.QtGui.QStyleOptionComboBox.currentText # -> just set it to None members[n] = None else: if inspect.isclass(scope): if is_in_base_classes(scope, n, exe): continue if inspect.isbuiltin(exe) or inspect.ismethod(exe) or inspect.ismethoddescriptor(exe): funcs[n] = exe elif inspect.isclass(exe): classes[n] = exe elif inspect.ismemberdescriptor(exe): members[n] = exe else: stmts[n] = exe return classes, funcs, stmts, members code = "" if inspect.ismodule(scope): # generate comment where the code's from. try: path = scope.__file__ except AttributeError: path = "?" code += "# Generated module %s from %s\n" % (scope.__name__, path) code += get_doc(scope) names = set(dir(scope)) - set(["__file__", "__name__", "__doc__", "__path__", "__package__"]) | set(["mro"]) classes, funcs, stmts, members = get_scope_objects(names) # classes for name, cl in classes.items(): bases = (c.__name__ for c in cl.__bases__) code += "class %s(%s):\n" % (name, ",".join(bases)) if depth == 0: try: mixin = mixin_funcs[name] except KeyError: mixin = {} cl_code = _generate_code(cl, mixin, depth + 1) code += common.indent_block(cl_code) code += "\n" # functions for name, func in funcs.items(): params, ret = _parse_function_doc(func) if depth > 0: params = "self, " + params doc_str = get_doc(func, indent=True) try: mixin = mixin_funcs[name] except KeyError: # normal code generation code += "def %s(%s):\n" % (name, params) code += doc_str code += common.indent_block("%s\n\n" % ret) else: # generation of code with mixins # the parser only supports basic functions with a newline after # the double dots # find doc_str place try: pos = re.search(r"\):\s*\n", mixin).end() except TypeError: # pypy uses a different reversed builtin if name == "reversed": mixin = "def reversed(sequence):\n" " for i in self.__sequence: yield i" pos = 24 else: debug.warning("mixin trouble in pypy: %s", name) raise if pos is None: raise Exception("Builtin function not parsed correctly") code += mixin[:pos] + doc_str + mixin[pos:] # class members (functions) properties? for name, func in members.items(): # recursion problem in properties TODO remove if name in ["fget", "fset", "fdel"]: continue ret = "pass" code += "@property\ndef %s(self):\n" % (name) code += common.indent_block(get_doc(func) + "%s\n\n" % ret) # variables for name, value in stmts.items(): if is_py3k: file_type = io.TextIOWrapper else: file_type = types.FileType if type(value) == file_type: value = "open()" elif name == "None": value = "" elif type(value).__name__ in ["int", "bool", "float", "dict", "list", "tuple"]: value = repr(value) else: # get the type, if the type is not simple. mod = type(value).__module__ value = type(value).__name__ + "()" if mod != "__builtin__": value = "%s.%s" % (mod, value) code += "%s = %s\n" % (name, value) if depth == 0: # with open('writeout.py', 'w') as f: # f.write(code) # import sys # sys.stdout.write(code) # exit() pass return code