def make_fakeimport(self, module, variable=None, alias=None): """ Make a fake import object. The following statements are created depending on what parameters are given: - only `module`: ``import <module>`` - `module` and `variable`: ``from <module> import <variable>`` - all: ``from <module> import <variable> as <alias>`` :type module: str :arg module: ``<module>`` part in ``from <module> import ...`` :type variable: str :arg variable: ``<variable>`` part in ``from ... import <variable>`` :type alias: str :arg alias: ``<alias>`` part in ``... import ... as <alias>``. :rtype: :class:`parsing_representation.Import` """ submodule = self.scope._sub_module if variable: varname = pr.Name(module=submodule, names=[(variable, (-1, 0))], start_pos=(-1, 0), end_pos=(None, None)) else: varname = None modname = pr.Name(module=submodule, names=[(module, (-1, 0))], start_pos=(-1, 0), end_pos=(None, None)) if alias: aliasname = pr.Name(module=submodule, names=[(alias, (-1, 0))], start_pos=(-1, 0), end_pos=(None, None)) else: aliasname = None if varname: fakeimport = pr.Import(module=submodule, namespace=varname, from_ns=modname, alias=aliasname, start_pos=(-1, 0), end_pos=(None, None)) else: fakeimport = pr.Import(module=submodule, namespace=modname, alias=aliasname, start_pos=(-1, 0), end_pos=(None, None)) return fakeimport
def _get_nested_import(self, parent): """ See documentation of `self._is_nested_import`. Generates an Import statement, that can be used to fake nested imports. """ i = self.import_stmt # This is not an existing Import statement. Therefore, set position to # 0 (0 is not a valid line number). zero = (0, 0) names = i.namespace.names[1:] n = pr.Name(i._sub_module, names, zero, zero, self.import_stmt) new = pr.Import(i._sub_module, zero, zero, n) new.parent = parent debug.dbg('Generated a nested import: %s' % new) return new
def _parse(self): """ The main part of the program. It analyzes the given code-text and returns a tree-like scope. For a more detailed description, see the class description. :param text: The code which should be parsed. :param type: str :raises: IndentationError """ extended_flow = ['else', 'elif', 'except', 'finally'] statement_toks = ['{', '[', '(', '`'] self._decorators = [] self.freshscope = True for tok in self._gen: token_type = tok.type tok_str = tok.string first_pos = tok.start_pos self.module.temp_used_names = [] # debug.dbg('main: tok=[%s] type=[%s] indent=[%s]', \ # tok, tokenize.tok_name[token_type], start_position[0]) # check again for unindented stuff. this is true for syntax # errors. only check for names, because thats relevant here. If # some docstrings are not indented, I don't care. while first_pos[1] <= self._scope.start_pos[1] \ and (token_type == tokenize.NAME or tok_str in ('(', '['))\ and self._scope != self.module: self._scope.end_pos = first_pos self._scope = self._scope.parent if isinstance(self._scope, pr.Module) \ and not isinstance(self._scope, pr.SubModule): self._scope = self.module if isinstance(self._scope, pr.SubModule): use_as_parent_scope = self._top_module else: use_as_parent_scope = self._scope if tok_str == 'def': func = self._parse_function() if func is None: debug.warning("function: syntax error@%s", first_pos[0]) continue self.freshscope = True self._scope = self._scope.add_scope(func, self._decorators) self._decorators = [] elif tok_str == 'class': cls = self._parse_class() if cls is None: debug.warning("class: syntax error@%s" % first_pos[0]) continue self.freshscope = True self._scope = self._scope.add_scope(cls, self._decorators) self._decorators = [] # import stuff elif tok_str == 'import': imports = self._parse_import_list() for count, (m, alias, defunct) in enumerate(imports): e = (alias or m or self._gen.previous).end_pos end_pos = self._gen.previous.end_pos if count + 1 == len( imports) else e i = pr.Import(self.module, first_pos, end_pos, m, alias, defunct=defunct) self._check_user_stmt(i) self._scope.add_import(i) if not imports: i = pr.Import(self.module, first_pos, self._gen.current.end_pos, None, defunct=True) self._check_user_stmt(i) self.freshscope = False elif tok_str == 'from': defunct = False # take care for relative imports relative_count = 0 while True: tok = next(self._gen) if tok.string != '.': break relative_count += 1 # the from import mod, tok = self._parse_dot_name(self._gen.current) tok_str = tok.string if str(mod) == 'import' and relative_count: self._gen.push_last_back() tok_str = 'import' mod = None if not mod and not relative_count or tok_str != "import": debug.warning("from: syntax error@%s", tok.start_pos[0]) defunct = True if tok_str != 'import': self._gen.push_last_back() names = self._parse_import_list() for count, (name, alias, defunct2) in enumerate(names): star = name is not None and unicode(name.names[0]) == '*' if star: name = None e = (alias or name or self._gen.previous).end_pos end_pos = self._gen.previous.end_pos if count + 1 == len( names) else e i = pr.Import(self.module, first_pos, end_pos, name, alias, mod, star, relative_count, defunct=defunct or defunct2) self._check_user_stmt(i) self._scope.add_import(i) self.freshscope = False # loops elif tok_str == 'for': set_stmt, tok = self._parse_statement(added_breaks=['in'], names_are_set_vars=True) if tok.string != 'in': debug.warning('syntax err, for flow incomplete @%s', tok.start_pos[0]) try: statement, tok = self._parse_statement() except StopIteration: statement, tok = None, None s = [] if statement is None else [statement] f = pr.ForFlow(self.module, s, first_pos, set_stmt) self._scope = self._scope.add_statement(f) if tok is None or tok.string != ':': debug.warning('syntax err, for flow started @%s', first_pos[0]) elif tok_str in ['if', 'while', 'try', 'with'] + extended_flow: added_breaks = [] command = tok_str if command in ('except', 'with'): added_breaks.append(',') # multiple inputs because of with inputs = [] first = True while first or command == 'with' and tok.string not in ( ':', '\n', '\r\n'): statement, tok = \ self._parse_statement(added_breaks=added_breaks) if command == 'except' and tok.string == ',': # the except statement defines a var # this is only true for python 2 n, tok = self._parse_dot_name() if n: n.parent = statement statement.as_names.append(n) if statement: inputs.append(statement) first = False f = pr.Flow(self.module, command, inputs, first_pos) if command in extended_flow: # the last statement has to be another part of # the flow statement, because a dedent releases the # main scope, so just take the last statement. try: s = self._scope.statements[-1].set_next(f) except (AttributeError, IndexError): # If set_next doesn't exist, just add it. s = self._scope.add_statement(f) else: s = self._scope.add_statement(f) self._scope = s if tok.string != ':': debug.warning('syntax err, flow started @%s', tok.start_pos[0]) # returns elif tok_str in ('return', 'yield'): s = tok.start_pos self.freshscope = False # add returns to the scope func = self._scope.get_parent_until(pr.Function) if tok_str == 'yield': func.is_generator = True stmt, tok = self._parse_statement() if stmt is not None: stmt.parent = use_as_parent_scope try: func.returns.append(stmt) # start_pos is the one of the return statement stmt.start_pos = s except AttributeError: debug.warning('return in non-function') elif tok_str == 'assert': stmt, tok = self._parse_statement() if stmt is not None: stmt.parent = use_as_parent_scope self._scope.asserts.append(stmt) elif tok_str in STATEMENT_KEYWORDS: stmt, _ = self._parse_statement() kw = pr.KeywordStatement(tok_str, tok.start_pos, use_as_parent_scope, stmt) self._scope.add_statement(kw) if stmt is not None and tok_str == 'global': for t in stmt._token_list: if isinstance(t, pr.Name): # Add the global to the top module, it counts there. self.module.add_global(t) # decorator elif tok_str == '@': stmt, tok = self._parse_statement() if stmt is not None: self._decorators.append(stmt) elif tok_str == 'pass': continue # default elif token_type in (tokenize.NAME, tokenize.STRING, tokenize.NUMBER, tokenize.OP) \ or tok_str in statement_toks: # this is the main part - a name can be a function or a # normal var, which can follow anything. but this is done # by the statement parser. stmt, tok = self._parse_statement(self._gen.current) if stmt: self._scope.add_statement(stmt) self.freshscope = False else: if token_type not in (tokenize.COMMENT, tokenize.NEWLINE, tokenize.ENDMARKER): debug.warning('Token not used: %s %s %s', tok_str, tokenize.tok_name[token_type], first_pos) continue self.no_docstr = False