Ejemplo n.º 1
0
def test_token_stream():
    stream = TokenStream(StringIO('hello#world'))
    assert stream.current.value == 'hello'
    assert stream.line == 1
    assert stream.move().value == 'hello'
    assert stream.current.value == '#world'
    assert stream.line == 1
Ejemplo n.º 2
0
def test_token_stream():
    stream = TokenStream(StringIO('hello#world'))
    assert stream.current.value == 'hello'
    assert stream.line == 1
    assert stream.move().value == 'hello'
    assert stream.current.value == '#world'
    assert stream.line == 1
Ejemplo n.º 3
0
class Parser(object):

    def __call__(self, filelike, filename):
        self.source = filelike.readlines()
        src = ''.join(self.source)
        self.stream = TokenStream(StringIO(src))
        self.filename = filename
        self.all = None
        return self.parse_module()

    current = property(lambda self: self.stream.current)
    line = property(lambda self: self.stream.line)

    def consume(self, kind):
        assert self.stream.move().kind == kind

    def leapfrog(self, kind):
        for token in self.stream:
            if token.kind == kind:
                self.consume(kind)
                return

    def parse_docstring(self):
        for token in self.stream:
            if token.kind in [tk.COMMENT, tk.NEWLINE, tk.NL]:
                continue
            elif token.kind == tk.STRING:
                return token.value
            else:
                return None

    def parse_definitions(self, class_, all=False):
        for token in self.stream:
            if all and token.value == '__all__':
                self.parse_all()
            if token.value in ['def', 'class']:
                yield self.parse_definition(class_._nest(token.value))
            if token.kind == tk.INDENT:
                self.consume(tk.INDENT)
                for definition in self.parse_definitions(class_):
                    yield definition
            if token.kind == tk.DEDENT:
                return

    def parse_all(self):
        assert self.current.value == '__all__'
        self.consume(tk.NAME)
        if self.current.value != '=':
            raise AllError('Could not evaluate contents of __all__. ')
        self.consume(tk.OP)
        if self.current.value not in '([':
            raise AllError('Could not evaluate contents of __all__. ')
        if self.current.value == '[':
            msg = ("%s WARNING: __all__ is defined as a list, this means "
                   "pep257 cannot reliably detect contents of the __all__ "
                   "variable, because it can be mutated. Change __all__ to be "
                   "an (immutable) tuple, to remove this warning. Note, "
                   "pep257 uses __all__ to detect which definitions are "
                   "public, to warn if public definitions are missing "
                   "docstrings. If __all__ is a (mutable) list, pep257 cannot "
                   "reliably assume its contents. pep257 will proceed "
                   "assuming __all__ is not mutated.\n" % self.filename)
            sys.stderr.write(msg)
        self.consume(tk.OP)
        s = '('
        while self.current.kind in (tk.NL, tk.COMMENT):
            self.stream.move()
        if self.current.kind != tk.STRING:
            raise AllError('Could not evaluate contents of __all__. ')
        while self.current.value not in ')]':
            s += self.current.value
            self.stream.move()
        s += ')'
        try:
            self.all = eval(s, {})
        except BaseException:
            raise AllError('Could not evaluate contents of __all__: %s. ' % s)

    def parse_module(self):
        start = self.line
        docstring = self.parse_docstring()
        children = list(self.parse_definitions(Module, all=True))
        assert self.current is None
        end = self.line
        module = Module(self.filename, self.source, start, end,
                        docstring, children, None, self.all)
        for child in module.children:
            child.parent = module
        return module

    def parse_definition(self, class_):
        start = self.line
        self.consume(tk.NAME)
        name = self.current.value
        self.leapfrog(tk.INDENT)
        assert self.current.kind != tk.INDENT
        docstring = self.parse_docstring()
        children = list(self.parse_definitions(class_))
        assert self.current.kind == tk.DEDENT
        end = self.line - 1
        definition = class_(name, self.source, start, end,
                            docstring, children, None)
        for child in definition.children:
            child.parent = definition
        return definition