def test_token_stream():
stream = TokenStream(StringIO('hello#world'))
assert stream.current.value == 'hello'
assert stream.line == 1
assert stream.move().value == 'hello'
assert stream.current.value == '#world'
assert stream.line == 1
def test_token_stream():
stream = TokenStream(StringIO('hello#world'))
assert stream.current.value == 'hello'
assert stream.line == 1
assert stream.move().value == 'hello'
assert stream.current.value == '#world'
assert stream.line == 1
class Parser(object):
def __call__(self, filelike, filename):
self.source = filelike.readlines()
src = ''.join(self.source)
self.stream = TokenStream(StringIO(src))
self.filename = filename
self.all = None
return self.parse_module()
current = property(lambda self: self.stream.current)
line = property(lambda self: self.stream.line)
def consume(self, kind):
assert self.stream.move().kind == kind
def leapfrog(self, kind):
for token in self.stream:
if token.kind == kind:
self.consume(kind)
return
def parse_docstring(self):
for token in self.stream:
if token.kind in [tk.COMMENT, tk.NEWLINE, tk.NL]:
continue
elif token.kind == tk.STRING:
return token.value
else:
return None
def parse_definitions(self, class_, all=False):
for token in self.stream:
if all and token.value == '__all__':
self.parse_all()
if token.value in ['def', 'class']:
yield self.parse_definition(class_._nest(token.value))
if token.kind == tk.INDENT:
self.consume(tk.INDENT)
for definition in self.parse_definitions(class_):
yield definition
if token.kind == tk.DEDENT:
return
def parse_all(self):
assert self.current.value == '__all__'
self.consume(tk.NAME)
if self.current.value != '=':
raise AllError('Could not evaluate contents of __all__. ')
self.consume(tk.OP)
if self.current.value not in '([':
raise AllError('Could not evaluate contents of __all__. ')
if self.current.value == '[':
msg = ("%s WARNING: __all__ is defined as a list, this means "
"pep257 cannot reliably detect contents of the __all__ "
"variable, because it can be mutated. Change __all__ to be "
"an (immutable) tuple, to remove this warning. Note, "
"pep257 uses __all__ to detect which definitions are "
"public, to warn if public definitions are missing "
"docstrings. If __all__ is a (mutable) list, pep257 cannot "
"reliably assume its contents. pep257 will proceed "
"assuming __all__ is not mutated.\n" % self.filename)
sys.stderr.write(msg)
self.consume(tk.OP)
s = '('
while self.current.kind in (tk.NL, tk.COMMENT):
self.stream.move()
if self.current.kind != tk.STRING:
raise AllError('Could not evaluate contents of __all__. ')
while self.current.value not in ')]':
s += self.current.value
self.stream.move()
s += ')'
try:
self.all = eval(s, {})
except BaseException:
raise AllError('Could not evaluate contents of __all__: %s. ' % s)
def parse_module(self):
start = self.line
docstring = self.parse_docstring()
children = list(self.parse_definitions(Module, all=True))
assert self.current is None
end = self.line
module = Module(self.filename, self.source, start, end,
docstring, children, None, self.all)
for child in module.children:
child.parent = module
return module
def parse_definition(self, class_):
start = self.line
self.consume(tk.NAME)
name = self.current.value
self.leapfrog(tk.INDENT)
assert self.current.kind != tk.INDENT
docstring = self.parse_docstring()
children = list(self.parse_definitions(class_))
assert self.current.kind == tk.DEDENT
end = self.line - 1
definition = class_(name, self.source, start, end,
docstring, children, None)
for child in definition.children:
child.parent = definition
return definition