def skip_spaces(self, pre_increment=0):
if pre_increment:
indent, chars = \
scanner.count_indent(self.line[self.column:
self.column+pre_increment], True)
self.indent += indent
self.column += chars
indent, chars = scanner.count_indent(self.line[self.column:])
self.indent += indent
self.column += chars
def skip_spaces(self, pre_increment=0):
if pre_increment:
indent, chars = \
scanner.count_indent(self.line[self.column:
self.column+pre_increment], True)
self.indent += indent
self.column += chars
indent, chars = scanner.count_indent(self.line[self.column:])
self.indent += indent
self.column += chars
def readline(self):
r'''
>>> from StringIO import StringIO
>>> p = kqb_parser(StringIO("""
... line 1
... # this should be ignored
... line 2
...
... line 3
... """))
>>> p.readline()
>>> p.indent, p.line, p.lineno, p.column
(0, 'line 1', 2, 0)
>>> p.readline()
>>> p.indent, p.line, p.lineno, p.column
(1, ' line 2', 4, 1)
>>> p.readline()
>>> p.indent, p.line, p.lineno, p.column
(2, ' line 3', 6, 2)
>>> p.eof
False
>>> p.readline()
>>> p.eof
True
'''
while True:
line = self.f.readline()
if line == '':
self.eof = True
break
self.lineno += 1
line = line.rstrip('\n')
if not self.blank_line.match(line):
self.indent, self.column = scanner.count_indent(line)
self.line = line
break
def readline(self):
r'''
>>> from StringIO import StringIO
>>> p = kqb_parser(StringIO("""
... line 1
... # this should be ignored
... line 2
...
... line 3
... """))
>>> p.readline()
>>> p.indent, p.line, p.lineno, p.column
(0, 'line 1', 2, 0)
>>> p.readline()
>>> p.indent, p.line, p.lineno, p.column
(1, ' line 2', 4, 1)
>>> p.readline()
>>> p.indent, p.line, p.lineno, p.column
(2, ' line 3', 6, 2)
>>> p.eof
False
>>> p.readline()
>>> p.eof
True
'''
while True:
line = self.f.readline()
if line == '':
self.eof = True
break
self.lineno += 1
line = line.rstrip('\n')
if not self.blank_line.match(line):
self.indent, self.column = scanner.count_indent(line)
self.line = line
break
def get_block_string(self, stop=None, hanging=False, ending_newlines=False):
r'''
>>> from StringIO import StringIO
>>> f = StringIO(r"""
... line 1 # comment
... more stuff
... last line
... blah hanging line 1
...
... line 2
... indented
... last line
... ! the end !
... """)
>>> f.name = 'StringIO'
>>> p = kqb_parser(f)
>>> p.get_block_string()
u'line 1 # comment\n more stuff\nlast line'
>>> p.column = 4
>>> p.indent = 4
>>> p.get_block_string('!', True)
u'hanging line 1\n\nline 2\n indented\nlast line'
>>> f = StringIO(r"""
... ! line 1 # comment
... more stuff
... last line
... blah
... """)
>>> f.name = 'StringIO'
>>> p = kqb_parser(f)
>>> p.readline()
>>> p.get_token('bang')
('bang', None)
>>> p.get_block_string(hanging=True)
u'line 1 # comment\n more stuff\nlast line'
'''
if hanging:
indent, more_chars = \
scanner.count_indent(self.line[self.column:])
self.indent += indent
self.column += more_chars
else:
self.readline()
indent = self.indent
if self.eof: self.SyntaxError("expected block string, got EOF", False)
rest_line = self.line[self.column:]
if self.blank_line.match(rest_line):
ans = []
else:
ans = [rest_line]
while not self.eof:
last_lineno = self.lineno
self.readline()
if ending_newlines:
for i in range(self.lineno - last_lineno - 1): ans.append('')
if self.eof or self.indent < indent or \
stop and self.line[self.column:].startswith(stop):
break
if not ending_newlines:
for i in range(self.lineno - last_lineno - 1): ans.append('')
ans.append(' ' * (self.indent - indent) + self.line[self.column:])
if not ans: self.SyntaxError("expected block string", False)
return u'\n'.join(scanner.unescape(str) for str in ans)
def get_token(self, check_token=None):
r'''
>>> from StringIO import StringIO
>>> f = StringIO(r"""
... line 1=2.5: ( /\s* /, $foo) # comment
... ,|!-True "hi\n" [mom]
... """)
>>> f.name = 'StringIO'
>>> p = kqb_parser(f)
>>> p.get_token()
('id', 'line')
>>> p.get_token()
('number', 1)
>>> p.get_token()
('equal', None)
>>> p.get_token()
('number', 2.5)
>>> p.get_token()
('colon', None)
>>> p.get_token()
('lparen', None)
>>> p.get_token()
('regexp', '\\s* ')
>>> p.get_token()
('comma', None)
>>> p.get_token()
('param', 'foo')
>>> p.get_token()
('rparen', None)
>>> p.get_token()
('comma', None)
>>> p.get_token()
('bar', None)
>>> p.get_token()
('bang', None)
>>> p.get_token()
('hyphen', None)
>>> p.get_token()
('const', True)
>>> p.get_token()
('str', 'hi\n')
>>> p.get_token()
('prompt', 'mom')
>>> p.get_token()
(None, None)
'''
if self.pushed_token:
ans = self.pushed_token
self.indent = self.pushed_indent
self.column = self.pushed_column
self.pushed_token = self.pushed_column = self.pushed_indent = None
if check_token and check_token != ans[0]:
self.SyntaxError("expected %s, got %s" % (check_token, ans[0]))
#print "get_token: returning pushed_token", ans # FIX
return ans
if self.column < len(self.line): self.skip_spaces()
if self.column >= len(self.line):
self.readline()
if self.eof:
self.last_token = None, None
#print "get_token: returning EOF" # FIX
return self.last_token
self.last_line = self.line
self.last_lineno = self.lineno
self.last_column = self.column
self.last_indent = self.indent
match = self.tokenizer.match(self.line, self.column)
if not match: self.SyntaxError("Scanner error: no legal token")
token = match.lastgroup
chars = match.group(token)
end = match.end()
indent, ignore = scanner.count_indent(self.line[self.column:end], True)
self.indent += indent
self.column = end
if token == 'str' or token == 'prompt':
value = scanner.unescape(chars)
elif token == 'const':
value = eval(chars)
elif token == 'number':
try:
value = int(chars)
except ValueError:
value = float(chars)
elif token == 'param' or token == 'id':
value = chars
elif token == 'regexp1' or token == 'regexp2':
# FIX
token = 'regexp'
value = chars
self.lineno += chars.count('\n')
last_nl = chars.rfind('\n')
if last_nl >= 0:
self.column = len(chars) - last_nl + 4
else:
value = None
if check_token and check_token != token:
self.SyntaxError("expected %s, got %s" % (check_token, token))
self.last_token = str(token), value
#print "get_token: returning", self.last_token # FIX
return self.last_token
def get_block_string(self,
stop=None,
hanging=False,
ending_newlines=False):
r'''
>>> from StringIO import StringIO
>>> f = StringIO(r"""
... line 1 # comment
... more stuff
... last line
... blah hanging line 1
...
... line 2
... indented
... last line
... ! the end !
... """)
>>> f.name = 'StringIO'
>>> p = kqb_parser(f)
>>> p.get_block_string()
u'line 1 # comment\n more stuff\nlast line'
>>> p.column = 4
>>> p.indent = 4
>>> p.get_block_string('!', True)
u'hanging line 1\n\nline 2\n indented\nlast line'
>>> f = StringIO(r"""
... ! line 1 # comment
... more stuff
... last line
... blah
... """)
>>> f.name = 'StringIO'
>>> p = kqb_parser(f)
>>> p.readline()
>>> p.get_token('bang')
('bang', None)
>>> p.get_block_string(hanging=True)
u'line 1 # comment\n more stuff\nlast line'
'''
if hanging:
indent, more_chars = \
scanner.count_indent(self.line[self.column:])
self.indent += indent
self.column += more_chars
else:
self.readline()
indent = self.indent
if self.eof: self.SyntaxError("expected block string, got EOF", False)
rest_line = self.line[self.column:]
if self.blank_line.match(rest_line):
ans = []
else:
ans = [rest_line]
while not self.eof:
last_lineno = self.lineno
self.readline()
if ending_newlines:
for i in range(self.lineno - last_lineno - 1):
ans.append('')
if self.eof or self.indent < indent or \
stop and self.line[self.column:].startswith(stop):
break
if not ending_newlines:
for i in range(self.lineno - last_lineno - 1):
ans.append('')
ans.append(' ' * (self.indent - indent) + self.line[self.column:])
if not ans: self.SyntaxError("expected block string", False)
return '\n'.join(scanner.unescape(str) for str in ans)
def get_token(self, check_token=None):
r'''
>>> from StringIO import StringIO
>>> f = StringIO(r"""
... line 1=2.5: ( /\s* /, $foo) # comment
... ,|!-True "hi\n" [mom]
... """)
>>> f.name = 'StringIO'
>>> p = kqb_parser(f)
>>> p.get_token()
('id', 'line')
>>> p.get_token()
('number', 1)
>>> p.get_token()
('equal', None)
>>> p.get_token()
('number', 2.5)
>>> p.get_token()
('colon', None)
>>> p.get_token()
('lparen', None)
>>> p.get_token()
('regexp', '\\s* ')
>>> p.get_token()
('comma', None)
>>> p.get_token()
('param', 'foo')
>>> p.get_token()
('rparen', None)
>>> p.get_token()
('comma', None)
>>> p.get_token()
('bar', None)
>>> p.get_token()
('bang', None)
>>> p.get_token()
('hyphen', None)
>>> p.get_token()
('const', True)
>>> p.get_token()
('str', 'hi\n')
>>> p.get_token()
('prompt', 'mom')
>>> p.get_token()
(None, None)
'''
if self.pushed_token:
ans = self.pushed_token
self.indent = self.pushed_indent
self.column = self.pushed_column
self.pushed_token = self.pushed_column = self.pushed_indent = None
if check_token and check_token != ans[0]:
self.SyntaxError("expected %s, got %s" % (check_token, ans[0]))
#print "get_token: returning pushed_token", ans # FIX
return ans
if self.column < len(self.line): self.skip_spaces()
if self.column >= len(self.line):
self.readline()
if self.eof:
self.last_token = None, None
#print "get_token: returning EOF" # FIX
return self.last_token
self.last_line = self.line
self.last_lineno = self.lineno
self.last_column = self.column
self.last_indent = self.indent
match = self.tokenizer.match(self.line, self.column)
if not match: self.SyntaxError("Scanner error: no legal token")
token = match.lastgroup
chars = match.group(token)
end = match.end()
indent, ignore = scanner.count_indent(self.line[self.column:end], True)
self.indent += indent
self.column = end
if token == 'str' or token == 'prompt':
value = scanner.unescape(chars)
elif token == 'const':
value = eval(chars)
elif token == 'number':
try:
value = int(chars)
except ValueError:
value = float(chars)
elif token == 'param' or token == 'id':
value = chars
elif token == 'regexp1' or token == 'regexp2':
# FIX
token = 'regexp'
value = chars
self.lineno += chars.count('\n')
last_nl = chars.rfind('\n')
if last_nl >= 0:
self.column = len(chars) - last_nl + 4
else:
value = None
if check_token and check_token != token:
self.SyntaxError("expected %s, got %s" % (check_token, token))
self.last_token = str(token), value
#print "get_token: returning", self.last_token # FIX
return self.last_token
