Beispiel #1
0
 def skip_spaces(self, pre_increment=0):
     if pre_increment:
         indent, chars = \
             scanner.count_indent(self.line[self.column:
                                            self.column+pre_increment], True)
         self.indent += indent
         self.column += chars
     indent, chars = scanner.count_indent(self.line[self.column:])
     self.indent += indent
     self.column += chars
Beispiel #2
0
 def skip_spaces(self, pre_increment=0):
     if pre_increment:
         indent, chars = \
             scanner.count_indent(self.line[self.column:
                                            self.column+pre_increment], True)
         self.indent += indent
         self.column += chars
     indent, chars = scanner.count_indent(self.line[self.column:])
     self.indent += indent
     self.column += chars
Beispiel #3
0
 def readline(self):
     r'''
         >>> from StringIO import StringIO
         >>> p = kqb_parser(StringIO("""
         ... line 1
         ...     # this should be ignored
         ...  line 2
         ...
         ...   line 3
         ... """))
         >>> p.readline()
         >>> p.indent, p.line, p.lineno, p.column
         (0, 'line 1', 2, 0)
         >>> p.readline()
         >>> p.indent, p.line, p.lineno, p.column
         (1, ' line 2', 4, 1)
         >>> p.readline()
         >>> p.indent, p.line, p.lineno, p.column
         (2, '  line 3', 6, 2)
         >>> p.eof
         False
         >>> p.readline()
         >>> p.eof
         True
     '''
     while True:
         line = self.f.readline()
         if line == '':
             self.eof = True
             break
         self.lineno += 1
         line = line.rstrip('\n')
         if not self.blank_line.match(line):
             self.indent, self.column = scanner.count_indent(line)
             self.line = line
             break
Beispiel #4
0
 def readline(self):
     r'''
         >>> from StringIO import StringIO
         >>> p = kqb_parser(StringIO("""
         ... line 1
         ...     # this should be ignored
         ...  line 2
         ...
         ...   line 3
         ... """))
         >>> p.readline()
         >>> p.indent, p.line, p.lineno, p.column
         (0, 'line 1', 2, 0)
         >>> p.readline()
         >>> p.indent, p.line, p.lineno, p.column
         (1, ' line 2', 4, 1)
         >>> p.readline()
         >>> p.indent, p.line, p.lineno, p.column
         (2, '  line 3', 6, 2)
         >>> p.eof
         False
         >>> p.readline()
         >>> p.eof
         True
     '''
     while True:
         line = self.f.readline()
         if line == '':
             self.eof = True
             break
         self.lineno += 1
         line = line.rstrip('\n')
         if not self.blank_line.match(line):
             self.indent, self.column = scanner.count_indent(line)
             self.line = line
             break
Beispiel #5
0
 def get_block_string(self, stop=None, hanging=False, ending_newlines=False):
     r'''
         >>> from StringIO import StringIO
         >>> f = StringIO(r"""
         ...     line 1 # comment
         ...        more stuff
         ...     last line
         ... blah    hanging line 1
         ...
         ...         line 2
         ...           indented
         ...         last line
         ...         ! the end !
         ... """)
         >>> f.name = 'StringIO'
         >>> p = kqb_parser(f)
         >>> p.get_block_string()
         u'line 1 # comment\n   more stuff\nlast line'
         >>> p.column = 4
         >>> p.indent = 4
         >>> p.get_block_string('!', True)
         u'hanging line 1\n\nline 2\n  indented\nlast line'
         >>> f = StringIO(r"""
         ...     ! line 1 # comment
         ...          more stuff
         ...       last line
         ... blah
         ... """)
         >>> f.name = 'StringIO'
         >>> p = kqb_parser(f)
         >>> p.readline()
         >>> p.get_token('bang')
         ('bang', None)
         >>> p.get_block_string(hanging=True)
         u'line 1 # comment\n   more stuff\nlast line'
     '''
     if hanging:
         indent, more_chars = \
             scanner.count_indent(self.line[self.column:])
         self.indent += indent
         self.column += more_chars
     else:
         self.readline()
     indent = self.indent
     if self.eof: self.SyntaxError("expected block string, got EOF", False)
     rest_line = self.line[self.column:]
     if self.blank_line.match(rest_line):
         ans = []
     else:
         ans = [rest_line]
     while not self.eof:
         last_lineno = self.lineno
         self.readline()
         if ending_newlines:
             for i in range(self.lineno - last_lineno - 1): ans.append('')
         if self.eof or self.indent < indent or \
            stop and self.line[self.column:].startswith(stop):
             break
         if not ending_newlines:
             for i in range(self.lineno - last_lineno - 1): ans.append('')
         ans.append(' ' * (self.indent - indent) + self.line[self.column:])
     if not ans: self.SyntaxError("expected block string", False)
     return u'\n'.join(scanner.unescape(str) for str in ans)
Beispiel #6
0
 def get_token(self, check_token=None):
     r'''
         >>> from StringIO import StringIO
         >>> f = StringIO(r"""
         ...   line 1=2.5: ( /\s* /, $foo) # comment
         ... ,|!-True  "hi\n"  [mom]
         ... """)
         >>> f.name = 'StringIO'
         >>> p = kqb_parser(f)
         >>> p.get_token()
         ('id', 'line')
         >>> p.get_token()
         ('number', 1)
         >>> p.get_token()
         ('equal', None)
         >>> p.get_token()
         ('number', 2.5)
         >>> p.get_token()
         ('colon', None)
         >>> p.get_token()
         ('lparen', None)
         >>> p.get_token()
         ('regexp', '\\s* ')
         >>> p.get_token()
         ('comma', None)
         >>> p.get_token()
         ('param', 'foo')
         >>> p.get_token()
         ('rparen', None)
         >>> p.get_token()
         ('comma', None)
         >>> p.get_token()
         ('bar', None)
         >>> p.get_token()
         ('bang', None)
         >>> p.get_token()
         ('hyphen', None)
         >>> p.get_token()
         ('const', True)
         >>> p.get_token()
         ('str', 'hi\n')
         >>> p.get_token()
         ('prompt', 'mom')
         >>> p.get_token()
         (None, None)
     '''
     if self.pushed_token:
         ans = self.pushed_token
         self.indent = self.pushed_indent
         self.column = self.pushed_column
         self.pushed_token = self.pushed_column = self.pushed_indent = None
         if check_token and check_token != ans[0]:
             self.SyntaxError("expected %s, got %s" % (check_token, ans[0]))
         #print "get_token: returning pushed_token", ans  # FIX
         return ans
     if self.column < len(self.line): self.skip_spaces()
     if self.column >= len(self.line):
         self.readline()
         if self.eof:
             self.last_token = None, None
             #print "get_token: returning EOF"  # FIX
             return self.last_token
     self.last_line = self.line
     self.last_lineno = self.lineno
     self.last_column = self.column
     self.last_indent = self.indent
     match = self.tokenizer.match(self.line, self.column)
     if not match: self.SyntaxError("Scanner error: no legal token")
     token = match.lastgroup
     chars = match.group(token)
     end = match.end()
     indent, ignore = scanner.count_indent(self.line[self.column:end], True)
     self.indent += indent
     self.column = end
     if token == 'str' or token == 'prompt':
         value = scanner.unescape(chars)
     elif token == 'const':
         value = eval(chars)
     elif token == 'number':
         try:
             value = int(chars)
         except ValueError:
             value = float(chars)
     elif token == 'param' or token == 'id':
         value = chars
     elif token == 'regexp1' or token == 'regexp2':
         # FIX
         token = 'regexp'
         value = chars
         self.lineno += chars.count('\n')
         last_nl = chars.rfind('\n')
         if last_nl >= 0:
             self.column = len(chars) - last_nl + 4
     else:
         value = None
     if check_token and check_token != token:
         self.SyntaxError("expected %s, got %s" % (check_token, token))
     self.last_token = str(token), value
     #print "get_token: returning", self.last_token  # FIX
     return self.last_token
Beispiel #7
0
 def get_block_string(self,
                      stop=None,
                      hanging=False,
                      ending_newlines=False):
     r'''
         >>> from StringIO import StringIO
         >>> f = StringIO(r"""
         ...     line 1 # comment
         ...        more stuff
         ...     last line
         ... blah    hanging line 1
         ...
         ...         line 2
         ...           indented
         ...         last line
         ...         ! the end !
         ... """)
         >>> f.name = 'StringIO'
         >>> p = kqb_parser(f)
         >>> p.get_block_string()
         u'line 1 # comment\n   more stuff\nlast line'
         >>> p.column = 4
         >>> p.indent = 4
         >>> p.get_block_string('!', True)
         u'hanging line 1\n\nline 2\n  indented\nlast line'
         >>> f = StringIO(r"""
         ...     ! line 1 # comment
         ...          more stuff
         ...       last line
         ... blah
         ... """)
         >>> f.name = 'StringIO'
         >>> p = kqb_parser(f)
         >>> p.readline()
         >>> p.get_token('bang')
         ('bang', None)
         >>> p.get_block_string(hanging=True)
         u'line 1 # comment\n   more stuff\nlast line'
     '''
     if hanging:
         indent, more_chars = \
             scanner.count_indent(self.line[self.column:])
         self.indent += indent
         self.column += more_chars
     else:
         self.readline()
     indent = self.indent
     if self.eof: self.SyntaxError("expected block string, got EOF", False)
     rest_line = self.line[self.column:]
     if self.blank_line.match(rest_line):
         ans = []
     else:
         ans = [rest_line]
     while not self.eof:
         last_lineno = self.lineno
         self.readline()
         if ending_newlines:
             for i in range(self.lineno - last_lineno - 1):
                 ans.append('')
         if self.eof or self.indent < indent or \
            stop and self.line[self.column:].startswith(stop):
             break
         if not ending_newlines:
             for i in range(self.lineno - last_lineno - 1):
                 ans.append('')
         ans.append(' ' * (self.indent - indent) + self.line[self.column:])
     if not ans: self.SyntaxError("expected block string", False)
     return '\n'.join(scanner.unescape(str) for str in ans)
Beispiel #8
0
 def get_token(self, check_token=None):
     r'''
         >>> from StringIO import StringIO
         >>> f = StringIO(r"""
         ...   line 1=2.5: ( /\s* /, $foo) # comment
         ... ,|!-True  "hi\n"  [mom]
         ... """)
         >>> f.name = 'StringIO'
         >>> p = kqb_parser(f)
         >>> p.get_token()
         ('id', 'line')
         >>> p.get_token()
         ('number', 1)
         >>> p.get_token()
         ('equal', None)
         >>> p.get_token()
         ('number', 2.5)
         >>> p.get_token()
         ('colon', None)
         >>> p.get_token()
         ('lparen', None)
         >>> p.get_token()
         ('regexp', '\\s* ')
         >>> p.get_token()
         ('comma', None)
         >>> p.get_token()
         ('param', 'foo')
         >>> p.get_token()
         ('rparen', None)
         >>> p.get_token()
         ('comma', None)
         >>> p.get_token()
         ('bar', None)
         >>> p.get_token()
         ('bang', None)
         >>> p.get_token()
         ('hyphen', None)
         >>> p.get_token()
         ('const', True)
         >>> p.get_token()
         ('str', 'hi\n')
         >>> p.get_token()
         ('prompt', 'mom')
         >>> p.get_token()
         (None, None)
     '''
     if self.pushed_token:
         ans = self.pushed_token
         self.indent = self.pushed_indent
         self.column = self.pushed_column
         self.pushed_token = self.pushed_column = self.pushed_indent = None
         if check_token and check_token != ans[0]:
             self.SyntaxError("expected %s, got %s" % (check_token, ans[0]))
         #print "get_token: returning pushed_token", ans  # FIX
         return ans
     if self.column < len(self.line): self.skip_spaces()
     if self.column >= len(self.line):
         self.readline()
         if self.eof:
             self.last_token = None, None
             #print "get_token: returning EOF"  # FIX
             return self.last_token
     self.last_line = self.line
     self.last_lineno = self.lineno
     self.last_column = self.column
     self.last_indent = self.indent
     match = self.tokenizer.match(self.line, self.column)
     if not match: self.SyntaxError("Scanner error: no legal token")
     token = match.lastgroup
     chars = match.group(token)
     end = match.end()
     indent, ignore = scanner.count_indent(self.line[self.column:end], True)
     self.indent += indent
     self.column = end
     if token == 'str' or token == 'prompt':
         value = scanner.unescape(chars)
     elif token == 'const':
         value = eval(chars)
     elif token == 'number':
         try:
             value = int(chars)
         except ValueError:
             value = float(chars)
     elif token == 'param' or token == 'id':
         value = chars
     elif token == 'regexp1' or token == 'regexp2':
         # FIX
         token = 'regexp'
         value = chars
         self.lineno += chars.count('\n')
         last_nl = chars.rfind('\n')
         if last_nl >= 0:
             self.column = len(chars) - last_nl + 4
     else:
         value = None
     if check_token and check_token != token:
         self.SyntaxError("expected %s, got %s" % (check_token, token))
     self.last_token = str(token), value
     #print "get_token: returning", self.last_token  # FIX
     return self.last_token