def p_data_string(p): ''' data : STRING_TOK ''' if goal_mode: if p[1].startswith("'''") or p[1].startswith('"""'): p[0] = scanner.unescape(p[1][3:-3]) else: p[0] = scanner.unescape(p[1][1:-1]) else: p[0] = p[1]
def get_block_string(self, stop=None, hanging=False, ending_newlines=False): r''' >>> from StringIO import StringIO >>> f = StringIO(r""" ... line 1 # comment ... more stuff ... last line ... blah hanging line 1 ... ... line 2 ... indented ... last line ... ! the end ! ... """) >>> f.name = 'StringIO' >>> p = kqb_parser(f) >>> p.get_block_string() u'line 1 # comment\n more stuff\nlast line' >>> p.column = 4 >>> p.indent = 4 >>> p.get_block_string('!', True) u'hanging line 1\n\nline 2\n indented\nlast line' >>> f = StringIO(r""" ... ! line 1 # comment ... more stuff ... last line ... blah ... """) >>> f.name = 'StringIO' >>> p = kqb_parser(f) >>> p.readline() >>> p.get_token('bang') ('bang', None) >>> p.get_block_string(hanging=True) u'line 1 # comment\n more stuff\nlast line' ''' if hanging: indent, more_chars = \ scanner.count_indent(self.line[self.column:]) self.indent += indent self.column += more_chars else: self.readline() indent = self.indent if self.eof: self.SyntaxError("expected block string, got EOF", False) rest_line = self.line[self.column:] if self.blank_line.match(rest_line): ans = [] else: ans = [rest_line] while not self.eof: last_lineno = self.lineno self.readline() if ending_newlines: for i in range(self.lineno - last_lineno - 1): ans.append('') if self.eof or self.indent < indent or \ stop and self.line[self.column:].startswith(stop): break if not ending_newlines: for i in range(self.lineno - last_lineno - 1): ans.append('') ans.append(' ' * (self.indent - indent) + self.line[self.column:]) if not ans: self.SyntaxError("expected block string", False) return u'\n'.join(scanner.unescape(str) for str in ans)
def get_token(self, check_token=None): r''' >>> from StringIO import StringIO >>> f = StringIO(r""" ... line 1=2.5: ( /\s* /, $foo) # comment ... ,|!-True "hi\n" [mom] ... """) >>> f.name = 'StringIO' >>> p = kqb_parser(f) >>> p.get_token() ('id', 'line') >>> p.get_token() ('number', 1) >>> p.get_token() ('equal', None) >>> p.get_token() ('number', 2.5) >>> p.get_token() ('colon', None) >>> p.get_token() ('lparen', None) >>> p.get_token() ('regexp', '\\s* ') >>> p.get_token() ('comma', None) >>> p.get_token() ('param', 'foo') >>> p.get_token() ('rparen', None) >>> p.get_token() ('comma', None) >>> p.get_token() ('bar', None) >>> p.get_token() ('bang', None) >>> p.get_token() ('hyphen', None) >>> p.get_token() ('const', True) >>> p.get_token() ('str', 'hi\n') >>> p.get_token() ('prompt', 'mom') >>> p.get_token() (None, None) ''' if self.pushed_token: ans = self.pushed_token self.indent = self.pushed_indent self.column = self.pushed_column self.pushed_token = self.pushed_column = self.pushed_indent = None if check_token and check_token != ans[0]: self.SyntaxError("expected %s, got %s" % (check_token, ans[0])) #print "get_token: returning pushed_token", ans # FIX return ans if self.column < len(self.line): self.skip_spaces() if self.column >= len(self.line): self.readline() if self.eof: self.last_token = None, None #print "get_token: returning EOF" # FIX return self.last_token self.last_line = self.line self.last_lineno = self.lineno self.last_column = self.column self.last_indent = self.indent match = self.tokenizer.match(self.line, self.column) if not match: self.SyntaxError("Scanner error: no legal token") token = match.lastgroup chars = match.group(token) end = match.end() indent, ignore = scanner.count_indent(self.line[self.column:end], True) self.indent += indent self.column = end if token == 'str' or token == 'prompt': value = scanner.unescape(chars) elif token == 'const': value = eval(chars) elif token == 'number': try: value = int(chars) except ValueError: value = float(chars) elif token == 'param' or token == 'id': value = chars elif token == 'regexp1' or token == 'regexp2': # FIX token = 'regexp' value = chars self.lineno += chars.count('\n') last_nl = chars.rfind('\n') if last_nl >= 0: self.column = len(chars) - last_nl + 4 else: value = None if check_token and check_token != token: self.SyntaxError("expected %s, got %s" % (check_token, token)) self.last_token = str(token), value #print "get_token: returning", self.last_token # FIX return self.last_token
def get_block_string(self, stop=None, hanging=False, ending_newlines=False): r''' >>> from StringIO import StringIO >>> f = StringIO(r""" ... line 1 # comment ... more stuff ... last line ... blah hanging line 1 ... ... line 2 ... indented ... last line ... ! the end ! ... """) >>> f.name = 'StringIO' >>> p = kqb_parser(f) >>> p.get_block_string() u'line 1 # comment\n more stuff\nlast line' >>> p.column = 4 >>> p.indent = 4 >>> p.get_block_string('!', True) u'hanging line 1\n\nline 2\n indented\nlast line' >>> f = StringIO(r""" ... ! line 1 # comment ... more stuff ... last line ... blah ... """) >>> f.name = 'StringIO' >>> p = kqb_parser(f) >>> p.readline() >>> p.get_token('bang') ('bang', None) >>> p.get_block_string(hanging=True) u'line 1 # comment\n more stuff\nlast line' ''' if hanging: indent, more_chars = \ scanner.count_indent(self.line[self.column:]) self.indent += indent self.column += more_chars else: self.readline() indent = self.indent if self.eof: self.SyntaxError("expected block string, got EOF", False) rest_line = self.line[self.column:] if self.blank_line.match(rest_line): ans = [] else: ans = [rest_line] while not self.eof: last_lineno = self.lineno self.readline() if ending_newlines: for i in range(self.lineno - last_lineno - 1): ans.append('') if self.eof or self.indent < indent or \ stop and self.line[self.column:].startswith(stop): break if not ending_newlines: for i in range(self.lineno - last_lineno - 1): ans.append('') ans.append(' ' * (self.indent - indent) + self.line[self.column:]) if not ans: self.SyntaxError("expected block string", False) return '\n'.join(scanner.unescape(str) for str in ans)