def testPeepAheadOnce(self): global tokenizer tokenizer=Tokenizer('a b') a=tokenizer.peepahead() self.assertEqual(valueof(a),'a') a=tokenizer.peepahead() self.assertEqual(valueof(a),'a') a=tokenizer.advance() self.assertEqual(valueof(a),'a') b=tokenizer.peepahead() self.assertEqual(valueof(b),'b') b=tokenizer.peepahead() self.assertEqual(valueof(b),'b') b=tokenizer.advance() self.assertEqual(valueof(b),'b')
def testAdvanceBeforePeepAheadShouldPass(self): global tokenizer tokenizer=Tokenizer('a b') a=tokenizer.advance() self.assertEqual(valueof(a),'a') b=tokenizer.peepahead() self.assertEqual(valueof(b),'b')
def testAdvanceBeforePeepAheadTwiceShouldPass(self): global tokenizer tokenizer=Tokenizer('a') a=tokenizer.advance() self.assertEqual(valueof(a),'a') end=tokenizer.advance() self.assertEqual(valueof(end),'(end)') end=tokenizer.peepahead() self.assertEqual(valueof(end),'(end)')
def parse(str): array=[] CKeyword.defineTable={} CKeyword.stringTable={} tokenizer=Tokenizer(str) token=tokenizer.peepahead() CExpression.configure_tokenizer_Expression(tokenizer) CKeyword.configure_tokenizer_Keyword(tokenizer) while(token.first == ';'): tokenizer.advance() token=tokenizer.peepahead() while(token.first != '(end)'): if hasattr(token,'std'): temp=CKeyword.parseStatement() else: temp=CExpression.expression(0) tokenizer.advance(';') token=tokenizer.peepahead() array.append(temp) return array