def testStringLiteral(self): lex = Lexer() lex.setSrc('"asdasd" "ss\\"" \'\\s\'') self.assertEqual(lex.getToken(), (TOK.STRING, 'asdasd')) self.assertEqual(lex.getToken(), (TOK.STRING, 'ss"')) self.assertEqual(lex.getToken(), (TOK.STRING, 's')) lex.setSrc('"\\\\" "\\\'" "\\"" "\\b" "\\t" "\\n" "\\v" "\\f" "\\r"') self.assertEqual(lex.getToken(), (TOK.STRING, '\\')) self.assertEqual(lex.getToken(), (TOK.STRING, '\'')) self.assertEqual(lex.getToken(), (TOK.STRING, '"')) self.assertEqual(lex.getToken(), (TOK.STRING, '\b')) self.assertEqual(lex.getToken(), (TOK.STRING, '\t')) self.assertEqual(lex.getToken(), (TOK.STRING, '\n')) self.assertEqual(lex.getToken(), (TOK.STRING, '\v')) self.assertEqual(lex.getToken(), (TOK.STRING, '\f')) self.assertEqual(lex.getToken(), (TOK.STRING, '\r')) lex.setSrc('"asd\\\ndd"') self.assertEqual(lex.getToken(), (TOK.STRING, 'asddd')) lex.setSrc('"dd\\u2345" \'"s\\x34\'') self.assertEqual(lex.getToken(), (TOK.STRING, 'dd\u2345')) self.assertEqual(lex.getToken(), (TOK.STRING, '"s\x34')) lex.setSrc('"\\x0A"') self.assertEqual(lex.getToken(), (TOK.STRING, '\x0A')) lex.setSrc("'\\0' '\\0a'") self.assertEqual(lex.getToken(), (TOK.STRING, '\0')) self.assertEqual(lex.getToken(), (TOK.STRING, '\0a'))
def testDivPunctuator(self): lex=Lexer() lex.setSrc('asd/333/=') lex.getToken() self.assertEqual(lex.getToken(), (TOK.DIV_PUNCTUATOR,'/')) lex.getToken() self.assertEqual(lex.getToken(), (TOK.DIV_PUNCTUATOR,'/='))
def testPos(self): lex = Lexer() lex.setSrc('\n123 {"asd"\n +\n/123/') lex.getToken() pos = lex.getLastTokenPos() self.assertEqual(pos['line'], 2) self.assertEqual(pos['column'], 1) lex.getToken() pos = lex.getLastTokenPos() self.assertEqual(pos['line'], 2) self.assertEqual(pos['column'], 5) lex.getToken() pos = lex.getLastTokenPos() self.assertEqual(pos['line'], 2) self.assertEqual(pos['column'], 6) lex.getToken() pos = lex.getLastTokenPos() self.assertEqual(pos['line'], 3) self.assertEqual(pos['column'], 2) lex.getToken() pos = lex.getLastTokenPos() self.assertEqual(pos['line'], 4) self.assertEqual(pos['column'], 1)
def testCommentToken(self): lex = Lexer() lex.setSrc('//asdasd') self.assertEqual(lex.getToken(), (TOK.EOF, '')) lex.setSrc('//') self.assertEqual(lex.getToken(), (TOK.EOF, ''))
def testNotIDStartAfterNumeric(self): lex = Lexer() lex.setSrc("0.233a") self.assertEqual(lex.getToken(), (TOK.ERROR , '0.233')) lex.setSrc("233b") self.assertEqual(lex.getToken(), (TOK.ERROR , '233')) lex.setSrc("0x233br") self.assertEqual(lex.getToken(), (TOK.ERROR , '0x233b'))
def testRewind(self): lex = Lexer() lex.setSrc('asd/ee/') self.assertEqual(lex.getToken(),(TOK.ID, 'asd')) lex.rewind() self.assertEqual(lex.getToken(),(TOK.ID, 'asd')) self.assertEqual(lex.getToken(True),(TOK.REGEXP, '/ee/')) lex.rewind() self.assertEqual(lex.getToken(False),(TOK.DIV_PUNCTUATOR, '/'))
def testExponentialNumbers(self): lex = Lexer() lex.setSrc("22e3 22.34e33 .232E23") self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22e3')) self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22.34e33')) self.assertEqual(lex.getToken(), (TOK.NUMERIC , '.232E23')) lex.setSrc("22.34e-33 .232E+23") self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22.34e-33')) self.assertEqual(lex.getToken(), (TOK.NUMERIC , '.232E+23')) lex.setSrc("0.e+0") self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0.e+0'))
def testFloatNumbers(self): lex = Lexer() lex.setSrc("22.34 .232") self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22.34')) self.assertEqual(lex.getToken(), (TOK.NUMERIC , '.232')) lex.setSrc("0. 0.3") self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0.')) self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0.3')) lex.setSrc("22. 44.") self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22.')) self.assertEqual(lex.getToken(), (TOK.NUMERIC , '44.'))
def testFutureStrictReservedWords(self): lex = Lexer() rw = ['implements','let','private','public','yield','interface','package','protected','static'] lex.strictMode = True lex.setSrc(' '.join(rw)) for w in rw: self.assertEqual(lex.getToken(), (TOK.FUTURE_RESERVED , w)) lex.strictMode = False lex.setSrc(' '.join(rw)) for w in rw: self.assertEqual(lex.getToken(), (TOK.ID , w))
def testRegExp(self): lex=Lexer() lex.setSrc('asd/asddd/iuy') lex.getToken() self.assertEqual(lex.getToken(True), (TOK.REGEXP,'/asddd/iuy')) lex.setSrc('/asddd/') self.assertEqual(lex.getToken(True), (TOK.REGEXP,'/asddd/')) lex.setSrc('/\\dasddd/') self.assertEqual(lex.getToken(True), (TOK.REGEXP,'/\\dasddd/')) lex.setSrc('/[asd\\sdd](e)?:(1)+[a-z,1-9]asddd/') self.assertEqual(lex.getToken(True), (TOK.REGEXP,'/[asd\\sdd](e)?:(1)+[a-z,1-9]asddd/'))
def testRawComment(self): lex = Lexer() lex.setSrc('//sdfsdff\n//asdasd') self.assertEqual(lex.getNext(), (TOK.SINGLE_COMMENT,'sdfsdff')) lex.getNext() self.assertEqual(lex.getNext(), (TOK.SINGLE_COMMENT,'asdasd')) lex.setSrc("""//sdfdff ttttt""") self.assertEqual(lex.getNext(), (TOK.SINGLE_COMMENT,'sdfdff')) lex.setSrc('/*asdasd*/') self.assertEqual(lex.getNext(), (TOK.MULTI_COMMENT, 'asdasd')) lex.setSrc('/*asd\nasd*/123123') self.assertEqual(lex.getNext(), (TOK.MULTINL_COMMENT, 'asd\nasd'))
def testHexNumbers(self): lex = Lexer() lex.setSrc("0x233aD") self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0x233aD'))
def testSimpleNumber(self): lex = Lexer() lex.setSrc("3213") self.assertEqual(lex.getToken(), (TOK.NUMERIC , '3213'))
def testZeroNumber(self): lex = Lexer() lex.setSrc("0") self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0'))
def testWhitespaceSkipping(self): lex = Lexer() lex.setSrc(" \t \v ") self.assertEqual(lex.getToken(), (TOK.EOF , ''))
def testID(self): lex = Lexer() lex.setSrc('aA$34_a bdd') self.assertEqual(lex.getToken(), (TOK.ID, 'aA$34_a')) self.assertEqual(lex.getToken(), (TOK.ID, 'bdd'))
def testFutureReservedWords(self): lex = Lexer() rw = ['class','enum','extends','super','const','export','import'] lex.setSrc(' '.join(rw)) for w in rw: self.assertEqual(lex.getToken(), (TOK.FUTURE_RESERVED , w))
def testPunctuator(self): lex = Lexer() ps = ['^=','{','}','(',')','[',']','.',';',',','<','>','<=','>=','==','!=','===','!==','+','-','*','%','++','--','<<','>>','>>>','&','|','^','!','~','&&','||','?',':','=','+=','-=','*=','%=','<<=','>>=','>>>=','&=','|=','^='] lex.setSrc(' '.join(ps)) for p in ps: self.assertEqual(lex.getToken(), (TOK.PUNCTUATOR , p))
def testBool(self): lex = Lexer() lex.setSrc('true false') self.assertEqual(lex.getToken(), (TOK.BOOL, 'true')) self.assertEqual(lex.getToken(), (TOK.BOOL, 'false'))
def testNull(self): lex = Lexer() lex.setSrc('null null') self.assertEqual(lex.getToken(), (TOK.NULL, 'null')) self.assertEqual(lex.getToken(), (TOK.NULL, 'null'))
def testReservedWords(self): lex = Lexer() rw = ['break','do','instanceof','typeof','case','else','new','var','catch','finally','return','void','continue','for','switch','while','debugger','function','this','with','default','if','throw','delete','in','try'] lex.setSrc(' '.join(rw)) for w in rw: self.assertEqual(lex.getToken(), (TOK.RESERVED , w))
def __init__(self): self.state = 0 self.src = '' self.lexer = Lexer()