def testTokenizingFuncDef(self):
self.assertEqual(list(Tokenize('写九九表咋整：整完了。')), [
IdentifierToken('写九九表'),
Keyword('咋整：'),
Keyword('整完了'),
Keyword('。'),
])
def testTokenizingFuncDef(self):
self.assertEqual(list(Tokenize('写九九表咋整：整完了。')), [
Token(TK_IDENTIFIER, '写九九表'),
Keyword('咋整：'),
Keyword('整完了'),
Keyword('。'),
])
def testParseArithmeticExpr(self):
self.assertEqual(ParseExprFromStr('5加六')[0],
ArithmeticExpr(
IntegerLiteralExpr(5),
Keyword('加'),
IntegerLiteralExpr(6)
))
self.assertEqual(ParseExprFromStr('5加六乘3')[0],
ArithmeticExpr(
IntegerLiteralExpr(5),
Keyword('加'),
ArithmeticExpr(
IntegerLiteralExpr(6),
Keyword('乘'),
IntegerLiteralExpr(3))))
self.assertEqual(ParseExprFromStr('5减六减老王')[0],
ArithmeticExpr(
ArithmeticExpr(
IntegerLiteralExpr(5),
Keyword('减'),
IntegerLiteralExpr(6)
),
Keyword('减'),
VariableExpr('老王'))
)
def testParseArithmeticExpr(self):
self.assertEqual(ParseExprFromStr(u'5加六')[0],
ArithmeticExpr(
LiteralExpr(Token(TK_INTEGER_LITERAL, 5)),
Keyword(u'加'),
LiteralExpr(Token(TK_INTEGER_LITERAL, 6))
))
self.assertEqual(ParseExprFromStr(u'5加六乘3')[0],
ArithmeticExpr(
LiteralExpr(Token(TK_INTEGER_LITERAL, 5)),
Keyword(u'加'),
ArithmeticExpr(
LiteralExpr(Token(TK_INTEGER_LITERAL, 6)),
Keyword(u'乘'),
LiteralExpr(Token(TK_INTEGER_LITERAL, 3)))))
self.assertEqual(ParseExprFromStr(u'5减六减老王')[0],
ArithmeticExpr(
ArithmeticExpr(
LiteralExpr(Token(TK_INTEGER_LITERAL, 5)),
Keyword(u'减'),
LiteralExpr(Token(TK_INTEGER_LITERAL, 6))
),
Keyword(u'减'),
VariableExpr(Token(TK_IDENTIFIER, u'老王')))
)
def testTokenizingDecrements(self):
self.assertEqual(list(Tokenize('老王稍稍')), [
IdentifierToken('老王'),
Keyword('稍稍'),
])
self.assertEqual(list(Tokenize('老王稍三步')), [
IdentifierToken('老王'),
Keyword('稍'),
Token(TK_INTEGER_LITERAL, 3),
Keyword('步'),
])
def testTokenizeLoop(self):
self.assertEqual(list(Tokenize('老王从1到9磨叽：磨叽完了。')), [
Token(TK_IDENTIFIER, '老王'),
Keyword('从'),
Token(TK_INTEGER_LITERAL, 1),
Keyword('到'),
Token(TK_INTEGER_LITERAL, 9),
Keyword('磨叽：'),
Keyword('磨叽完了'),
Keyword('。'),
])
def testTokenizingIncrements(self):
self.assertEqual(list(Tokenize('老王走走')), [
Token(TK_IDENTIFIER, '老王'),
Keyword('走走'),
])
self.assertEqual(list(Tokenize('老王走两步')), [
Token(TK_IDENTIFIER, '老王'),
Keyword('走'),
Token(TK_INTEGER_LITERAL, 2),
Keyword('步'),
])
def testTokenizingDecrements(self):
self.assertEqual(list(Tokenize('老王退退')), [
Token(TK_IDENTIFIER, '老王'),
Keyword('退退'),
])
self.assertEqual(list(Tokenize('老王退三步')), [
Token(TK_IDENTIFIER, '老王'),
Keyword('退'),
Token(TK_INTEGER_LITERAL, 3),
Keyword('步'),
])
def testTokenizeArithmetic(self):
self.assertEqual(list(Tokenize('250加13减二乘五除以九')), [
Token(TK_INTEGER_LITERAL, 250),
Keyword('加'),
Token(TK_INTEGER_LITERAL, 13),
Keyword('减'),
Token(TK_INTEGER_LITERAL, 2),
Keyword('乘'),
Token(TK_INTEGER_LITERAL, 5),
Keyword('除以'),
Token(TK_INTEGER_LITERAL, 9),
])
def testParseTermExpr(self):
self.assertEqual(
ParseExprFromStr('老王乘五')[0],
ArithmeticExpr(VariableExpr('老王'), Keyword('乘'),
IntegerLiteralExpr(5)))
self.assertEqual(
ParseExprFromStr('五除以老王')[0],
ArithmeticExpr(IntegerLiteralExpr(5), Keyword('除以'),
VariableExpr('老王')))
self.assertEqual(
ParseExprFromStr('五除以老王乘老刘')[0],
ArithmeticExpr(
ArithmeticExpr(IntegerLiteralExpr(5), Keyword('除以'),
VariableExpr('老王')), Keyword('乘'),
VariableExpr('老刘')))
def testParsingComparison(self):
self.assertEquals(ParseToAst('唠唠：2比5大。'), [
Statement(
STMT_SAY,
ComparisonExpr(IntegerLiteralExpr(2), Keyword('大'),
IntegerLiteralExpr(5)))
])
def testParsingComparison(self):
self.assertEquals(ParseToAst('唠唠：2比5大。'), [
Statement(
STMT_SAY,
ComparisonExpr(LiteralExpr(Token(TK_INTEGER_LITERAL, 2)),
Keyword('大'),
LiteralExpr(Token(TK_INTEGER_LITERAL, 5))))
])
def testParseConcatExpr(self):
self.assertEqual(
ParseExprFromStr('老王加油、2、“哈”')[0],
ConcatExpr([
ArithmeticExpr(VariableExpr('老王'), Keyword('加'),
VariableExpr('油')),
IntegerLiteralExpr(2),
StringLiteralExpr('哈')
]))
def testParseTermExpr(self):
self.assertEqual(
ParseExprFromStr('老王乘五')[0],
ArithmeticExpr(VariableExpr(Token(TK_IDENTIFIER, '老王')),
Keyword('乘'),
LiteralExpr(Token(TK_INTEGER_LITERAL, 5))))
self.assertEqual(
ParseExprFromStr('五除以老王')[0],
ArithmeticExpr(LiteralExpr(Token(TK_INTEGER_LITERAL, 5)),
Keyword('除以'),
VariableExpr(Token(TK_IDENTIFIER, '老王'))))
self.assertEqual(
ParseExprFromStr('五除以老王乘老刘')[0],
ArithmeticExpr(
ArithmeticExpr(LiteralExpr(Token(TK_INTEGER_LITERAL, 5)),
Keyword('除以'),
VariableExpr(Token(TK_IDENTIFIER, '老王'))),
Keyword('乘'), VariableExpr(Token(TK_IDENTIFIER, '老刘'))))
def testParseConcatExpr(self):
self.assertEqual(
ParseExprFromStr('老王加油、2、“哈”')[0],
ConcatExpr([
ArithmeticExpr(VariableExpr(Token(TK_IDENTIFIER, '老王')),
Keyword('加'),
VariableExpr(Token(TK_IDENTIFIER, '油'))),
LiteralExpr(Token(TK_INTEGER_LITERAL, 2)),
LiteralExpr(Token(TK_STRING_LITERAL, '哈'))
]))
def testParseConditional(self):
self.assertEqual(
ParseStmtFromStr('寻思：老王比五大？要行咧就唠唠：老王。')[0],
Statement(
STMT_CONDITIONAL,
(
ComparisonExpr(VariableExpr('老王'), Keyword('大'),
IntegerLiteralExpr(5)),
# then-branch
Statement(STMT_SAY, VariableExpr('老王')),
# else-branch
None)))
def testParseComparisonExpr(self):
self.assertEqual(ParseExprFromStr('5比6大')[0],
ComparisonExpr(
IntegerLiteralExpr(5),
Keyword('大'),
IntegerLiteralExpr(6)
))
self.assertEqual(ParseExprFromStr('老王加5比6小')[0],
ComparisonExpr(
ArithmeticExpr(
VariableExpr('老王'),
Keyword('加'),
IntegerLiteralExpr(5)),
Keyword('小'),
IntegerLiteralExpr(6)
))
self.assertEqual(ParseExprFromStr('老王跟老刘一样一样的')[0],
ComparisonExpr(
VariableExpr('老王'),
Keyword('一样一样的'),
VariableExpr('老刘')
))
self.assertEqual(ParseExprFromStr('老王加5跟6不是一样一样的')[0],
ComparisonExpr(
ArithmeticExpr(
VariableExpr('老王'),
Keyword('加'),
IntegerLiteralExpr(5)),
Keyword('不是一样一样的'),
IntegerLiteralExpr(6)
))
def testParseComparisonExpr(self):
self.assertEqual(
ParseExprFromStr('5比6大')[0],
ComparisonExpr(LiteralExpr(Token(TK_INTEGER_LITERAL, 5)),
Keyword('大'),
LiteralExpr(Token(TK_INTEGER_LITERAL, 6))))
self.assertEqual(
ParseExprFromStr('老王加5比6小')[0],
ComparisonExpr(
ArithmeticExpr(VariableExpr(Token(TK_IDENTIFIER, '老王')),
Keyword('加'),
LiteralExpr(Token(TK_INTEGER_LITERAL, 5))),
Keyword('小'), LiteralExpr(Token(TK_INTEGER_LITERAL, 6))))
self.assertEqual(
ParseExprFromStr('老王跟老刘一样一样的')[0],
ComparisonExpr(VariableExpr(Token(TK_IDENTIFIER, '老王')),
Keyword('一样一样的'),
VariableExpr(Token(TK_IDENTIFIER, '老刘'))))
self.assertEqual(
ParseExprFromStr('老王加5跟6不是一样一样的')[0],
ComparisonExpr(
ArithmeticExpr(VariableExpr(Token(TK_IDENTIFIER, '老王')),
Keyword('加'),
LiteralExpr(Token(TK_INTEGER_LITERAL, 5))),
Keyword('不是一样一样的'), LiteralExpr(Token(TK_INTEGER_LITERAL, 6))))
def testParseConditional(self):
self.assertEqual(
ParseStmtFromStr('寻思：老王比五大？要行咧就唠唠：老王。')[0],
Statement(
STMT_CONDITIONAL,
(
ComparisonExpr(VariableExpr(Token(TK_IDENTIFIER, '老王')),
Keyword('大'),
LiteralExpr(Token(TK_INTEGER_LITERAL, 5))),
# then-branch
Statement(STMT_SAY, VariableExpr(Token(
TK_IDENTIFIER, '老王'))),
# else-branch
None)))
def testTokenizeCompound(self):
self.assertEqual(list(Tokenize('开整：\n 唠唠：老王。\n整完了。')), [
Keyword('开整：'),
Keyword('唠唠'),
Keyword('：'),
Token(TK_IDENTIFIER, '老王'),
Keyword('。'),
Keyword('整完了'),
Keyword('。'),
])
def testTokenizeCompound(self):
self.assertEqual(list(Tokenize('开整：\n 唠唠：老王。\n整完了。')), [
Keyword('开整：'),
Keyword('唠唠'),
Keyword('：'),
IdentifierToken('老王'),
Keyword('。'),
Keyword('整完了'),
Keyword('。'),
])
def testTokenizingFuncCall(self):
self.assertEqual(list(Tokenize('整写九九表')), [
Keyword('整'),
Token(TK_IDENTIFIER, '写九九表'),
])
def testTokenizingConcat(self):
self.assertEqual(list(Tokenize('老刘、二')), [
Token(TK_IDENTIFIER, '老刘'),
Keyword('、'),
Token(TK_INTEGER_LITERAL, 2),
])
def testTokenizingConcat(self):
self.assertEqual(list(Tokenize('老刘、二')), [
IdentifierToken('老刘'),
Keyword('、'),
Token(TK_INTEGER_LITERAL, 2),
])
def testTokenize(self):
self.assertEqual(list(BasicTokenize('【阶乘】')), [
Token(TK_IDENTIFIER, '阶乘'),
])
self.assertEqual(list(BasicTokenize('【 阶 乘 】')), [
Token(TK_IDENTIFIER, '阶乘'),
])
self.assertEqual(list(BasicTokenize('【阶乘】（那啥）')), [
Token(TK_IDENTIFIER, '阶乘'),
Keyword('（'),
Token(TK_CHAR, '那'),
Token(TK_CHAR, '啥'),
Keyword('）'),
])
self.assertEqual(list(BasicTokenize('“ ”')), [
Keyword('“'),
Token(TK_STRING_LITERAL, ' '),
Keyword('”'),
])
self.assertEqual(list(BasicTokenize('“”')), [
Keyword('“'),
Token(TK_STRING_LITERAL, ''),
Keyword('”'),
])
self.assertEqual(list(BasicTokenize('“ A B ”')), [
Keyword('“'),
Token(TK_STRING_LITERAL, ' A B '),
Keyword('”'),
])
self.assertEqual(list(BasicTokenize('老张')), [
Token(TK_CHAR, '老'),
Token(TK_CHAR, '张'),
])
self.assertEqual(list(BasicTokenize(' 老 张 ')), [
Token(TK_CHAR, '老'),
Token(TK_CHAR, '张'),
])
self.assertEqual(list(Tokenize('# 123456\n老张')),
[Token(TK_IDENTIFIER, '老张')])
self.assertEqual(list(Tokenize('老张')), [Token(TK_IDENTIFIER, '老张')])
self.assertEqual(ParseInteger('老张'), (None, '老张'))
self.assertEqual(list(ParseChars('老张')), [Token(TK_IDENTIFIER, '老张')])
self.assertEqual(list(Tokenize('老张是活雷锋')),
[Token(TK_IDENTIFIER, '老张'),
Keyword('是活雷锋')])
self.assertEqual(list(Tokenize('老张是 活雷\n锋 。 ')), [
Token(TK_IDENTIFIER, '老张'),
Keyword('是活雷锋'),
Keyword('。'),
])
self.assertEqual(list(Tokenize('老张是活雷锋。\n老王是活雷锋。\n')), [
Token(TK_IDENTIFIER, '老张'),
Keyword('是活雷锋'),
Keyword('。'),
Token(TK_IDENTIFIER, '老王'),
Keyword('是活雷锋'),
Keyword('。'),
])
self.assertEqual(list(Tokenize('老张装250。\n老王装老张。\n')), [
Token(TK_IDENTIFIER, '老张'),
Keyword('装'),
Token(TK_INTEGER_LITERAL, 250),
Keyword('。'),
Token(TK_IDENTIFIER, '老王'),
Keyword('装'),
Token(TK_IDENTIFIER, '老张'),
Keyword('。')
])
self.assertEqual(list(Tokenize('唠唠：“你好”。')), [
Keyword('唠唠'),
Keyword('：'),
Keyword('“'),
Token(TK_STRING_LITERAL, '你好'),
Keyword('”'),
Keyword('。')
])
def testTokenize(self):
self.assertEqual(list(BasicTokenize(u'【阶乘】')), [
Token(TK_IDENTIFIER, u'阶乘'),
])
self.assertEqual(list(BasicTokenize(u'【 阶 乘 】')), [
Token(TK_IDENTIFIER, u'阶乘'),
])
self.assertEqual(list(BasicTokenize(u'【阶乘】（几）')), [
Token(TK_IDENTIFIER, u'阶乘'),
Keyword(u'（'),
Token(TK_CHAR, u'几'),
Keyword(u'）'),
])
self.assertEqual(list(BasicTokenize(u'“ ”')), [
Keyword(u'“'),
Token(TK_STRING_LITERAL, u' '),
Keyword(u'”'),
])
self.assertEqual(list(BasicTokenize(u'“”')), [
Keyword(u'“'),
Token(TK_STRING_LITERAL, u''),
Keyword(u'”'),
])
self.assertEqual(list(BasicTokenize(u'“ A B ”')), [
Keyword(u'“'),
Token(TK_STRING_LITERAL, u' A B '),
Keyword(u'”'),
])
self.assertEqual(list(BasicTokenize(u'老张')), [
Token(TK_CHAR, u'老'),
Token(TK_CHAR, u'张'),
])
self.assertEqual(list(BasicTokenize(u' 老 张 ')), [
Token(TK_CHAR, u'老'),
Token(TK_CHAR, u'张'),
])
self.assertEqual(list(Tokenize(u'# 123456\n老张')),
[Token(TK_IDENTIFIER, u'老张')])
self.assertEqual(list(Tokenize(u'老张')), [Token(TK_IDENTIFIER, u'老张')])
self.assertEqual(ParseInteger(u'老张'), (None, u'老张'))
self.assertEqual(list(ParseChars(u'老张')),
[Token(TK_IDENTIFIER, u'老张')])
self.assertEqual(list(Tokenize(u'老张是活雷锋')),
[Token(TK_IDENTIFIER, u'老张'),
Keyword(u'是活雷锋')])
self.assertEqual(list(Tokenize(u'老张是 活雷\n锋 。 ')), [
Token(TK_IDENTIFIER, u'老张'),
Keyword(u'是活雷锋'),
Keyword(u'。'),
])
self.assertEqual(list(Tokenize(u'老张是活雷锋。\n老王是活雷锋。\n')), [
Token(TK_IDENTIFIER, u'老张'),
Keyword(u'是活雷锋'),
Keyword(u'。'),
Token(TK_IDENTIFIER, u'老王'),
Keyword(u'是活雷锋'),
Keyword(u'。'),
])
self.assertEqual(list(Tokenize(u'老张装250。\n老王装老张。\n')), [
Token(TK_IDENTIFIER, u'老张'),
Keyword(u'装'),
Token(TK_INTEGER_LITERAL, 250),
Keyword(u'。'),
Token(TK_IDENTIFIER, u'老王'),
Keyword(u'装'),
Token(TK_IDENTIFIER, u'老张'),
Keyword(u'。')
])
self.assertEqual(list(Tokenize(u'唠唠：“你好”。')), [
Keyword(u'唠唠'),
Keyword(u'：'),
Keyword(u'“'),
Token(TK_STRING_LITERAL, u'你好'),
Keyword(u'”'),
Keyword(u'。')
])
def testTokenizingFuncCall(self):
self.assertEqual(list(Tokenize('整写九九表')), [
Keyword('整'),
IdentifierToken('写九九表'),
])