def setUp(self):
self.lexer = CrystalLexer()
self.maxDiff = None
def lexer():
yield CrystalLexer()
class CrystalTest(unittest.TestCase):
def setUp(self):
self.lexer = CrystalLexer()
self.maxDiff = None
def testRangeSyntax1(self):
fragment = '1...3\n'
tokens = [
(Number.Integer, '1'),
(Operator, '...'),
(Number.Integer, '3'),
(Text, '\n'),
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
def testRangeSyntax2(self):
fragment = '1 .. 3\n'
tokens = [
(Number.Integer, '1'),
(Text, ' '),
(Operator, '..'),
(Text, ' '),
(Number.Integer, '3'),
(Text, '\n'),
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
def testInterpolationNestedCurly(self):
fragment = (
'"A#{ (3..5).group_by { |x| x/2}.map '
'do |k,v| "#{k}" end.join }" + "Z"\n')
tokens = [
(String.Double, '"'),
(String.Double, 'A'),
(String.Interpol, '#{'),
(Text, ' '),
(Punctuation, '('),
(Number.Integer, '3'),
(Operator, '..'),
(Number.Integer, '5'),
(Punctuation, ')'),
(Operator, '.'),
(Name, 'group_by'),
(Text, ' '),
(String.Interpol, '{'),
(Text, ' '),
(Operator, '|'),
(Name, 'x'),
(Operator, '|'),
(Text, ' '),
(Name, 'x'),
(Operator, '/'),
(Number.Integer, '2'),
(String.Interpol, '}'),
(Operator, '.'),
(Name, 'map'),
(Text, ' '),
(Keyword, 'do'),
(Text, ' '),
(Operator, '|'),
(Name, 'k'),
(Punctuation, ','),
(Name, 'v'),
(Operator, '|'),
(Text, ' '),
(String.Double, '"'),
(String.Interpol, '#{'),
(Name, 'k'),
(String.Interpol, '}'),
(String.Double, '"'),
(Text, ' '),
(Keyword, 'end'),
(Operator, '.'),
(Name, 'join'),
(Text, ' '),
(String.Interpol, '}'),
(String.Double, '"'),
(Text, ' '),
(Operator, '+'),
(Text, ' '),
(String.Double, '"'),
(String.Double, 'Z'),
(String.Double, '"'),
(Text, '\n'),
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
def testOperatorMethods(self):
fragment = '([] of Int32).[]?(5)\n'
tokens = [
(Punctuation, '('),
(Operator, '['),
(Operator, ']'),
(Text, ' '),
(Keyword, 'of'),
(Text, ' '),
(Name.Builtin, 'Int32'),
(Punctuation, ')'),
(Operator, '.'),
(Name.Operator, '[]?'),
(Punctuation, '('),
(Number.Integer, '5'),
(Punctuation, ')'),
(Text, '\n')
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
def testArrayAccess(self):
fragment = '[5][5]?\n'
tokens = [
(Operator, '['),
(Number.Integer, '5'),
(Operator, ']'),
(Operator, '['),
(Number.Integer, '5'),
(Operator, ']?'),
(Text, '\n')
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
def testNumbers(self):
for kind, testset in [
(Number.Integer, '0 1 1_000_000 1u8 11231231231121312i64'),
(Number.Float, '0.0 1.0_f32 1_f32 0f64 1e+4 1e111 1_234.567_890'),
(Number.Bin, '0b1001_0110 0b0u8'),
(Number.Oct, '0o17 0o7_i32'),
(Number.Hex, '0xdeadBEEF'),
]:
for fragment in testset.split():
self.assertEqual([(kind, fragment), (Text, '\n')],
list(self.lexer.get_tokens(fragment + '\n')))
for fragment in '01 0b2 0x129g2 0o12358'.split():
self.assertEqual(next(self.lexer.get_tokens(fragment + '\n'))[0],
Error)
def testChars(self):
for fragment in ["'a'", "'я'", "'\\u{1234}'", "'\n'"]:
self.assertEqual([(String.Char, fragment), (Text, '\n')],
list(self.lexer.get_tokens(fragment + '\n')))
self.assertEqual(next(self.lexer.get_tokens("'abc'"))[0], Error)
def testMacro(self):
fragment = (
'def<=>(other : self) : Int\n'
'{%for field in %w(first_name middle_name last_name)%}\n'
'cmp={{field.id}}<=>other.{{field.id}}\n'
'return cmp if cmp!=0\n'
'{%end%}\n'
'0\n'
'end\n')
tokens = [
(Keyword, 'def'),
(Name.Function, '<=>'),
(Punctuation, '('),
(Name, 'other'),
(Text, ' '),
(Punctuation, ':'),
(Text, ' '),
(Keyword.Pseudo, 'self'),
(Punctuation, ')'),
(Text, ' '),
(Punctuation, ':'),
(Text, ' '),
(Name.Builtin, 'Int'),
(Text, '\n'),
(String.Interpol, '{%'),
(Keyword, 'for'),
(Text, ' '),
(Name, 'field'),
(Text, ' '),
(Keyword, 'in'),
(Text, ' '),
(String.Other, '%w('),
(String.Other, 'first_name middle_name last_name'),
(String.Other, ')'),
(String.Interpol, '%}'),
(Text, '\n'),
(Name, 'cmp'),
(Operator, '='),
(String.Interpol, '{{'),
(Name, 'field'),
(Operator, '.'),
(Name, 'id'),
(String.Interpol, '}}'),
(Operator, '<=>'),
(Name, 'other'),
(Operator, '.'),
(String.Interpol, '{{'),
(Name, 'field'),
(Operator, '.'),
(Name, 'id'),
(String.Interpol, '}}'),
(Text, '\n'),
(Keyword, 'return'),
(Text, ' '),
(Name, 'cmp'),
(Text, ' '),
(Keyword, 'if'),
(Text, ' '),
(Name, 'cmp'),
(Operator, '!='),
(Number.Integer, '0'),
(Text, '\n'),
(String.Interpol, '{%'),
(Keyword, 'end'),
(String.Interpol, '%}'),
(Text, '\n'),
(Number.Integer, '0'),
(Text, '\n'),
(Keyword, 'end'),
(Text, '\n')
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
def testLib(self):
fragment = (
'@[Link("some")]\nlib LibSome\n'
'@[CallConvention("X86_StdCall")]\nfun foo="some.foo"(thing : Void*) : LibC::Int\n'
'end\n')
tokens = [
(Operator, '@['),
(Name.Decorator, 'Link'),
(Punctuation, '('),
(String.Double, '"'),
(String.Double, 'some'),
(String.Double, '"'),
(Punctuation, ')'),
(Operator, ']'),
(Text, '\n'),
(Keyword, 'lib'),
(Text, ' '),
(Name.Namespace, 'LibSome'),
(Text, '\n'),
(Operator, '@['),
(Name.Decorator, 'CallConvention'),
(Punctuation, '('),
(String.Double, '"'),
(String.Double, 'X86_StdCall'),
(String.Double, '"'),
(Punctuation, ')'),
(Operator, ']'),
(Text, '\n'),
(Keyword, 'fun'),
(Text, ' '),
(Name.Function, 'foo'),
(Operator, '='),
(String.Double, '"'),
(String.Double, 'some.foo'),
(String.Double, '"'),
(Punctuation, '('),
(Name, 'thing'),
(Text, ' '),
(Punctuation, ':'),
(Text, ' '),
(Name.Builtin, 'Void'),
(Operator, '*'),
(Punctuation, ')'),
(Text, ' '),
(Punctuation, ':'),
(Text, ' '),
(Name, 'LibC'),
(Operator, '::'),
(Name.Builtin, 'Int'),
(Text, '\n'),
(Keyword, 'end'),
(Text, '\n')
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
def testEscapedBracestring(self):
fragment = 'str.gsub(%r{\\\\\\\\}, "/")\n'
tokens = [
(Name, 'str'),
(Operator, '.'),
(Name, 'gsub'),
(Punctuation, '('),
(String.Regex, '%r{'),
(String.Regex, '\\\\'),
(String.Regex, '\\\\'),
(String.Regex, '}'),
(Punctuation, ','),
(Text, ' '),
(String.Double, '"'),
(String.Double, '/'),
(String.Double, '"'),
(Punctuation, ')'),
(Text, '\n'),
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))