def setup(self): self.lexer = MathematicaLexer()
class TestMathematicaLexer: def setup(self): self.lexer = MathematicaLexer() def verify(self, code, expected): expected.append((Token.Text.Whitespace, '\n')) returned = list(self.lexer.get_tokens(code)) assert_equal(expected, returned) def verify_all(self, code_list, expected_list): for code, expected in zip(code_list, expected_list): self.verify(code, expected) def test_comments(self): code = '(* a comment *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' a comment '), (MToken.COMMENT, '*)') ] self.verify(code, expected) def test_comments_with_code(self): code = '(* Plot[Sin[x], {x, 0, 2 Pi}] *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' Plot[Sin[x], {x, 0, 2 Pi}] '), (MToken.COMMENT, '*)') ] self.verify(code, expected) def test_nested_comments(self): code = '(* foo (* bar *) baz *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' foo '), (MToken.COMMENT, '(*'), (MToken.COMMENT, ' bar '), (MToken.COMMENT, '*)'), (MToken.COMMENT, ' baz '), (MToken.COMMENT, '*)'), ] self.verify(code, expected) def test_multiline_comment(self): code = '(* a comment\non two lines *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' a comment\non two lines '), (MToken.COMMENT, '*)'), ] self.verify(code, expected) def test_strings(self): code = [ '"a string"', '"a string \\" with a quote"', '"a string with a newline\\n"' ] expected = [ [ (MToken.STRING, '"'), (MToken.STRING, 'a string'), (MToken.STRING, '"'), ], [ (MToken.STRING, '"'), (MToken.STRING, 'a string '), (MToken.STRING, '\\"'), (MToken.STRING, ' with a quote'), (MToken.STRING, '"'), ], [ (MToken.STRING, '"'), (MToken.STRING, 'a string with a newline'), (MToken.STRING, '\\n'), (MToken.STRING, '"'), ] ] self.verify_all(code, expected) def test_integers(self): code = '123' expected = [(MToken.NUMBER, '123')] self.verify(code, expected) def test_floats(self): code = ['1.23', '10.1', '.123'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_precision_numbers(self): code = ['1`', '1.2`', '1.23`30', '20`20'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_base_numbers(self): code = ['2^^101', '8 ^^ 17', '10^^ 3.4'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_scientific_number(self): code = ['1*^3', '2 *^23', '1.23*^4'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_patterns(self): code = [ '_Head', '__Head', '___Head', 'x_Head', 'x__Head', 'x___Head', 'Foo`Bar_Head', 'Foo`Bar__Integer', 'Foo`Bar___Baz', 'Foo`Bar_Ctx`Baz', 'Foo`Bar__Ctx`Baz', 'Foo`Bar___Ctx`Baz`Qux', ] expected = [[(MToken.PATTERN, pat)] for pat in code] self.verify_all(code, expected) def test_slots(self): code = ['#', '#1', '#234'] expected = [[(MToken.SLOT, st)] for st in code] self.verify_all(code, expected) def test_slot_sequences(self): code = ['##', '##2', '##23'] expected = [[(MToken.SLOT, st)] for st in code] self.verify_all(code, expected) def test_association_slots(self): code = ['#foo', '#"foo"', '#foo`bar', '#Foo$1`Bar2$'] expected = [[(MToken.SLOT, st)] for st in code] self.verify_all(code, expected) def test_operators(self): code = mma.OPERATORS expected = [[(MToken.OPERATOR, op)] for op in code] self.verify_all(code, expected) def test_messages(self): code = ['General::foo', 'Foo::bar', 'Foo`Bar::baz'] expected = [ [ (MToken.BUILTIN, 'General'), (MToken.OPERATOR, '::'), (MToken.MESSAGE, 'foo') ], [ (MToken.SYMBOL, 'Foo'), (MToken.OPERATOR, '::'), (MToken.MESSAGE, 'bar') ], [ (MToken.SYMBOL, 'Foo`Bar'), (MToken.OPERATOR, '::'), (MToken.MESSAGE, 'baz') ], ] self.verify_all(code, expected) def test_symbols(self): code = ['foo', 'Foo', 'camelCase', 'Context`symbol', '`symbol', '$foo`bar', '$Bar`Baz`Qux'] expected = [[(MToken.SYMBOL, sym)] for sym in code] self.verify_all(code, expected) def test_builtins(self): code = list(mma.SYSTEM_SYMBOLS) expected = [[(MToken.BUILTIN, sym)] for sym in code] self.verify_all(code, expected) def test_unicode_builtins(self): code = list(mma.UNICODE_SYSTEM_SYMBOLS) expected = [[(MToken.BUILTIN, sym)] for sym in code] self.verify_all(code, expected) def test_unicode_groupings(self): code = list(mma.UNICODE_GROUPINGS) expected = [[(MToken.GROUP, grp)] for grp in code] self.verify_all(code, expected) def test_unicode_operators(self): code = list(mma.UNICODE_OPERATORS) expected = [[(MToken.OPERATOR, op)] for op in code] self.verify_all(code, expected) def test_unicode_undefined(self): code = list(mma.UNICODE_SYSTEM_UNDEFINED_SYMBOLS) expected = [[(MToken.SYMBOL, sym)] for sym in code] self.verify_all(code, expected) def test_lexical_scope_simple(self): code = [ 'Block[{x = 1}, Sin[x]]', 'Module[{y := Cos[x]}, x + y]', 'With[{$foo = 1}, f[$foo]]' ] expected = [ [ (MToken.BUILTIN, 'Block'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'x'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '='), (MToken.WHITESPACE, ' '), (MToken.NUMBER, '1'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.BUILTIN, 'Sin'), (MToken.GROUP, '['), (MToken.LOCAL_SCOPE, 'x'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), ], [ (MToken.BUILTIN, 'Module'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'y'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, ':='), (MToken.WHITESPACE, ' '), (MToken.BUILTIN, 'Cos'), (MToken.GROUP, '['), (MToken.SYMBOL, 'x'), (MToken.GROUP, ']'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.SYMBOL, 'x'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '+'), (MToken.WHITESPACE, ' '), (MToken.LOCAL_SCOPE, 'y'), (MToken.GROUP, ']'), ], [ (MToken.BUILTIN, 'With'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, '$foo'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '='), (MToken.WHITESPACE, ' '), (MToken.NUMBER, '1'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.SYMBOL, 'f'), (MToken.GROUP, '['), (MToken.LOCAL_SCOPE, '$foo'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), ], ] self.verify_all(code, expected) def test_lexical_scope_nested(self): code = 'Block[{Plus = Times}, x + With[{y = 1}, 3 * y]]' expected = [ (MToken.BUILTIN, 'Block'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'Plus'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '='), (MToken.WHITESPACE, ' '), (MToken.BUILTIN, 'Times'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.SYMBOL, 'x'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '+'), (MToken.WHITESPACE, ' '), (MToken.BUILTIN, 'With'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'y'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '='), (MToken.WHITESPACE, ' '), (MToken.NUMBER, '1'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.NUMBER, '3'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '*'), (MToken.WHITESPACE, ' '), (MToken.LOCAL_SCOPE, 'y'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), ] self.verify(code, expected) def test_lexical_scope_nasty(self): code = 'Block[{x=Module[{y=<|a->1,b->2|>},y],z=With[{k={1,2}},k*3]}, x+y*Block[{k=3},f[k]]]' expected = [ (MToken.BUILTIN, 'Block'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'x'), (MToken.OPERATOR, '='), (MToken.BUILTIN, 'Module'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'y'), (MToken.OPERATOR, '='), (MToken.GROUP, '<|'), (MToken.SYMBOL, 'a'), (MToken.OPERATOR, '->'), (MToken.NUMBER, '1'), (MToken.GROUP, ','), (MToken.SYMBOL, 'b'), (MToken.OPERATOR, '->'), (MToken.NUMBER, '2'), (MToken.GROUP, '|>'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.LOCAL_SCOPE, 'y'), (MToken.GROUP, ']'), (MToken.GROUP, ','), (MToken.LOCAL_SCOPE, 'z'), (MToken.OPERATOR, '='), (MToken.BUILTIN, 'With'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'k'), (MToken.OPERATOR, '='), (MToken.GROUP, '{'), (MToken.NUMBER, '1'), (MToken.GROUP, ','), (MToken.NUMBER, '2'), (MToken.GROUP, '}'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.LOCAL_SCOPE, 'k'), (MToken.OPERATOR, '*'), (MToken.NUMBER, '3'), (MToken.GROUP, ']'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.LOCAL_SCOPE, 'x'), (MToken.OPERATOR, '+'), (MToken.SYMBOL, 'y'), (MToken.OPERATOR, '*'), (MToken.BUILTIN, 'Block'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'k'), (MToken.OPERATOR, '='), (MToken.NUMBER, '3'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.SYMBOL, 'f'), (MToken.GROUP, '['), (MToken.LOCAL_SCOPE, 'k'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), ] self.verify(code, expected)
class TestMathematicaLexer: def setup(self): self.lexer = MathematicaLexer() def verify(self, code, expected): expected.append((Token.Text.Whitespace, '\n')) returned = list(self.lexer.get_tokens(code)) assert_equal(expected, returned) def verify_all(self, code_list, expected_list): for code, expected in zip(code_list, expected_list): self.verify(code, expected) def test_comments(self): code = '(* a comment *)' expected = [(MToken.COMMENT, '(*'), (MToken.COMMENT, ' a comment '), (MToken.COMMENT, '*)')] self.verify(code, expected) def test_comments_with_code(self): code = '(* Plot[Sin[x], {x, 0, 2 Pi}] *)' expected = [(MToken.COMMENT, '(*'), (MToken.COMMENT, ' Plot[Sin[x], {x, 0, 2 Pi}] '), (MToken.COMMENT, '*)')] self.verify(code, expected) def test_nested_comments(self): code = '(* foo (* bar *) baz *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' foo '), (MToken.COMMENT, '(*'), (MToken.COMMENT, ' bar '), (MToken.COMMENT, '*)'), (MToken.COMMENT, ' baz '), (MToken.COMMENT, '*)'), ] self.verify(code, expected) def test_multiline_comment(self): code = '(* a comment\non two lines *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' a comment\non two lines '), (MToken.COMMENT, '*)'), ] self.verify(code, expected) def test_strings(self): code = [ '"a string"', '"a string \\" with a quote"', '"a string with a newline\\n"', '"a string with \\ two backslashes"', ] expected = [[ (MToken.STRING, '"'), (MToken.STRING, 'a string'), (MToken.STRING, '"'), ], [ (MToken.STRING, '"'), (MToken.STRING, 'a string '), (MToken.STRING, '\\"'), (MToken.STRING, ' with a quote'), (MToken.STRING, '"'), ], [ (MToken.STRING, '"'), (MToken.STRING, 'a string with a newline'), (MToken.STRING, '\\n'), (MToken.STRING, '"'), ], [ (MToken.STRING, '"'), (MToken.STRING, 'a string with '), (MToken.STRING, '\\'), (MToken.STRING, ' two backslashes'), (MToken.STRING, '"'), ]] self.verify_all(code, expected) def test_integers(self): code = '123' expected = [(MToken.NUMBER, '123')] self.verify(code, expected) def test_floats(self): code = ['1.23', '10.1', '.123'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_precision_numbers(self): code = ['1`', '1.2`', '1.23`30', '20`20'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_base_numbers(self): code = ['2^^101', '8 ^^ 17', '10^^ 3.4'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_scientific_number(self): code = ['1*^3', '2 *^23', '1.23*^4'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_patterns(self): code = [ '_Head', '__Head', '___Head', 'x_Head', 'x__Head', 'x___Head', 'Foo`Bar_Head', 'Foo`Bar__Integer', 'Foo`Bar___Baz', 'Foo`Bar_Ctx`Baz', 'Foo`Bar__Ctx`Baz', 'Foo`Bar___Ctx`Baz`Qux', ] expected = [[(MToken.PATTERN, pat)] for pat in code] self.verify_all(code, expected) def test_slots(self): code = ['#', '#1', '#234'] expected = [[(MToken.SLOT, st)] for st in code] self.verify_all(code, expected) def test_slot_sequences(self): code = ['##', '##2', '##23'] expected = [[(MToken.SLOT, st)] for st in code] self.verify_all(code, expected) def test_association_slots(self): code = ['#foo', '#"foo"', '#foo`bar', '#Foo$1`Bar2$'] expected = [[(MToken.SLOT, st)] for st in code] self.verify_all(code, expected) def test_operators(self): code = mma.OPERATORS expected = [[(MToken.OPERATOR, op)] for op in code] self.verify_all(code, expected) def test_messages(self): code = ['General::foo', 'Foo::bar', 'Foo`Bar::baz'] expected = [ [(MToken.BUILTIN, 'General'), (MToken.OPERATOR, '::'), (MToken.MESSAGE, 'foo')], [(MToken.SYMBOL, 'Foo'), (MToken.OPERATOR, '::'), (MToken.MESSAGE, 'bar')], [(MToken.SYMBOL, 'Foo`Bar'), (MToken.OPERATOR, '::'), (MToken.MESSAGE, 'baz')], ] self.verify_all(code, expected) def test_symbols(self): code = [ 'foo', 'Foo', 'camelCase', 'Context`symbol', '`symbol', '$foo`bar', '$Bar`Baz`Qux' ] expected = [[(MToken.SYMBOL, sym)] for sym in code] self.verify_all(code, expected) def test_get(self): code = ['<<Foo`', '<<Foo`Bar`'] expected = [[ (MToken.OPERATOR, '<<'), (MToken.SYMBOL, 'Foo`'), ], [ (MToken.OPERATOR, '<<'), (MToken.SYMBOL, 'Foo`Bar`'), ]] self.verify_all(code, expected) def test_builtins(self): code = list(mma.SYSTEM_SYMBOLS) expected = [[(MToken.BUILTIN, sym)] for sym in code] self.verify_all(code, expected) def test_unicode_builtins(self): code = list(mma.UNICODE_SYSTEM_SYMBOLS) expected = [[(MToken.BUILTIN, sym)] for sym in code] self.verify_all(code, expected) def test_unicode_groupings(self): code = list(mma.UNICODE_GROUPINGS) expected = [[(MToken.GROUP, grp)] for grp in code] self.verify_all(code, expected) def test_unicode_operators(self): code = list(mma.UNICODE_OPERATORS) expected = [[(MToken.OPERATOR, op)] for op in code] self.verify_all(code, expected) def test_unicode_undefined(self): code = list(mma.UNICODE_SYSTEM_UNDEFINED_SYMBOLS) expected = [[(MToken.SYMBOL, sym)] for sym in code] self.verify_all(code, expected) def test_lexical_scope_simple(self): code = [ 'Block[{x = 1}, Sin[x]]', 'Module[{y := Cos[x]}, x + y]', 'With[{$foo = 1}, f[$foo]]' ] expected = [ [ (MToken.BUILTIN, 'Block'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'x'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '='), (MToken.WHITESPACE, ' '), (MToken.NUMBER, '1'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.BUILTIN, 'Sin'), (MToken.GROUP, '['), (MToken.LOCAL_SCOPE, 'x'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), ], [ (MToken.BUILTIN, 'Module'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'y'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, ':='), (MToken.WHITESPACE, ' '), (MToken.BUILTIN, 'Cos'), (MToken.GROUP, '['), (MToken.SYMBOL, 'x'), (MToken.GROUP, ']'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.SYMBOL, 'x'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '+'), (MToken.WHITESPACE, ' '), (MToken.LOCAL_SCOPE, 'y'), (MToken.GROUP, ']'), ], [ (MToken.BUILTIN, 'With'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, '$foo'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '='), (MToken.WHITESPACE, ' '), (MToken.NUMBER, '1'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.SYMBOL, 'f'), (MToken.GROUP, '['), (MToken.LOCAL_SCOPE, '$foo'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), ], ] self.verify_all(code, expected) def test_lexical_scope_nested(self): code = 'Block[{Plus = Times}, x + With[{y = 1}, 3 * y]]' expected = [ (MToken.BUILTIN, 'Block'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'Plus'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '='), (MToken.WHITESPACE, ' '), (MToken.BUILTIN, 'Times'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.SYMBOL, 'x'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '+'), (MToken.WHITESPACE, ' '), (MToken.BUILTIN, 'With'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'y'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '='), (MToken.WHITESPACE, ' '), (MToken.NUMBER, '1'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.NUMBER, '3'), (MToken.WHITESPACE, ' '), (MToken.OPERATOR, '*'), (MToken.WHITESPACE, ' '), (MToken.LOCAL_SCOPE, 'y'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), ] self.verify(code, expected) def test_lexical_scope_nasty(self): code = 'Block[{x=Module[{y=<|a->1,b->2|>},y],z=With[{k={1,2}},k*3]}, x+y*Block[{k=3},f[k]]]' expected = [ (MToken.BUILTIN, 'Block'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'x'), (MToken.OPERATOR, '='), (MToken.BUILTIN, 'Module'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'y'), (MToken.OPERATOR, '='), (MToken.GROUP, '<|'), (MToken.SYMBOL, 'a'), (MToken.OPERATOR, '->'), (MToken.NUMBER, '1'), (MToken.GROUP, ','), (MToken.SYMBOL, 'b'), (MToken.OPERATOR, '->'), (MToken.NUMBER, '2'), (MToken.GROUP, '|>'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.LOCAL_SCOPE, 'y'), (MToken.GROUP, ']'), (MToken.GROUP, ','), (MToken.LOCAL_SCOPE, 'z'), (MToken.OPERATOR, '='), (MToken.BUILTIN, 'With'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'k'), (MToken.OPERATOR, '='), (MToken.GROUP, '{'), (MToken.NUMBER, '1'), (MToken.GROUP, ','), (MToken.NUMBER, '2'), (MToken.GROUP, '}'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.LOCAL_SCOPE, 'k'), (MToken.OPERATOR, '*'), (MToken.NUMBER, '3'), (MToken.GROUP, ']'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.WHITESPACE, ' '), (MToken.LOCAL_SCOPE, 'x'), (MToken.OPERATOR, '+'), (MToken.SYMBOL, 'y'), (MToken.OPERATOR, '*'), (MToken.BUILTIN, 'Block'), (MToken.GROUP, '['), (MToken.GROUP, '{'), (MToken.LOCAL_SCOPE, 'k'), (MToken.OPERATOR, '='), (MToken.NUMBER, '3'), (MToken.GROUP, '}'), (MToken.GROUP, ','), (MToken.SYMBOL, 'f'), (MToken.GROUP, '['), (MToken.LOCAL_SCOPE, 'k'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), (MToken.GROUP, ']'), ] self.verify(code, expected)
class TestMathematicaLexer: def setup(self): self.lexer = MathematicaLexer() def verify(self, code, expected): expected.append((Token.Text.Whitespace, '\n')) returned = list(self.lexer.get_tokens(code)) assert_equal(expected, returned) def verify_all(self, code_list, expected_list): for code, expected in zip(code_list, expected_list): self.verify(code, expected) def test_comments(self): code = '(* a comment *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' a comment '), (MToken.COMMENT, '*)') ] self.verify(code, expected) def test_comments_with_code(self): code = '(* Plot[Sin[x], {x, 0, 2 Pi}] *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' Plot[Sin[x], {x, 0, 2 Pi}] '), (MToken.COMMENT, '*)') ] self.verify(code, expected) def test_nested_comments(self): code = '(* foo (* bar *) baz *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' foo '), (MToken.COMMENT, '(*'), (MToken.COMMENT, ' bar '), (MToken.COMMENT, '*)'), (MToken.COMMENT, ' baz '), (MToken.COMMENT, '*)'), ] self.verify(code, expected) def test_multiline_comment(self): code = '(* a comment\non two lines *)' expected = [ (MToken.COMMENT, '(*'), (MToken.COMMENT, ' a comment\non two lines '), (MToken.COMMENT, '*)'), ] self.verify(code, expected) def test_strings(self): code = [ '"a string"', '"a string \\" with a quote"', '"a string with a newline\\n"' ] expected = [ [ (MToken.STRING, '"'), (MToken.STRING, 'a string'), (MToken.STRING, '"'), ], [ (MToken.STRING, '"'), (MToken.STRING, 'a string '), (MToken.STRING, '\\"'), (MToken.STRING, ' with a quote'), (MToken.STRING, '"'), ], [ (MToken.STRING, '"'), (MToken.STRING, 'a string with a newline'), (MToken.STRING, '\\n'), (MToken.STRING, '"'), ] ] self.verify_all(code, expected) def test_integers(self): code = '123' expected = [(MToken.NUMBER, '123')] self.verify(code, expected) def test_floats(self): code = ['1.23', '10.1', '.123'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_precision_numbers(self): code = ['1`', '1.2`', '1.23`30', '20`20'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_base_numbers(self): code = ['2^^101', '8 ^^ 17', '10^^ 3.4'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_scientific_number(self): code = ['1*^3', '2 *^23', '1.23*^4'] expected = [[(MToken.NUMBER, num)] for num in code] self.verify_all(code, expected) def test_patterns(self): code = [ '_Head', '__Head', '___Head', 'x_Head', 'x__Head', 'x___Head', 'Foo`Bar_Head', 'Foo`Bar__Integer', 'Foo`Bar___Baz', 'Foo`Bar_Ctx`Baz', 'Foo`Bar__Ctx`Baz', 'Foo`Bar___Ctx`Baz`Qux', ] expected = [[(MToken.PATTERN, pat)] for pat in code] self.verify_all(code, expected) def test_slots(self): code = ['#', '#1', '#234'] expected = [[(MToken.SLOT, st)] for st in code] self.verify_all(code, expected) def test_slot_sequences(self): code = ['##', '##2', '##23'] expected = [[(MToken.SLOT, st)] for st in code] self.verify_all(code, expected) def test_association_slots(self): code = ['#foo', '#"foo"', '#foo`bar', '#Foo$1`Bar2$'] expected = [[(MToken.SLOT, st)] for st in code] self.verify_all(code, expected) def test_operators(self): code = mma.OPERATORS expected = [[(MToken.OPERATOR, op)] for op in code] self.verify_all(code, expected) def test_messages(self): code = ['General::foo', 'Foo::bar', 'Foo`Bar::baz'] expected = [ [ (MToken.BUILTIN, 'General'), (MToken.OPERATOR, '::'), (MToken.MESSAGE, 'foo') ], [ (MToken.SYMBOL, 'Foo'), (MToken.OPERATOR, '::'), (MToken.MESSAGE, 'bar') ], [ (MToken.SYMBOL, 'Foo`Bar'), (MToken.OPERATOR, '::'), (MToken.MESSAGE, 'baz') ], ] self.verify_all(code, expected) def test_symbols(self): code = ['foo', 'Foo', 'camelCase', 'Context`symbol', '`symbol', '$foo`bar', '$Bar`Baz`Qux'] expected = [[(MToken.SYMBOL, sym)] for sym in code] self.verify_all(code, expected) def test_builtins(self): code = list(mma.SYSTEM_SYMBOLS) expected = [[(MToken.BUILTIN, sym)] for sym in code] self.verify_all(code, expected)