Exemplo n.º 1
0
class Sequence_5(TestCase, ExpectedDataMixin, TokenSequence):
	code = """abc/* comment */123/* comment */456.789/* comment */'Z'/* comment */"Hallo"/* comment */\\foo\\/* comment */-- comment\n/* comment *//* comment */;/* comment */  /* comment */"""
	tokenstream = ExpectedTokenStream(
		tokenList=[
			(StartOfDocumentToken,   None),
			(WordToken,              "abc"),
			(MultiLineCommentToken,  "/* comment */"),
			(IntegerLiteralToken,    "123"),
			(MultiLineCommentToken,  "/* comment */"),
			(RealLiteralToken,       "456.789"),
			(MultiLineCommentToken,  "/* comment */"),
			(CharacterLiteralToken,  "Z"),
			(MultiLineCommentToken,  "/* comment */"),
			(StringLiteralToken,     "Hallo"),
			(MultiLineCommentToken,  "/* comment */"),
			(ExtendedIdentifier,     "\\foo\\"),
			(MultiLineCommentToken,  "/* comment */"),
			(SingleLineCommentToken, "-- comment\n"),
			(MultiLineCommentToken,  "/* comment */"),
			(MultiLineCommentToken,  "/* comment */"),
			(CharacterToken,         ";"),
			(MultiLineCommentToken,  "/* comment */"),
			(SpaceToken,             "  "),
			(MultiLineCommentToken,  "/* comment */"),
			(EndOfDocumentToken,     None)
		]
	)
Exemplo n.º 2
0
class SimpleArchitecture_OneLine_EndWithKeywordAndName(TestCase,
                                                       ExpectedDataMixin,
                                                       LinkingTests,
                                                       TokenSequence,
                                                       BlockSequence):
    code = "architecture a of e is begin end architecture a;"
    tokenstream = ExpectedTokenStream([(StartOfDocumentToken, None),
                                       (WordToken, "architecture"),
                                       (SpaceToken, " "), (WordToken, "a"),
                                       (SpaceToken, " "), (WordToken, "of"),
                                       (SpaceToken, " "), (WordToken, "e"),
                                       (SpaceToken, " "), (WordToken, "is"),
                                       (SpaceToken, " "), (WordToken, "begin"),
                                       (SpaceToken, " "), (WordToken, "end"),
                                       (SpaceToken, " "),
                                       (WordToken, "architecture"),
                                       (SpaceToken, " "), (WordToken, "a"),
                                       (CharacterToken, ";"),
                                       (EndOfDocumentToken, None)])
    blockstream = ExpectedBlockStream([
        (StartOfDocumentBlock, None),
        (Architecture.NameBlock, "architecture a of e is"),
        (WhitespaceBlock, " "), (Architecture.BeginBlock, "begin"),
        (WhitespaceBlock, " "), (Architecture.EndBlock, "end architecture a;"),
        (EndOfDocumentBlock, None)
    ])
Exemplo n.º 3
0
class SimpleEntity_MultiLine_LongForm_WithSingleGeneric_TypoInGeneric(
        TestCase, ExpectedDataMixin, LinkingTests, TokenSequence,
        BlockSequence):
    code = dedent("""\
		entity e is
			gen (
				G : integer
			;
		end	entity e;
		""")
    tokenstream = ExpectedTokenStream([
        (StartOfDocumentToken, None), (WordToken, "entity"), (SpaceToken, " "),
        (WordToken, "e"), (SpaceToken, " "), (WordToken, "is"),
        (LinebreakToken, None), (IndentationToken, "\t"), (WordToken, "gen"),
        (SpaceToken, " "), (CharacterToken, "("), (LinebreakToken, None),
        (IndentationToken, "\\t\\t"), (WordToken, "G"), (SpaceToken, " "),
        (CharacterToken, ":"), (SpaceToken, " "), (WordToken, "integer"),
        (LinebreakToken, None), (IndentationToken, "\\t"),
        (CharacterToken, ")"), (CharacterToken, ";"), (LinebreakToken, None),
        (WordToken, "end"), (SpaceToken, " "), (WordToken, "entity"),
        (SpaceToken, " "), (WordToken, "a"), (CharacterToken, ";"),
        (EndOfDocumentToken, None)
    ])
    blockstream = ExpectedBlockStream([
        (StartOfDocumentBlock, None),  #
        (Entity.NameBlock, "entity e is"),  # entity e is
        (WhitespaceBlock, " "),  #
        (Entity.EndBlock, "end entity a;"),  # end entity a;
        (EndOfDocumentBlock, None)  #
    ])
Exemplo n.º 4
0
class Sequence_4(TestCase, ExpectedDataMixin, TokenSequence):
	code = """abc-- comment\n123-- comment\n456.789-- comment\n'Z'-- comment\n"Hallo"-- comment\n\\foo\\-- comment\n-- comment\n/* comment */-- comment\n;-- comment\n  -- comment\n"""
	tokenstream = ExpectedTokenStream(
		tokenList=[
			(StartOfDocumentToken,    None),
			(WordToken,               "abc"),
			(SingleLineCommentToken,  "-- comment\n"),
			(IntegerLiteralToken,     "123"),
			(SingleLineCommentToken,  "-- comment\n"),
			(RealLiteralToken,        "456.789"),
			(SingleLineCommentToken,  "-- comment\n"),
			(CharacterLiteralToken,   "Z"),
			(SingleLineCommentToken,  "-- comment\n"),
			(StringLiteralToken,      "Hallo"),
			(SingleLineCommentToken,  "-- comment\n"),
			(ExtendedIdentifier,      "\\foo\\"),
			(SingleLineCommentToken,  "-- comment\n"),
			(SingleLineCommentToken,  "-- comment\n"),
			(MultiLineCommentToken,   "/* comment */"),
			(SingleLineCommentToken,  "-- comment\n"),
			(CharacterToken,          ";"),
			(SingleLineCommentToken,  "-- comment\n"),
			(IndentationToken,        "  "),
			(SingleLineCommentToken,  "-- comment\n"),
			(EndOfDocumentToken,      None)
		]
	)
Exemplo n.º 5
0
class SimpleEntity_OneLine_EndWithKeywordAndName(TestCase, ExpectedDataMixin,
                                                 LinkingTests, TokenSequence,
                                                 BlockSequence):
    code = "entity e is end entity e;"
    tokenstream = ExpectedTokenStream([
        (StartOfDocumentToken, None),  #
        (WordToken, "entity"),  # entity
        (SpaceToken, " "),  #
        (WordToken, "e"),  # e
        (SpaceToken, " "),  #
        (WordToken, "is"),  # is
        (SpaceToken, " "),  #
        (WordToken, "end"),  # end
        (SpaceToken, " "),  #
        (WordToken, "entity"),  # entity
        (SpaceToken, " "),  #
        (WordToken, "e"),  # e
        (CharacterToken, ";"),  # ;
        (EndOfDocumentToken, None)  #
    ])
    blockstream = ExpectedBlockStream([
        (StartOfDocumentBlock, None),  #
        (Entity.NameBlock, "entity e is"),  # entity e is
        (WhitespaceBlock, " "),  #
        (Entity.EndBlock, "end entity e;"),  # end entity e;
        (EndOfDocumentBlock, None)  #
    ])
Exemplo n.º 6
0
class SimpleEntity_AllLine_LongForm(TestCase, ExpectedDataMixin, LinkingTests,
                                    TokenSequence, BlockSequence):
    code = dedent("""\
		entity
		e
		is
		end
		entity
		e
		;
		""")
    tokenstream = ExpectedTokenStream([
        (StartOfDocumentToken, None), (WordToken, "entity"),
        (LinebreakToken, "\n"), (WordToken, "e"), (LinebreakToken, "\n"),
        (WordToken, "is"), (LinebreakToken, "\n"), (WordToken, "end"),
        (LinebreakToken, "\n"), (WordToken, "entity"), (LinebreakToken, "\n"),
        (WordToken, "e"), (LinebreakToken, "\n"), (CharacterToken, ";"),
        (LinebreakToken, "\n"), (EndOfDocumentToken, None)
    ])
    blockstream = ExpectedBlockStream([
        (StartOfDocumentBlock, None),  #
        (Entity.NameBlock, "entity e is"),  # entity e is
        (WhitespaceBlock, " "),  #
        (Entity.EndBlock, "end entity a;"),  # end entity a;
        (EndOfDocumentBlock, None)  #
    ])
Exemplo n.º 7
0
class SimpleEntity_MultiLine_LongForm_WithSingleGeneric(TestCase, ExpectedDataMixin, LinkingTests, TokenSequence, BlockSequence):
	code = dedent("""\
		entity e is
			generic (
				G : integer
			);
		end	entity e;
		""")
	tokenStream = ExpectedTokenStream(
		[ (StartOfDocumentToken, None),
			(WordToken,            "entity"),
			(SpaceToken,           " "),
			(WordToken,            "e"),
			(SpaceToken,           " "),
			(WordToken,            "is"),
			(LinebreakToken,       None),
			(IndentationToken,     "\t"),
			(WordToken,            "generic"),
			(SpaceToken,           " "),
			(CharacterToken,       "("),
			(LinebreakToken,       None),
			(IndentationToken,     "\\t\\t"),
			(WordToken,            "G"),
			(SpaceToken,           " "),
			(CharacterToken,       ":"),
			(SpaceToken,           " "),
			(WordToken,            "integer"),
			(LinebreakToken,       None),
			(IndentationToken,     "\\t"),
			(CharacterToken,       ")"),
			(CharacterToken,       ";"),
			(LinebreakToken,       None),
			(WordToken,            "end"),
			(SpaceToken,           " "),
			(WordToken,            "entity"),
			(SpaceToken,           " "),
			(WordToken,            "a"),
			(CharacterToken,       ";"),
			(EndOfDocumentToken,   None)
		]
	)
	blockStream = ExpectedBlockStream(
		[ (StartOfDocumentBlock,    None),
			(Entity.NameBlock,        "entity e is"),
			(LinebreakBlock,          "\\n"),
			(IndentationBlock,        "\\t"),
			(GenericList.OpenBlock,   "generic ("),
			(LinebreakBlock,          "\\n"),
			(IndentationBlock,        "\\t\\t"),
			(GenericList.GenericListInterfaceConstantBlock, "G : integer"),
			(LinebreakBlock,          "\\n"),
			(GenericList.GenericListInterfaceConstantBlock, "\\t"),
			(GenericList.CloseBlock,  ");"),
			(LinebreakBlock,          "\\n"),
			(Entity.EndBlock,         "end entity e;"),
			(EndOfDocumentBlock,      None)
		]
	)
Exemplo n.º 8
0
class Sequence_3(TestCase, ExpectedDataMixin, TokenSequence):
	code = """abc\n123\n456.789\n'Z'\n"Hallo"\n\\foo\\\n-- comment\n/* comment */\n;\n  \nabc\r\n123\r\n456.789\r\n'Z'\r\n"Hallo"\r\n\\foo\\\r\n-- comment\r\n/* comment */\r\n;\r\n  \r\n\tabc """
	tokenstream = ExpectedTokenStream(
		tokenList=[
			(StartOfDocumentToken, None),
			(WordToken,               "abc"),
			(LinebreakToken,          None),
			(IntegerLiteralToken,     "123"),
			(LinebreakToken,          None),
			(RealLiteralToken,        "456.789"),
			(LinebreakToken,          None),
			(CharacterLiteralToken,   "Z"),
			(LinebreakToken,          None),
			(StringLiteralToken,      "Hallo"),
			(LinebreakToken,          None),
			(ExtendedIdentifier,      "\\foo\\"),
			(LinebreakToken,          None),
			(SingleLineCommentToken,  "-- comment\n"),
#			(LinebreakToken,          None),
			(MultiLineCommentToken,   "/* comment */"),
			(LinebreakToken,          None),
			(CharacterToken,          ";"),
			(LinebreakToken,          None),
			(IndentationToken,        "  "),
			(LinebreakToken,          None),
			(WordToken,               "abc"),
			(LinebreakToken,          None),
			(IntegerLiteralToken,     "123"),
			(LinebreakToken,          None),
			(RealLiteralToken,        "456.789"),
			(LinebreakToken,          None),
			(CharacterLiteralToken,   "Z"),
			(LinebreakToken,          None),
			(StringLiteralToken,      "Hallo"),
			(LinebreakToken,          None),
			(ExtendedIdentifier,      "\\foo\\"),
			(LinebreakToken,          None),
			(SingleLineCommentToken,  "-- comment\r\n"),
#			(LinebreakToken,          None),
			(MultiLineCommentToken,   "/* comment */"),
			(LinebreakToken,          None),
			(CharacterToken,          ";"),
			(LinebreakToken,          None),
			(IndentationToken,        "  "),
			(LinebreakToken,          None),
			(IndentationToken,        "\t"),
			(WordToken,               "abc"),
			(EndOfDocumentToken, None)
		]
	)
Exemplo n.º 9
0
class Sequence_2(TestCase, ExpectedDataMixin, TokenSequence):
    code = """abc   \def\ \t 'a' "abc" /* help */ -- foo\n"""
    tokenStream = ExpectedTokenStream([(StartOfDocumentToken, None),
                                       (WordToken, "abc"), (SpaceToken, "   "),
                                       (ExtendedIdentifier, "\\def\\"),
                                       (SpaceToken, " \t "),
                                       (CharacterLiteralToken, "a"),
                                       (SpaceToken, " "),
                                       (StringLiteralToken, "abc"),
                                       (SpaceToken, " "),
                                       (MultiLineCommentToken, "/* help */"),
                                       (SpaceToken, " "),
                                       (SingleLineCommentToken, "-- foo\n"),
                                       (EndOfDocumentToken, None)])
Exemplo n.º 10
0
class SimpleArchitecture_MultiLineIndented_EndWithKeywordAndName(
        TestCase, ExpectedDataMixin, LinkingTests, TokenSequence,
        BlockSequence):
    code = "\tarchitecture\n\ta\n\tof\n\te\n\tis\n\tbegin\n\tend\n\tarchitecture\n\ta\n\t;\n"
    tokenstream = ExpectedTokenStream([
        (StartOfDocumentToken, None), (IndentationToken, "\t"),
        (WordToken, "architecture"), (LinebreakToken, None),
        (IndentationToken, "\t"), (WordToken, "a"), (LinebreakToken, None),
        (IndentationToken, "\t"), (WordToken, "of"), (LinebreakToken, None),
        (IndentationToken, "\t"), (WordToken, "e"), (LinebreakToken, None),
        (IndentationToken, "\t"), (WordToken, "is"), (LinebreakToken, None),
        (IndentationToken, "\t"), (WordToken, "begin"), (LinebreakToken, None),
        (IndentationToken, "\t"), (WordToken, "end"), (LinebreakToken, None),
        (IndentationToken, "\t"), (WordToken, "architecture"),
        (LinebreakToken, None), (IndentationToken, "\t"), (WordToken, "a"),
        (LinebreakToken, None), (IndentationToken, "\t"),
        (CharacterToken, ";"), (LinebreakToken, None),
        (EndOfDocumentToken, None)
    ])
    blockstream = ExpectedBlockStream([
        (StartOfDocumentBlock, None),
        (IndentationBlock, "\\t"),
        (Architecture.NameBlock, "architecture"),
        (LinebreakBlock, "\\n"),
        #			(IndentationBlock,        "\\t"),
        (Architecture.NameBlock, "\\ta"),
        (LinebreakBlock, "\\n"),
        (Architecture.NameBlock, "\\tof"),
        (LinebreakBlock, "\\n"),
        (Architecture.NameBlock, "\\te"),
        (LinebreakBlock, "\\n"),
        (Architecture.NameBlock, "\\tis"),
        (LinebreakBlock, "\\n"),
        (IndentationBlock, "\\t"),
        (Architecture.BeginBlock, "begin"),
        (LinebreakBlock, "\\n"),
        (IndentationBlock, "\\t"),
        (Architecture.EndBlock, "end\\n"),
        #			(LinebreakBlock,          "\\n"),
        (Architecture.EndBlock, "\\tarchitecture\\n"),
        #			(LinebreakBlock,          "\\n"),
        (Architecture.EndBlock, "\\ta\\n"),
        #			(LinebreakBlock,          "\\n"),
        (Architecture.EndBlock, "\\t;"),
        (LinebreakBlock, "\\n"),
        (EndOfDocumentBlock, None)
    ])
Exemplo n.º 11
0
class SimpleEntity_AllLine_LongForm(TestCase, ExpectedDataMixin, LinkingTests, TokenSequence, BlockSequence):
	code = "entity\ne\nis\nend\nentity\ne\n;\n"
	tokenStream = ExpectedTokenStream(
		[ (StartOfDocumentToken, None),
			(WordToken,            "entity"),
			(LinebreakToken,       "\n"),
			(WordToken,            "e"),
			(LinebreakToken,       "\n"),
			(WordToken,            "is"),
			(LinebreakToken,       "\n"),
			(WordToken,            "end"),
			(LinebreakToken,       "\n"),
			(WordToken,            "entity"),
			(LinebreakToken,       "\n"),
			(WordToken,            "e"),
			(LinebreakToken,       "\n"),
			(CharacterToken,       ";"),
			(LinebreakToken,       "\n"),
			(EndOfDocumentToken,   None)
		]
	)
	blockStream = ExpectedBlockStream(
		[ (StartOfDocumentBlock,  None),
			(Entity.NameBlock,      "entity"),
			(LinebreakBlock,        "\\n"),
#			(IndentationBlock,      "\\t"),
			(Entity.NameBlock,      "e"),
			(LinebreakBlock,        "\\n"),
			(Entity.NameBlock,      "is"),
			(LinebreakBlock,        "\\n"),
			(Entity.EndBlock,       "end\\n"),
#			(LinebreakBlock,        "\\n"),
			(Entity.EndBlock,       "entity\\n"),
#			(LinebreakBlock,        "\\n"),
			(Entity.EndBlock,       "e\\n"),
#			(LinebreakBlock,        "\\n"),
			(Entity.EndBlock,       ";"),
			(LinebreakBlock,        "\\n"),
			(EndOfDocumentBlock,    None)
		]
	)
Exemplo n.º 12
0
class SimpleArchitecture_SingleLineComments_EndWithKeywordAndName(
        TestCase, ExpectedDataMixin, LinkingTests, TokenSequence,
        BlockSequence):
    code = "architecture-- comment\na-- comment\nof-- comment\ne-- comment\nis-- comment\nbegin-- comment\nend-- comment\narchitecture-- comment\na-- comment\n;-- comment\n"
    tokenstream = ExpectedTokenStream([
        (StartOfDocumentToken, None), (WordToken, "architecture"),
        (SingleLineCommentToken, "-- comment\n"), (WordToken, "a"),
        (SingleLineCommentToken, "-- comment\n"), (WordToken, "of"),
        (SingleLineCommentToken, "-- comment\n"), (WordToken, "e"),
        (SingleLineCommentToken, "-- comment\n"), (WordToken, "is"),
        (SingleLineCommentToken, "-- comment\n"), (WordToken, "begin"),
        (SingleLineCommentToken, "-- comment\n"), (WordToken, "end"),
        (SingleLineCommentToken, "-- comment\n"), (WordToken, "architecture"),
        (SingleLineCommentToken, "-- comment\n"), (WordToken, "a"),
        (SingleLineCommentToken, "-- comment\n"), (CharacterToken, ";"),
        (SingleLineCommentToken, "-- comment\n"), (EndOfDocumentToken, None)
    ])
    blockstream = ExpectedBlockStream([(StartOfDocumentBlock, None),
                                       (Architecture.NameBlock,
                                        "architecture"),
                                       (CommentBlock, "-- comment\\n"),
                                       (Architecture.NameBlock, "a"),
                                       (CommentBlock, "-- comment\\n"),
                                       (Architecture.NameBlock, "of"),
                                       (CommentBlock, "-- comment\\n"),
                                       (Architecture.NameBlock, "e"),
                                       (CommentBlock, "-- comment\\n"),
                                       (Architecture.NameBlock, "is"),
                                       (CommentBlock, "-- comment\\n"),
                                       (Architecture.BeginBlock, "begin"),
                                       (CommentBlock, "-- comment\\n"),
                                       (Architecture.EndBlock, "end"),
                                       (CommentBlock, "-- comment\\n"),
                                       (Architecture.EndBlock, "architecture"),
                                       (CommentBlock, "-- comment\\n"),
                                       (Architecture.EndBlock, "a"),
                                       (CommentBlock, "-- comment\\n"),
                                       (Architecture.EndBlock, ";"),
                                       (CommentBlock, "-- comment\\n"),
                                       (EndOfDocumentBlock, None)])
Exemplo n.º 13
0
class SimpleArchitecture_MultilineComments_EndWithKeywordAndName(
        TestCase, ExpectedDataMixin, LinkingTests, TokenSequence,
        BlockSequence):
    code = "architecture/* comment */a/* comment */of/* comment */e/* comment */is/* comment */begin/* comment */end/* comment */architecture/* comment */a/* comment */;/* comment */"
    tokenstream = ExpectedTokenStream([
        (StartOfDocumentToken, None), (WordToken, "architecture"),
        (MultiLineCommentToken, "/* comment */"), (WordToken, "a"),
        (MultiLineCommentToken, "/* comment */"), (WordToken, "of"),
        (MultiLineCommentToken, "/* comment */"), (WordToken, "e"),
        (MultiLineCommentToken, "/* comment */"), (WordToken, "is"),
        (MultiLineCommentToken, "/* comment */"), (WordToken, "begin"),
        (MultiLineCommentToken, "/* comment */"), (WordToken, "end"),
        (MultiLineCommentToken, "/* comment */"), (WordToken, "architecture"),
        (MultiLineCommentToken, "/* comment */"), (WordToken, "a"),
        (MultiLineCommentToken, "/* comment */"), (CharacterToken, ";"),
        (MultiLineCommentToken, "/* comment */"), (EndOfDocumentToken, None)
    ])
    blockstream = ExpectedBlockStream([(StartOfDocumentBlock, None),
                                       (Architecture.NameBlock,
                                        "architecture"),
                                       (CommentBlock, "/* comment */"),
                                       (Architecture.NameBlock, "a"),
                                       (CommentBlock, "/* comment */"),
                                       (Architecture.NameBlock, "of"),
                                       (CommentBlock, "/* comment */"),
                                       (Architecture.NameBlock, "e"),
                                       (CommentBlock, "/* comment */"),
                                       (Architecture.NameBlock, "is"),
                                       (CommentBlock, "/* comment */"),
                                       (Architecture.BeginBlock, "begin"),
                                       (CommentBlock, "/* comment */"),
                                       (Architecture.EndBlock, "end"),
                                       (CommentBlock, "/* comment */"),
                                       (Architecture.EndBlock, "architecture"),
                                       (CommentBlock, "/* comment */"),
                                       (Architecture.EndBlock, "a"),
                                       (CommentBlock, "/* comment */"),
                                       (Architecture.EndBlock, ";"),
                                       (CommentBlock, "/* comment */"),
                                       (EndOfDocumentBlock, None)])
Exemplo n.º 14
0
class SimpleEntity_OneLine_OnlyEnd(TestCase, ExpectedDataMixin, LinkingTests, TokenSequence, BlockSequence):
	code = "entity e is end;"
	tokenStream = ExpectedTokenStream(
		[ (StartOfDocumentToken, None),
			(WordToken,            "entity"),
			(SpaceToken,           " "),
			(WordToken,            "e"),
			(SpaceToken,           " "),
			(WordToken,            "is"),
			(SpaceToken,           " "),
			(WordToken,            "end"),
			(CharacterToken,       ";"),
			(EndOfDocumentToken,   None)
		]
	)
	blockStream = ExpectedBlockStream(
		[ (StartOfDocumentBlock, None),           #
			(Entity.NameBlock,     "entity e is"),  # entity e is
			(WhitespaceBlock,      " "),            #
			(Entity.EndBlock,      "end;"),         # end;
			(EndOfDocumentBlock,   None)            #
		]
	)
Exemplo n.º 15
0
class Sequence_1(TestCase, ExpectedDataMixin, TokenSequence):
#	code = "a bbb 1 23 3.4 45.6 5.67 67.89 .7 .89 ( ) < > = . , ; & / + - * << >> /= <= >= => := ** ?= ?/= ?< ?> ?<= ?>= <>"
	code = "a bbb 1 23 3.4 45.6 5.67 67.89 .7 .89 ( ) < > = . , ; & / + - * << >> /= <= >= => := ** ?= ?/= ?< ?> <>"
	tokenstream = ExpectedTokenStream(
		tokenList=[
			(StartOfDocumentToken, None),
			(WordToken,           "a"),
			(SpaceToken,          " "),
			(WordToken,           "bbb"),
			(SpaceToken,          " "),
			(IntegerLiteralToken, "1"),
			(SpaceToken,          " "),
			(IntegerLiteralToken, "23"),
			(SpaceToken,          " "),
			(RealLiteralToken,    "3.4"),
			(SpaceToken,          " "),
			(RealLiteralToken,    "45.6"),
			(SpaceToken,          " "),
			(RealLiteralToken,    "5.67"),
			(SpaceToken,          " "),
			(RealLiteralToken,    "67.89"),
			(SpaceToken,          " "),
			(RealLiteralToken,    ".7"),
			(SpaceToken,          " "),
			(RealLiteralToken,    ".89"),
			(SpaceToken,          " "),
			(CharacterToken,      "("),
			(SpaceToken,          " "),
			(CharacterToken,      ")"),
			(SpaceToken,          " "),
			(CharacterToken,      "<"),
			(SpaceToken,          " "),
			(CharacterToken,      ">"),
			(SpaceToken,          " "),
			(CharacterToken,      "="),
			(SpaceToken,          " "),
			(CharacterToken,      "."),
			(SpaceToken,          " "),
			(CharacterToken,      ","),
			(SpaceToken,          " "),
			(CharacterToken,      ";"),
			(SpaceToken,          " "),
			(CharacterToken,      "&"),
			(SpaceToken,          " "),
			(CharacterToken,      "/"),
			(SpaceToken,          " "),
			(CharacterToken,      "+"),
			(SpaceToken,          " "),
			(CharacterToken,      "-"),
			(SpaceToken,          " "),
			(CharacterToken,      "*"),
			(SpaceToken,          " "),
#			(CharacterToken,      "'"),
#			(SpaceToken,          " "),
			(FusedCharacterToken, "<<"),
			(SpaceToken,          " "),
			(FusedCharacterToken, ">>"),
			(SpaceToken,          " "),
			(FusedCharacterToken, "/="),
			(SpaceToken,          " "),
			(FusedCharacterToken, "<="),
			(SpaceToken,          " "),
			(FusedCharacterToken, ">="),
			(SpaceToken,          " "),
			(FusedCharacterToken, "=>"),
			(SpaceToken,          " "),
			(FusedCharacterToken, ":="),
			(SpaceToken,          " "),
			(FusedCharacterToken, "**"),
			(SpaceToken,          " "),
			(FusedCharacterToken, "?="),
			(SpaceToken,          " "),
			(FusedCharacterToken, "?/="),
			(SpaceToken,          " "),
			(FusedCharacterToken, "?<"),
			(SpaceToken,          " "),
			(FusedCharacterToken, "?>"),
#			(SpaceToken,          " "),
#			(FusedCharacterToken, "?<="),
#			(SpaceToken,          " "),
#			(FusedCharacterToken, "?>="),
			(SpaceToken,          " "),
			(FusedCharacterToken, "<>"),
			(EndOfDocumentToken, None)
		]
	)