def __init__(self): RMTCParser.__init__(self, TokenizationContext("Lyc Tokenization Context"), default_lyc_readtable, IndentationReadtableTokenizer, default_lyc_transducer_chain) self.tokenization_context.set( # Tokenizer classes DelimiterTokenizer = LycDelimiterTokenizer, StringTokenizer = LycStringTokenizer, DelimitedSymbolTokenizer = DelimitedIdentifierTokenizer, CommentTokenizer = CommentTokenizer, RawCommentTokenizer = RawCommentTokenizer, )
def __init__(self): RMTCParser.__init__(self, TokenizationContext("Lyc Tokenization Context"), default_lyc_readtable, IndentationReadtableTokenizer, default_lyc_transducer_chain) self.tokenization_context.set( # Tokenizer classes DelimiterTokenizer=LycDelimiterTokenizer, StringTokenizer=LycStringTokenizer, DelimitedSymbolTokenizer=DelimitedIdentifierTokenizer, CommentTokenizer=CommentTokenizer, RawCommentTokenizer=RawCommentTokenizer, )
def __init__(self): RMTCParser.__init__(self, TokenizationContext("Anoky Tokenization Context"), default_readtable, IndentationReadtableTokenizer, default_python_transducer_chain) self.tokenization_context.set( # .. DelimiterTokenizer = AnokyDelimiterTokenizer, SharpDelimiterTokenizer = AnokySharpDelimiterTokenizer, SingleQuoteStringTokenizer = AnokySingleQuoteStringTokenizer, DoubleQuoteStringTokenizer = AnokyDoubleQuoteStringTokenizer, TripleSingleQuoteStringTokenizer = AnokyTripleSingleQuoteStringTokenizer, TripleDoubleQuoteStringTokenizer = AnokyTripleDoubleQuoteStringTokenizer, CommentTokenizer = AnokyCommentTokenizer, LispModeTokenizer = LispModeTokenizer, CodeQuoteTokenizer = SingleUseTokenizer, DelimitedIdentifier = DelimitedIdentifierTokenizer, MetaImporter = MetaImporter )