def __init__(self):
RMTCParser.__init__(self, TokenizationContext("Lyc Tokenization Context"),
default_lyc_readtable,
IndentationReadtableTokenizer,
default_lyc_transducer_chain)
self.tokenization_context.set(
# Tokenizer classes
DelimiterTokenizer = LycDelimiterTokenizer,
StringTokenizer = LycStringTokenizer,
DelimitedSymbolTokenizer = DelimitedIdentifierTokenizer,
CommentTokenizer = CommentTokenizer,
RawCommentTokenizer = RawCommentTokenizer,
)
def __init__(self):
RMTCParser.__init__(self,
TokenizationContext("Lyc Tokenization Context"),
default_lyc_readtable,
IndentationReadtableTokenizer,
default_lyc_transducer_chain)
self.tokenization_context.set(
# Tokenizer classes
DelimiterTokenizer=LycDelimiterTokenizer,
StringTokenizer=LycStringTokenizer,
DelimitedSymbolTokenizer=DelimitedIdentifierTokenizer,
CommentTokenizer=CommentTokenizer,
RawCommentTokenizer=RawCommentTokenizer,
)
def __init__(self):
RMTCParser.__init__(self, TokenizationContext("Anoky Tokenization Context"),
default_readtable,
IndentationReadtableTokenizer,
default_python_transducer_chain)
self.tokenization_context.set(
# ..
DelimiterTokenizer = AnokyDelimiterTokenizer,
SharpDelimiterTokenizer = AnokySharpDelimiterTokenizer,
SingleQuoteStringTokenizer = AnokySingleQuoteStringTokenizer,
DoubleQuoteStringTokenizer = AnokyDoubleQuoteStringTokenizer,
TripleSingleQuoteStringTokenizer = AnokyTripleSingleQuoteStringTokenizer,
TripleDoubleQuoteStringTokenizer = AnokyTripleDoubleQuoteStringTokenizer,
CommentTokenizer = AnokyCommentTokenizer,
LispModeTokenizer = LispModeTokenizer,
CodeQuoteTokenizer = SingleUseTokenizer,
DelimitedIdentifier = DelimitedIdentifierTokenizer,
MetaImporter = MetaImporter
)
def _get_stream(self, code_or_stream:Union[str, CharacterStream]):
stream = RMTCParser._get_stream(self, code_or_stream)
if not isinstance(stream, IndentedCharacterStream):
stream = IndentedCharacterStream(stream)
return stream
def _get_stream(self, code_or_stream:Union[str, CharacterStream]):
stream = RMTCParser._get_stream(self, code_or_stream)
if not isinstance(stream, IndentedCharacterStream):
stream = IndentedCharacterStream(stream)
return stream
