def scan_multilang(tokens, module_elem): """Build the Tcl module CIX element tree. "tokens" is a generator of UDL tokens for this UDL-based multi-lang document. "module_elem" is the <module> element of a CIX element tree on which the Tcl module should be built. This should return a list of the CSL tokens in the token stream. """ tokenizer = tcl_lexer.TclMultiLangLexer(tokens) parser = tcl_parser.Parser(tokenizer, "AOL") # TODO: What is AOL here? parse_tree = parser.parse() parser_cix.produce_elementTree_contents_cix(parse_tree, module_elem) csl_tokens = tokenizer.get_csl_tokens() return csl_tokens
def scan_multilang(tokens, module_elem): """Build the Tcl module CIX element tree. "tokens" is a generator of UDL tokens for this UDL-based multi-lang document. "module_elem" is the <module> element of a CIX element tree on which the Tcl module should be built. This should return a list of the CSL tokens in the token stream. """ tokenizer = tcl_lexer.TclMultiLangLexer(tokens) parser = tcl_parser.Parser(tokenizer, "AOL") # TODO: What is AOL here? parse_tree = parser.parse() parser_cix.produce_elementTree_contents_cix(parse_tree, module_elem) csl_tokens = tokenizer.get_csl_tokens() return csl_tokens
def scan_multilang(tokens, module_elem): """Build the Ruby module CIX element tree. "tokens" is a generator of UDL tokens for this UDL-based multi-lang document. "module_elem" is the <module> element of a CIX element tree on which the Ruby module should be built. This should return a tuple of: * the list of the CSL tokens in the token stream, * whether or not the document contains any Ruby tokens (style UDL_SSL...) """ tokenizer = ruby_lexer.RubyMultiLangLexer(tokens) parser = ruby_parser.Parser(tokenizer, "RHTML") parse_tree = parser.parse() parser_cix.produce_elementTree_contents_cix(parse_tree, module_elem) csl_tokens = tokenizer.get_csl_tokens() return csl_tokens, tokenizer.has_ruby_code()
def scan_multilang(tokens, module_elem): """Build the Ruby module CIX element tree. "tokens" is a generator of UDL tokens for this UDL-based multi-lang document. "module_elem" is the <module> element of a CIX element tree on which the Ruby module should be built. This should return a tuple of: * the list of the CSL tokens in the token stream, * whether or not the document contains any Ruby tokens (style UDL_SSL...) """ tokenizer = ruby_lexer.RubyMultiLangLexer(tokens) parser = ruby_parser.Parser(tokenizer, "RHTML") parse_tree = parser.parse() parser_cix.produce_elementTree_contents_cix(parse_tree, module_elem) csl_tokens = tokenizer.get_csl_tokens() return csl_tokens, tokenizer.has_ruby_code()