def scan_purelang(content, filename): content = content.expandtabs(8) tokenizer = tcl_lexer.TclLexer(content) parser = tcl_parser.Parser(tokenizer, "Tcl") parse_tree = parser.parse() #XXX Change last arg from "Tcl" to "tclcile"? tree = parser_cix.produce_elementTree_cix(parse_tree, filename, "Tcl", "Tcl") return tree
def scan_multilang(tokens, module_elem): """Build the Tcl module CIX element tree. "tokens" is a generator of UDL tokens for this UDL-based multi-lang document. "module_elem" is the <module> element of a CIX element tree on which the Tcl module should be built. This should return a list of the CSL tokens in the token stream. """ tokenizer = tcl_lexer.TclMultiLangLexer(tokens) parser = tcl_parser.Parser(tokenizer, "AOL") # TODO: What is AOL here? parse_tree = parser.parse() parser_cix.produce_elementTree_contents_cix(parse_tree, module_elem) csl_tokens = tokenizer.get_csl_tokens() return csl_tokens