Exemplo n.º 1
0
def scan_multilang(tokens, module_elem):
    """Build the Tcl module CIX element tree.

        "tokens" is a generator of UDL tokens for this UDL-based
            multi-lang document.
        "module_elem" is the <module> element of a CIX element tree on
            which the Tcl module should be built.

    This should return a list of the CSL tokens in the token stream.
    """
    tokenizer = tcl_lexer.TclMultiLangLexer(tokens)
    parser = tcl_parser.Parser(tokenizer, "AOL")  # TODO: What is AOL here?
    parse_tree = parser.parse()
    parser_cix.produce_elementTree_contents_cix(parse_tree, module_elem)
    csl_tokens = tokenizer.get_csl_tokens()
    return csl_tokens
Exemplo n.º 2
0
def scan_multilang(tokens, module_elem):
    """Build the Tcl module CIX element tree.

        "tokens" is a generator of UDL tokens for this UDL-based
            multi-lang document.
        "module_elem" is the <module> element of a CIX element tree on
            which the Tcl module should be built.

    This should return a list of the CSL tokens in the token stream.
    """
    tokenizer = tcl_lexer.TclMultiLangLexer(tokens)
    parser = tcl_parser.Parser(tokenizer, "AOL")  # TODO: What is AOL here?
    parse_tree = parser.parse()
    parser_cix.produce_elementTree_contents_cix(parse_tree, module_elem)
    csl_tokens = tokenizer.get_csl_tokens()
    return csl_tokens
Exemplo n.º 3
0
def scan_multilang(tokens, module_elem):
    """Build the Ruby module CIX element tree.

        "tokens" is a generator of UDL tokens for this UDL-based
            multi-lang document.
        "module_elem" is the <module> element of a CIX element tree on
            which the Ruby module should be built.

    This should return a tuple of:
    * the list of the CSL tokens in the token stream,
    * whether or not the document contains any Ruby tokens (style UDL_SSL...)
    """

    tokenizer = ruby_lexer.RubyMultiLangLexer(tokens)
    parser = ruby_parser.Parser(tokenizer, "RHTML")
    parse_tree = parser.parse()
    parser_cix.produce_elementTree_contents_cix(parse_tree, module_elem)
    csl_tokens = tokenizer.get_csl_tokens()
    return csl_tokens, tokenizer.has_ruby_code()
Exemplo n.º 4
0
def scan_multilang(tokens, module_elem):
    """Build the Ruby module CIX element tree.

        "tokens" is a generator of UDL tokens for this UDL-based
            multi-lang document.
        "module_elem" is the <module> element of a CIX element tree on
            which the Ruby module should be built.

    This should return a tuple of:
    * the list of the CSL tokens in the token stream,
    * whether or not the document contains any Ruby tokens (style UDL_SSL...)
    """

    tokenizer = ruby_lexer.RubyMultiLangLexer(tokens)
    parser = ruby_parser.Parser(tokenizer, "RHTML")
    parse_tree = parser.parse()
    parser_cix.produce_elementTree_contents_cix(parse_tree, module_elem)
    csl_tokens = tokenizer.get_csl_tokens()
    return csl_tokens, tokenizer.has_ruby_code()