def scan_multilang(tokens, module_elem):
"""Build the Perl module CIX element tree.
"tokens" is a generator of UDL tokens for this UDL-based
multi-lang document.
"module_elem" is the <module> element of a CIX element tree on
which the Perl module should be built.
This should return a list of the CSL tokens in the token stream.
"""
tokenizer = perl_lexer.PerlMultiLangLexer(tokens)
# "PerlHTML" is about all we need for whichever Perl-based
# template language is being used. This could just as easily be a
# boolean that indicates whether we're processing a pure language
# or a multi-lang one.
parser = perl_parser.Parser(tokenizer,
lang="PerlHTML",
provide_full_docs=gProvideFullDocs)
parser.moduleName = "" #Unknown
parser.parse()
parse_tree = parser.produce_CIX_NoHeader(module_elem)
csl_tokens = tokenizer.get_csl_tokens()
css_tokens = tokenizer.get_css_tokens()
return csl_tokens, css_tokens, tokenizer.has_perl_code()
def scan_purelang(buf):
content = buf.accessor.text.expandtabs(8)
tokenizer = perl_lexer.PerlLexer(content, gProvideFullDocs)
parser = perl_parser.Parser(tokenizer, provide_full_docs=gProvideFullDocs)
parser.moduleName = buf.path
parse_tree = parser.parse()
tree = parser.produce_CIX()
return tree
