def output(self, handler):
if self.__double_quotes:
handler('"')
Token.output(self, handler)
if self.__double_quotes:
handler('"')
def output(self, handler):
# Blocktrans output
handler(u'{%blocktrans ');
for p in self.params:
p.output(handler)
handler(u' ')
handler(u'%}')
Token.output(self, handler)
handler(u'{%endblocktrans%}')
def output(self, handler):
# Blocktrans output
handler(u"{%blocktrans ")
for p in self.params:
p.output(handler)
handler(u" ")
handler(u"%}")
Token.output(self, handler)
handler(u"{%endblocktrans%}")
def output(self, handler):
# Yield this node's content, or if the variable name
# has been changed, use the modified name.
if self.__varname:
handler(self.__varname)
elif self.__link_to:
self.__link_to.output(handler)
else:
Token.output(self, handler)
def output(self, handler):
handler("<script ")
handler(u" ".join([u"%s%s" % (a, self.__attrs[a]) for a in self.__attrs.keys()]))
handler(">")
if not self.is_external:
handler("//<![CDATA[\n")
Token.output(self, handler)
if not self.is_external:
handler(u"//]]>\n")
handler(u"</script>")
def output(self, handler):
handler('<script ')
handler(u' '.join([ u'%s%s' % (a, self.__attrs[a]) for a in self.__attrs.keys() ]))
handler('>')
if not self.is_external:
handler('//<![CDATA[\n')
Token.output(self, handler)
if not self.is_external:
handler(u'//]]>\n')
handler(u'</script>')
def fix_whitespace_bug(js_node):
"""
Fixes the following case in js code:
<script type="text/javascript"> if { {% if test %} ... {% endif %} } </script>
The lexer above would remove the space between the first '{' and '{%'. This collision
would make Django think it's the start of a variable.
"""
# For every scope (starting with '{')
for scope in js_node.child_nodes_of_class(JavascriptScope):
# Look if the first child inside this scope also renders to a '{'
if scope.children and scope.children[0].output_as_string()[0:1] == '{':
# If so, insert a whitespace token in between.
space = Token(name='required-whitespace')
space.children = [' ']
scope.children.insert(0, space)
def compile_javascript_string(js_string, context, path=''):
"""
Compile JS code (can be used for external javascript files)
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ js_string ]
# Tokenize
tokenize(tree, __JS_STATES, Token)
# Compile
_compile(tree, context)
# Output
return tree.output_as_string()
def compile_html_string(html_string, path=''):
"""
Compile a html string
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ html_string ]
# Tokenize
tokenize(tree, __HTML_STATES, Token)
from template_preprocessor.core.context import Context
context = Context(path)
_process_html_tree(tree, context)
# Output
return tree.output_as_string()
def compile_html_string(html_string, path=""):
"""
Compile a html string
"""
# First, create a tree to begin with
tree = Token(name="root", line=1, column=1, path=path)
tree.children = [html_string]
# Tokenize
tokenize(tree, __HTML_STATES, [Token])
from template_preprocessor.core.django_processor import PreProcessSettings
options = PreProcessSettings()
_process_html_tree(tree, options)
# Output
return tree.output_as_string()
def compile_javascript_string(js_string, context, path=''):
"""
Compile JS code (can be used for external javascript files)
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ js_string ]
# Tokenize
with context.time_operation("tokenize string"):
tokenize(tree, __JS_STATES, Token)
# Compile
_compile(tree, context, already_minified=path.endswith(".min.js"))
# Output
return tree.output_as_string()
def compile_css_string(css_string, context, path='', url=None):
"""
Compile CSS code
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ css_string ]
# Tokenize
tokenize(tree, __CSS_STATES, Token)
_add_css_parser_extensions(tree)
# Rewrite url() in external css files
if url:
_rewrite_urls(tree, url)
# Compile
_compress_css_whitespace(tree)
# Output
return u''.join([o for o in tree.output_as_string() ])
def compile_css_string(css_string, context, path='', url=None):
"""
Compile CSS code
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [css_string]
# Tokenize
tokenize(tree, __CSS_STATES, Token)
_add_css_parser_extensions(tree)
# Rewrite url() in external css files
if url:
_rewrite_urls(tree, url)
# Compile
_compress_css_whitespace(tree)
# Output
return u''.join([o for o in tree.output_as_string()])
def compile_javascript_string(js_string, path=''):
"""
Compile JS code (can be used for external javascript files)
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ js_string ]
# Tokenize
tokenize(tree, __JS_STATES, [Token] )
# Compile
_add_javascript_parser_extensions(tree)
_validate_javascript(tree)
_compress_javascript_whitespace(tree)
_minify_variable_names(tree)
fix_whitespace_bug(tree)
# Output
return tree.output_as_string()
def output(self, handler):
handler(u'(')
Token.output(self, handler)
handler(u')')
def output(self, handler):
if self.__show_cdata_signs: handler('<![CDATA[')
Token.output(self, handler)
if self.__show_cdata_signs: handler(']]>')
def output(self, handler):
handler(u"{%block ")
handler(self.block_name)
handler(u"%}")
Token.output(self, handler)
handler(u"{%endblock%}")
def output(self, handler):
handler(u'{%macro "')
handler(self.macro_name)
handler(u'"%}')
Token.output(self, handler)
handler(u"{%endmacro%}")
def init_extension(self):
self.__varname = Token.output_as_string(self, True)
def output(self, handler):
self.__open_tag.output(handler)
Token.output(self, handler)
handler("</%s>" % name)
def output(self, handler):
self.__open_tag.output(handler)
Token.output(self, handler)
handler("</pre>")
def convert_variable(self, name):
"""
Convert a template variable to a Python variable.
"""
# 88 -> 88
# a.1.b -> a[1].b
# 8.a -> CompileException
# "..." -> "..."
# var|filter:"..."|filter2:value -> _filters['filter2'](_filters['filter'](var))
# Parse the variable
tree = Token(name='root', line=1, column=1, path='django variable')
tree.children = [name]
tokenize(tree, _DJANGO_VARIABLE_STATES, [Token])
#print tree._print()
def handle_filter(subject, children):
filter_name = None
filter_option = None
in_filter_option = False
def result():
assert filter_name
if filter_name in _native_filters:
return _native_filters[filter_name](self, subject,
filter_option)
elif filter_option:
return '_f["%s"](%s, %s)' % (filter_name, subject,
filter_option)
else:
return '_f["%s"](%s)' % (filter_name, subject)
for i in range(0, len(children)):
part = children[i].output_as_string()
c = children[i]
if c.name == 'digits':
if not filter_option and in_filter_option:
filter_option = part
else:
raise CompileException(self._tag, 'Invalid variable')
elif c.name == 'name':
if not filter_option and in_filter_option:
filter_option = part
elif not filter_name:
filter_name = part
else:
raise CompileException(self._tag, 'Invalid variable')
elif c.name == 'string':
if not filter_option and in_filter_option:
filter_option = part
else:
raise CompileException(self._tag, 'Invalid variable')
elif c.name == 'trans':
if not filter_option and in_filter_option:
filter_option = '_(%s)' % c.output_as_string()
else:
raise CompileException(self._tag, 'Invalid variable')
if c.name == 'filter-option' and filter_name:
# Entered the colon ':'
in_filter_option = True
elif c.name == 'pipe':
# | is the start of a following filter
return handle_filter(result(), children[i + 1:])
return result()
def handle_var(children):
out = []
for i in range(0, len(children)):
part = children[i].output_as_string()
c = children[i]
if c.name == 'digits':
# First digits are literals, following digits are indexers
out.append('[%s]' % part if out else part)
elif c.name == 'dot':
#out.append('.') # assume last is not a dot
pass
elif c.name == 'string':
out.append(part)
elif c.name == 'name':
if out:
out.append('.%s' % part)
else:
if not self.variable_in_current_scope(part):
# If variable is not found in current or one of the parents'
# scopes, then prefix variable with "_c."
out.append('_c.%s' % part)
else:
out.append(part)
elif c.name == 'trans':
if out:
raise CompileException(self._tag, 'Invalid variable')
else:
out.append('_(%s)' % handle_var(c.children))
elif c.name == 'pipe':
# | is the start of a filter
return handle_filter(''.join(out), children[i + 1:])
return ''.join(out)
return handle_var(tree.children)
def output(self, handler):
handler(u'{%macro "'); handler(self.macro_name); handler(u'"%}')
Token.output(self, handler)
handler(u'{%endmacro%}')
def init_extension(self):
self.__varname = Token.output_as_string(self, True)
def parse(source_code, path, context, main_template=False):
"""
Parse the code.
- source_code: string
- path: for attaching meta information to the tree.
- context: preprocess context (holding the settings/dependecies/warnings, ...)
- main_template: False for includes/extended templates. True for the
original path that was called.
"""
# To start, create the root node of a tree.
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ source_code ]
# Lex Django tags
tokenize(tree, __DJANGO_STATES, Token)
# Phase I: add parser extensions
_add_parser_extensions(tree)
# Phase II: process inline tags
_process_inline_tags(tree)
# Phase III: create recursive structure for block level tags.
nest_block_level_elements(tree, __DJANGO_BLOCK_ELEMENTS, DjangoTag, lambda c: c.tagname)
# === Actions ===
if main_template:
_find_first_level_dependencies(tree, context)
# Extend parent template and process includes
tree = _process_extends(tree, context) # NOTE: this returns a new tree!
_preprocess_includes(tree, context)
_preprocess_decorate_tags(tree, context)
# Following actions only need to be applied if this is the 'main' tree.
# It does not make sense to apply it on every include, and then again
# on the complete tree.
if main_template:
_update_preprocess_settings(tree, context)
options = context.options
# Remember translations in context (form PO-file generation)
remember_gettext_entries(tree, context)
# Do translations
if options.preprocess_translations:
_preprocess_trans_tags(tree)
# Reverse URLS
if options.preprocess_urls:
_preprocess_urls(tree)
# Do variable lookups
if options.preprocess_variables:
sites_enabled = 'django.contrib.sites' in settings.INSTALLED_APPS
_preprocess_variables(tree,
{
'MEDIA_URL': getattr(settings, 'MEDIA_URL', ''),
'STATIC_URL': getattr(settings, 'STATIC_URL', ''),
})
if sites_enabled:
from django.contrib.sites.models import Site
try:
# Don't preprocess anything when we don't have a Site
# instance yet.
site = Site.objects.get_current()
_preprocess_variables(tree,
{
'SITE_DOMAIN': site.domain,
'SITE_NAME': site.name,
'SITE_URL': 'http://%s' % site.domain,
})
except Site.DoesNotExist, e:
pass
# Don't output {% block %} tags in the compiled file.
if options.remove_block_tags:
tree.collapse_nodes_of_class(DjangoBlockTag)
# Preprocess {% callmacro %} tags
if options.preprocess_macros:
_preprocess_macros(tree)
# Group all {% load %} statements
if options.merge_all_load_tags:
_group_all_loads(tree)
# Preprocessable tags
if options.execute_preprocessable_tags:
_execute_preprocessable_tags(tree)
# HTML compiler
if options.is_html:
compile_html(tree, context)
def output(self, handler):
handler(self._open_tag)
Token.output(self, handler)
handler(self._end_tag)
def output(self, handler):
handler(u'{')
Token.output(self, handler)
handler(u'}')
def output(self, handler):
handler(u'{%block '); handler(self.block_name); handler(u'%}')
Token.output(self, handler)
handler(u'{%endblock%}')
def output(self, handler):
handler(u'<style type="text/css"><!--')
Token.output(self, handler)
handler(u"--></style>")
def output(self, handler):
handler(u'[')
Token.output(self, handler)
handler(u']')
def output(self, handler):
self.__open_tag.output(handler)
Token.output(self, handler)
handler("</textarea>")
def output(self, handler):
handler(self.__start)
Token.output(self, handler)
handler(u"<![endif]-->")
def parse(source_code, path, context, main_template=False):
"""
Parse the code.
- source_code: string
- path: for attaching meta information to the tree.
- context: preprocess context (holding the settings/dependecies/warnings, ...)
- main_template: False for includes/extended templates. True for the
original path that was called.
"""
# To start, create the root node of a tree.
tree = Token(name="root", line=1, column=1, path=path)
tree.children = [source_code]
# Lex Django tags
tokenize(tree, __DJANGO_STATES, Token)
# Phase I: add parser extensions
_add_parser_extensions(tree)
# Phase II: process inline tags
_process_inline_tags(tree)
# Phase III: create recursive structure for block level tags.
nest_block_level_elements(tree, __DJANGO_BLOCK_ELEMENTS, DjangoTag, lambda c: c.tagname)
# === Actions ===
if main_template:
_find_first_level_dependencies(tree, context)
# Extend parent template and process includes
tree = _process_extends(tree, context) # NOTE: this returns a new tree!
_preprocess_includes(tree, context)
_preprocess_decorate_tags(tree, context)
# Following actions only need to be applied if this is the 'main' tree.
# It does not make sense to apply it on every include, and then again
# on the complete tree.
if main_template:
_update_preprocess_settings(tree, context)
options = context.options
# Remember translations in context (form PO-file generation)
remember_gettext_entries(tree, context)
# Do translations
if options.preprocess_translations:
_preprocess_trans_tags(tree)
# Reverse URLS
if options.preprocess_urls:
_preprocess_urls(tree)
# Do variable lookups
if options.preprocess_variables:
sites_enabled = "django.contrib.sites" in settings.INSTALLED_APPS
_preprocess_variables(
tree,
{"MEDIA_URL": getattr(settings, "MEDIA_URL", ""), "STATIC_URL": getattr(settings, "STATIC_URL", "")},
)
if sites_enabled:
from django.contrib.sites.models import Site
try:
# Don't preprocess anything when we don't have a Site
# instance yet.
site = Site.objects.get_current()
_preprocess_variables(
tree,
{"SITE_DOMAIN": site.domain, "SITE_NAME": site.name, "SITE_URL": "http://%s" % site.domain},
)
except Site.DoesNotExist, e:
pass
# Don't output {% block %} tags in the compiled file.
if options.remove_block_tags:
tree.collapse_nodes_of_class(DjangoBlockTag)
# Preprocess {% callmacro %} tags
if options.preprocess_macros:
_preprocess_macros(tree)
# Group all {% load %} statements
if options.merge_all_load_tags:
_group_all_loads(tree)
# Preprocessable tags
if options.execute_preprocessable_tags:
_execute_preprocessable_tags(tree)
# HTML compiler
if options.is_html:
compile_html(tree, context)
def output(self, handler):
if self.__show_comment_signs:
handler("<!--")
Token.output(self, handler)
if self.__show_comment_signs:
handler("-->")
def _tokenize(node, nodelist, state_stack, token_stack, root=False):
"""
node: The current parse node that we are lexing. We are lexing
tokens in a parse tree of another language, and this node
is the parse node of the other language where we are now.
state_stack: The current state in our lexing 'grammar'
token_stack: The output token to where we are moving nodes right now.
This is a stack of childnodes lists
root: True when this is the main call.
"""
# Copy input nodes to new list, and clear nodelist
input_nodes = nodelist[:]
nodelist.__init__()
# Position TODO: this information is only right for the first children-list, not
# for the others!!!
line = node.line
column = node.column
path = node.path
# As long as we have input nodes
while input_nodes:
# Pop input node
current_input_node = input_nodes[0]
del input_nodes[0]
if isinstance(current_input_node, basestring):
# Tokenize content
string = current_input_node
# When the string starts with a BOM_UTF8 character, remove it.
string = string.lstrip(unicode(codecs.BOM_UTF8, 'utf8'))
# We want the regex to be able to match as much as possible,
# So, if several basestring nodes, are following each other,
# concatenate as one.
while input_nodes and isinstance(input_nodes[0], basestring):
# Pop another input node
string += input_nodes[0]
del input_nodes[0]
# Parse position
position = 0
while position < len(string):
for compiled_regex, action_list in states[ state_stack[-1] ].transitions():
match = compiled_regex.match(string[position:])
#print state_stack, string[position:position+10]
if match:
(start, count) = match.span()
# Read content
content = string[position : position + count]
# Execute actions for this match
for action in action_list:
if isinstance(action, Record):
if action.value:
token_stack[-1].append(action.value)
else:
token_stack[-1].append(content)
elif isinstance(action, Shift):
position += count
count = 0
# Update row/column
f = content.find('\n')
while f >= 0:
line += 1
column = 1
content = content[f+1:]
f = content.find('\n')
column += len(content)
elif isinstance(action, Push):
state_stack.append(action.state_name)
elif isinstance(action, Pop):
del state_stack[-1]
elif isinstance(action, StartToken):
token = Token(action.state_name, line, column, path)
token_stack[-1].append(token)
token_stack.append(token.children)
elif isinstance(action, StopToken):
# TODO: check following constraint!
# token_stack[-1] is a childnode list now instead of a node. it does no longer
# have an attribute name!
# if action.state_name and token_stack[-1].name != action.state_name:
# raise CompileException(line, column, path, 'Token mismatch')
del token_stack[-1]
elif isinstance(action, Error):
raise CompileException(line, column, path, action.message +
"; near: '%s'" % string[max(0,position-20):position+20])
break # Out of for
# Not a DjangoContent node? Copy in current position.
else:
# Recursively tokenize in this node (continue with states, token will be replaced by parsed content)
if isinstance(current_input_node, classes_to_replace_by_parsed_content):
for l in current_input_node.children_lists:
_tokenize(current_input_node, l, state_stack, token_stack)
# Recursively tokenize in this node (start parsing again in nested node)
elif isinstance(current_input_node, classes_to_enter):
for l in current_input_node.children_lists:
_tokenize(current_input_node, l, state_stack, [ l ], True)
token_stack[-1].append(current_input_node)
# Any other class, copy in current token
else:
token_stack[-1].append(current_input_node)
if root and token_stack != [ nodelist ]:
top = token_stack[-1]
raise CompileException(top.line, top.column, top.path, '%s not terminated' % top.name)
def output(self, handler):
if self.__show_cdata_signs:
handler("<![CDATA[")
Token.output(self, handler)
if self.__show_cdata_signs:
handler("]]>")
def output(self, handler):
# Don't output the template tags.
# (these are hints to the preprocessor only.)
Token.output(self, handler)
def output(self, handler):
handler("</")
Token.output(self, handler)
handler(">")
def output(self, handler):
# Don't output the template tags.
# (these are hints to the preprocessor only.)
Token.output(self, handler)
def output(self, handler):
handler('</')
Token.output(self, handler)
handler('>')
