def fix_whitespace_bug(js_node):
"""
Fixes the following case in js code:
<script type="text/javascript"> if { {% if test %} ... {% endif %} } </script>
The lexer above would remove the space between the first '{' and '{%'. This collision
would make Django think it's the start of a variable.
"""
# For every scope (starting with '{')
for scope in js_node.child_nodes_of_class(JavascriptScope):
# Look if the first child inside this scope also renders to a '{'
if scope.children and scope.children[0].output_as_string()[0:1] == '{':
# If so, insert a whitespace token in between.
space = Token(name='required-whitespace')
space.children = [' ']
scope.children.insert(0, space)
def compile_javascript_string(js_string, context, path=''):
"""
Compile JS code (can be used for external javascript files)
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ js_string ]
# Tokenize
tokenize(tree, __JS_STATES, Token)
# Compile
_compile(tree, context)
# Output
return tree.output_as_string()
def compile_html_string(html_string, path=''):
"""
Compile a html string
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ html_string ]
# Tokenize
tokenize(tree, __HTML_STATES, Token)
from template_preprocessor.core.context import Context
context = Context(path)
_process_html_tree(tree, context)
# Output
return tree.output_as_string()
def compile_html_string(html_string, path=""):
"""
Compile a html string
"""
# First, create a tree to begin with
tree = Token(name="root", line=1, column=1, path=path)
tree.children = [html_string]
# Tokenize
tokenize(tree, __HTML_STATES, [Token])
from template_preprocessor.core.django_processor import PreProcessSettings
options = PreProcessSettings()
_process_html_tree(tree, options)
# Output
return tree.output_as_string()
def compile_javascript_string(js_string, context, path=''):
"""
Compile JS code (can be used for external javascript files)
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ js_string ]
# Tokenize
with context.time_operation("tokenize string"):
tokenize(tree, __JS_STATES, Token)
# Compile
_compile(tree, context, already_minified=path.endswith(".min.js"))
# Output
return tree.output_as_string()
def compile_css_string(css_string, context, path='', url=None):
"""
Compile CSS code
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ css_string ]
# Tokenize
tokenize(tree, __CSS_STATES, Token)
_add_css_parser_extensions(tree)
# Rewrite url() in external css files
if url:
_rewrite_urls(tree, url)
# Compile
_compress_css_whitespace(tree)
# Output
return u''.join([o for o in tree.output_as_string() ])
def compile_javascript_string(js_string, path=''):
"""
Compile JS code (can be used for external javascript files)
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ js_string ]
# Tokenize
tokenize(tree, __JS_STATES, [Token] )
# Compile
_add_javascript_parser_extensions(tree)
_validate_javascript(tree)
_compress_javascript_whitespace(tree)
_minify_variable_names(tree)
fix_whitespace_bug(tree)
# Output
return tree.output_as_string()
def compile_css_string(css_string, context, path='', url=None):
"""
Compile CSS code
"""
# First, create a tree to begin with
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [css_string]
# Tokenize
tokenize(tree, __CSS_STATES, Token)
_add_css_parser_extensions(tree)
# Rewrite url() in external css files
if url:
_rewrite_urls(tree, url)
# Compile
_compress_css_whitespace(tree)
# Output
return u''.join([o for o in tree.output_as_string()])
def parse(source_code, path, context, main_template=False):
"""
Parse the code.
- source_code: string
- path: for attaching meta information to the tree.
- context: preprocess context (holding the settings/dependecies/warnings, ...)
- main_template: False for includes/extended templates. True for the
original path that was called.
"""
# To start, create the root node of a tree.
tree = Token(name="root", line=1, column=1, path=path)
tree.children = [source_code]
# Lex Django tags
tokenize(tree, __DJANGO_STATES, Token)
# Phase I: add parser extensions
_add_parser_extensions(tree)
# Phase II: process inline tags
_process_inline_tags(tree)
# Phase III: create recursive structure for block level tags.
nest_block_level_elements(tree, __DJANGO_BLOCK_ELEMENTS, DjangoTag, lambda c: c.tagname)
# === Actions ===
if main_template:
_find_first_level_dependencies(tree, context)
# Extend parent template and process includes
tree = _process_extends(tree, context) # NOTE: this returns a new tree!
_preprocess_includes(tree, context)
_preprocess_decorate_tags(tree, context)
# Following actions only need to be applied if this is the 'main' tree.
# It does not make sense to apply it on every include, and then again
# on the complete tree.
if main_template:
_update_preprocess_settings(tree, context)
options = context.options
# Remember translations in context (form PO-file generation)
remember_gettext_entries(tree, context)
# Do translations
if options.preprocess_translations:
_preprocess_trans_tags(tree)
# Reverse URLS
if options.preprocess_urls:
_preprocess_urls(tree)
# Do variable lookups
if options.preprocess_variables:
sites_enabled = "django.contrib.sites" in settings.INSTALLED_APPS
_preprocess_variables(
tree,
{"MEDIA_URL": getattr(settings, "MEDIA_URL", ""), "STATIC_URL": getattr(settings, "STATIC_URL", "")},
)
if sites_enabled:
from django.contrib.sites.models import Site
try:
# Don't preprocess anything when we don't have a Site
# instance yet.
site = Site.objects.get_current()
_preprocess_variables(
tree,
{"SITE_DOMAIN": site.domain, "SITE_NAME": site.name, "SITE_URL": "http://%s" % site.domain},
)
except Site.DoesNotExist, e:
pass
# Don't output {% block %} tags in the compiled file.
if options.remove_block_tags:
tree.collapse_nodes_of_class(DjangoBlockTag)
# Preprocess {% callmacro %} tags
if options.preprocess_macros:
_preprocess_macros(tree)
# Group all {% load %} statements
if options.merge_all_load_tags:
_group_all_loads(tree)
# Preprocessable tags
if options.execute_preprocessable_tags:
_execute_preprocessable_tags(tree)
# HTML compiler
if options.is_html:
compile_html(tree, context)
def parse(source_code, path, context, main_template=False):
"""
Parse the code.
- source_code: string
- path: for attaching meta information to the tree.
- context: preprocess context (holding the settings/dependecies/warnings, ...)
- main_template: False for includes/extended templates. True for the
original path that was called.
"""
# To start, create the root node of a tree.
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ source_code ]
# Lex Django tags
tokenize(tree, __DJANGO_STATES, Token)
# Phase I: add parser extensions
_add_parser_extensions(tree)
# Phase II: process inline tags
_process_inline_tags(tree)
# Phase III: create recursive structure for block level tags.
nest_block_level_elements(tree, __DJANGO_BLOCK_ELEMENTS, DjangoTag, lambda c: c.tagname)
# === Actions ===
if main_template:
_find_first_level_dependencies(tree, context)
# Extend parent template and process includes
tree = _process_extends(tree, context) # NOTE: this returns a new tree!
_preprocess_includes(tree, context)
_preprocess_decorate_tags(tree, context)
# Following actions only need to be applied if this is the 'main' tree.
# It does not make sense to apply it on every include, and then again
# on the complete tree.
if main_template:
_update_preprocess_settings(tree, context)
options = context.options
# Remember translations in context (form PO-file generation)
remember_gettext_entries(tree, context)
# Do translations
if options.preprocess_translations:
_preprocess_trans_tags(tree)
# Reverse URLS
if options.preprocess_urls:
_preprocess_urls(tree)
# Do variable lookups
if options.preprocess_variables:
sites_enabled = 'django.contrib.sites' in settings.INSTALLED_APPS
_preprocess_variables(tree,
{
'MEDIA_URL': getattr(settings, 'MEDIA_URL', ''),
'STATIC_URL': getattr(settings, 'STATIC_URL', ''),
})
if sites_enabled:
from django.contrib.sites.models import Site
try:
# Don't preprocess anything when we don't have a Site
# instance yet.
site = Site.objects.get_current()
_preprocess_variables(tree,
{
'SITE_DOMAIN': site.domain,
'SITE_NAME': site.name,
'SITE_URL': 'http://%s' % site.domain,
})
except Site.DoesNotExist, e:
pass
# Don't output {% block %} tags in the compiled file.
if options.remove_block_tags:
tree.collapse_nodes_of_class(DjangoBlockTag)
# Preprocess {% callmacro %} tags
if options.preprocess_macros:
_preprocess_macros(tree)
# Group all {% load %} statements
if options.merge_all_load_tags:
_group_all_loads(tree)
# Preprocessable tags
if options.execute_preprocessable_tags:
_execute_preprocessable_tags(tree)
# HTML compiler
if options.is_html:
compile_html(tree, context)
def convert_variable(self, name):
"""
Convert a template variable to a Python variable.
"""
# 88 -> 88
# a.1.b -> a[1].b
# 8.a -> CompileException
# "..." -> "..."
# var|filter:"..."|filter2:value -> _filters['filter2'](_filters['filter'](var))
# Parse the variable
tree = Token(name='root', line=1, column=1, path='django variable')
tree.children = [name]
tokenize(tree, _DJANGO_VARIABLE_STATES, [Token])
#print tree._print()
def handle_filter(subject, children):
filter_name = None
filter_option = None
in_filter_option = False
def result():
assert filter_name
if filter_name in _native_filters:
return _native_filters[filter_name](self, subject,
filter_option)
elif filter_option:
return '_f["%s"](%s, %s)' % (filter_name, subject,
filter_option)
else:
return '_f["%s"](%s)' % (filter_name, subject)
for i in range(0, len(children)):
part = children[i].output_as_string()
c = children[i]
if c.name == 'digits':
if not filter_option and in_filter_option:
filter_option = part
else:
raise CompileException(self._tag, 'Invalid variable')
elif c.name == 'name':
if not filter_option and in_filter_option:
filter_option = part
elif not filter_name:
filter_name = part
else:
raise CompileException(self._tag, 'Invalid variable')
elif c.name == 'string':
if not filter_option and in_filter_option:
filter_option = part
else:
raise CompileException(self._tag, 'Invalid variable')
elif c.name == 'trans':
if not filter_option and in_filter_option:
filter_option = '_(%s)' % c.output_as_string()
else:
raise CompileException(self._tag, 'Invalid variable')
if c.name == 'filter-option' and filter_name:
# Entered the colon ':'
in_filter_option = True
elif c.name == 'pipe':
# | is the start of a following filter
return handle_filter(result(), children[i + 1:])
return result()
def handle_var(children):
out = []
for i in range(0, len(children)):
part = children[i].output_as_string()
c = children[i]
if c.name == 'digits':
# First digits are literals, following digits are indexers
out.append('[%s]' % part if out else part)
elif c.name == 'dot':
#out.append('.') # assume last is not a dot
pass
elif c.name == 'string':
out.append(part)
elif c.name == 'name':
if out:
out.append('.%s' % part)
else:
if not self.variable_in_current_scope(part):
# If variable is not found in current or one of the parents'
# scopes, then prefix variable with "_c."
out.append('_c.%s' % part)
else:
out.append(part)
elif c.name == 'trans':
if out:
raise CompileException(self._tag, 'Invalid variable')
else:
out.append('_(%s)' % handle_var(c.children))
elif c.name == 'pipe':
# | is the start of a filter
return handle_filter(''.join(out), children[i + 1:])
return ''.join(out)
return handle_var(tree.children)
def convert_variable(self, name):
"""
Convert a template variable to a Python variable.
"""
# 88 -> 88
# a.1.b -> a[1].b
# 8.a -> CompileException
# "..." -> "..."
# var|filter:"..."|filter2:value -> _filters['filter2'](_filters['filter'](var))
# Parse the variable
tree = Token(name='root', line=1, column=1, path='django variable')
tree.children = [ name ]
tokenize(tree, _DJANGO_VARIABLE_STATES, [Token])
#print tree._print()
def handle_filter(subject, children):
filter_name = None
filter_option = None
in_filter_option = False
def result():
assert filter_name
if filter_name in _native_filters:
return _native_filters[filter_name](self, subject, filter_option)
elif filter_option:
return '_f["%s"](%s, %s)' % (filter_name, subject, filter_option)
else:
return '_f["%s"](%s)' % (filter_name, subject)
for i in range(0,len(children)):
part = children[i].output_as_string()
c = children[i]
if c.name == 'digits':
if not filter_option and in_filter_option:
filter_option = part
else:
raise CompileException(self._tag, 'Invalid variable')
elif c.name == 'name':
if not filter_option and in_filter_option:
filter_option = part
elif not filter_name:
filter_name = part
else:
raise CompileException(self._tag, 'Invalid variable')
elif c.name == 'string':
if not filter_option and in_filter_option:
filter_option = part
else:
raise CompileException(self._tag, 'Invalid variable')
elif c.name == 'trans':
if not filter_option and in_filter_option:
filter_option = '_(%s)' % c.output_as_string()
else:
raise CompileException(self._tag, 'Invalid variable')
if c.name == 'filter-option' and filter_name:
# Entered the colon ':'
in_filter_option = True
elif c.name == 'pipe':
# | is the start of a following filter
return handle_filter(result(), children[i+1:])
return result()
def handle_var(children):
out = []
for i in range(0,len(children)):
part = children[i].output_as_string()
c = children[i]
if c.name == 'digits':
# First digits are literals, following digits are indexers
out.append('[%s]' % part if out else part)
elif c.name == 'dot':
#out.append('.') # assume last is not a dot
pass
elif c.name == 'string':
out.append(part)
elif c.name == 'name':
if out:
out.append('.%s' % part)
else:
if not self.variable_in_current_scope(part):
# If variable is not found in current or one of the parents'
# scopes, then prefix variable with "_c."
out.append('_c.%s' % part)
else:
out.append(part)
elif c.name == 'trans':
if out:
raise CompileException(self._tag, 'Invalid variable')
else:
out.append('_(%s)' % handle_var(c.children))
elif c.name == 'pipe':
# | is the start of a filter
return handle_filter(''.join(out), children[i+1:])
return ''.join(out)
return handle_var(tree.children)
def parse(source_code, path, loader, main_template=False, options=None):
"""
Parse the code.
- source_code: string
- loader: method to be called to include other templates.
- path: for attaching meta information to the tree.
- main_template: False for includes/extended templates. True for the
original path that was called.
"""
# To start, create the root node of a tree.
tree = Token(name='root', line=1, column=1, path=path)
tree.children = [ source_code ]
# Lex Django tags
tokenize(tree, __DJANGO_STATES, [Token])
# Phase I: add parser extensions
_add_parser_extensions(tree)
# Phase II: process inline tags
_process_inline_tags(tree)
# Phase III: create recursive structure for block level tags.
nest_block_level_elements(tree, __DJANGO_BLOCK_ELEMENTS, [DjangoTag], lambda c: c.tagname)
# === Actions ===
# Extend parent template and process includes
tree = _process_extends(tree, loader) # NOTE: this returns a new tree!
_preprocess_includes(tree, loader)
_preprocess_decorate_tags(tree, loader)
# Following actions only need to be applied if this is the 'main' tree.
# It does not make sense to apply it on every include, and then again
# on the complete tree.
if main_template:
options = _get_preprocess_settings(tree, options)
# Do translations
if options.preprocess_translations:
_preprocess_trans_tags(tree)
# Reverse URLS
if options.preprocess_urls:
_preprocess_urls(tree)
# Do variable lookups
if options.preprocess_variables:
from django.contrib.sites.models import Site
_preprocess_variables(tree,
{
'MEDIA_URL': settings.MEDIA_URL,
'SITE_DOMAIN': Site.objects.get_current().domain,
'SITE_NAME': Site.objects.get_current().name,
'SITE_URL': 'http://%s' % Site.objects.get_current().domain,
})
# Don't output {% block %} tags in the compiled file.
if options.remove_block_tags:
tree.collapse_nodes_of_class(DjangoBlockTag)
# Preprocess {% callmacro %} tags
if options.preprocess_macros:
_preprocess_macros(tree)
if options.preprocess_ifdebug:
_preprocess_ifdebug(tree)
# Group all {% load %} statements
if options.merge_all_load_tags:
_group_all_loads(tree)
# Preprocessable tags
if options.execute_preprocessable_tags:
_execute_preprocessable_tags(tree)
# HTML compiler
if options.is_html:
compile_html(tree, options)
return tree