def output(self, handler): if self.__double_quotes: handler('"') Token.output(self, handler) if self.__double_quotes: handler('"')
def output(self, handler): # Blocktrans output handler(u'{%blocktrans '); for p in self.params: p.output(handler) handler(u' ') handler(u'%}') Token.output(self, handler) handler(u'{%endblocktrans%}')
def output(self, handler): # Blocktrans output handler(u"{%blocktrans ") for p in self.params: p.output(handler) handler(u" ") handler(u"%}") Token.output(self, handler) handler(u"{%endblocktrans%}")
def output(self, handler): # Yield this node's content, or if the variable name # has been changed, use the modified name. if self.__varname: handler(self.__varname) elif self.__link_to: self.__link_to.output(handler) else: Token.output(self, handler)
def output(self, handler): handler("<script ") handler(u" ".join([u"%s%s" % (a, self.__attrs[a]) for a in self.__attrs.keys()])) handler(">") if not self.is_external: handler("//<![CDATA[\n") Token.output(self, handler) if not self.is_external: handler(u"//]]>\n") handler(u"</script>")
def output(self, handler): handler('<script ') handler(u' '.join([ u'%s%s' % (a, self.__attrs[a]) for a in self.__attrs.keys() ])) handler('>') if not self.is_external: handler('//<![CDATA[\n') Token.output(self, handler) if not self.is_external: handler(u'//]]>\n') handler(u'</script>')
def fix_whitespace_bug(js_node): """ Fixes the following case in js code: <script type="text/javascript"> if { {% if test %} ... {% endif %} } </script> The lexer above would remove the space between the first '{' and '{%'. This collision would make Django think it's the start of a variable. """ # For every scope (starting with '{') for scope in js_node.child_nodes_of_class(JavascriptScope): # Look if the first child inside this scope also renders to a '{' if scope.children and scope.children[0].output_as_string()[0:1] == '{': # If so, insert a whitespace token in between. space = Token(name='required-whitespace') space.children = [' '] scope.children.insert(0, space)
def compile_javascript_string(js_string, context, path=''): """ Compile JS code (can be used for external javascript files) """ # First, create a tree to begin with tree = Token(name='root', line=1, column=1, path=path) tree.children = [ js_string ] # Tokenize tokenize(tree, __JS_STATES, Token) # Compile _compile(tree, context) # Output return tree.output_as_string()
def compile_html_string(html_string, path=''): """ Compile a html string """ # First, create a tree to begin with tree = Token(name='root', line=1, column=1, path=path) tree.children = [ html_string ] # Tokenize tokenize(tree, __HTML_STATES, Token) from template_preprocessor.core.context import Context context = Context(path) _process_html_tree(tree, context) # Output return tree.output_as_string()
def compile_html_string(html_string, path=""): """ Compile a html string """ # First, create a tree to begin with tree = Token(name="root", line=1, column=1, path=path) tree.children = [html_string] # Tokenize tokenize(tree, __HTML_STATES, [Token]) from template_preprocessor.core.django_processor import PreProcessSettings options = PreProcessSettings() _process_html_tree(tree, options) # Output return tree.output_as_string()
def compile_javascript_string(js_string, context, path=''): """ Compile JS code (can be used for external javascript files) """ # First, create a tree to begin with tree = Token(name='root', line=1, column=1, path=path) tree.children = [ js_string ] # Tokenize with context.time_operation("tokenize string"): tokenize(tree, __JS_STATES, Token) # Compile _compile(tree, context, already_minified=path.endswith(".min.js")) # Output return tree.output_as_string()
def compile_css_string(css_string, context, path='', url=None): """ Compile CSS code """ # First, create a tree to begin with tree = Token(name='root', line=1, column=1, path=path) tree.children = [ css_string ] # Tokenize tokenize(tree, __CSS_STATES, Token) _add_css_parser_extensions(tree) # Rewrite url() in external css files if url: _rewrite_urls(tree, url) # Compile _compress_css_whitespace(tree) # Output return u''.join([o for o in tree.output_as_string() ])
def compile_css_string(css_string, context, path='', url=None): """ Compile CSS code """ # First, create a tree to begin with tree = Token(name='root', line=1, column=1, path=path) tree.children = [css_string] # Tokenize tokenize(tree, __CSS_STATES, Token) _add_css_parser_extensions(tree) # Rewrite url() in external css files if url: _rewrite_urls(tree, url) # Compile _compress_css_whitespace(tree) # Output return u''.join([o for o in tree.output_as_string()])
def compile_javascript_string(js_string, path=''): """ Compile JS code (can be used for external javascript files) """ # First, create a tree to begin with tree = Token(name='root', line=1, column=1, path=path) tree.children = [ js_string ] # Tokenize tokenize(tree, __JS_STATES, [Token] ) # Compile _add_javascript_parser_extensions(tree) _validate_javascript(tree) _compress_javascript_whitespace(tree) _minify_variable_names(tree) fix_whitespace_bug(tree) # Output return tree.output_as_string()
def output(self, handler): handler(u'(') Token.output(self, handler) handler(u')')
def output(self, handler): if self.__show_cdata_signs: handler('<![CDATA[') Token.output(self, handler) if self.__show_cdata_signs: handler(']]>')
def output(self, handler): handler(u"{%block ") handler(self.block_name) handler(u"%}") Token.output(self, handler) handler(u"{%endblock%}")
def output(self, handler): handler(u'{%macro "') handler(self.macro_name) handler(u'"%}') Token.output(self, handler) handler(u"{%endmacro%}")
def init_extension(self): self.__varname = Token.output_as_string(self, True)
def output(self, handler): self.__open_tag.output(handler) Token.output(self, handler) handler("</%s>" % name)
def output(self, handler): self.__open_tag.output(handler) Token.output(self, handler) handler("</pre>")
def convert_variable(self, name): """ Convert a template variable to a Python variable. """ # 88 -> 88 # a.1.b -> a[1].b # 8.a -> CompileException # "..." -> "..." # var|filter:"..."|filter2:value -> _filters['filter2'](_filters['filter'](var)) # Parse the variable tree = Token(name='root', line=1, column=1, path='django variable') tree.children = [name] tokenize(tree, _DJANGO_VARIABLE_STATES, [Token]) #print tree._print() def handle_filter(subject, children): filter_name = None filter_option = None in_filter_option = False def result(): assert filter_name if filter_name in _native_filters: return _native_filters[filter_name](self, subject, filter_option) elif filter_option: return '_f["%s"](%s, %s)' % (filter_name, subject, filter_option) else: return '_f["%s"](%s)' % (filter_name, subject) for i in range(0, len(children)): part = children[i].output_as_string() c = children[i] if c.name == 'digits': if not filter_option and in_filter_option: filter_option = part else: raise CompileException(self._tag, 'Invalid variable') elif c.name == 'name': if not filter_option and in_filter_option: filter_option = part elif not filter_name: filter_name = part else: raise CompileException(self._tag, 'Invalid variable') elif c.name == 'string': if not filter_option and in_filter_option: filter_option = part else: raise CompileException(self._tag, 'Invalid variable') elif c.name == 'trans': if not filter_option and in_filter_option: filter_option = '_(%s)' % c.output_as_string() else: raise CompileException(self._tag, 'Invalid variable') if c.name == 'filter-option' and filter_name: # Entered the colon ':' in_filter_option = True elif c.name == 'pipe': # | is the start of a following filter return handle_filter(result(), children[i + 1:]) return result() def handle_var(children): out = [] for i in range(0, len(children)): part = children[i].output_as_string() c = children[i] if c.name == 'digits': # First digits are literals, following digits are indexers out.append('[%s]' % part if out else part) elif c.name == 'dot': #out.append('.') # assume last is not a dot pass elif c.name == 'string': out.append(part) elif c.name == 'name': if out: out.append('.%s' % part) else: if not self.variable_in_current_scope(part): # If variable is not found in current or one of the parents' # scopes, then prefix variable with "_c." out.append('_c.%s' % part) else: out.append(part) elif c.name == 'trans': if out: raise CompileException(self._tag, 'Invalid variable') else: out.append('_(%s)' % handle_var(c.children)) elif c.name == 'pipe': # | is the start of a filter return handle_filter(''.join(out), children[i + 1:]) return ''.join(out) return handle_var(tree.children)
def output(self, handler): handler(u'{%macro "'); handler(self.macro_name); handler(u'"%}') Token.output(self, handler) handler(u'{%endmacro%}')
def parse(source_code, path, context, main_template=False): """ Parse the code. - source_code: string - path: for attaching meta information to the tree. - context: preprocess context (holding the settings/dependecies/warnings, ...) - main_template: False for includes/extended templates. True for the original path that was called. """ # To start, create the root node of a tree. tree = Token(name='root', line=1, column=1, path=path) tree.children = [ source_code ] # Lex Django tags tokenize(tree, __DJANGO_STATES, Token) # Phase I: add parser extensions _add_parser_extensions(tree) # Phase II: process inline tags _process_inline_tags(tree) # Phase III: create recursive structure for block level tags. nest_block_level_elements(tree, __DJANGO_BLOCK_ELEMENTS, DjangoTag, lambda c: c.tagname) # === Actions === if main_template: _find_first_level_dependencies(tree, context) # Extend parent template and process includes tree = _process_extends(tree, context) # NOTE: this returns a new tree! _preprocess_includes(tree, context) _preprocess_decorate_tags(tree, context) # Following actions only need to be applied if this is the 'main' tree. # It does not make sense to apply it on every include, and then again # on the complete tree. if main_template: _update_preprocess_settings(tree, context) options = context.options # Remember translations in context (form PO-file generation) remember_gettext_entries(tree, context) # Do translations if options.preprocess_translations: _preprocess_trans_tags(tree) # Reverse URLS if options.preprocess_urls: _preprocess_urls(tree) # Do variable lookups if options.preprocess_variables: sites_enabled = 'django.contrib.sites' in settings.INSTALLED_APPS _preprocess_variables(tree, { 'MEDIA_URL': getattr(settings, 'MEDIA_URL', ''), 'STATIC_URL': getattr(settings, 'STATIC_URL', ''), }) if sites_enabled: from django.contrib.sites.models import Site try: # Don't preprocess anything when we don't have a Site # instance yet. site = Site.objects.get_current() _preprocess_variables(tree, { 'SITE_DOMAIN': site.domain, 'SITE_NAME': site.name, 'SITE_URL': 'http://%s' % site.domain, }) except Site.DoesNotExist, e: pass # Don't output {% block %} tags in the compiled file. if options.remove_block_tags: tree.collapse_nodes_of_class(DjangoBlockTag) # Preprocess {% callmacro %} tags if options.preprocess_macros: _preprocess_macros(tree) # Group all {% load %} statements if options.merge_all_load_tags: _group_all_loads(tree) # Preprocessable tags if options.execute_preprocessable_tags: _execute_preprocessable_tags(tree) # HTML compiler if options.is_html: compile_html(tree, context)
def output(self, handler): handler(self._open_tag) Token.output(self, handler) handler(self._end_tag)
def output(self, handler): handler(u'{') Token.output(self, handler) handler(u'}')
def output(self, handler): handler(u'{%block '); handler(self.block_name); handler(u'%}') Token.output(self, handler) handler(u'{%endblock%}')
def output(self, handler): handler(u'<style type="text/css"><!--') Token.output(self, handler) handler(u"--></style>")
def output(self, handler): handler(u'[') Token.output(self, handler) handler(u']')
def output(self, handler): self.__open_tag.output(handler) Token.output(self, handler) handler("</textarea>")
def output(self, handler): handler(self.__start) Token.output(self, handler) handler(u"<![endif]-->")
def parse(source_code, path, context, main_template=False): """ Parse the code. - source_code: string - path: for attaching meta information to the tree. - context: preprocess context (holding the settings/dependecies/warnings, ...) - main_template: False for includes/extended templates. True for the original path that was called. """ # To start, create the root node of a tree. tree = Token(name="root", line=1, column=1, path=path) tree.children = [source_code] # Lex Django tags tokenize(tree, __DJANGO_STATES, Token) # Phase I: add parser extensions _add_parser_extensions(tree) # Phase II: process inline tags _process_inline_tags(tree) # Phase III: create recursive structure for block level tags. nest_block_level_elements(tree, __DJANGO_BLOCK_ELEMENTS, DjangoTag, lambda c: c.tagname) # === Actions === if main_template: _find_first_level_dependencies(tree, context) # Extend parent template and process includes tree = _process_extends(tree, context) # NOTE: this returns a new tree! _preprocess_includes(tree, context) _preprocess_decorate_tags(tree, context) # Following actions only need to be applied if this is the 'main' tree. # It does not make sense to apply it on every include, and then again # on the complete tree. if main_template: _update_preprocess_settings(tree, context) options = context.options # Remember translations in context (form PO-file generation) remember_gettext_entries(tree, context) # Do translations if options.preprocess_translations: _preprocess_trans_tags(tree) # Reverse URLS if options.preprocess_urls: _preprocess_urls(tree) # Do variable lookups if options.preprocess_variables: sites_enabled = "django.contrib.sites" in settings.INSTALLED_APPS _preprocess_variables( tree, {"MEDIA_URL": getattr(settings, "MEDIA_URL", ""), "STATIC_URL": getattr(settings, "STATIC_URL", "")}, ) if sites_enabled: from django.contrib.sites.models import Site try: # Don't preprocess anything when we don't have a Site # instance yet. site = Site.objects.get_current() _preprocess_variables( tree, {"SITE_DOMAIN": site.domain, "SITE_NAME": site.name, "SITE_URL": "http://%s" % site.domain}, ) except Site.DoesNotExist, e: pass # Don't output {% block %} tags in the compiled file. if options.remove_block_tags: tree.collapse_nodes_of_class(DjangoBlockTag) # Preprocess {% callmacro %} tags if options.preprocess_macros: _preprocess_macros(tree) # Group all {% load %} statements if options.merge_all_load_tags: _group_all_loads(tree) # Preprocessable tags if options.execute_preprocessable_tags: _execute_preprocessable_tags(tree) # HTML compiler if options.is_html: compile_html(tree, context)
def output(self, handler): if self.__show_comment_signs: handler("<!--") Token.output(self, handler) if self.__show_comment_signs: handler("-->")
def _tokenize(node, nodelist, state_stack, token_stack, root=False): """ node: The current parse node that we are lexing. We are lexing tokens in a parse tree of another language, and this node is the parse node of the other language where we are now. state_stack: The current state in our lexing 'grammar' token_stack: The output token to where we are moving nodes right now. This is a stack of childnodes lists root: True when this is the main call. """ # Copy input nodes to new list, and clear nodelist input_nodes = nodelist[:] nodelist.__init__() # Position TODO: this information is only right for the first children-list, not # for the others!!! line = node.line column = node.column path = node.path # As long as we have input nodes while input_nodes: # Pop input node current_input_node = input_nodes[0] del input_nodes[0] if isinstance(current_input_node, basestring): # Tokenize content string = current_input_node # When the string starts with a BOM_UTF8 character, remove it. string = string.lstrip(unicode(codecs.BOM_UTF8, 'utf8')) # We want the regex to be able to match as much as possible, # So, if several basestring nodes, are following each other, # concatenate as one. while input_nodes and isinstance(input_nodes[0], basestring): # Pop another input node string += input_nodes[0] del input_nodes[0] # Parse position position = 0 while position < len(string): for compiled_regex, action_list in states[ state_stack[-1] ].transitions(): match = compiled_regex.match(string[position:]) #print state_stack, string[position:position+10] if match: (start, count) = match.span() # Read content content = string[position : position + count] # Execute actions for this match for action in action_list: if isinstance(action, Record): if action.value: token_stack[-1].append(action.value) else: token_stack[-1].append(content) elif isinstance(action, Shift): position += count count = 0 # Update row/column f = content.find('\n') while f >= 0: line += 1 column = 1 content = content[f+1:] f = content.find('\n') column += len(content) elif isinstance(action, Push): state_stack.append(action.state_name) elif isinstance(action, Pop): del state_stack[-1] elif isinstance(action, StartToken): token = Token(action.state_name, line, column, path) token_stack[-1].append(token) token_stack.append(token.children) elif isinstance(action, StopToken): # TODO: check following constraint! # token_stack[-1] is a childnode list now instead of a node. it does no longer # have an attribute name! # if action.state_name and token_stack[-1].name != action.state_name: # raise CompileException(line, column, path, 'Token mismatch') del token_stack[-1] elif isinstance(action, Error): raise CompileException(line, column, path, action.message + "; near: '%s'" % string[max(0,position-20):position+20]) break # Out of for # Not a DjangoContent node? Copy in current position. else: # Recursively tokenize in this node (continue with states, token will be replaced by parsed content) if isinstance(current_input_node, classes_to_replace_by_parsed_content): for l in current_input_node.children_lists: _tokenize(current_input_node, l, state_stack, token_stack) # Recursively tokenize in this node (start parsing again in nested node) elif isinstance(current_input_node, classes_to_enter): for l in current_input_node.children_lists: _tokenize(current_input_node, l, state_stack, [ l ], True) token_stack[-1].append(current_input_node) # Any other class, copy in current token else: token_stack[-1].append(current_input_node) if root and token_stack != [ nodelist ]: top = token_stack[-1] raise CompileException(top.line, top.column, top.path, '%s not terminated' % top.name)
def output(self, handler): if self.__show_cdata_signs: handler("<![CDATA[") Token.output(self, handler) if self.__show_cdata_signs: handler("]]>")
def output(self, handler): # Don't output the template tags. # (these are hints to the preprocessor only.) Token.output(self, handler)
def output(self, handler): handler("</") Token.output(self, handler) handler(">")
def output(self, handler): handler('</') Token.output(self, handler) handler('>')