def __get_parser(self, engine):
if engine == "jinja2":
from compressor.offline.jinja2 import Jinja2Parser
env = settings.COMPRESS_JINJA2_GET_ENVIRONMENT()
parser = Jinja2Parser(charset=settings.FILE_CHARSET, env=env)
elif engine == "django":
from compressor.offline.django import DjangoParser
parser = DjangoParser(charset=settings.FILE_CHARSET)
else:
raise OfflineGenerationError("Invalid templating engine specified.")
return parser
def render_offline(self, context):
"""
If enabled and in offline mode, and not forced check the offline cache
and return the result if given
"""
key = get_offline_hexdigest(self.get_original_content(context))
offline_manifest = get_offline_manifest()
if key in offline_manifest:
return offline_manifest[key]
else:
raise OfflineGenerationError('You have offline compression '
'enabled but key "%s" is missing from offline manifest. '
'You may need to run "python manage.py compress".' % key)
def render_offline(self, kind, mode, compressor, forced, nodelist_string):
"""
If enabled and in offline mode, and not forced or in debug mode
check the offline cache and return the result if given
"""
if (settings.COMPRESS_ENABLED
and settings.COMPRESS_OFFLINE) and not forced:
key = get_offline_jinja_hexdigest(nodelist_string)
offline_manifest = get_offline_manifest()
if key in offline_manifest:
return offline_manifest[key]
else:
raise OfflineGenerationError(
'You have offline compression enabled but key "%s" is missing from offline manifest. You may need to run "python manage.py compress".'
% key)
def render_offline(self, context):
"""
If enabled and in offline mode, and not forced check the offline cache
and return the result if given
"""
original_content = self.get_original_content(context)
key = get_offline_hexdigest(original_content)
offline_manifest = get_offline_manifest()
if key in offline_manifest:
return offline_manifest[key].replace(
settings.COMPRESS_URL_PLACEHOLDER, settings.COMPRESS_URL)
else:
raise OfflineGenerationError(
'You have offline compression '
'enabled but key "%s" is missing from offline manifest. '
'You may need to run "python manage.py compress". Here '
'is the original content:\n\n%s' % (key, original_content))
def render_offline(self, context, mode):
"""
If enabled and in offline mode, and not forced check the offline cache
and return the result if given
"""
original_content = self.get_original_content(context)
key = get_offline_hexdigest(original_content)
offline_manifest = get_offline_manifest()
if key in offline_manifest:
return offline_manifest[key][mode].replace(
settings.COMPRESS_URL_PLACEHOLDER,
# Cast ``settings.COMPRESS_URL`` to a string to allow it to be
# a string-alike object to e.g. add ``SCRIPT_NAME`` WSGI param
# as a *path prefix* to the output URL.
# See https://code.djangoproject.com/ticket/25598.
six.text_type(settings.COMPRESS_URL))
else:
raise OfflineGenerationError(
'You have offline compression '
'enabled but key "%s" is missing from offline manifest. '
'You may need to run "python manage.py compress". Here '
'is the original content:\n\n%s' % (key, original_content))
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
engine = options.get("engine", "django")
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not self.get_loaders():
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings or set 'loaders' in your "
"TEMPLATES dictionary.")
templates = set()
if engine == 'django':
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module,
'get_template_sources', None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(smart_text(origin) for origin in get_template_sources(''))
except (ImportError, AttributeError, TypeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"https://docs.djangoproject.com/en/1.8/topics/templates/ "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
for path in paths:
for root, dirs, files in os.walk(path,
followlinks=options.get('followlinks', False)):
templates.update(os.path.join(root, name)
for name in files if not name.startswith('.') and
any(fnmatch(name, "*%s" % glob) for glob in extensions))
elif engine == 'jinja2':
env = settings.COMPRESS_JINJA2_GET_ENVIRONMENT()
if env and hasattr(env, 'list_templates'):
templates |= set([env.loader.get_source(env, template)[1] for template in
env.list_templates(filter_func=lambda _path:
os.path.splitext(_path)[-1] in extensions)])
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
contexts = settings.COMPRESS_OFFLINE_CONTEXT
if isinstance(contexts, six.string_types):
try:
module, function = get_mod_func(contexts)
contexts = getattr(import_module(module), function)()
except (AttributeError, ImportError, TypeError) as e:
raise ImportError("Couldn't import offline context function %s: %s" %
(settings.COMPRESS_OFFLINE_CONTEXT, e))
elif not isinstance(contexts, (list, tuple)):
contexts = [contexts]
contexts = list(contexts) # evaluate generator
parser = self.__get_parser(engine)
compressor_nodes = OrderedDict()
for template_name in templates:
try:
template = parser.parse(template_name)
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError as e: # broken template -> ignore
if verbosity > 0:
log.write("Invalid template %s: %s\n" % (template_name, smart_text(e)))
continue
except TemplateDoesNotExist: # non existent template -> ignore
if verbosity > 0:
log.write("Non-existent template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
continue
for context_dict in contexts:
context = parser.get_init_context(context_dict)
context = Context(context)
try:
nodes = list(parser.walk_nodes(template, context=context))
except (TemplateDoesNotExist, TemplateSyntaxError) as e:
# Could be an error in some base template
if verbosity > 0:
log.write("Error parsing template %s: %s\n" %
(template_name, smart_text(e)))
continue
if nodes:
template.template_name = template_name
template_nodes = compressor_nodes.setdefault(template, OrderedDict())
for node in nodes:
template_nodes.setdefault(node, []).append(context)
if not compressor_nodes:
raise OfflineGenerationError(
"No 'compress' template tags found in templates."
"Try running compress command with --follow-links and/or"
"--extension=EXTENSIONS")
if verbosity > 0:
log.write("Found 'compress' tags in:\n\t" +
"\n\t".join((t.template_name
for t in compressor_nodes.keys())) + "\n")
log.write("Compressing... ")
block_count = 0
compressed_contexts = []
results = []
offline_manifest = OrderedDict()
for template, nodes in compressor_nodes.items():
template._log = log
template._log_verbosity = verbosity
for node, contexts in nodes.items():
for context in contexts:
if context not in compressed_contexts:
compressed_contexts.append(context)
context.push()
if not parser.process_template(template, context):
continue
parser.process_node(template, context, node)
rendered = parser.render_nodelist(template, context, node)
key = get_offline_hexdigest(rendered)
if key in offline_manifest:
continue
try:
result = parser.render_node(template, context, node)
except Exception as e:
raise CommandError("An error occurred during rendering %s: "
"%s" % (template.template_name, smart_text(e)))
result = result.replace(
settings.COMPRESS_URL, settings.COMPRESS_URL_PLACEHOLDER
)
offline_manifest[key] = result
context.pop()
results.append(result)
block_count += 1
write_offline_manifest(offline_manifest)
context_count = len(compressed_contexts)
log.write("done\nCompressed %d block(s) from %d template(s) for %d context(s).\n" %
(block_count, len(compressor_nodes), context_count))
return block_count, results
class Command(NoArgsCommand):
help = "Compress content outside of the request/response cycle"
option_list = NoArgsCommand.option_list + (
make_option('--extension',
'-e',
action='append',
dest='extensions',
help='The file extension(s) to examine (default: ".html", '
'separate multiple extensions with commas, or use -e '
'multiple times)'),
make_option(
'-f',
'--force',
default=False,
action='store_true',
help="Force the generation of compressed content even if the "
"COMPRESS_ENABLED setting is not True.",
dest='force'),
make_option(
'--follow-links',
default=False,
action='store_true',
help="Follow symlinks when traversing the COMPRESS_ROOT "
"(which defaults to MEDIA_ROOT). Be aware that using this "
"can lead to infinite recursion if a link points to a parent "
"directory of itself.",
dest='follow_links'),
)
requires_model_validation = False
def get_loaders(self):
from django.template.loader import template_source_loaders
if template_source_loaders is None:
try:
from django.template.loader import (find_template as
finder_func)
except ImportError:
from django.template.loader import (find_template_source as
finder_func)
try:
source, name = finder_func('test')
except TemplateDoesNotExist:
pass
from django.template.loader import template_source_loaders
loaders = []
# If template loader is CachedTemplateLoader, return the loaders
# that it wraps around. So if we have
# TEMPLATE_LOADERS = (
# ('django.template.loaders.cached.Loader', (
# 'django.template.loaders.filesystem.Loader',
# 'django.template.loaders.app_directories.Loader',
# )),
# )
# The loaders will return django.template.loaders.filesystem.Loader
# and django.template.loaders.app_directories.Loader
for loader in template_source_loaders:
if CachedLoader is not None and isinstance(loader, CachedLoader):
loaders.extend(loader.loaders)
else:
loaders.append(loader)
return loaders
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not settings.TEMPLATE_LOADERS:
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings.")
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module, 'get_template_sources',
None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(list(get_template_sources('')))
except (ImportError, AttributeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"http://django.me/template-loaders "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
templates = set()
for path in paths:
for root, dirs, files in walk(path,
followlinks=options.get(
'followlinks', False)):
templates.update(
os.path.join(root, name) for name in files
if not name.startswith('.') and any(
fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
compressor_nodes = SortedDict()
for template_name in templates:
try:
template_file = open(template_name)
try:
template = Template(template_file.read().decode(
settings.FILE_CHARSET))
finally:
template_file.close()
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError: # broken template -> try jinja -> ignore if still broken
try:
template_file = open(template_name)
template = jinja_env.parse(template_file.read().decode(
settings.FILE_CHARSET))
template.is_jinja = True
template.name = template_name
except jinja2.exceptions.TemplateSyntaxError:
if verbosity > 0:
log.write("Invalid template at: %s\n" % template_name)
continue
finally:
template_file.close()
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
if getattr(template, 'is_jinja', False):
nodes = template.find_all(jinja2.nodes.CallBlock)
for node in nodes:
try:
compress_node = node.call.node
if (compress_node.identifier ==
'compressor.contrib.jinja2ext.CompressorExtension'
and compress_node.name == '_compress'):
template.template_name = template_name
compressor_nodes.setdefault(template,
[]).append(node)
except AttributeError, IndexError:
pass
else:
nodes = list(self.walk_nodes(template))
if nodes:
template.template_name = template_name
compressor_nodes.setdefault(template, []).extend(nodes)
if not compressor_nodes:
raise OfflineGenerationError(
"No 'compress' template tags found in templates.")
if verbosity > 0:
log.write("Found 'compress' tags in:\n\t" +
"\n\t".join((t.template_name
for t in compressor_nodes.keys())) + "\n")
log.write("Compressing... ")
count = 0
results = []
offline_manifest = {}
for template, nodes in compressor_nodes.iteritems():
if getattr(template, 'is_jinja', False):
for node in nodes:
context = settings.COMPRESS_OFFLINE_CONTEXT.copy()
old_forced = getattr(jinja_env,
'_django_compressor_offline_forced',
None)
jinja_env._django_compressor_offline_forced = True
nodelist = node.body
key = get_offline_jinja_hexdigest(nodelist)
result = render_jinja_node(node, context, jinja_env)
if old_forced is not None:
jinja_env._django_compressor_offline_forced = old_forced
offline_manifest[key] = result
results.append(result)
count += 1
continue
context = Context(settings.COMPRESS_OFFLINE_CONTEXT)
extra_context = {}
firstnode = template.nodelist[0]
if isinstance(firstnode, ExtendsNode):
# If this template has a ExtendsNode, we apply our patch to
# generate the necessary context, and then use it for all the
# nodes in it, just in case (we don't know which nodes were
# in a block)
firstnode._old_get_parent = firstnode.get_parent
firstnode.get_parent = MethodType(patched_get_parent,
firstnode)
extra_context = firstnode.render(context)
context.render_context = extra_context.render_context
for node in nodes:
context.push()
if extra_context and node._block_name:
context['block'] = context.render_context[
BLOCK_CONTEXT_KEY].pop(node._block_name)
if context['block']:
context['block'].context = context
key = get_offline_hexdigest(node.nodelist)
result = node.render(context, forced=True)
offline_manifest[key] = result
context.pop()
results.append(result)
count += 1
write_offline_manifest(offline_manifest)
log.write("done\nCompressed %d block(s) from %d template(s).\n" %
(count, len(compressor_nodes)))
return count, results
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not settings.TEMPLATE_LOADERS:
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings.")
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module, 'get_template_sources',
None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(list(get_template_sources('')))
except (ImportError, AttributeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"http://django.me/template-loaders "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
templates = set()
for path in paths:
for root, dirs, files in walk(path,
followlinks=options.get(
'followlinks', False)):
templates.update(
os.path.join(root, name) for name in files
if not name.startswith('.') and any(
fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
compressor_nodes = SortedDict()
for template_name in templates:
try:
template_file = open(template_name)
try:
template = Template(template_file.read().decode(
settings.FILE_CHARSET))
finally:
template_file.close()
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError: # broken template -> try jinja -> ignore if still broken
try:
template_file = open(template_name)
template = jinja_env.parse(template_file.read().decode(
settings.FILE_CHARSET))
template.is_jinja = True
template.name = template_name
except jinja2.exceptions.TemplateSyntaxError:
if verbosity > 0:
log.write("Invalid template at: %s\n" % template_name)
continue
finally:
template_file.close()
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
if getattr(template, 'is_jinja', False):
nodes = template.find_all(jinja2.nodes.CallBlock)
for node in nodes:
try:
compress_node = node.call.node
if (compress_node.identifier ==
'compressor.contrib.jinja2ext.CompressorExtension'
and compress_node.name == '_compress'):
template.template_name = template_name
compressor_nodes.setdefault(template,
[]).append(node)
except AttributeError, IndexError:
pass
else:
nodes = list(self.walk_nodes(template))
if nodes:
template.template_name = template_name
compressor_nodes.setdefault(template, []).extend(nodes)
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not settings.TEMPLATE_LOADERS:
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings.")
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module,
'get_template_sources', None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(list(get_template_sources('')))
except (ImportError, AttributeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"http://django.me/template-loaders "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
templates = set()
for path in paths:
for root, dirs, files in walk(path,
followlinks=options.get('followlinks', False)):
templates.update(os.path.join(root, name)
for name in files if not name.startswith('.') and
any(fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
compressor_nodes = SortedDict()
for template_name in templates:
try:
template_file = open(template_name)
try:
template = Template(template_file.read().decode(
settings.FILE_CHARSET))
finally:
template_file.close()
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError: # broken template -> ignore
if verbosity > 0:
log.write("Invalid template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
nodes = list(self.walk_nodes(template))
if nodes:
compressor_nodes.setdefault(template_name, []).extend(nodes)
if not compressor_nodes:
raise OfflineGenerationError(
"No 'compress' template tags found in templates.")
if verbosity > 0:
log.write("Found 'compress' tags in:\n\t" +
"\n\t".join(compressor_nodes.keys()) + "\n")
log.write("Compressing... ")
count = 0
results = []
context = Context(settings.COMPRESS_OFFLINE_CONTEXT)
for nodes in compressor_nodes.values():
for node in nodes:
key = get_offline_cachekey(node.nodelist)
try:
result = node.render(context, forced=True)
except Exception, e:
raise CommandError("An error occured during rending: "
"%s" % e)
cache.set(key, result, settings.COMPRESS_OFFLINE_TIMEOUT)
results.append(result)
count += 1
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not settings.TEMPLATE_LOADERS:
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings.")
#from jingo import env
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module, 'get_template_sources',
None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(list(get_template_sources('')))
except (ImportError, AttributeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"http://django.me/template-loaders "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
templates = set()
for path in paths:
for root, dirs, files in walk(path,
followlinks=options.get(
'followlinks', False)):
templates.update(
os.path.join(root, name) for name in files
if not name.startswith('.') and any(
fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
#from compressor.contrib.jinja2ext import CompressorExtension
from jingo_offline_compressor.jinja2ext import CompressorExtension
env.add_extension(CompressorExtension)
compressor_nodes = SortedDict()
for template_name in templates:
try:
#template_content = env.loader.get_source(env, template_name)[0]
template_content = open(template_name).read()
template = env.parse(template_content)
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError, e: # broken template -> ignore
if verbosity > 0:
log.write("Invalid template %s: %s\n" % (template_name, e))
continue
except TemplateDoesNotExist: # non existent template -> ignore
if verbosity > 0:
log.write("Non-existent template at: %s\n" % template_name)
continue
except TemplateDoesNotExist: # non existent template -> ignore
if verbosity > 0:
log.write("Non-existent template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
nodes = list(self.walk_nodes(template))
if nodes:
template.template_name = template_name
compressor_nodes.setdefault(template, []).extend(nodes)
if not compressor_nodes:
raise OfflineGenerationError(
"No 'compress' template tags found in templates."
"Try running compress command with --follow-links and/or"
"--extension=EXTENSIONS")
if verbosity > 0:
log.write("Found 'compress' tags in:\n\t" +
"\n\t".join((t.template_name
for t in compressor_nodes.keys())) + "\n")
log.write("Compressing... ")
count = 0
results = []
offline_manifest = SortedDict()
for template, nodes in compressor_nodes.iteritems():
context = Context(settings.COMPRESS_OFFLINE_CONTEXT)
template._log = log
template._log_verbosity = verbosity
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not settings.TEMPLATE_LOADERS:
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings.")
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module, 'get_template_sources',
None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(list(get_template_sources('')))
except (ImportError, AttributeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"http://django.me/template-loaders "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
templates = set()
for path in paths:
for root, dirs, files in os.walk(path,
followlinks=options.get(
'followlinks', False)):
templates.update(
os.path.join(root, name) for name in files
if not name.startswith('.') and any(
fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
engine = options.get("engine", "django")
parser = self.__get_parser(engine)
compressor_nodes = SortedDict()
for template_name in templates:
try:
template = parser.parse(template_name)
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError as e: # broken template -> ignore
if verbosity > 0:
log.write("Invalid template %s: %s\n" % (template_name, e))
continue
except TemplateDoesNotExist: # non existent template -> ignore
if verbosity > 0:
log.write("Non-existent template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
try:
nodes = list(parser.walk_nodes(template))
except (TemplateDoesNotExist, TemplateSyntaxError) as e:
# Could be an error in some base template
if verbosity > 0:
log.write("Error parsing template %s: %s\n" %
(template_name, e))
continue
if nodes:
template.template_name = template_name
compressor_nodes.setdefault(template, []).extend(nodes)
if not compressor_nodes:
raise OfflineGenerationError(
"No 'compress' template tags found in templates."
"Try running compress command with --follow-links and/or"
"--extension=EXTENSIONS")
if verbosity > 0:
log.write("Found 'compress' tags in:\n\t" +
"\n\t".join((t.template_name
for t in compressor_nodes.keys())) + "\n")
log.write("Compressing... ")
count = 0
results = []
offline_manifest = SortedDict()
init_context = parser.get_init_context(
settings.COMPRESS_OFFLINE_CONTEXT)
for template, nodes in compressor_nodes.items():
context = Context(init_context)
template._log = log
template._log_verbosity = verbosity
if not parser.process_template(template, context):
continue
for node in nodes:
context.push()
parser.process_node(template, context, node)
rendered = parser.render_nodelist(template, context, node)
key = get_offline_hexdigest(rendered)
if key in offline_manifest:
continue
try:
result = parser.render_node(template, context, node)
except Exception as e:
raise CommandError("An error occured during rendering %s: "
"%s" % (template.template_name, e))
offline_manifest[key] = result
context.pop()
results.append(result)
count += 1
write_offline_manifest(offline_manifest)
log.write("done\nCompressed %d block(s) from %d template(s).\n" %
(count, len(compressor_nodes)))
return count, results
def compress(self, engine, extensions, verbosity, follow_links, log):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
if not self.get_loaders():
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings or set 'loaders' in your "
"TEMPLATES dictionary.")
templates = set()
if engine == 'django':
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module,
'get_template_sources',
None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(
smart_str(origin)
for origin in get_template_sources(''))
except (ImportError, AttributeError, TypeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError(
"No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"https://docs.djangoproject.com/en/2.1/topics/templates/ "
"for more information on template "
"loaders.")
if verbosity >= 2:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
for path in paths:
for root, dirs, files in os.walk(path,
followlinks=follow_links):
templates.update(
os.path.relpath(os.path.join(root, name), path)
for name in files if not name.startswith('.') and any(
fnmatch(name, "*%s" % glob)
for glob in extensions))
elif engine == 'jinja2':
env = settings.COMPRESS_JINJA2_GET_ENVIRONMENT()
if env and hasattr(env, 'list_templates'):
templates |= set([
env.loader.get_source(env, template)[1]
for template in env.list_templates(
filter_func=lambda _path: os.path.splitext(_path)[
-1] in extensions)
])
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity >= 2:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
contexts = settings.COMPRESS_OFFLINE_CONTEXT
if isinstance(contexts, str):
try:
module, function = get_mod_func(contexts)
contexts = getattr(import_module(module), function)()
except (AttributeError, ImportError, TypeError) as e:
raise ImportError(
"Couldn't import offline context function %s: %s" %
(settings.COMPRESS_OFFLINE_CONTEXT, e))
elif not isinstance(contexts, (list, tuple)):
contexts = [contexts]
parser = self.__get_parser(engine)
fine_templates = []
if verbosity >= 1:
log.write("Compressing... ")
for template_name in templates:
try:
template = parser.parse(template_name)
template.template_name = template_name
fine_templates.append(template)
except IOError: # unreadable file -> ignore
if verbosity >= 1:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError as e: # broken template -> ignore
if verbosity >= 1:
log.write("Invalid template %s: %s\n" %
(template_name, smart_str(e)))
continue
except TemplateDoesNotExist: # non existent template -> ignore
if verbosity >= 1:
log.write("Non-existent template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity >= 1:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
continue
contexts_count = 0
nodes_count = 0
offline_manifest = OrderedDict()
errors = []
for context_dict in contexts:
compressor_nodes = OrderedDict()
for template in fine_templates:
context = Context(parser.get_init_context(context_dict))
try:
nodes = list(parser.walk_nodes(template, context=context))
except (TemplateDoesNotExist, TemplateSyntaxError) as e:
# Could be an error in some base template
if verbosity >= 1:
log.write("Error parsing template %s: %s\n" %
(template.template_name, smart_str(e)))
continue
if nodes:
template_nodes = compressor_nodes.setdefault(
template, OrderedDict())
for node in nodes:
nodes_count += 1
template_nodes.setdefault(node, []).append(context)
pool = concurrent.futures.ThreadPoolExecutor(max_workers=4)
for template, nodes in compressor_nodes.items():
template._log = log
template._log_verbosity = verbosity
pool.submit(self._compress_template, offline_manifest, nodes,
parser, template, errors)
pool.shutdown(wait=True)
contexts_count += 1
# If errors exist, raise the first one in the list
if errors:
raise errors[0]
elif not nodes_count:
raise OfflineGenerationError(
"No 'compress' template tags found in templates."
"Try running compress command with --follow-links and/or"
"--extension=EXTENSIONS")
if verbosity >= 1:
log.write(
"done\nCompressed %d block(s) from %d template(s) for %d context(s).\n"
% (len(offline_manifest), nodes_count, contexts_count))
return offline_manifest, len(
offline_manifest), offline_manifest.values()
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not settings.TEMPLATE_LOADERS:
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings.")
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module, 'get_template_sources',
None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(list(get_template_sources('')))
except (ImportError, AttributeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"http://django.me/template-loaders "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
templates = set()
for path in paths:
for root, dirs, files in walk(path,
followlinks=options.get(
'followlinks', False)):
templates.update(
os.path.join(root, name) for name in files
if not name.startswith('.') and any(
fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
compressor_nodes = SortedDict()
for template_name in templates:
try:
template_file = open(template_name)
try:
template = Template(template_file.read().decode(
settings.FILE_CHARSET))
finally:
template_file.close()
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError: # broken template -> ignore
if verbosity > 0:
log.write("Invalid template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
nodes = list(self.walk_nodes(template))
if nodes:
template.template_name = template_name
compressor_nodes.setdefault(template, []).extend(nodes)
if not compressor_nodes:
raise OfflineGenerationError(
"No 'compress' template tags found in templates.")
if verbosity > 0:
log.write("Found 'compress' tags in:\n\t" +
"\n\t".join((t.template_name
for t in compressor_nodes.keys())) + "\n")
log.write("Compressing... ")
count = 0
results = []
offline_manifest = {}
for template, nodes in compressor_nodes.iteritems():
context = Context(settings.COMPRESS_OFFLINE_CONTEXT)
extra_context = {}
firstnode = template.nodelist[0]
if isinstance(firstnode, ExtendsNode):
# If this template has a ExtendsNode, we apply our patch to
# generate the necessary context, and then use it for all the
# nodes in it, just in case (we don't know which nodes were
# in a block)
firstnode._old_get_parent = firstnode.get_parent
firstnode.get_parent = MethodType(patched_get_parent,
firstnode)
extra_context = firstnode.render(context)
context.render_context = extra_context.render_context
for node in nodes:
context.push()
if extra_context and node._block_name:
context['block'] = context.render_context[
BLOCK_CONTEXT_KEY].pop(node._block_name)
if context['block']:
context['block'].context = context
key = get_offline_hexdigest(node.nodelist)
try:
result = node.render(context, forced=True)
except Exception, e:
raise CommandError("An error occured during rendering: "
"%s" % e)
offline_manifest[key] = result
context.pop()
results.append(result)
count += 1
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['mako'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not self.get_loaders():
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings or set 'loaders' in your "
"TEMPLATES dictionary.")
templates = set()
for path in self.get_loaders()[0].directories:
for root, dirs, files in os.walk(path,
followlinks=options.get(
'followlinks', False)):
templates.update(
os.path.join(root, name)[len(path):] for name in files
if not name.startswith('.') and any(
fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
contexts = settings.COMPRESS_OFFLINE_CONTEXT
if isinstance(contexts, six.string_types):
try:
module, function = get_mod_func(contexts)
contexts = getattr(import_module(module), function)()
except (AttributeError, ImportError, TypeError) as e:
raise ImportError(
"Couldn't import offline context function %s: %s" %
(settings.COMPRESS_OFFLINE_CONTEXT, e))
elif not isinstance(contexts, (list, tuple)):
contexts = [contexts]
contexts = list(contexts) # evaluate generator
parser = self.__get_parser()
for template_name in templates:
try:
template = parser.get_template(template_name)
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError as e: # broken template -> ignore
if verbosity > 0:
log.write("Invalid template %s: %s\n" %
(template_name, smart_text(e)))
continue
except TemplateDoesNotExist: # non existent template -> ignore
if verbosity > 0:
log.write("Non-existent template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
continue
for context_dict in contexts:
try:
from django.test.client import RequestFactory
request = RequestFactory().get('/')
context_dict['request'] = request
# mako compress tag will compress and upadate manifest
template.render(context=context_dict, request=request)
except (TemplateDoesNotExist, TemplateSyntaxError) as e:
# Could be an error in some base template
if verbosity > 0:
log.write("Error parsing template %s: %s\n" %
(template_name, smart_text(e)))
continue