def test_precompiler_cache_issue750(self, mock_cache):
# emulate memcached and return string
mock_cache.side_effect = (lambda key: str("body { color:#990; }"))
command = '%s %s -f {infile} -o {outfile}' % (sys.executable,
self.test_precompiler)
compiler = CachedCompilerFilter(command=command,
**self.cached_precompiler_args)
self.assertEqual("body { color:#990; }", compiler.input())
self.assertEqual(type(compiler.input()),
type(smart_text("body { color:#990; }")))
def input(self, **kwargs):
if self.mimetype in settings.COMPRESS_CACHEABLE_PRECOMPILERS:
key = self.get_cache_key()
data = cache.get(key)
if data is not None:
return smart_text(data)
filtered = super(CachedCompilerFilter, self).input(**kwargs)
cache.set(key, filtered, settings.COMPRESS_REBUILD_TIMEOUT)
return filtered
else:
return super(CachedCompilerFilter, self).input(**kwargs)
def elem_content(self, elem):
return smart_text(elem['text'])
def elem_str(self, elem):
elem_as_string = smart_text(elem)
if elem.name == 'link':
# This makes testcases happy
elem_as_string = elem_as_string.replace('/>', '>')
return elem_as_string
def elem_str(self, elem):
# This method serializes HTML in a way that does not pass all tests.
# However, this method is only called in tests anyway, so it doesn't
# really matter.
return smart_text(self._serialize(elem))
def elem_content(self, elem):
return smart_text(elem.text)
def compress(self, engine, extensions, verbosity, follow_links, log):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
if not self.get_loaders():
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings or set 'loaders' in your "
"TEMPLATES dictionary.")
templates = set()
if engine == 'django':
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module,
'get_template_sources', None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(smart_text(origin) for origin in get_template_sources(''))
except (ImportError, AttributeError, TypeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"https://docs.djangoproject.com/en/2.1/topics/templates/ "
"for more information on template "
"loaders.")
if verbosity >= 2:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
for path in paths:
for root, dirs, files in os.walk(path, followlinks=follow_links):
templates.update(os.path.join(root, name)
for name in files if not name.startswith('.') and
any(fnmatch(name, "*%s" % glob) for glob in extensions))
elif engine == 'jinja2':
env = settings.COMPRESS_JINJA2_GET_ENVIRONMENT()
if env and hasattr(env, 'list_templates'):
templates |= set([env.loader.get_source(env, template)[1] for template in
env.list_templates(filter_func=lambda _path:
os.path.splitext(_path)[-1] in extensions)])
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity >= 2:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
contexts = settings.COMPRESS_OFFLINE_CONTEXT
if isinstance(contexts, six.string_types):
try:
module, function = get_mod_func(contexts)
contexts = getattr(import_module(module), function)()
except (AttributeError, ImportError, TypeError) as e:
raise ImportError("Couldn't import offline context function %s: %s" %
(settings.COMPRESS_OFFLINE_CONTEXT, e))
elif not isinstance(contexts, (list, tuple)):
contexts = [contexts]
parser = self.__get_parser(engine)
fine_templates = []
if verbosity >= 1:
log.write("Compressing... ")
for template_name in templates:
try:
template = parser.parse(template_name)
template.template_name = template_name
fine_templates.append(template)
except IOError: # unreadable file -> ignore
if verbosity >= 1:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError as e: # broken template -> ignore
if verbosity >= 1:
log.write("Invalid template %s: %s\n" % (template_name, smart_text(e)))
continue
except TemplateDoesNotExist: # non existent template -> ignore
if verbosity >= 1:
log.write("Non-existent template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity >= 1:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
continue
contexts_count = 0
nodes_count = 0
block_count = 0
offline_manifest = OrderedDict()
results = []
for context_dict in contexts:
compressor_nodes = OrderedDict()
for template in fine_templates:
context = Context(parser.get_init_context(context_dict))
try:
nodes = list(parser.walk_nodes(template, context=context))
except (TemplateDoesNotExist, TemplateSyntaxError) as e:
# Could be an error in some base template
if verbosity >= 1:
log.write("Error parsing template %s: %s\n" %
(template.template_name, smart_text(e)))
continue
if nodes:
template_nodes = compressor_nodes.setdefault(template, OrderedDict())
for node in nodes:
nodes_count += 1
template_nodes.setdefault(node, []).append(context)
for template, nodes in compressor_nodes.items():
template._log = log
template._log_verbosity = verbosity
for node, node_contexts in nodes.items():
for context in node_contexts:
context.push()
if not parser.process_template(template, context):
continue
parser.process_node(template, context, node)
rendered = parser.render_nodelist(template, context, node)
key = get_offline_hexdigest(rendered)
if key in offline_manifest:
continue
try:
result = parser.render_node(template, context, node)
except Exception as e:
raise CommandError("An error occurred during rendering %s: "
"%s" % (template.template_name, smart_text(e)))
result = result.replace(
settings.COMPRESS_URL, settings.COMPRESS_URL_PLACEHOLDER
)
offline_manifest[key] = result
context.pop()
results.append(result)
block_count += 1
if not nodes_count:
raise OfflineGenerationError(
"No 'compress' template tags found in templates."
"Try running compress command with --follow-links and/or"
"--extension=EXTENSIONS")
if verbosity >= 1:
log.write("done\nCompressed %d block(s) from %d template(s) for %d context(s).\n" %
(block_count, nodes_count, contexts_count))
return offline_manifest, block_count, results
def input(self, **kwargs):
encoding = self.default_encoding
options = dict(self.options)
if self.infile is None and "{infile}" in self.command:
# create temporary input file if needed
if self.filename is None:
self.infile = NamedTemporaryFile(mode='wb')
self.infile.write(self.content.encode(encoding))
self.infile.flush()
options["infile"] = self.infile.name
else:
# we use source file directly, which may be encoded using
# something different than utf8. If that's the case file will
# be included with charset="something" html attribute and
# charset will be available as filter's charset attribute
encoding = self.charset # or self.default_encoding
self.infile = open(self.filename)
options["infile"] = self.filename
if "{outfile}" in self.command and "outfile" not in options:
# create temporary output file if needed
ext = self.type and ".%s" % self.type or ""
self.outfile = NamedTemporaryFile(mode='r+', suffix=ext)
options["outfile"] = self.outfile.name
# Quote infile and outfile for spaces etc.
if "infile" in options:
options["infile"] = shell_quote(options["infile"])
if "outfile" in options:
options["outfile"] = shell_quote(options["outfile"])
try:
command = self.command.format(**options)
proc = subprocess.Popen(command,
shell=True,
cwd=self.cwd,
stdout=self.stdout,
stdin=self.stdin,
stderr=self.stderr)
if self.infile is None:
# if infile is None then send content to process' stdin
filtered, err = proc.communicate(self.content.encode(encoding))
else:
filtered, err = proc.communicate()
filtered, err = filtered.decode(encoding), err.decode(encoding)
except (IOError, OSError) as e:
raise FilterError('Unable to apply %s (%r): %s' %
(self.__class__.__name__, self.command, e))
else:
if proc.wait() != 0:
# command failed, raise FilterError exception
if not err:
err = ('Unable to apply %s (%s)' %
(self.__class__.__name__, self.command))
if filtered:
err += '\n%s' % filtered
raise FilterError(err)
if self.verbose:
self.logger.debug(err)
outfile_path = options.get('outfile')
if outfile_path:
with io.open(outfile_path, 'r', encoding=encoding) as file:
filtered = file.read()
finally:
if self.infile is not None:
self.infile.close()
if self.outfile is not None:
self.outfile.close()
return smart_text(filtered)
def elem_str(self, elem):
return smart_text(self.tostring(elem, method='html', encoding=six.text_type))