def handle_inner(self, **options): if not settings.COMPRESS_ENABLED and not options.get("force"): raise CommandError( "Compressor is disabled. Set the COMPRESS_ENABLED " "setting or use --force to override.") if not settings.COMPRESS_OFFLINE: if not options.get("force"): raise CommandError( "Offline compression is disabled. Set " "COMPRESS_OFFLINE or use the --force to override.") log = options.get("log", sys.stdout) verbosity = options.get("verbosity", 1) follow_links = options.get("follow_links", False) extensions = self.handle_extensions( options.get("extensions") or ["html"]) engines = [e.strip() for e in options.get("engines", [])] or ["django"] final_offline_manifest = {} final_block_count = 0 final_results = [] for engine in engines: offline_manifest, block_count, results = self.compress( engine, extensions, verbosity, follow_links, log) final_results.extend(results) final_block_count += block_count final_offline_manifest.update(offline_manifest) write_offline_manifest(final_offline_manifest) return final_block_count, final_results
def handle(self, **options): if not settings.COMPRESS_ENABLED and not options.get("force"): raise CommandError( "Compressor is disabled. Set the COMPRESS_ENABLED " "setting or use --force to override.") if not settings.COMPRESS_OFFLINE: if not options.get("force"): raise CommandError( "Offline compression is disabled. Set " "COMPRESS_OFFLINE or use the --force to override.") options.setdefault("log", sys.stdout) manifest = {} engines = [e.strip() for e in options.get("engines", [])] or ["django"] for engine in engines: opts = options.copy() opts["engine"] = engine self.compress(**opts) manifest.update(get_offline_manifest()) write_offline_manifest(manifest)
def _compress(context, **kwargs): try: # Those arguments are mandatory to parse template fragment # and should be provided by mako capture = context['capture'] caller = context['caller'] except KeyError: return '' # print('||>>>', dict(context)) parser = ParserMock(capture(caller.body)) # `kind` is mandatory for django-compressor. but `kwargs.get` will return # None if not provided, so `django-compressor` can handle the error args = ['compress', kwargs.get('kind')] if 'mode' in kwargs: args.append(kwargs['mode']) if 'name' in kwargs: raise NotImplementedError('"name" is not supported') token = TokenMock(*args) try: return compressor(parser, token).render({}) except OfflineGenerationError: # we prefer doing a try except rather than an if for performance issue key = get_offline_hexdigest(parser.body) offline_manifest = get_offline_manifest() if key not in offline_manifest: print('compressing:\n\n{}\n'.format(parser.body.strip())) compressed = offline_compress(parser, token, {}) offline_manifest[key] = compressed write_offline_manifest(offline_manifest) print('compressed:\n\tkey: {}\n\thtml: {}\n'.format( key, compressed))
def compress(self, log=None, **options): """ Searches templates containing 'compress' nodes and compresses them "offline" -- outside of the request/response cycle. The result is cached with a cache-key derived from the content of the compress nodes (not the content of the possibly linked files!). """ engine = options.get("engine", "django") extensions = options.get('extensions') extensions = self.handle_extensions(extensions or ['html']) verbosity = int(options.get("verbosity", 0)) if not log: log = StringIO() if not self.get_loaders(): raise OfflineGenerationError("No template loaders defined. You " "must set TEMPLATE_LOADERS in your " "settings or set 'loaders' in your " "TEMPLATES dictionary.") templates = set() if engine == 'django': paths = set() for loader in self.get_loaders(): try: module = import_module(loader.__module__) get_template_sources = getattr(module, 'get_template_sources', None) if get_template_sources is None: get_template_sources = loader.get_template_sources paths.update(smart_text(origin) for origin in get_template_sources('')) except (ImportError, AttributeError, TypeError): # Yeah, this didn't work out so well, let's move on pass if not paths: raise OfflineGenerationError("No template paths found. None of " "the configured template loaders " "provided template paths. See " "https://docs.djangoproject.com/en/1.8/topics/templates/ " "for more information on template " "loaders.") if verbosity > 1: log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n") for path in paths: for root, dirs, files in os.walk(path, followlinks=options.get('followlinks', False)): templates.update(os.path.join(root, name) for name in files if not name.startswith('.') and any(fnmatch(name, "*%s" % glob) for glob in extensions)) elif engine == 'jinja2': env = settings.COMPRESS_JINJA2_GET_ENVIRONMENT() if env and hasattr(env, 'list_templates'): templates |= set([env.loader.get_source(env, template)[1] for template in env.list_templates(filter_func=lambda _path: os.path.splitext(_path)[-1] in extensions)]) if not templates: raise OfflineGenerationError("No templates found. Make sure your " "TEMPLATE_LOADERS and TEMPLATE_DIRS " "settings are correct.") if verbosity > 1: log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n") contexts = settings.COMPRESS_OFFLINE_CONTEXT if isinstance(contexts, six.string_types): try: module, function = get_mod_func(contexts) contexts = getattr(import_module(module), function)() except (AttributeError, ImportError, TypeError) as e: raise ImportError("Couldn't import offline context function %s: %s" % (settings.COMPRESS_OFFLINE_CONTEXT, e)) elif not isinstance(contexts, (list, tuple)): contexts = [contexts] contexts = list(contexts) # evaluate generator parser = self.__get_parser(engine) compressor_nodes = OrderedDict() for template_name in templates: try: template = parser.parse(template_name) except IOError: # unreadable file -> ignore if verbosity > 0: log.write("Unreadable template at: %s\n" % template_name) continue except TemplateSyntaxError as e: # broken template -> ignore if verbosity > 0: log.write("Invalid template %s: %s\n" % (template_name, smart_text(e))) continue except TemplateDoesNotExist: # non existent template -> ignore if verbosity > 0: log.write("Non-existent template at: %s\n" % template_name) continue except UnicodeDecodeError: if verbosity > 0: log.write("UnicodeDecodeError while trying to read " "template %s\n" % template_name) continue for context_dict in contexts: context = parser.get_init_context(context_dict) context = Context(context) try: nodes = list(parser.walk_nodes(template, context=context)) except (TemplateDoesNotExist, TemplateSyntaxError) as e: # Could be an error in some base template if verbosity > 0: log.write("Error parsing template %s: %s\n" % (template_name, smart_text(e))) continue if nodes: template.template_name = template_name template_nodes = compressor_nodes.setdefault(template, OrderedDict()) for node in nodes: template_nodes.setdefault(node, []).append(context) if not compressor_nodes: raise OfflineGenerationError( "No 'compress' template tags found in templates." "Try running compress command with --follow-links and/or" "--extension=EXTENSIONS") if verbosity > 0: log.write("Found 'compress' tags in:\n\t" + "\n\t".join((t.template_name for t in compressor_nodes.keys())) + "\n") log.write("Compressing... ") block_count = 0 compressed_contexts = [] results = [] offline_manifest = OrderedDict() for template, nodes in compressor_nodes.items(): template._log = log template._log_verbosity = verbosity for node, contexts in nodes.items(): for context in contexts: if context not in compressed_contexts: compressed_contexts.append(context) context.push() if not parser.process_template(template, context): continue parser.process_node(template, context, node) rendered = parser.render_nodelist(template, context, node) key = get_offline_hexdigest(rendered) if key in offline_manifest: continue try: result = parser.render_node(template, context, node) except Exception as e: raise CommandError("An error occurred during rendering %s: " "%s" % (template.template_name, smart_text(e))) result = result.replace( settings.COMPRESS_URL, settings.COMPRESS_URL_PLACEHOLDER ) offline_manifest[key] = result context.pop() results.append(result) block_count += 1 write_offline_manifest(offline_manifest) context_count = len(compressed_contexts) log.write("done\nCompressed %d block(s) from %d template(s) for %d context(s).\n" % (block_count, len(compressor_nodes), context_count)) return block_count, results
try: context['compress_forced'] = True compiled_node = env.compile(jinja2.nodes.Template([node])) result = Template.from_code( env, compiled_node, {}).render(context) except Exception, e: raise CommandError("An error occured during rendering %s: " "%s" % (template.template_name, e)) offline_manifest[key] = result context.pop() results.append(result) count += 1 write_offline_manifest(offline_manifest) log.write("done\nCompressed %d block(s) from %d template(s).\n" % (count, len(compressor_nodes))) return count, results def get_nodelist(self, node): if (isinstance(node, IfNode) and hasattr(node, 'nodelist_true') and hasattr(node, 'nodelist_false')): return node.nodelist_true + node.nodelist_false return getattr(node, "body", getattr(node, "nodes", [])) def walk_nodes(self, node, block_name=None): for node in self.get_nodelist(node): if (isinstance(node, CallBlock) and
class Command(NoArgsCommand): help = "Compress content outside of the request/response cycle" option_list = NoArgsCommand.option_list + ( make_option('--extension', '-e', action='append', dest='extensions', help='The file extension(s) to examine (default: ".html", ' 'separate multiple extensions with commas, or use -e ' 'multiple times)'), make_option( '-f', '--force', default=False, action='store_true', help="Force the generation of compressed content even if the " "COMPRESS_ENABLED setting is not True.", dest='force'), make_option( '--follow-links', default=False, action='store_true', help="Follow symlinks when traversing the COMPRESS_ROOT " "(which defaults to MEDIA_ROOT). Be aware that using this " "can lead to infinite recursion if a link points to a parent " "directory of itself.", dest='follow_links'), ) requires_model_validation = False def get_loaders(self): from django.template.loader import template_source_loaders if template_source_loaders is None: try: from django.template.loader import (find_template as finder_func) except ImportError: from django.template.loader import (find_template_source as finder_func) try: source, name = finder_func('test') except TemplateDoesNotExist: pass from django.template.loader import template_source_loaders loaders = [] # If template loader is CachedTemplateLoader, return the loaders # that it wraps around. So if we have # TEMPLATE_LOADERS = ( # ('django.template.loaders.cached.Loader', ( # 'django.template.loaders.filesystem.Loader', # 'django.template.loaders.app_directories.Loader', # )), # ) # The loaders will return django.template.loaders.filesystem.Loader # and django.template.loaders.app_directories.Loader for loader in template_source_loaders: if CachedLoader is not None and isinstance(loader, CachedLoader): loaders.extend(loader.loaders) else: loaders.append(loader) return loaders def compress(self, log=None, **options): """ Searches templates containing 'compress' nodes and compresses them "offline" -- outside of the request/response cycle. The result is cached with a cache-key derived from the content of the compress nodes (not the content of the possibly linked files!). """ extensions = options.get('extensions') extensions = self.handle_extensions(extensions or ['html']) verbosity = int(options.get("verbosity", 0)) if not log: log = StringIO() if not settings.TEMPLATE_LOADERS: raise OfflineGenerationError("No template loaders defined. You " "must set TEMPLATE_LOADERS in your " "settings.") paths = set() for loader in self.get_loaders(): try: module = import_module(loader.__module__) get_template_sources = getattr(module, 'get_template_sources', None) if get_template_sources is None: get_template_sources = loader.get_template_sources paths.update(list(get_template_sources(''))) except (ImportError, AttributeError): # Yeah, this didn't work out so well, let's move on pass if not paths: raise OfflineGenerationError("No template paths found. None of " "the configured template loaders " "provided template paths. See " "http://django.me/template-loaders " "for more information on template " "loaders.") if verbosity > 1: log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n") templates = set() for path in paths: for root, dirs, files in walk(path, followlinks=options.get( 'followlinks', False)): templates.update( os.path.join(root, name) for name in files if not name.startswith('.') and any( fnmatch(name, "*%s" % glob) for glob in extensions)) if not templates: raise OfflineGenerationError("No templates found. Make sure your " "TEMPLATE_LOADERS and TEMPLATE_DIRS " "settings are correct.") if verbosity > 1: log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n") compressor_nodes = SortedDict() for template_name in templates: try: template_file = open(template_name) try: template = Template(template_file.read().decode( settings.FILE_CHARSET)) finally: template_file.close() except IOError: # unreadable file -> ignore if verbosity > 0: log.write("Unreadable template at: %s\n" % template_name) continue except TemplateSyntaxError: # broken template -> try jinja -> ignore if still broken try: template_file = open(template_name) template = jinja_env.parse(template_file.read().decode( settings.FILE_CHARSET)) template.is_jinja = True template.name = template_name except jinja2.exceptions.TemplateSyntaxError: if verbosity > 0: log.write("Invalid template at: %s\n" % template_name) continue finally: template_file.close() except UnicodeDecodeError: if verbosity > 0: log.write("UnicodeDecodeError while trying to read " "template %s\n" % template_name) if getattr(template, 'is_jinja', False): nodes = template.find_all(jinja2.nodes.CallBlock) for node in nodes: try: compress_node = node.call.node if (compress_node.identifier == 'compressor.contrib.jinja2ext.CompressorExtension' and compress_node.name == '_compress'): template.template_name = template_name compressor_nodes.setdefault(template, []).append(node) except AttributeError, IndexError: pass else: nodes = list(self.walk_nodes(template)) if nodes: template.template_name = template_name compressor_nodes.setdefault(template, []).extend(nodes) if not compressor_nodes: raise OfflineGenerationError( "No 'compress' template tags found in templates.") if verbosity > 0: log.write("Found 'compress' tags in:\n\t" + "\n\t".join((t.template_name for t in compressor_nodes.keys())) + "\n") log.write("Compressing... ") count = 0 results = [] offline_manifest = {} for template, nodes in compressor_nodes.iteritems(): if getattr(template, 'is_jinja', False): for node in nodes: context = settings.COMPRESS_OFFLINE_CONTEXT.copy() old_forced = getattr(jinja_env, '_django_compressor_offline_forced', None) jinja_env._django_compressor_offline_forced = True nodelist = node.body key = get_offline_jinja_hexdigest(nodelist) result = render_jinja_node(node, context, jinja_env) if old_forced is not None: jinja_env._django_compressor_offline_forced = old_forced offline_manifest[key] = result results.append(result) count += 1 continue context = Context(settings.COMPRESS_OFFLINE_CONTEXT) extra_context = {} firstnode = template.nodelist[0] if isinstance(firstnode, ExtendsNode): # If this template has a ExtendsNode, we apply our patch to # generate the necessary context, and then use it for all the # nodes in it, just in case (we don't know which nodes were # in a block) firstnode._old_get_parent = firstnode.get_parent firstnode.get_parent = MethodType(patched_get_parent, firstnode) extra_context = firstnode.render(context) context.render_context = extra_context.render_context for node in nodes: context.push() if extra_context and node._block_name: context['block'] = context.render_context[ BLOCK_CONTEXT_KEY].pop(node._block_name) if context['block']: context['block'].context = context key = get_offline_hexdigest(node.nodelist) result = node.render(context, forced=True) offline_manifest[key] = result context.pop() results.append(result) count += 1 write_offline_manifest(offline_manifest) log.write("done\nCompressed %d block(s) from %d template(s).\n" % (count, len(compressor_nodes))) return count, results
def compress(self, log=None, **options): """ Searches templates containing 'compress' nodes and compresses them "offline" -- outside of the request/response cycle. The result is cached with a cache-key derived from the content of the compress nodes (not the content of the possibly linked files!). """ extensions = options.get('extensions') extensions = self.handle_extensions(extensions or ['html']) verbosity = int(options.get("verbosity", 0)) if not log: log = StringIO() if not settings.TEMPLATE_LOADERS: raise OfflineGenerationError("No template loaders defined. You " "must set TEMPLATE_LOADERS in your " "settings.") paths = set() for loader in self.get_loaders(): try: module = import_module(loader.__module__) get_template_sources = getattr(module, 'get_template_sources', None) if get_template_sources is None: get_template_sources = loader.get_template_sources paths.update(list(get_template_sources(''))) except (ImportError, AttributeError, TypeError): # Yeah, this didn't work out so well, let's move on pass if not paths: raise OfflineGenerationError("No template paths found. None of " "the configured template loaders " "provided template paths. See " "http://django.me/template-loaders " "for more information on template " "loaders.") if verbosity > 1: log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n") templates = set() for path in paths: for root, dirs, files in os.walk(path, followlinks=options.get('followlinks', False)): templates.update(os.path.join(root, name) for name in files if not name.startswith('.') and any(fnmatch(name, "*%s" % glob) for glob in extensions)) if not templates: raise OfflineGenerationError("No templates found. Make sure your " "TEMPLATE_LOADERS and TEMPLATE_DIRS " "settings are correct.") if verbosity > 1: log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n") engine = options.get("engine", "django") parser = self.__get_parser(engine) compressor_nodes = SortedDict() for template_name in templates: try: template = parser.parse(template_name) except IOError: # unreadable file -> ignore if verbosity > 0: log.write("Unreadable template at: %s\n" % template_name) continue except TemplateSyntaxError as e: # broken template -> ignore if verbosity > 0: log.write("Invalid template %s: %s\n" % (template_name, e)) continue except TemplateDoesNotExist: # non existent template -> ignore if verbosity > 0: log.write("Non-existent template at: %s\n" % template_name) continue except UnicodeDecodeError: if verbosity > 0: log.write("UnicodeDecodeError while trying to read " "template %s\n" % template_name) try: nodes = list(parser.walk_nodes(template)) except (TemplateDoesNotExist, TemplateSyntaxError) as e: # Could be an error in some base template if verbosity > 0: log.write("Error parsing template %s: %s\n" % (template_name, e)) continue if nodes: template.template_name = template_name compressor_nodes.setdefault(template, []).extend(nodes) if not compressor_nodes: raise OfflineGenerationError( "No 'compress' template tags found in templates." "Try running compress command with --follow-links and/or" "--extension=EXTENSIONS") if verbosity > 0: log.write("Found 'compress' tags in:\n\t" + "\n\t".join((t.template_name for t in compressor_nodes.keys())) + "\n") log.write("Compressing... ") count = 0 results = [] offline_manifest = SortedDict() init_context = parser.get_init_context(settings.COMPRESS_OFFLINE_CONTEXT) for template, nodes in compressor_nodes.items(): context = Context(init_context) template._log = log template._log_verbosity = verbosity if not parser.process_template(template, context): continue for node in nodes: context.push() parser.process_node(template, context, node) rendered = parser.render_nodelist(template, context, node) key = get_offline_hexdigest(rendered) if key in offline_manifest: continue try: result = parser.render_node(template, context, node) except Exception as e: raise CommandError("An error occured during rendering %s: " "%s" % (template.template_name, e)) offline_manifest[key] = result context.pop() results.append(result) count += 1 write_offline_manifest(offline_manifest) log.write("done\nCompressed %d block(s) from %d template(s).\n" % (count, len(compressor_nodes))) return count, results
Template.from_code(jingo.register.env, compiled_node, {}).render(context)) try: context['compress_forced'] = True compiled_node = env.compile(jinja2.nodes.Template([node])) result = Template.from_code(env, compiled_node, {}).render(context) except Exception, e: raise CommandError("An error occured during rendering %s: " "%s" % (template.template_name, e)) offline_manifest[key] = result context.pop() results.append(result) count += 1 write_offline_manifest(offline_manifest) log.write("done\nCompressed %d block(s) from %d template(s).\n" % (count, len(compressor_nodes))) return count, results def get_nodelist(self, node): if (isinstance(node, IfNode) and hasattr(node, 'nodelist_true') and hasattr(node, 'nodelist_false')): return node.nodelist_true + node.nodelist_false return getattr(node, "body", getattr(node, "nodes", [])) def walk_nodes(self, node, block_name=None): for node in self.get_nodelist(node): if (isinstance(node, CallBlock) and isinstance(node.call, Call) and isinstance(node.call.node, ExtensionAttribute)
def compress(self, log=None, **options): """ Searches templates containing 'compress' nodes and compresses them "offline" -- outside of the request/response cycle. The result is cached with a cache-key derived from the content of the compress nodes (not the content of the possibly linked files!). """ extensions = options.get('extensions') extensions = self.handle_extensions(extensions or ['html']) verbosity = int(options.get("verbosity", 0)) if not log: log = StringIO() if not settings.TEMPLATE_LOADERS: raise OfflineGenerationError("No template loaders defined. You " "must set TEMPLATE_LOADERS in your " "settings.") paths = set() for loader in self.get_loaders(): try: module = import_module(loader.__module__) get_template_sources = getattr(module, 'get_template_sources', None) if get_template_sources is None: get_template_sources = loader.get_template_sources paths.update(list(get_template_sources(''))) except (ImportError, AttributeError): # Yeah, this didn't work out so well, let's move on pass if not paths: raise OfflineGenerationError("No template paths found. None of " "the configured template loaders " "provided template paths. See " "http://django.me/template-loaders " "for more information on template " "loaders.") if verbosity > 1: log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n") templates = set() for path in paths: for root, dirs, files in os.walk(path, followlinks=options.get( 'followlinks', False)): templates.update( os.path.join(root, name) for name in files if not name.startswith('.') and any( fnmatch(name, "*%s" % glob) for glob in extensions)) if not templates: raise OfflineGenerationError("No templates found. Make sure your " "TEMPLATE_LOADERS and TEMPLATE_DIRS " "settings are correct.") if verbosity > 1: log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n") engine = options.get("engine", "django") parser = self.__get_parser(engine) compressor_nodes = SortedDict() for template_name in templates: try: template = parser.parse(template_name) except IOError: # unreadable file -> ignore if verbosity > 0: log.write("Unreadable template at: %s\n" % template_name) continue except TemplateSyntaxError as e: # broken template -> ignore if verbosity > 0: log.write("Invalid template %s: %s\n" % (template_name, e)) continue except TemplateDoesNotExist: # non existent template -> ignore if verbosity > 0: log.write("Non-existent template at: %s\n" % template_name) continue except UnicodeDecodeError: if verbosity > 0: log.write("UnicodeDecodeError while trying to read " "template %s\n" % template_name) try: nodes = list(parser.walk_nodes(template)) except (TemplateDoesNotExist, TemplateSyntaxError) as e: # Could be an error in some base template if verbosity > 0: log.write("Error parsing template %s: %s\n" % (template_name, e)) continue if nodes: template.template_name = template_name compressor_nodes.setdefault(template, []).extend(nodes) if not compressor_nodes: raise OfflineGenerationError( "No 'compress' template tags found in templates." "Try running compress command with --follow-links and/or" "--extension=EXTENSIONS") if verbosity > 0: log.write("Found 'compress' tags in:\n\t" + "\n\t".join((t.template_name for t in compressor_nodes.keys())) + "\n") log.write("Compressing... ") count = 0 results = [] offline_manifest = SortedDict() init_context = parser.get_init_context( settings.COMPRESS_OFFLINE_CONTEXT) for template, nodes in compressor_nodes.items(): context = Context(init_context) template._log = log template._log_verbosity = verbosity if not parser.process_template(template, context): continue for node in nodes: context.push() parser.process_node(template, context, node) rendered = parser.render_nodelist(template, context, node) key = get_offline_hexdigest(rendered) if key in offline_manifest: continue try: result = parser.render_node(template, context, node) except Exception as e: raise CommandError("An error occured during rendering %s: " "%s" % (template.template_name, e)) offline_manifest[key] = result context.pop() results.append(result) count += 1 write_offline_manifest(offline_manifest) log.write("done\nCompressed %d block(s) from %d template(s).\n" % (count, len(compressor_nodes))) return count, results