class PipelineManifest(Manifest): def __init__(self): self.packager = Packager() self.packages = self.collect_packages() self.finder = DefaultStorageFinder() def collect_packages(self): packages = [] for package_name in self.packager.packages['css']: package = self.packager.package_for('css', package_name) if package.manifest: packages.append(package) for package_name in self.packager.packages['js']: package = self.packager.package_for('js', package_name) if package.manifest: packages.append(package) return packages def cache(self): ignore_patterns = getattr(settings, "STATICFILES_IGNORE_PATTERNS", None) if settings.PIPELINE: for package in self.packages: yield str(self.packager.individual_url(package.output_filename)) else: for package in self.packages: for path in self.packager.compile(package.paths): yield str(self.packager.individual_url(path)) for path in self.finder.list(ignore_patterns): yield str(self.packager.individual_url(path))
def handle(self, *args, **options): from pipeline.packager import Packager force = options.get('force', False) verbose = int(options.get('verbosity', 1)) >= 2 sync = options.get('dry_run', True) packager = Packager(verbose=verbose) for package_name in packager.packages['css']: if args and package_name not in args: continue package = packager.package_for('css', package_name) if verbose: print message = "CSS Group '%s'" % package_name print message print len(message) * '-' packager.pack_stylesheets(package, sync=sync, force=force) for package_name in packager.packages['js']: if args and package_name not in args: continue package = packager.package_for('js', package_name) if verbose: print message = "JS Group '%s'" % package_name print message print len(message) * '-' packager.pack_javascripts(package, sync=sync, force=force)
def post_process(self, paths, dry_run=False, **options): if dry_run: return [] from pipeline.packager import Packager packager = Packager(storage=self) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) output_file = package.output_filename if self.packing: packager.pack_stylesheets(package) paths[output_file] = (self, output_file) for package_name in packager.packages['js']: package = packager.package_for('js', package_name) output_file = package.output_filename if self.packing: packager.pack_javascripts(package) paths[output_file] = (self, output_file) super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'): return super_class.post_process(paths, dry_run, **options) return [ (path, path, True) for path in paths ]
def post_process(self, paths, dry_run=False, **options): if dry_run: return [] from pipeline.packager import Packager packager = Packager(storage=self) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) if self.packing: paths_written = packager.pack_stylesheets(package) for path in paths_written: paths[path] = (self, path) else: # TODO: bcooksey 5/15/13. Not sure why we pretend we packed if packing is false...will this mess up source maps output_file = package.output_filename paths[output_file] = (self, output_file) for package_name in packager.packages['js']: package = packager.package_for('js', package_name) if self.packing: paths_written = packager.pack_javascripts(package) for path in paths_written: paths[path] = (self, path) else: # TODO: bcooksey 5/15/13. Not sure why we pretend we packed if packing is false...will this mess up source maps output_file = package.output_filename paths[output_file] = (self, output_file) super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'): return super_class.post_process(paths, dry_run, **options) return [ (path, path, True) for path in paths ]
def post_process(self, paths, dry_run=False, **options): """ This post_process hook is used to package all themed assets. """ if dry_run: return themes = get_themes() for theme in themes: css_packages = self.get_themed_packages(theme.theme_dir_name, settings.PIPELINE_CSS) js_packages = self.get_themed_packages(theme.theme_dir_name, settings.PIPELINE_JS) from pipeline.packager import Packager packager = Packager(storage=self, css_packages=css_packages, js_packages=js_packages) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) output_file = package.output_filename if self.packing: packager.pack_stylesheets(package) paths[output_file] = (self, output_file) yield output_file, output_file, True for package_name in packager.packages['js']: package = packager.package_for('js', package_name) output_file = package.output_filename if self.packing: packager.pack_javascripts(package) paths[output_file] = (self, output_file) yield output_file, output_file, True super_class = super(ThemePipelineMixin, self) if hasattr(super_class, 'post_process'): for name, hashed_name, processed in super_class.post_process(paths.copy(), dry_run, **options): yield name, hashed_name, processed
def post_process(self, paths, dry_run=False, **options): if dry_run: return from pipeline.packager import Packager packager = Packager(storage=self) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) output_file = package.output_filename if self.packing: packager.pack_stylesheets(package) paths[output_file] = (self, output_file) yield output_file, output_file, True for package_name in packager.packages['js']: package = packager.package_for('js', package_name) output_file = package.output_filename if self.packing: packager.pack_javascripts(package) paths[output_file] = (self, output_file) yield output_file, output_file, True super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'): for name, hashed_name, processed in super_class.post_process(paths.copy(), dry_run, **options): yield name, hashed_name, processed
def post_process(self, paths, dry_run=False, **options): if dry_run: return packager = Packager(storage=self) for _abs_path, rel_path in paths: files_to_process = OrderedDict() files_to_process[rel_path] = (self, rel_path) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) output_file = package.output_filename if rel_path in package.paths: if self.packing: packager.pack_stylesheets(package) files_to_process[output_file] = (self, output_file) yield output_file, output_file, True for package_name in packager.packages['js']: package = packager.package_for('js', package_name) output_file = package.output_filename if rel_path in package.paths: if self.packing: packager.pack_javascripts(package) files_to_process[output_file] = (self, output_file) yield output_file, output_file, True super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'): for name, hashed_name, processed in super_class.post_process( files_to_process.copy(), dry_run, **options): yield name, hashed_name, processed
class PipelineManifest(Manifest): def __init__(self): self.packager = Packager() self.packages = self.collect_packages() self.finder = DefaultStorageFinder() def collect_packages(self): packages = [] for package_name in self.packager.packages["css"]: package = self.packager.package_for("css", package_name) if package.manifest: packages.append(package) for package_name in self.packager.packages["js"]: package = self.packager.package_for("js", package_name) if package.manifest: packages.append(package) return packages def cache(self): if settings.PIPELINE: for package in self.packages: yield str(self.packager.individual_url(package.output_filename)) else: for package in self.packages: for path in self.packager.compile(package.paths): yield str(self.packager.individual_url(path)) for path in self.finder.list(): yield str(self.packager.individual_url(path))
def test_package_for(self): packager = Packager() packager.packages["js"] = packager.create_packages( {"application": {"source_filenames": ("js/application.js",), "output_filename": "application.r?.js"}} ) try: packager.package_for("js", "application") except PackageNotFound: self.fail() try: packager.package_for("js", "broken") self.fail() except PackageNotFound: pass
class PipelineManifest(Manifest): def __init__(self): self.packager = Packager() self.packages = self.collect_packages() self.finders = get_finders() self.package_files = self.get_package_files() def get_package_files(self): files = list() for package in self.packages: files.append(package.output_filename) for path in self.packager.compile(package.paths): files.append(path) return files def collect_packages(self): packages = [] for package_name in self.packager.packages['css']: package = self.packager.package_for('css', package_name) if package.manifest: packages.append(package) for package_name in self.packager.packages['js']: package = self.packager.package_for('js', package_name) if package.manifest: packages.append(package) return packages def cache(self): ignore_patterns = getattr(settings, "STATICFILES_IGNORE_PATTERNS", None) if PIPELINE: for package in self.packages: yield str(self.packager.individual_url(package.output_filename)) else: for package in self.packages: for path in self.packager.compile(package.paths): yield str(self.packager.individual_url(path)) for finder in self.finders: for path, storage in finder.list(ignore_patterns): # Prefix the relative path if the source storage contains it if getattr(storage, 'prefix', None): prefixed_path = os.path.join(storage.prefix, path) else: prefixed_path = path # Dont add any doubles if prefixed_path not in self.package_files: yield str(self.packager.individual_url(prefixed_path))
def compressed_css(ctx, name): package = settings.PIPELINE_CSS.get(name, {}) if package: package = {name: package} packager = Packager(css_packages=package, js_packages={}) try: package = packager.package_for('css', name) except PackageNotFound: return "" def _render_css(path): template_name = package.template_name or "pipeline/css.jinja" context = package.extra_context context.update({ 'type': guess_type(path, 'text/css'), 'url': staticfiles_storage.url(path) }) template = ctx.environment.get_template(template_name) return template.render(context) if settings.PIPELINE_ENABLED: return _render_css(package.output_filename) else: default_collector.collect() paths = packager.compile(package.paths) tags = [_render_css(path) for path in paths] return '\n'.join(tags)
class CompressedCSSNode(template.Node): def __init__(self, name): self.name = name def render(self, context): package_name = template.Variable(self.name).resolve(context) package = settings.PIPELINE_CSS.get(package_name, {}) if package: package = {package_name: package} self.packager = Packager(css_packages=package, js_packages={}) try: package = self.packager.package_for('css', package_name) except PackageNotFound: return '' # fail silently, do not return anything if an invalid group is specified if settings.PIPELINE: return self.render_css(package, package.output_filename) else: paths = self.packager.compile(package.paths) return self.render_individual(package, paths) def render_css(self, package, path): template_name = package.template_name or "pipeline/css.html" context = package.extra_context context.update({ 'url': staticfiles_storage.url(path) }) return render_to_string(template_name, context) def render_individual(self, package, paths): tags = [self.render_css(package, path) for path in paths] return '\n'.join(tags)
class CommonNode(template.Node): def __init__(self, name): self.name = name def render(self, context): package_name = template.Variable(self.name).resolve(context) package = self.packages.get(package_name, {}) if package: package = {package_name: package} self.packager = Packager(css_packages=package, js_packages=package) try: package = self.packager.package_for(self.type, package_name) except PackageNotFound: return '' # fail silently, do not return anything if an invalid group is specified if settings.PIPELINE: return self.render_type_specific(package, package.output_filename) else: paths = self.packager.compile(package.paths) return self.render_individual(package, paths) def render_individual(self, package, paths): tags = [self.render_type_specific(package, path) for path in paths] return '\n'.join(tags) def get_url(self, path): if path_is_url(path): return path return default_storage.url(path)
def compressed_css(name): package = settings.PIPELINE_CSS.get(name, {}) if package: package = {name: package} packager = Packager(css_packages=package, js_packages={}) try: package = packager.package_for('css', name) except PackageNotFound: return "" def _render_css(path): template_name = package.template_name or "pipeline/css.jinja" context = package.extra_context context.update({ 'type': guess_type(path, 'text/css'), 'url': staticfiles_storage.url(path) }) return render_to_string(template_name, context) if not settings.DEBUG: return _render_css(package.output_filename) paths = packager.compile(package.paths) tags = [_render_css(path) for path in paths] return '\n'.join(tags)
class CompressedCSSNode(template.Node): def __init__(self, name): self.name = name self.packager = Packager() def render(self, context): package_name = template.Variable(self.name).resolve(context) try: package = self.packager.package_for('css', package_name) except PackageNotFound: return '' # fail silently, do not return anything if an invalid group is specified if settings.PIPELINE: compressed_path = self.packager.pack_stylesheets(package) return self.render_css(package, compressed_path) else: package['paths'] = self.packager.compile(package['paths']) return self.render_individual(package) def render_css(self, package, path): context = {} if not 'template' in package: package['template'] = "pipeline/css.html" if not 'context' in package: context = package['context'] context.update({ 'url': self.packager.individual_url(path) }) return render_to_string(package['template'], context) def render_individual(self, package): tags = [self.render_css(package, path) for path in package['paths']] return '\n'.join(tags)
class CompressedJSNode(template.Node): def __init__(self, name): self.name = name def render(self, context): package_name = template.Variable(self.name).resolve(context) package = settings.PIPELINE_JS.get(package_name, {}) if package: package = {package_name: package} self.packager = Packager(css_packages={}, js_packages=package) try: package = self.packager.package_for('js', package_name) except PackageNotFound: return '' # fail silently, do not return anything if an invalid group is specified if 'externals' in package: return '\n'.join([self.render_external(package, url) for url in package['externals']]) if settings.PIPELINE: compressed_path = self.packager.pack_javascripts(package) return self.render_js(package, compressed_path) else: package['paths'] = self.packager.compile(package['paths']) templates = self.packager.pack_templates(package) return self.render_individual(package, templates) def render_js(self, package, path): context = {} if not 'template' in package: package['template'] = "pipeline/js.html" if 'context' in package: context = package['context'] context.update({ 'url': self.packager.individual_url(path) }) return render_to_string(package['template'], context) def render_external(self, package, url): if not 'template' in package: package['template'] = "pipeline/js.html" return render_to_string(package['template'], { 'url': url }) def render_inline(self, package, js): context = {} if 'context' in package: context = package['context'] context.update({ 'source': js }) return render_to_string("pipeline/inline_js.html", context) def render_individual(self, package, templates=None): tags = [self.render_js(package, js) for js in package['paths']] if templates: tags.append(self.render_inline(package, templates)) return '\n'.join(tags)
def test_package_for(self): packager = Packager() packager.packages['js'] = packager.create_packages({ 'application': { 'source_filenames': ( 'js/application.js', ), 'output_filename': 'application.r?.js' } }) try: packager.package_for('js', 'application') except PackageNotFound: self.fail() try: packager.package_for('js', 'broken') self.fail() except PackageNotFound: pass
class PipelineManifest(Manifest): def __init__(self): self.packager = Packager() self.packages = self.collect_packages() def collect_packages(self): packages = [] for package_name in self.packager.packages['css']: package = self.packager.package_for('css', package_name) if package['manifest']: packages.append(package) for package_name in self.packager.packages['js']: package = self.packager.package_for('js', package_name) if package['manifest']: packages.append(package) return packages def cache(self): if settings.PIPELINE: for package in self.packages: filename_base, filename = os.path.split(package['output']) version = self.packager.versioning.version_from_file(filename_base, filename) output_filename = self.packager.versioning.output_filename(package['output'], version) yield str(self.packager.individual_url(output_filename)) else: for package in self.packages: for path in self.packager.compile(package['paths']): yield str(self.packager.individual_url(path)) def revision(self): versions = [] if settings.PIPELINE: for package in self.packages: filename_base, filename = os.path.split(package['output']) version = self.packager.versioning.version_from_file(filename_base, filename) versions.append(str(version)) else: for package in self.packages: version = self.packager.versioning.version(package['paths']) versions.append(str(version)) versions.sort() return versions[-1]
def tabzilla_css_redirect(r): packer = Packager() tabzilla_package = packer.package_for('css', 'tabzilla') if not settings.DEBUG: file_path = tabzilla_package.output_filename else: default_collector.collect() paths = packer.compile(tabzilla_package.paths) file_path = paths[0] return static(file_path)
def compressed_css(package_name, raw=False): # lint-amnesty, pylint: disable=missing-function-docstring package = settings.STYLESHEETS.get(package_name, {}) if package: package = {package_name: package} packager = Packager(css_packages=package, js_packages={}) package = packager.package_for('css', package_name) if settings.PIPELINE_ENABLED: return render_css(package, package.output_filename, raw=raw) else: paths = packager.compile(package.paths) return render_individual_css(package, paths, raw=raw)
def compressed_css(package_name, raw=False): package = settings.PIPELINE_CSS.get(package_name, {}) if package: package = {package_name: package} packager = Packager(css_packages=package, js_packages={}) package = packager.package_for('css', package_name) if settings.PIPELINE_ENABLED: return render_css(package, package.output_filename, raw=raw) else: paths = packager.compile(package.paths) return render_individual_css(package, paths, raw=raw)
def compressed_css(package_name, raw=False): package = settings.STYLESHEETS.get(package_name, {}) if package: package = {package_name: package} packager = Packager(css_packages=package, js_packages={}) package = packager.package_for('css', package_name) if settings.PIPELINE_ENABLED: return render_css(package, package.output_filename, raw=raw) else: paths = packager.compile(package.paths) return render_individual_css(package, paths, raw=raw)
def post_process(self, paths, dry_run=False, **options): """ This post_process hook is used to package all themed assets. """ if dry_run: return themes = get_themes() for theme in themes: css_packages = self.get_themed_packages(theme.theme_dir_name, settings.PIPELINE_CSS) js_packages = self.get_themed_packages(theme.theme_dir_name, settings.PIPELINE_JS) from pipeline.packager import Packager packager = Packager(storage=self, css_packages=css_packages, js_packages=js_packages) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) output_file = package.output_filename if self.packing: packager.pack_stylesheets(package) paths[output_file] = (self, output_file) yield output_file, output_file, True for package_name in packager.packages['js']: package = packager.package_for('js', package_name) output_file = package.output_filename if self.packing: packager.pack_javascripts(package) paths[output_file] = (self, output_file) yield output_file, output_file, True super_class = super(ThemePipelineMixin, self) if hasattr(super_class, 'post_process'): for name, hashed_name, processed in super_class.post_process( paths.copy(), dry_run, **options): yield name, hashed_name, processed
def compressed_js(package_name): package = settings.PIPELINE_JS.get(package_name, {}) if package: package = {package_name: package} packager = Packager(css_packages={}, js_packages=package) package = packager.package_for('js', package_name) if settings.PIPELINE_ENABLED: return render_js(package, package.output_filename) else: paths = packager.compile(package.paths) templates = packager.pack_templates(package) return render_individual_js(package, paths, templates)
def compressed_js(package_name): package = settings.JAVASCRIPT.get(package_name, {}) if package: package = {package_name: package} packager = Packager(css_packages={}, js_packages=package) package = packager.package_for('js', package_name) if settings.PIPELINE_ENABLED: return render_js(package, package.output_filename) else: paths = packager.compile(package.paths) templates = packager.pack_templates(package) return render_individual_js(package, paths, templates)
def compressed_js(package_name): # lint-amnesty, pylint: disable=missing-function-docstring package = settings.JAVASCRIPT.get(package_name, {}) if package: package = {package_name: package} packager = Packager(css_packages={}, js_packages=package) package = packager.package_for('js', package_name) if settings.PIPELINE_ENABLED: return render_js(package, package.output_filename) else: paths = packager.compile(package.paths) templates = packager.pack_templates(package) return render_individual_js(package, paths, templates)
def handle_noargs(self, **options): from pipeline.packager import Packager packager = Packager( force=options.get('force', False), verbose=int(options.get('verbosity', 1)) >= 2 ) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) if packager.verbose or packager.force: print message = "CSS Group '%s'" % package_name print message print len(message) * '-' packager.pack_stylesheets(package) for package_name in packager.packages['js']: package = packager.package_for('js', package_name) if packager.verbose or packager.force: print message = "JS Group '%s'" % package_name print message print len(message) * '-' packager.pack_javascripts(package)
def post_process(self, paths, dry_run=False, **options): if dry_run: return [] from pipeline.packager import Packager packager = Packager(storage=self) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) output_file = package.output_filename if self.packing: packager.pack_stylesheets(package) paths[output_file] = (self, output_file) for package_name in packager.packages['js']: package = packager.package_for('js', package_name) output_file = package.output_filename if self.packing: packager.pack_javascripts(package) paths[output_file] = (self, output_file) super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'): return super_class.post_process(paths, dry_run, **options) return [(path, path, True) for path in paths]
class CompressedJSNode(template.Node): def __init__(self, name): self.name = name def render(self, context): package_name = template.Variable(self.name).resolve(context) package = settings.PIPELINE_JS.get(package_name, {}) if package: package = {package_name: package} self.packager = Packager(css_packages={}, js_packages=package) try: package = self.packager.package_for('js', package_name) except PackageNotFound: return '' # fail silently, do not return anything if an invalid group is specified if settings.PIPELINE: return self.render_js(package, package.output_filename) else: paths = self.packager.compile(package.paths) templates = self.packager.pack_templates(package) return self.render_individual(package, paths, templates) def render_js(self, package, path): template_name = package.template_name or "pipeline/js.html" context = package.extra_context context.update({ 'type': guess_type(path, 'text/javascript'), 'url': staticfiles_storage.url(path) }) return render_to_string(template_name, context) def render_inline(self, package, js): context = package.extra_context context.update({ 'source': js }) return render_to_string("pipeline/inline_js.html", context) def render_individual(self, package, paths, templates=None): tags = [self.render_js(package, js) for js in paths] if templates: tags.append(self.render_inline(package, templates)) return '\n'.join(tags)
def compressed_js(ctx, name): package = settings.PIPELINE_JS.get(name, {}) if package: package = {name: package} packager = Packager(css_packages={}, js_packages=package) try: package = packager.package_for("js", name) except PackageNotFound: return "" def _render_js(path): template_name = package.template_name or "pipeline/js.jinja" context = package.extra_context context.update({ 'type': guess_type(path, 'text/javascript'), 'url': staticfiles_storage.url(path), }) template = ctx.environment.get_template(template_name) return template.render(context) def _render_inline_js(js): context = package.extra_context context.update({ 'source': js }) template = ctx.environment.get_template("pipeline/inline_js.jinja") return template.render(context) # Render a optimized one if settings.PIPELINE_ENABLED: return _render_js(package.output_filename) else: default_collector.collect() paths = packager.compile(package.paths) templates = packager.pack_templates(package) tags = [_render_js(js) for js in paths] if templates: tags.append(_render_inline_js(templates)) return '\n'.join(tags)
class CompressedCSSNode(template.Node): def __init__(self, name): self.name = name def render(self, context): package_name = template.Variable(self.name).resolve(context) package = settings.PIPELINE_CSS.get(package_name, {}) if package: package = {package_name: package} self.packager = Packager(css_packages=package, js_packages={}) try: package = self.packager.package_for('css', package_name) except PackageNotFound: return '' # fail silently, do not return anything if an invalid group is specified if settings.PIPELINE: compressed_path = self.packager.pack_stylesheets(package) return self.render_css(package, compressed_path) else: package['paths'] = self.packager.compile(package['paths']) return self.render_individual(package) def render_css(self, package, path): context = {} if not 'template' in package: package['template'] = "pipeline/css.html" if 'context' in package: context = package['context'] url = self.packager.individual_url(path) context.update({'url': url}) if url.endswith('.less'): context.update({'rel': 'stylesheet/less'}) else: context.update({'rel': 'stylesheet'}) return render_to_string(package['template'], context) def render_individual(self, package): tags = [self.render_css(package, path) for path in package['paths']] return '\n'.join(tags)
def compressed_js(name): package = settings.PIPELINE_JS.get(name, {}) if package: package = {name: package} packager = Packager(css_packages={}, js_packages=package) try: package = packager.package_for('js', name) except PackageNotFound: return "" def _render_js(path): template_name = package.template_name or "pipeline/js.jinja" context = package.extra_context context.update({ 'type': guess_type(path, 'text/javascript'), 'url': staticfiles_storage.url(path), }) return render_to_string(template_name, context) def _render_inline_js(js): context = package.extra_context context.update({ 'source': js }) return render_to_string("pipeline/inline_js.jinja", context) if not settings.DEBUG: return _render_js(package.output_filename) paths = packager.compile(package.paths) templates = packager.pack_templates(package) tags = [_render_js(js) for js in paths] if templates: tags.append(_render_inline(templates)) return '\n'.join(tags)
def compressed_js(name): package = settings.PIPELINE_JS.get(name, {}) if package: package = {name: package} packager = Packager(css_packages={}, js_packages=package) try: package = packager.package_for('js', name) except PackageNotFound: return "" def _render_js(path): template_name = package.template_name or "pipeline/js.jinja" context = package.extra_context context.update({ 'type': guess_type(path, 'text/javascript'), 'url': staticfiles_storage.url(path), }) return render_to_string(template_name, context) def _render_inline_js(js): context = package.extra_context context.update({'source': js}) return render_to_string("pipeline/inline_js.jinja", context) if not settings.DEBUG: return _render_js(package.output_filename) paths = packager.compile(package.paths) templates = packager.pack_templates(package) tags = [_render_js(js) for js in paths] if templates: tags.append(_render_inline(templates)) return '\n'.join(tags)
class OptimizedPipelineStorage(PipelineMixin, StaticFilesStorage): """This storage compresses only the packages which had modifications in their source files, or that have not been compressed yet. This speeds up the collectstatic process, since must of the time we modify only a few javascript/css files at a time. It also appends the a md5 hash to the compressed files' url so any existing cache mechanisms are naturally invalidated.""" compressed_packages = [] unchanged_packages = [] packager = None HASH_CACHE_KEY = 'pipeline_compressed_hash_key' SOURCES_DUMP_KEY = 'pipeline_dumped_sources_key' def url(self, name): """Append the produced hash to the resource url so existing cache mechanisms are naturally invalidated.""" url = super(OptimizedPipelineStorage, self).url(name) _hash = self.get_compressed_files_hash() if _hash and name: return '{url}?{_hash}'.format(url=url, _hash=_hash) else: return url def post_process(self, paths, dry_run=False, **options): if dry_run: return from pipeline.packager import Packager self.packager = Packager(storage=self) for package_name in self.packager.packages['css']: package = self.packager.package_for('css', package_name) output_file = package.output_filename if self.packing and self._is_outdated(package_name, package): print('COMPRESSING {} package...'.format(package_name)) self.packager.pack_stylesheets(package) self.compressed_packages.append(package_name) else: self.unchanged_packages.append(package_name) paths[output_file] = (self, output_file) yield output_file, output_file, True for package_name in self.packager.packages['js']: package = self.packager.package_for('js', package_name) output_file = package.output_filename if self.packing and self._is_outdated(package_name, package): print('COMPRESSING {} package...'.format(package_name)) self.packager.pack_javascripts(package) self.compressed_packages.append(package_name) else: self.unchanged_packages.append(package_name) paths[output_file] = (self, output_file) yield output_file, output_file, True super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'): for name, hashed_name, processed in super_class.post_process( paths.copy(), dry_run, **options): yield name, hashed_name, processed self._finalize() def _is_outdated(self, package_name, package): outdated = False for path in package.paths: # Needs to run for every path in order to generate the individual # file hashes. if self._is_content_changed(path) and not outdated: outdated = True if not outdated: previous_paths = self._get_previous_compressed_sources( package_name) if not previous_paths or set(previous_paths) != set(package.paths): outdated = True from django.conf import settings output_path = os.path.join(settings.STATIC_ROOT, package.output_filename) return outdated or not os.path.exists(output_path) def _is_content_changed(self, path): """Verifies if the content of :path change based on the hash that was produced during the last collecstatic run.""" from django.conf import settings changed = True infile_path = os.path.join(self.location, path) outfile_path = os.path.join(settings.STATIC_ROOT, path) infile_hash_path = outfile_path + '.hash' with open(infile_path, 'rb') as infile_file: current_hash = hashlib.md5(infile_file.read()).hexdigest() from django.core.cache import caches DEFAULT_CACHE = caches['default'] old_hash = DEFAULT_CACHE.get(infile_hash_path) changed = current_hash != old_hash DEFAULT_CACHE.set(infile_hash_path, current_hash, None) return changed def _finalize(self): self._dump_sources() print('\n=== {} results ==='.format(self.__class__.__name__)) total_removed = self._remove_sources() self._write_hash() print('{} removed files used in the compressing'.format(total_removed)) print('{} new compressed packages: {}'.format( len(self.compressed_packages), self.compressed_packages)) print('{} unchanged packages: {}'.format(len(self.unchanged_packages), self.unchanged_packages)) print('=== End {} results ==='.format(self.__class__.__name__)) def _remove_sources(self): """We do not want to expose our source files, thus they are removed from the STATIC_ROOT directory, keeping only the compressed files.""" from django.conf import settings sources = [] for package_name in self.packager.packages['js']: package = self.packager.package_for('js', package_name) sources.extend(package.paths) for package_name in self.packager.packages['css']: package = self.packager.package_for('css', package_name) sources.extend(package.paths) removed = 0 for source in sources: source_path = os.path.join(settings.STATIC_ROOT, source) if os.path.exists(source_path): os.remove(source_path) removed += 1 return removed def _dump_sources(self): """We dump the list of compressed source files so we can compare if there is any difference (new files or removed files) in the next collectstatic run.""" from django.core.cache import caches DEFAULT_CACHE = caches['default'] packages = {} for package_name in self.packager.packages['js']: package = self.packager.package_for('js', package_name) packages[package_name] = package.paths for package_name in self.packager.packages['css']: package = self.packager.package_for('css', package_name) packages[package_name] = package.paths # cache forever DEFAULT_CACHE.set(self.SOURCES_DUMP_KEY, packages, None) def _get_previous_compressed_sources(self, package_name): from django.core.cache import caches DEFAULT_CACHE = caches['default'] return DEFAULT_CACHE.get(self.SOURCES_DUMP_KEY, {}).\ get(package_name) def _write_hash(self): """Writes a single md5 hash considering all the content from the source files. This is useful to force any cache mechanism to update their registries.""" from django.conf import settings from django.core.cache import caches DEFAULT_CACHE = caches['default'] output_filenames = [] for package_name in self.packager.packages['js']: package = self.packager.package_for('js', package_name) output_filenames.append(package.output_filename) for package_name in self.packager.packages['css']: package = self.packager.package_for('css', package_name) output_filenames.append(package.output_filename) contents = [] for output_filename in output_filenames: abs_path = os.path.join(settings.STATIC_ROOT, output_filename) with io.open(abs_path, 'rb') as output_file: contents.append(output_file.read()) digest = hashlib.md5(b''.join(contents)).hexdigest() print('New hash: {}'.format(digest)) DEFAULT_CACHE.set(self.HASH_CACHE_KEY, digest, None) # cache forever @staticmethod def get_compressed_files_hash(): from django.core.cache import caches DEFAULT_CACHE = caches['default'] return DEFAULT_CACHE.get(OptimizedPipelineStorage.HASH_CACHE_KEY)