Exemple #1
0
    def post_process(self, paths, dry_run=False, **options):
        if dry_run:
            return

        from pipeline.packager import Packager
        packager = Packager(storage=self)
        for package_name in packager.packages['css']:
            package = packager.package_for('css', package_name)
            output_file = package.output_filename
            if self.packing:
                packager.pack_stylesheets(package)
            paths[output_file] = (self, output_file)
            yield output_file, output_file, True
        for package_name in packager.packages['js']:
            package = packager.package_for('js', package_name)
            output_file = package.output_filename
            if self.packing:
                packager.pack_javascripts(package)
            paths[output_file] = (self, output_file)
            yield output_file, output_file, True

        super_class = super(PipelineMixin, self)
        if hasattr(super_class, 'post_process'):
            for name, hashed_name, processed in super_class.post_process(paths.copy(), dry_run, **options):
                yield name, hashed_name, processed
    def post_process(self, paths, dry_run=False, **options):
        """
        This post_process hook is used to package all themed assets.
        """
        if dry_run:
            return
        themes = get_themes()

        for theme in themes:
            css_packages = self.get_themed_packages(
                theme.theme_dir_name, settings.PIPELINE['STYLESHEETS'])

            from pipeline.packager import Packager
            packager = Packager(storage=self, css_packages=css_packages)
            for package_name in packager.packages['css']:
                package = packager.package_for('css', package_name)
                output_file = package.output_filename
                if self.packing:
                    packager.pack_stylesheets(package)
                paths[output_file] = (self, output_file)
                yield output_file, output_file, True

        super_class = super(ThemePipelineMixin, self)
        if hasattr(super_class, 'post_process'):
            for name, hashed_name, processed in super_class.post_process(
                    paths.copy(), dry_run, **options):
                yield name, hashed_name, processed
Exemple #3
0
    def post_process(self, paths, dry_run=False, **options):
        """
        This post_process hook is used to package all themed assets.
        """
        if dry_run:
            return
        themes = get_themes()

        for theme in themes:
            css_packages = self.get_themed_packages(theme.theme_dir_name, settings.PIPELINE_CSS)

            from pipeline.packager import Packager
            packager = Packager(storage=self, css_packages=css_packages)
            for package_name in packager.packages['css']:
                package = packager.package_for('css', package_name)
                output_file = package.output_filename
                if self.packing:
                    packager.pack_stylesheets(package)
                paths[output_file] = (self, output_file)
                yield output_file, output_file, True

        super_class = super(ThemePipelineMixin, self)
        if hasattr(super_class, 'post_process'):
            for name, hashed_name, processed in super_class.post_process(paths.copy(), dry_run, **options):
                yield name, hashed_name, processed
    def post_process(self, paths, dry_run=False, **options):
        if dry_run:
            return []

        from pipeline.packager import Packager
        packager = Packager(storage=self)
        for package_name in packager.packages['css']:
            package = packager.package_for('css', package_name)
            output_file = package.output_filename
            if self.packing:
                packager.pack_stylesheets(package)
            paths[output_file] = (self, output_file)
        for package_name in packager.packages['js']:
            package = packager.package_for('js', package_name)
            output_file = package.output_filename
            if self.packing:
                packager.pack_javascripts(package)
            paths[output_file] = (self, output_file)

        super_class = super(PipelineMixin, self)
        if hasattr(super_class, 'post_process'):
            return super_class.post_process(paths, dry_run, **options)

        return [
            (path, path, True)
            for path in paths
        ]
Exemple #5
0
    def post_process(self, paths, dry_run=False, **options):
        if dry_run:
            return []

        from pipeline.packager import Packager
        packager = Packager(storage=self)
        for package_name in packager.packages['css']:
            package = packager.package_for('css', package_name)
            output_file = package.output_filename
            if self.packing:
                packager.pack_stylesheets(package)
            paths[output_file] = (self, output_file)
        for package_name in packager.packages['js']:
            package = packager.package_for('js', package_name)
            output_file = package.output_filename
            if self.packing:
                packager.pack_javascripts(package)
            paths[output_file] = (self, output_file)

        super_class = super(PipelineMixin, self)
        if hasattr(super_class, 'post_process'):
            return super_class.post_process(paths, dry_run, **options)

        return [
            (path, path, True)
            for path in paths
        ]
    def post_process(self, paths, dry_run=False, **options):
        if dry_run:
            return

        packager = Packager(storage=self)
        for _abs_path, rel_path in paths:
            files_to_process = OrderedDict()
            files_to_process[rel_path] = (self, rel_path)
            for package_name in packager.packages['css']:
                package = packager.package_for('css', package_name)
                output_file = package.output_filename

                if rel_path in package.paths:
                    if self.packing:
                        packager.pack_stylesheets(package)
                    files_to_process[output_file] = (self, output_file)
                    yield output_file, output_file, True
            for package_name in packager.packages['js']:
                package = packager.package_for('js', package_name)
                output_file = package.output_filename
                if rel_path in package.paths:
                    if self.packing:
                        packager.pack_javascripts(package)
                    files_to_process[output_file] = (self, output_file)
                    yield output_file, output_file, True

            super_class = super(PipelineMixin, self)

            if hasattr(super_class, 'post_process'):
                for name, hashed_name, processed in super_class.post_process(
                        files_to_process.copy(), dry_run, **options):
                    yield name, hashed_name, processed
    def handle(self, *args, **options):
        from pipeline.packager import Packager

        force = options.get('force', False)
        verbose = int(options.get('verbosity', 1)) >= 2
        sync = options.get('dry_run', True)

        packager = Packager(verbose=verbose)
        for package_name in packager.packages['css']:
            if args and package_name not in args:
                continue
            package = packager.package_for('css', package_name)
            if verbose:
                print
                message = "CSS Group '%s'" % package_name
                print message
                print len(message) * '-'
            packager.pack_stylesheets(package, sync=sync, force=force)

        for package_name in packager.packages['js']:
            if args and package_name not in args:
                continue
            package = packager.package_for('js', package_name)
            if verbose:
                print
                message = "JS Group '%s'" % package_name
                print message
                print len(message) * '-'
            packager.pack_javascripts(package, sync=sync, force=force)
Exemple #8
0
    def handle(self, *args, **options):
        from pipeline.packager import Packager

        force = options.get('force', False)
        verbose = int(options.get('verbosity', 1)) >= 2
        sync = options.get('dry_run', True)

        packager = Packager(verbose=verbose)
        for package_name in packager.packages['css']:
            if args and package_name not in args:
                continue
            package = packager.package_for('css', package_name)
            if verbose:
                print
                message = "CSS Group '%s'" % package_name
                print message
                print len(message) * '-'
            packager.pack_stylesheets(package, sync=sync, force=force)

        for package_name in packager.packages['js']:
            if args and package_name not in args:
                continue
            package = packager.package_for('js', package_name)
            if verbose:
                print
                message = "JS Group '%s'" % package_name
                print message
                print len(message) * '-'
            packager.pack_javascripts(package, sync=sync, force=force)
Exemple #9
0
class CompressedCSSNode(template.Node):
    def __init__(self, name):
        self.name = name

    def render(self, context):
        package_name = template.Variable(self.name).resolve(context)
        package = settings.PIPELINE_CSS.get(package_name, {})
        if package:
            package = {package_name: package}
        self.packager = Packager(css_packages=package, js_packages={})

        try:
            package = self.packager.package_for('css', package_name)
        except PackageNotFound:
            return ''  # fail silently, do not return anything if an invalid group is specified

        if settings.PIPELINE:
            compressed_path = self.packager.pack_stylesheets(package)
            return self.render_css(package, compressed_path)
        else:
            package['paths'] = self.packager.compile(package['paths'])
            return self.render_individual(package)

    def render_css(self, package, path):
        context = {}
        if not 'template' in package:
            package['template'] = "pipeline/css.html"
        if 'context' in package:
            context = package['context']
        context.update({'url': self.packager.individual_url(path)})
        return render_to_string(package['template'], context)

    def render_individual(self, package):
        tags = [self.render_css(package, path) for path in package['paths']]
        return '\n'.join(tags)
class CompressedCSSNode(template.Node):
    def __init__(self, name):
        self.name = name
        self.packager = Packager()

    def render(self, context):
        package_name = template.Variable(self.name).resolve(context)
        try:
            package = self.packager.package_for('css', package_name)
        except PackageNotFound:
            return ''  # fail silently, do not return anything if an invalid group is specified

        if settings.PIPELINE:
            compressed_path = self.packager.pack_stylesheets(package)
            return self.render_css(package, compressed_path)
        else:
            package['paths'] = self.packager.compile(package['paths'])
            return self.render_individual(package)

    def render_css(self, package, path):
        context = {}
        if not 'template' in package:
            package['template'] = "pipeline/css.html"
        if not 'context' in package:
            context = package['context']
        context.update({
            'url': self.packager.individual_url(path)
        })
        return render_to_string(package['template'], context)

    def render_individual(self, package):
        tags = [self.render_css(package, path) for path in package['paths']]
        return '\n'.join(tags)
Exemple #11
0
    def post_process(self, paths, dry_run=False, **options):
        if dry_run:
            return []

        from pipeline.packager import Packager
        packager = Packager(storage=self)
        for package_name in packager.packages['css']:
            package = packager.package_for('css', package_name)
            if self.packing:
                paths_written = packager.pack_stylesheets(package)
                for path in paths_written:
                    paths[path] = (self, path)
            else:
                # TODO: bcooksey 5/15/13. Not sure why we pretend we packed if packing is false...will this mess up source maps
                output_file = package.output_filename
                paths[output_file] = (self, output_file)
        for package_name in packager.packages['js']:
            package = packager.package_for('js', package_name)
            if self.packing:
                paths_written = packager.pack_javascripts(package)
                for path in paths_written:
                    paths[path] = (self, path)
            else:
                # TODO: bcooksey 5/15/13. Not sure why we pretend we packed if packing is false...will this mess up source maps
                output_file = package.output_filename
                paths[output_file] = (self, output_file)

        super_class = super(PipelineMixin, self)
        if hasattr(super_class, 'post_process'):
            return super_class.post_process(paths, dry_run, **options)

        return [
            (path, path, True)
            for path in paths
        ]
    def handle_noargs(self, **options):
        from pipeline.packager import Packager
        packager = Packager(
            force=options.get('force', False),
            verbose=int(options.get('verbosity', 1)) >= 2
        )

        for package_name in packager.packages['css']:
            package = packager.package_for('css', package_name)
            if packager.verbose or packager.force:
                print
                message = "CSS Group '%s'" % package_name
                print message
                print len(message) * '-'
            packager.pack_stylesheets(package)

        for package_name in packager.packages['js']:
            package = packager.package_for('js', package_name)
            if packager.verbose or packager.force:
                print
                message = "JS Group '%s'" % package_name
                print message
                print len(message) * '-'
            packager.pack_javascripts(package)
Exemple #13
0
class OptimizedPipelineStorage(PipelineMixin, StaticFilesStorage):
    """This storage compresses only the packages which had modifications in
    their source files, or that have not been compressed yet. This speeds up
    the collectstatic process, since must of the time we modify only a few
    javascript/css files at a time.

    It also appends the a md5 hash to the compressed files' url so any existing
    cache mechanisms are naturally invalidated."""

    compressed_packages = []
    unchanged_packages = []
    packager = None
    HASH_CACHE_KEY = 'pipeline_compressed_hash_key'
    SOURCES_DUMP_KEY = 'pipeline_dumped_sources_key'

    def url(self, name):
        """Append the produced hash to the resource url so existing cache
        mechanisms are naturally invalidated."""
        url = super(OptimizedPipelineStorage, self).url(name)
        _hash = self.get_compressed_files_hash()
        if _hash and name:
            return '{url}?{_hash}'.format(url=url, _hash=_hash)
        else:
            return url

    def post_process(self, paths, dry_run=False, **options):
        if dry_run:
            return

        from pipeline.packager import Packager
        self.packager = Packager(storage=self)

        for package_name in self.packager.packages['css']:
            package = self.packager.package_for('css', package_name)
            output_file = package.output_filename

            if self.packing and self._is_outdated(package_name, package):
                print('COMPRESSING {} package...'.format(package_name))
                self.packager.pack_stylesheets(package)
                self.compressed_packages.append(package_name)
            else:
                self.unchanged_packages.append(package_name)

            paths[output_file] = (self, output_file)
            yield output_file, output_file, True

        for package_name in self.packager.packages['js']:
            package = self.packager.package_for('js', package_name)
            output_file = package.output_filename

            if self.packing and self._is_outdated(package_name, package):
                print('COMPRESSING {} package...'.format(package_name))
                self.packager.pack_javascripts(package)
                self.compressed_packages.append(package_name)
            else:
                self.unchanged_packages.append(package_name)

            paths[output_file] = (self, output_file)
            yield output_file, output_file, True

        super_class = super(PipelineMixin, self)
        if hasattr(super_class, 'post_process'):
            for name, hashed_name, processed in super_class.post_process(
                    paths.copy(), dry_run, **options):
                yield name, hashed_name, processed

        self._finalize()

    def _is_outdated(self, package_name, package):
        outdated = False

        for path in package.paths:
            # Needs to run for every path in order to generate the individual
            # file hashes.
            if self._is_content_changed(path) and not outdated:
                outdated = True

        if not outdated:
            previous_paths = self._get_previous_compressed_sources(
                package_name)
            if not previous_paths or set(previous_paths) != set(package.paths):
                outdated = True

        from django.conf import settings
        output_path = os.path.join(settings.STATIC_ROOT,
                                   package.output_filename)
        return outdated or not os.path.exists(output_path)

    def _is_content_changed(self, path):
        """Verifies if the content of :path change based on the hash that was
        produced during the last collecstatic run."""
        from django.conf import settings
        changed = True
        infile_path = os.path.join(self.location, path)
        outfile_path = os.path.join(settings.STATIC_ROOT, path)
        infile_hash_path = outfile_path + '.hash'

        with open(infile_path, 'rb') as infile_file:
            current_hash = hashlib.md5(infile_file.read()).hexdigest()

        from django.core.cache import caches
        DEFAULT_CACHE = caches['default']
        old_hash = DEFAULT_CACHE.get(infile_hash_path)
        changed = current_hash != old_hash
        DEFAULT_CACHE.set(infile_hash_path, current_hash, None)
        return changed

    def _finalize(self):
        self._dump_sources()
        print('\n=== {} results ==='.format(self.__class__.__name__))
        total_removed = self._remove_sources()
        self._write_hash()
        print('{} removed files used in the compressing'.format(total_removed))
        print('{} new compressed packages: {}'.format(
            len(self.compressed_packages), self.compressed_packages))
        print('{} unchanged packages: {}'.format(len(self.unchanged_packages),
                                                 self.unchanged_packages))
        print('=== End {} results ==='.format(self.__class__.__name__))

    def _remove_sources(self):
        """We do not want to expose our source files, thus they are removed
        from the STATIC_ROOT directory, keeping only the compressed files."""
        from django.conf import settings
        sources = []

        for package_name in self.packager.packages['js']:
            package = self.packager.package_for('js', package_name)
            sources.extend(package.paths)

        for package_name in self.packager.packages['css']:
            package = self.packager.package_for('css', package_name)
            sources.extend(package.paths)

        removed = 0
        for source in sources:
            source_path = os.path.join(settings.STATIC_ROOT, source)
            if os.path.exists(source_path):
                os.remove(source_path)
                removed += 1

        return removed

    def _dump_sources(self):
        """We dump the list of compressed source files so we can compare if
        there is any difference (new files or removed files) in the next
        collectstatic run."""
        from django.core.cache import caches
        DEFAULT_CACHE = caches['default']

        packages = {}

        for package_name in self.packager.packages['js']:
            package = self.packager.package_for('js', package_name)
            packages[package_name] = package.paths

        for package_name in self.packager.packages['css']:
            package = self.packager.package_for('css', package_name)
            packages[package_name] = package.paths
            # cache forever
            DEFAULT_CACHE.set(self.SOURCES_DUMP_KEY, packages, None)

    def _get_previous_compressed_sources(self, package_name):
        from django.core.cache import caches
        DEFAULT_CACHE = caches['default']
        return DEFAULT_CACHE.get(self.SOURCES_DUMP_KEY, {}).\
            get(package_name)

    def _write_hash(self):
        """Writes a single md5 hash considering all the content from the
        source files. This is useful to force any cache mechanism to update
        their registries."""
        from django.conf import settings
        from django.core.cache import caches
        DEFAULT_CACHE = caches['default']
        output_filenames = []

        for package_name in self.packager.packages['js']:
            package = self.packager.package_for('js', package_name)
            output_filenames.append(package.output_filename)

        for package_name in self.packager.packages['css']:
            package = self.packager.package_for('css', package_name)
            output_filenames.append(package.output_filename)

        contents = []
        for output_filename in output_filenames:
            abs_path = os.path.join(settings.STATIC_ROOT, output_filename)
            with io.open(abs_path, 'rb') as output_file:
                contents.append(output_file.read())

        digest = hashlib.md5(b''.join(contents)).hexdigest()
        print('New hash: {}'.format(digest))
        DEFAULT_CACHE.set(self.HASH_CACHE_KEY, digest, None)  # cache forever

    @staticmethod
    def get_compressed_files_hash():
        from django.core.cache import caches
        DEFAULT_CACHE = caches['default']
        return DEFAULT_CACHE.get(OptimizedPipelineStorage.HASH_CACHE_KEY)