Example #1
0
def get_files(storage, ignore_patterns=None, location=''):
    """
    Recursively walk the storage directories yielding the paths
    of all files that should be copied.
    """
    if ignore_patterns is None:
        ignore_patterns = []
    directories, files = storage.listdir(location)
    for fn in files:
        if matches_patterns(fn, ignore_patterns) or (
                location and matches_patterns(os.path.join(location, fn),
                                              ignore_patterns)):
            continue
        if location:
            fn = os.path.join(location, fn)
        yield fn
    for dir in directories:
        if matches_patterns(dir, ignore_patterns) or (
                location and matches_patterns(os.path.join(location, dir),
                                              ignore_patterns)):
            continue
        if location:
            dir = os.path.join(location, dir)
        for fn in get_files(storage, ignore_patterns, dir):
            yield fn
Example #2
0
    def collect(self) -> Dict[str, Any]:
        result = super().collect()

        try:
            destination_dir = os.path.join(settings.STATIC_ROOT, 'js')
        except IndexError:
            # If the user does not want do have staticfiles, he should not get
            # the webclient files either.
            pass
        else:
            if self.dry_run:
                self.log('Pretending to write WebclientJavaScriptView for all realms.', level=1)
            else:
                if not os.path.exists(destination_dir):
                    os.makedirs(destination_dir)

                for realm in self.realms:
                    filename = self.js_filename.format(realm)
                    # Matches only the basename.
                    if matches_patterns(filename, self.ignore_patterns):
                        continue
                    path = os.path.join(destination_dir, filename)
                    if matches_patterns(path, self.ignore_patterns):
                        continue

                    content = self.view.get(realm=realm).content
                    with open(path, 'wb+') as f:
                        f.write(content)
                    message = "Written WebclientJavaScriptView for realm {} to '{}'".format(
                        realm,
                        path)
                    self.log(message, level=1)
                    result['modified'].append(path)

        return result
Example #3
0
 def glob(cls, files=None):
     '''
     Glob a pattern or a list of pattern static storage relative(s).
     '''
     files = files or []
     if isinstance(files, str):
         matches = lambda path: matches_patterns(path, [files])
     elif isinstance(files, (list, tuple)):
         matches = lambda path: matches_patterns(path, files)
     return [path for path in cls.get_static_files() if matches(path)]
Example #4
0
 def glob(cls, files=None):
     '''
     Glob a pattern or a list of pattern static storage relative(s).
     '''
     files = files or []
     if isinstance(files, str):
         matches = lambda path: matches_patterns(path, [files])
         return [path for path in cls.get_static_files() if matches(path)]
     elif isinstance(files, (list, tuple)):
         all_files = cls.get_static_files()
         sorted_result = []
         for pattern in files:
             sorted_result.extend([f for f in all_files if matches_patterns(f, [pattern])])
         return sorted_result
Example #5
0
    def handle_noargs(self, **options):
        self.set_options(**options)
        if not os.path.isdir(self.source):
            raise CommandError('Non existing local path: %s' % self.source)
        if not hasattr(settings, 'AZURE_ACCOUNT_NAME'):
            raise CommandError('AZURE_ACCOUNT_NAME setting is missing')
        if not hasattr(settings, 'AZURE_ACCOUNT_KEY'):
            raise CommandError('AZURE_ACCOUNT_KEY setting is missing')
        if self.container is None and not hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
            raise CommandError('AZURE_DEFAULT_CONTAINER setting is missing')

        self.log('Starting uploading from "%s" to Azure Storage '
                 'container "%s"' % (self.source, self.container))

        storage = AzureStorage(container=self.container)
        uploaded_files = []
        for root, dirs, files in os.walk(self.source): #@UnusedVariable
            for f in files:
                if matches_patterns(f, self.ignore_patterns):
                    continue
                path = os.path.join(root, f)
                blob_name = os.path.relpath(path, self.source).replace('\\', '/')
                if self.dir:
                    blob_name = os.path.join(self.dir, blob_name)
                self.log('uploading %s...' % blob_name)
                try:
                    with open(path, 'rb') as source_file:
                        storage.save(blob_name, source_file)
                except Exception as e:
                    self.log('upload aborted...')
                    self.log(str(e), 3)
                    return
                else:
                    uploaded_files.append(blob_name)
        self.stdout.write('%s files uploaded.' % len(uploaded_files))
Example #6
0
    def list(self, ignore_patterns):
        # While ``StaticFileStorage`` itself is smart enough not to stumble
        # over this finder returning the full contents of STATIC_ROOT via
        # ``AssetsFileStorage``, ``CachedAssetsFileStorage`` is not. It would
        # create hashed versions of already hashed files.
        #
        # Since the development ``serve`` view will not use this ``list()``
        # method, but the ``collectstatic`` command does, we can customize
        # it to deal with ``CachedAssetsFileStorage``.
        #
        # We restrict the files returned to known bundle output files. Those
        # will then be post-processed by ``CachedAssetsFileStorage`` and
        # properly hashed and rewritten.
        #
        # See also this discussion:
        #    https://github.com/miracle2k/webassets/issues/114

        env = get_env()
        if env.directory == getattr(settings, "STATIC_ROOT"):
            for bundle in env:
                try:
                    output = bundle.resolve_output(env)
                except BundleError:
                    # We don't have a version for this bundle
                    continue

                if not matches_patterns(output, ignore_patterns) and self.storage.exists(output):
                    yield output, self.storage
        else:
            # When ASSETS_ROOT is a separate directory independent of
            # STATIC_ROOT, we're good just letting all files be collected.
            for output in super(AssetsFinder, self).list(ignore_patterns):
                yield output
Example #7
0
    def post_process(self, paths, dry_run=False, **options):
        super_class = super(GZIPMixin, self)
        if hasattr(super_class, 'post_process'):
            for name, hashed_name, processed in super_class.post_process(paths.copy(), dry_run, **options):
                if hashed_name != name:
                    paths[hashed_name] = (self, hashed_name)
                yield name, hashed_name, processed

        if dry_run:
            return

        for path in paths:
            if path:
                if not matches_patterns(path, self.gzip_patterns):
                    continue
                original_file = self.open(path)
                if path.endswith('.css'):
                    gzipped_path = '.'.join((path[:-4], 'gz', 'css'))
                elif path.endswith('.js'):
                    gzipped_path = '.'.join((path[:-3], 'gz', 'js'))

                if self.exists(gzipped_path):
                    self.delete(gzipped_path)
                gzipped_file = self._compress(original_file)
                gzipped_path = self.save(gzipped_path, gzipped_file)
                yield gzipped_path, gzipped_path, True
    def list(self, ignore_patterns):
        """
        Delegate the work of this method to the wrapped finder, but filter its
        results.

        """
        for path, storage in self.wrapped.list(ignore_patterns):
            if utils.matches_patterns(path, self.include_patterns):
                yield path, storage
Example #9
0
def should_save_gzipped_copy(path):

    """
    returns True if the file specified by path should
    also have a copy gzipped and saved as get_gzipped_name(path)

    """

    return matches_patterns(path, GZIP_PATTERNS)
Example #10
0
	def list(self, ignore_patterns):
		for package, data in ASSETS.items():
			if "cache" in data:
				for src, dest in data["cache"]["paths"].items():
					dest = self.to_cache_path(package, dest)
					if matches_patterns(dest, ignore_patterns):
						continue
					dest_dir = os.path.dirname(dest)
					if not os.path.exists(self.storage.path(dest_dir)):
						os.makedirs(self.storage.path(dest_dir))
					if not os.path.exists(self.storage.path(dest)):
						with open(self.storage.path(dest), 'wb') as fp:
							fp.write(urlopen(self.to_url(src)).read())
					yield dest, self.storage
Example #11
0
def get_files(storage, match_patterns='*', ignore_patterns=None, location=''):
    if ignore_patterns is None:
        ignore_patterns = []
    if match_patterns is None:
        match_patterns = []

    directories, files = storage.listdir(location)
    for fn in files:
        if django_utils.matches_patterns(fn, ignore_patterns):
            continue
        if location:
            fn = os.path.join(location, fn)
        if not django_utils.matches_patterns(fn, match_patterns):
            continue
        yield fn
    for dir in directories:
        if django_utils.matches_patterns(dir, ignore_patterns):
            continue
        if location:
            dir = os.path.join(location, dir)
        if may_contain_match(dir, match_patterns) or django_utils.matches_patterns(dir, match_patterns):
            for fn in get_files(storage, match_patterns, ignore_patterns, dir):
                yield fn
Example #12
0
    def post_process(self, paths, dry_run=False, **options):
        """
        Post process the given OrderedDict of files (called from collectstatic).

        Processing is actually two separate operations:

        1. renaming files to include a hash of their content for cache-busting,
           and copying those files to the target storage.
        2. adjusting files which contain references to other files so they
           refer to the cache-busting filenames.

        If either of these are performed on a file, then that file is considered
        post-processed.
        """
        # don't even dare to process the files if we're in dry run mode
        if dry_run:
            return

        # where to store the new paths
        hashed_files = OrderedDict()

        # build a list of adjustable files
        adjustable_paths = [
            path for path in paths
            if matches_patterns(path, self._patterns.keys())
        ]
        # Do a single pass first. Post-process all files once, then repeat for
        # adjustable files.
        for name, hashed_name, processed, _ in self._post_process(paths, adjustable_paths, hashed_files):
            yield name, hashed_name, processed

        paths = {path: paths[path] for path in adjustable_paths}

        for i in range(self.max_post_process_passes):
            substitutions = False
            for name, hashed_name, processed, subst in self._post_process(paths, adjustable_paths, hashed_files):
                yield name, hashed_name, processed
                substitutions = substitutions or subst

            if not substitutions:
                break

        if substitutions:
            yield 'All', None, RuntimeError('Max post-process passes exceeded.')

        # Store the processed paths
        self.hashed_files.update(hashed_files)
Example #13
0
    def post_process(self, paths, dry_run=False, **options):
        """
        Post process the given list of files (called from collectstatic).
        """
        processed_files = []
        # don't even dare to process the files if we're in dry run mode
        if dry_run:
            return processed_files

        # delete cache of all handled paths
        self.cache.delete_many([self.cache_key(path) for path in paths])

        # only try processing the files we have patterns for
        matches = lambda path: matches_patterns(path, self._patterns.keys())
        processing_paths = [path for path in paths if matches(path)]

        # then sort the files by the directory level
        path_level = lambda name: len(name.split(os.sep))
        for name in sorted(paths, key=path_level, reverse=True):

            # first get a hashed name for the given file
            hashed_name = self.hashed_name(name)

            with self.open(name) as original_file:
                # then get the original's file content
                content = original_file.read()

                # to apply each replacement pattern on the content
                if name in processing_paths:
                    converter = self.url_converter(name)
                    for patterns in self._patterns.values():
                        for pattern in patterns:
                            content = pattern.sub(converter, content)

                # then save the processed result
                if self.exists(hashed_name):
                    self.delete(hashed_name)

                content_file = ContentFile(smart_str(content))
                saved_name = self._save(hashed_name, content_file)
                hashed_name = force_unicode(saved_name.replace('\\', '/'))
                processed_files.append(hashed_name)

                # and then set the cache accordingly
                self.cache.set(self.cache_key(name), hashed_name)

        return processed_files
Example #14
0
 def _get_files(self, storage, ignore_patterns=None, location=''):
     """
     Recursively walk the storage directories yielding the paths
     of all files that should be copied.
     """
     if ignore_patterns is None:
         ignore_patterns = []
     directories, files = storage.listdir(location)
     for fn in files:
         if utils.matches_patterns(fn, ignore_patterns):
             continue
         if location:
             fn = os.path.join(location, fn)
         yield fn
     for dir in directories:
         if self.match_ignored_dir('/'.join((storage.path(location), dir)), ignore_patterns):
             continue
         if location:
             dir = os.path.join(location, dir)
         for fn in self._get_files(storage, ignore_patterns, dir):
             yield fn
Example #15
0
    def _post_process(self, paths, adjustable_paths, hashed_files):
        # Sort the files by directory level
        def path_level(name):
            return len(name.split(os.sep))

        for name in sorted(paths.keys(), key=path_level, reverse=True):
            substitutions = True
            # use the original, local file, not the copied-but-unprocessed
            # file, which might be somewhere far away, like S3
            storage, path = paths[name]
            with storage.open(path) as original_file:
                cleaned_name = self.clean_name(name)
                hash_key = self.hash_key(cleaned_name)

                # generate the hash with the original content, even for
                # adjustable files.
                if hash_key not in hashed_files:
                    hashed_name = self.hashed_name(name, original_file)
                else:
                    hashed_name = hashed_files[hash_key]

                # then get the original's file content..
                if hasattr(original_file, 'seek'):
                    original_file.seek(0)

                hashed_file_exists = self.exists(hashed_name)
                processed = False

                # ..to apply each replacement pattern to the content
                if name in adjustable_paths:
                    old_hashed_name = hashed_name
                    content = original_file.read().decode(settings.FILE_CHARSET)
                    for extension, patterns in self._patterns.items():
                        if matches_patterns(path, (extension,)):
                            for pattern, template in patterns:
                                converter = self.url_converter(name, hashed_files, template)
                                try:
                                    content = pattern.sub(converter, content)
                                except ValueError as exc:
                                    yield name, None, exc, False
                    if hashed_file_exists:
                        self.delete(hashed_name)
                    # then save the processed result
                    content_file = ContentFile(force_bytes(content))
                    # Save intermediate file for reference
                    saved_name = self._save(hashed_name, content_file)
                    hashed_name = self.hashed_name(name, content_file)

                    if self.exists(hashed_name):
                        self.delete(hashed_name)

                    saved_name = self._save(hashed_name, content_file)
                    hashed_name = self.clean_name(saved_name)
                    # If the file hash stayed the same, this file didn't change
                    if old_hashed_name == hashed_name:
                        substitutions = False
                    processed = True

                if not processed:
                    # or handle the case in which neither processing nor
                    # a change to the original file happened
                    if not hashed_file_exists:
                        processed = True
                        saved_name = self._save(hashed_name, original_file)
                        hashed_name = self.clean_name(saved_name)

                # and then set the cache accordingly
                hashed_files[hash_key] = hashed_name

                yield name, hashed_name, processed, substitutions
    def post_process(self, paths, dry_run=False, **options):
        """ Copy from CachedFilesMixin
            https://code.djangoproject.com/ticket/19670
        """
        # don't even dare to process the files if we're in dry run mode
        if dry_run:
            return

        # where to store the new paths
        hashed_files = OrderedDict()

        # build a list of adjustable files
        matches = lambda path: matches_patterns(path, self._patterns.keys())
        adjustable_paths = [path for path in paths if matches(path)]

        # then sort the files by the directory level
        path_level = lambda name: len(name.split(os.sep))
        for name in sorted(paths.keys(), key=path_level, reverse=True):

            # use the original, local file, not the copied-but-unprocessed
            # file, which might be somewhere far away, like S3
            storage, path = paths[name]
            with storage.open(path) as original_file:

                # generate the hash with the original content, even for
                # adjustable files.
                hashed_name = self.hashed_name(name, original_file)

                # then get the original's file content..
                if hasattr(original_file, 'seek'):
                    original_file.seek(0)

                hashed_file_exists = self.exists(hashed_name)
                processed = False

                # ..to apply each replacement pattern to the content
                if name in adjustable_paths:
                    content = original_file.read().decode(settings.FILE_CHARSET)
                    for extension, patterns in self._patterns.items():
                        if matches_patterns(path, (extension,)):
                            for pattern, template in patterns:
                                converter = self.url_converter(name, template)
                                try:
                                    content = pattern.sub(converter, content)
                                except ValueError as exc:
                                    yield name, None, exc
                    if hashed_file_exists:
                        self.delete(hashed_name)
                    # then save the processed result
                    content_file = ContentFile(force_bytes(content))
                    saved_name = self._save(hashed_name, content_file)
                    hashed_name = force_text(self.clean_name(saved_name))
                    processed = True
                else:
                    # or handle the case in which neither processing nor
                    # a change to the original file happened
                    if not hashed_file_exists:
                        processed = True
                        saved_name = self._save(hashed_name, original_file)
                        hashed_name = force_text(self.clean_name(saved_name))

                # and then set the cache accordingly
                hashed_files[self.hash_key(name)] = hashed_name
                yield name, hashed_name, processed

        # Finally store the processed paths
        self.hashed_files.update(hashed_files)
Example #17
0
 def list(self, ignore_patterns):
     for path, resource_info in self.resources.items():
         if matches_patterns(path, ignore_patterns):
             continue
         self.fetch(path, resource_info)
         yield path, self.storage
    def _post_process(self, paths, adjustable_paths, hashed_files):
        # Sort the files by directory level
        def path_level(name):
            return len(name.split(os.sep))

        for name in sorted(paths.keys(), key=path_level, reverse=True):
            # Added by Rockallite: check whether hashing should be ignored
            cleaned_name = self.clean_name(name)
            # Ignore hashing by short-circuited the logic
            if cleaned_name in self.hashing_ignored_files:
                yield name, cleaned_name, False, False
                continue
            hash_key = self.hash_key(cleaned_name)
            if self.re_ignore_hashing is not None and self.re_ignore_hashing.search(
                    cleaned_name):
                hashed_files[hash_key] = cleaned_name
                self.hashing_ignored_files.add(cleaned_name)
                yield name, cleaned_name, False, False
                continue

            substitutions = True

            # Commented out by Rockallite: old code always use original file
            # # use the original, local file, not the copied-but-unprocessed
            # # file, which might be somewhere far away, like S3
            # storage, path = paths[name]
            # with storage.open(path) as original_file:

            # Added by Rockallite: new code which checks whether we should use
            # cached minified content
            storage, path = paths[name]
            cached_content = self.get_minified_content_file(name, paths=paths)
            if cached_content is None:
                # use the original, local file, not the copied-but-unprocessed
                # file, which might be somewhere far away, like S3
                open_original_file = lambda: storage.open(path)
            else:
                # Use the cached and minified content
                open_original_file = lambda: cached_content

            with open_original_file() as original_file:
                # Added by Rockallite: end of new code

                # Commited out by Rockallite: cleaned name and hash key already
                # generated
                # cleaned_name = self.clean_name(name)
                # hash_key = self.hash_key(cleaned_name)

                # generate the hash with the original content, even for
                # adjustable files.
                if hash_key not in hashed_files:
                    # Modified by Rockallite: use cleaned name (with backslashes
                    # replaced by slashes) instead of "raw" name for hashing.
                    # hashed_name = self.hashed_name(name, original_file)
                    hashed_name = self.hashed_name(cleaned_name, original_file)
                else:
                    hashed_name = hashed_files[hash_key]

                # then get the original's file content..
                if hasattr(original_file, 'seek'):
                    original_file.seek(0)

                hashed_file_exists = self.exists(hashed_name)
                processed = False

                # ..to apply each replacement pattern to the content
                if name in adjustable_paths:
                    old_hashed_name = hashed_name
                    content = original_file.read().decode(
                        settings.FILE_CHARSET)
                    # Added by Rockallite: flag indicating content substitution
                    content_sub = False
                    for extension, patterns in iteritems(self._patterns):
                        if matches_patterns(path, (extension, )):
                            for pattern, template in patterns:
                                converter = self.url_converter(
                                    name, hashed_files, template)
                                try:
                                    # Modified by Rockallite: get number of sub
                                    # content = pattern.sub(converter, content)
                                    content, num_sub = pattern.subn(
                                        converter, content)
                                except ValueError as exc:
                                    yield name, None, exc, False
                                # Added by Rockallite: check content subsitution
                                else:
                                    if num_sub > 0:
                                        content_sub = True
                    # Commented out by Rockallite: original code is a bit messy
                    # if hashed_file_exists:
                    #     self.delete(hashed_name)
                    # # then save the processed result
                    # content_file = ContentFile(force_bytes(content))
                    # # Save intermediate file for reference
                    # saved_name = self._save(hashed_name, content_file)
                    # hashed_name = self.hashed_name(name, content_file)
                    #
                    # if self.exists(hashed_name):
                    #     self.delete(hashed_name)
                    #
                    # saved_name = self._save(hashed_name, content_file)
                    # hashed_name = force_text(self.clean_name(saved_name))
                    # # If the file hash stayed the same, this file didn't change
                    # if old_hashed_name == hashed_name:
                    #     substitutions = False
                    #
                    # processed = True

                    # Added by Rockallite: new code begins here
                    if content_sub:
                        # Content is substituted. Re-calculate file hash
                        content_file = ContentFile(force_bytes(content))
                        hashed_name = self.hashed_name(cleaned_name,
                                                       content_file)
                        if hashed_name == old_hashed_name:
                            # The file didn't change
                            substitutions = False
                        else:
                            # The file changed
                            if not self.exists(hashed_name):
                                # Save the file only if it doesn't exist
                                saved_name = self._save(
                                    hashed_name,
                                    content_file,
                                    disable_minified_cache=True)
                                hashed_name = force_text(
                                    self.clean_name(saved_name))
                            processed = True
                    else:
                        # The file didn't get substituted, thus didn't change.
                        # Avoid unnecessary hashing calculation.
                        substitutions = False
                    # Comment by Rockallite: end of new code

                if not processed:
                    # or handle the case in which neither processing nor
                    # a change to the original file happened
                    if not hashed_file_exists:
                        processed = True
                        saved_name = self._save(hashed_name,
                                                original_file,
                                                disable_minified_cache=True)
                        hashed_name = force_text(self.clean_name(saved_name))

                # Added by Rockallite: remember intermediate file
                if hash_key in hashed_files:
                    old_hashed_name = hashed_files[hash_key]
                    if old_hashed_name != hashed_name:
                        self.intermediate_files.add(old_hashed_name)

                # and then set the cache accordingly
                hashed_files[hash_key] = hashed_name

                yield name, hashed_name, processed, substitutions
Example #19
0
    def post_process(self, paths, dry_run=False, **options):
        """
        Post process the given OrderedDict of files (called from collectstatic).

        Processing is actually two separate operations:

        1. renaming files to include a hash of their content for cache-busting,
           and copying those files to the target storage.
        2. adjusting files which contain references to other files so they
           refer to the cache-busting filenames.

        If either of these are performed on a file, then that file is considered
        post-processed.
        """
        # don't even dare to process the files if we're in dry run mode
        if dry_run:
            return

        # where to store the new paths
        hashed_files = OrderedDict()

        # build a list of adjustable files
        adjustable_paths = [
            path for path in paths
            if matches_patterns(path, self._patterns.keys())
        ]

        # then sort the files by the directory level
        def path_level(name):
            return len(name.split(os.sep))

        for name in sorted(paths.keys(), key=path_level, reverse=True):

            # use the original, local file, not the copied-but-unprocessed
            # file, which might be somewhere far away, like S3
            storage, path = paths[name]
            with storage.open(path) as original_file:

                # generate the hash with the original content, even for
                # adjustable files.
                hashed_name = self.hashed_name(name, original_file)

                # then get the original's file content..
                if hasattr(original_file, 'seek'):
                    original_file.seek(0)

                hashed_file_exists = self.exists(hashed_name)
                processed = False

                # ..to apply each replacement pattern to the content
                if name in adjustable_paths:
                    content = original_file.read().decode(settings.FILE_CHARSET)
                    for extension, patterns in iteritems(self._patterns):
                        if matches_patterns(path, (extension,)):
                            for pattern, template in patterns:
                                converter = self.url_converter(name, template)
                                try:
                                    content = pattern.sub(converter, content)
                                except ValueError as exc:
                                    yield name, None, exc
                    if hashed_file_exists:
                        self.delete(hashed_name)
                    # then save the processed result
                    content_file = ContentFile(force_bytes(content))
                    saved_name = self._save(hashed_name, content_file)
                    hashed_name = force_text(self.clean_name(saved_name))
                    processed = True
                else:
                    # or handle the case in which neither processing nor
                    # a change to the original file happened
                    if not hashed_file_exists:
                        processed = True
                        saved_name = self._save(hashed_name, original_file)
                        hashed_name = force_text(self.clean_name(saved_name))

                # and then set the cache accordingly
                hashed_files[self.hash_key(name)] = hashed_name
                yield name, hashed_name, processed

        # Finally store the processed paths
        self.hashed_files.update(hashed_files)
Example #20
0
 def url(self, name, force=False):
     url = super(GZIPMixin, self).url(name, force)
     if matches_patterns(name, self.gzip_patterns):
         return "{0}.gz".format(url)
     return url
 def ignore(self, path):
     for segment in self.split_path(path):
         if matches_patterns(segment, self.ignore_patterns):
             return True
     return False
Example #22
0
    def post_process(self, paths, dry_run=False, **options):
        """
        Post process the given list of files (called from collectstatic).

        Processing is actually two separate operations:

        1. renaming files to include a hash of their content for cache-busting,
           and copying those files to the target storage.
        2. adjusting files which contain references to other files so they
           refer to the cache-busting filenames.

        If either of these are performed on a file, then that file is considered
        post-processed.
        """
        # don't even dare to process the files if we're in dry run mode
        if dry_run:
            return

        # delete cache of all handled paths
        self.cache.delete_many([self.cache_key(path) for path in paths])

        # build a list of adjustable files
        matches = lambda path: matches_patterns(path, self._patterns.keys())
        adjustable_paths = [path for path in paths if matches(path)]

        # then sort the files by the directory level
        path_level = lambda name: len(name.split(os.sep))
        for name in sorted(paths.keys(), key=path_level, reverse=True):

            # use the original, local file, not the copied-but-unprocessed
            # file, which might be somewhere far away, like S3
            storage, path = paths[name]
            with storage.open(path) as original_file:

                # generate the hash with the original content, even for
                # adjustable files.
                hashed_name = self.hashed_name(name, original_file)

                # then get the original's file content..
                if hasattr(original_file, 'seek'):
                    original_file.seek(0)

                hashed_file_exists = self.exists(hashed_name)
                processed = False

                # ..to apply each replacement pattern to the content
                if name in adjustable_paths:
                    content = original_file.read()
                    converter = self.url_converter(name)
                    for patterns in self._patterns.values():
                        for pattern in patterns:
                            content = pattern.sub(converter, content)
                    if hashed_file_exists:
                        self.delete(hashed_name)
                    # then save the processed result
                    content_file = ContentFile(smart_str(content))
                    saved_name = self._save(hashed_name, content_file)
                    hashed_name = force_unicode(saved_name.replace('\\', '/'))
                    processed = True
                else:
                    # or handle the case in which neither processing nor
                    # a change to the original file happened
                    if not hashed_file_exists:
                        processed = True
                        saved_name = self._save(hashed_name, original_file)
                        hashed_name = force_unicode(saved_name.replace('\\', '/'))

                # and then set the cache accordingly
                self.cache.set(self.cache_key(name), hashed_name)
                yield name, hashed_name, processed
 def list(self, ignore_patterns):
     for filename in settings.COMPRESS_SETS.keys():
         if not matches_patterns(filename, ignore_patterns):
             yield filename, self.storage
Example #24
0
 def find(self, path, all=False):
     relpath = os.path.relpath(path, self.destination)
     if not django_utils.matches_patterns(relpath, self.match_patterns):
         return []
     return super(NpmFinder, self).find(path, all=all)
Example #25
0
 def find(self, path, all=False):
     patterns = flatten_patterns(getattr(settings, 'NPM_FILE_PATTERNS', None))
     relpath = os.path.relpath(path, getattr(settings, 'NPM_DESTINATION_PREFIX', ''))
     if not django_utils.matches_patterns(patterns, relpath):
         return []
     return super(NpmFinder, self).find(path, all=all)
Example #26
0
    def _post_process(self, paths, adjustable_paths, hashed_files):
        # Sort the files by directory level
        def path_level(name):
            return len(name.split(os.sep))

        for name in sorted(paths, key=path_level, reverse=True):
            substitutions = True
            # use the original, local file, not the copied-but-unprocessed
            # file, which might be somewhere far away, like S3
            storage, path = paths[name]
            with storage.open(path) as original_file:
                cleaned_name = self.clean_name(name)
                hash_key = self.hash_key(cleaned_name)

                # generate the hash with the original content, even for
                # adjustable files.
                if hash_key not in hashed_files:
                    hashed_name = self.hashed_name(name, original_file)
                else:
                    hashed_name = hashed_files[hash_key]

                # then get the original's file content..
                if hasattr(original_file, 'seek'):
                    original_file.seek(0)

                hashed_file_exists = self.exists(hashed_name)
                processed = False

                # ..to apply each replacement pattern to the content
                if name in adjustable_paths:
                    old_hashed_name = hashed_name
                    content = original_file.read().decode(settings.FILE_CHARSET)
                    for extension, patterns in self._patterns.items():
                        if matches_patterns(path, (extension,)):
                            for pattern, template in patterns:
                                converter = self.url_converter(name, hashed_files, template)
                                try:
                                    content = pattern.sub(converter, content)
                                except ValueError as exc:
                                    yield name, None, exc, False
                    if hashed_file_exists:
                        self.delete(hashed_name)
                    # then save the processed result
                    content_file = ContentFile(content.encode())
                    if self.keep_intermediate_files:
                        # Save intermediate file for reference
                        self._save(hashed_name, content_file)
                    hashed_name = self.hashed_name(name, content_file)

                    if self.exists(hashed_name):
                        self.delete(hashed_name)

                    saved_name = self._save(hashed_name, content_file)
                    hashed_name = self.clean_name(saved_name)
                    # If the file hash stayed the same, this file didn't change
                    if old_hashed_name == hashed_name:
                        substitutions = False
                    processed = True

                if not processed:
                    # or handle the case in which neither processing nor
                    # a change to the original file happened
                    if not hashed_file_exists:
                        processed = True
                        saved_name = self._save(hashed_name, original_file)
                        hashed_name = self.clean_name(saved_name)

                # and then set the cache accordingly
                hashed_files[hash_key] = hashed_name

                yield name, hashed_name, processed, substitutions
Example #27
0
    def post_process(self, paths, dry_run=False, **options):
        super_class = super()
        processed_hash_names = []
        if hasattr(super_class, "post_process"):
            for name, hashed_name, processed in super_class.post_process(
                paths.copy(), dry_run, **options
            ):
                if hashed_name != name:
                    paths[hashed_name] = (self, hashed_name)
                    processed_hash_names.append(hashed_name)

                yield name, hashed_name, processed

        if dry_run:
            return

        for path in processed_hash_names:
            if not matches_patterns(path, self.name_patterns):
                continue

            original_file = self.open(path)
            if original_file.size < self.minimum_size_bytes:
                continue

            gzipped_path = "{0}.gz".format(path)
            if not self.exists(gzipped_path):
                # This is the beauty of using django_pipeline.
                # If the .gz file exists, it means the source of it hasn't changed
                # even though it was re-written to disk, because we only bother
                # with files with hashed in the name.
                compressed_file = self._zopfli_compress(original_file)
                gzipped_path = self.save(gzipped_path, compressed_file)
                abs_path = os.path.join(settings.STATIC_ROOT, gzipped_path)
                if os.getenv("CI") or os.stat(abs_path).st_size > 1:
                    yield gzipped_path, gzipped_path, True
                else:
                    # Something went very wrong!
                    size_before = os.stat(abs_path).st_size
                    os.remove(abs_path)
                    print(
                        "The file {} was too small! ({} bytes)".format(
                            abs_path, size_before
                        )
                    )

            brotli_path = "{0}.br".format(path)
            if not self.exists(brotli_path):
                # This is the beauty of using django_pipeline.
                # If the .gz file exists, it means the source of it hasn't changed
                # even though it was re-written to disk, because we only bother
                # with files with hashed in the name.
                compressed_file = self._brotli_compress(original_file)
                brotli_path = self.save(brotli_path, compressed_file)
                abs_path = os.path.join(settings.STATIC_ROOT, brotli_path)
                if os.getenv("CI") or os.stat(abs_path).st_size > 1:
                    yield brotli_path, brotli_path, True
                else:
                    # Something went very wrong!
                    size_before = os.stat(abs_path).st_size
                    os.remove(abs_path)
                    print(
                        "The file {} was too small! ({} bytes)".format(
                            abs_path, size_before
                        )
                    )
Example #28
0
from __future__ import unicode_literals
Example #29
0
    def post_process(self, paths, dry_run=False, **options):
        """
        Post process the given dictionary of files (called from collectstatic).

        Processing is actually two separate operations:

        1. renaming files to include a hash of their content for cache-busting,
           and copying those files to the target storage.
        2. adjusting files which contain references to other files so they
           refer to the cache-busting filenames.

        If either of these are performed on a file, then that file is considered
        post-processed.
        """
        # don't even dare to process the files if we're in dry run mode
        if dry_run:
            return

        # where to store the new paths
        hashed_files = {}

        # build a list of adjustable files
        adjustable_paths = [
            path for path in paths if matches_patterns(path, self._patterns)
        ]

        # Adjustable files to yield at end, keyed by the original path.
        processed_adjustable_paths = {}

        # Do a single pass first. Post-process all files once, yielding not
        # adjustable files and exceptions, and collecting adjustable files.
        for name, hashed_name, processed, _ in self._post_process(
                paths, adjustable_paths, hashed_files):
            if name not in adjustable_paths or isinstance(
                    processed, Exception):
                yield name, hashed_name, processed
            else:
                processed_adjustable_paths[name] = (name, hashed_name,
                                                    processed)

        paths = {path: paths[path] for path in adjustable_paths}
        substitutions = False

        for i in range(self.max_post_process_passes):
            substitutions = False
            for name, hashed_name, processed, subst in self._post_process(
                    paths, adjustable_paths, hashed_files):
                # Overwrite since hashed_name may be newer.
                processed_adjustable_paths[name] = (name, hashed_name,
                                                    processed)
                substitutions = substitutions or subst

            if not substitutions:
                break

        if substitutions:
            yield 'All', None, RuntimeError(
                'Max post-process passes exceeded.')

        # Store the processed paths
        self.hashed_files.update(hashed_files)

        # Yield adjustable files with final, hashed name.
        yield from processed_adjustable_paths.values()
    def get_minified_content_file(self, name, content=None, paths=None):
        if settings.DEBUG:
            # Return no cached minifiable file when debug mode is on
            return

        cleaned_name = self.clean_name(name)
        if cleaned_name in self.minified_files:
            # There is cached minified file. Return it
            return File(open(self.minified_files[cleaned_name], 'rb'))
        else:
            # No cached minified content. Check whether we should minify the
            # file content.
            pm_name = self.get_pre_minified_name(cleaned_name)
            if pm_name is None:
                # File already minified
                return

            min_func = None
            min_func_kwargs = None
            if (self.css_min_enabled or self.js_min_enabled) and \
                    not (self.re_ignore_min and
                         self.re_ignore_min.search(cleaned_name)):
                # Minification mode is on and file isn't ignored
                if self.css_min_enabled and \
                        matches_patterns(cleaned_name, self.css_file_patterns):
                    # Minify CSS
                    min_func = self.css_min_func
                    min_func_kwargs = self.css_min_func_kwargs
                elif self.js_min_enabled and \
                        matches_patterns(cleaned_name, self.js_file_patterns):
                    # Minify JavaScript
                    min_func = self.js_min_func
                    min_func_kwargs = self.js_min_func_kwargs

            if min_func:
                # File content needs to be minified
                assert content is not None or paths is not None, \
                    '"content" and "paths" argument can\'t be both None'
                opened = False
                if content is None:
                    storage, path = paths[name]
                    content = storage.open(path)
                    opened = True
                try:
                    # content_text = content.read().decode(settings.FILE_CHARSET)
                    content_text = content.read()
                finally:
                    if opened:
                        content.close()
                # Minify the content
                if min_func_kwargs is None:
                    min_func_kwargs = {}
                content_text = min_func(content_text, **min_func_kwargs)
                # Convert to bytes and save it to a temporary file
                with NamedTemporaryFile(delete=False) as temp_file:
                    temp_file.write(force_bytes(content_text))
                # Cache the temp file path
                temp_file_path = temp_file.name
                self.minified_files[cleaned_name] = temp_file_path
                # Return minified file
                return File(open(temp_file_path, 'rb'))
 def include(self, path):
     return matches_patterns(path, self.include_patterns)
 def url(self, name, force=False):
     url = super(GZIPMixin, self).url(name, force)
     if matches_patterns(name, self.gzip_patterns):
         return "{0}.gz".format(url)
     return url
 def exclude(self, path):
     return matches_patterns(path, self.exclude_patterns)
    def post_process(self, paths, dry_run=False, **options):
        """
        Post process the given SortedDict of files (called from collectstatic).

        Processing is actually two separate operations:

        1. renaming files to include a hash of their content for cache-busting,
           and copying those files to the target storage.
        2. adjusting files which contain references to other files so they
           refer to the cache-busting filenames.

        If either of these are performed on a file, then that file is considered
        post-processed.
        """
        # don't even dare to process the files if we're in dry run mode
        if dry_run:
            return

        # where to store the new paths
        hashed_paths = {}

        # build a list of adjustable files
        matches = lambda path: matches_patterns(path, self._patterns.keys())
        adjustable_paths = [path for path in paths if matches(path)]

        # then sort the files by the directory level
        path_level = lambda name: len(name.split(os.sep))
        for name in sorted(paths.keys(), key=path_level, reverse=True):

            # use the original, local file, not the copied-but-unprocessed
            # file, which might be somewhere far away, like S3
            storage, path = paths[name]
            with storage.open(path) as original_file:

                # generate the hash with the original content, even for
                # adjustable files.
                hashed_name = self.hashed_name(name, original_file)

                # then get the original's file content..
                if hasattr(original_file, 'seek'):
                    original_file.seek(0)

                hashed_file_exists = self.exists(hashed_name)
                processed = False

                try:
                    print(path)
                except:
                    pass
                try:
                    print(entry)
                except:
                    pass

                # ..to apply each replacement pattern to the content
                if name in adjustable_paths:
                    content = original_file.read().decode(
                        settings.FILE_CHARSET)
                    for patterns in self._patterns.values():
                        for pattern, template in patterns:
                            converter = self.url_converter(name, template)
                            try:
                                content = pattern.sub(converter, content)
                            except ValueError as exc:
                                yield name, None, exc
                    if hashed_file_exists:
                        self.delete(hashed_name)
                    # then save the processed result
                    content_file = ContentFile(force_bytes(content))
                    saved_name = self._save(hashed_name, content_file)
                    hashed_name = force_text(saved_name.replace('\\', '/'))
                    processed = True
                else:
                    # or handle the case in which neither processing nor
                    # a change to the original file happened
                    if not hashed_file_exists:
                        processed = True
                        saved_name = self._save(hashed_name, original_file)
                        hashed_name = force_text(saved_name.replace('\\', '/'))

                # and then set the cache accordingly
                hashed_paths[self.cache_key(name.replace('\\',
                                                         '/'))] = hashed_name
                yield name, hashed_name, processed

        # Finally set the cache
        self.cache.set_many(hashed_paths)
Example #35
0
 def find(self, path, all=False):
     relpath = os.path.relpath(path, self.destination)
     if not django_utils.matches_patterns(relpath, self.match_patterns):
         return []
     return super(YarnFinder, self).find(path, all=all)
Example #36
0
    def post_process(self, paths, dry_run=False, **options):
        print "min enter super", len(paths)
        super_class = super(MinifyMixin, self)
        if hasattr(super_class, 'post_process'):
            for name, hashed_name, processed in super_class.post_process(
                    paths.copy(), dry_run, **options):
                if hashed_name != name:
                    if paths.has_key(name):
                        del paths[name]
                    paths[hashed_name] = (self, hashed_name)
                yield name, hashed_name, processed
        print "min leave super", len(paths)

        if not self._should_minify:
            return
        if dry_run:
            return
        hashed_files = OrderedDict()

        print "starting minify step"
        for path in paths:
            if path:
                if not matches_patterns(path, self.min_patterns):
                    continue
                if self.min_anti_pattern in str(path):
                    continue
                original_file = self.open(path)
                convoy_split_path = path.split(".")
                convoy_split_path.insert(-1, "cmin")
                convoy_min_path = ".".join(convoy_split_path)

                min_contents = False
                if CONVOY_USE_EXISTING_MIN_FILES:
                    #This works best if minification is FIRST OR SECOND in the pipeline
                    # if a minified file exists from the distribution, use it
                    # we want all bugs in minified code to match the distributed bugs 1 to 1
                    split_path = path.split(".")
                    # Kludge for if there is a hash in the filename
                    # Tolerable because we falback to minifying it ourselves
                    # TODO: break this out into a has_hash or has_fingerprint method
                    #       that looks at the filesystem for the un-hashed file
                    # TODO: write a test that fails if django increases the hash length
                    if len(split_path[-2]) == 12 and len(split_path) > 2:
                        split_path.pop(-2)
                    split_path.insert(-1, "min")
                    dist_min_path = ".".join(split_path)
                    if self.exists(dist_min_path):
                        print "Using existing minified file %s" % dist_min_path
                        #Copy the existing minified file into our name scheme
                        f = self.open(dist_min_path)
                        min_contents = f.read()
                        f.close()
                if not min_contents:
                    min_contents = self._min_compress(original_file,
                                                      convoy_split_path[-1])

                if self.exists(convoy_min_path):
                    self.delete(convoy_min_path)
                saved_name = self.save(convoy_min_path,
                                       ContentFile(min_contents))
                hashed_files[self.hash_key(path)] = convoy_min_path
                yield path, convoy_min_path, True

        self.hashed_files.update(hashed_files)