def get_files(storage, ignore_patterns=None, location=''): """ Recursively walk the storage directories yielding the paths of all files that should be copied. """ if ignore_patterns is None: ignore_patterns = [] directories, files = storage.listdir(location) for fn in files: if matches_patterns(fn, ignore_patterns) or ( location and matches_patterns(os.path.join(location, fn), ignore_patterns)): continue if location: fn = os.path.join(location, fn) yield fn for dir in directories: if matches_patterns(dir, ignore_patterns) or ( location and matches_patterns(os.path.join(location, dir), ignore_patterns)): continue if location: dir = os.path.join(location, dir) for fn in get_files(storage, ignore_patterns, dir): yield fn
def collect(self) -> Dict[str, Any]: result = super().collect() try: destination_dir = os.path.join(settings.STATIC_ROOT, 'js') except IndexError: # If the user does not want do have staticfiles, he should not get # the webclient files either. pass else: if self.dry_run: self.log('Pretending to write WebclientJavaScriptView for all realms.', level=1) else: if not os.path.exists(destination_dir): os.makedirs(destination_dir) for realm in self.realms: filename = self.js_filename.format(realm) # Matches only the basename. if matches_patterns(filename, self.ignore_patterns): continue path = os.path.join(destination_dir, filename) if matches_patterns(path, self.ignore_patterns): continue content = self.view.get(realm=realm).content with open(path, 'wb+') as f: f.write(content) message = "Written WebclientJavaScriptView for realm {} to '{}'".format( realm, path) self.log(message, level=1) result['modified'].append(path) return result
def glob(cls, files=None): ''' Glob a pattern or a list of pattern static storage relative(s). ''' files = files or [] if isinstance(files, str): matches = lambda path: matches_patterns(path, [files]) elif isinstance(files, (list, tuple)): matches = lambda path: matches_patterns(path, files) return [path for path in cls.get_static_files() if matches(path)]
def glob(cls, files=None): ''' Glob a pattern or a list of pattern static storage relative(s). ''' files = files or [] if isinstance(files, str): matches = lambda path: matches_patterns(path, [files]) return [path for path in cls.get_static_files() if matches(path)] elif isinstance(files, (list, tuple)): all_files = cls.get_static_files() sorted_result = [] for pattern in files: sorted_result.extend([f for f in all_files if matches_patterns(f, [pattern])]) return sorted_result
def handle_noargs(self, **options): self.set_options(**options) if not os.path.isdir(self.source): raise CommandError('Non existing local path: %s' % self.source) if not hasattr(settings, 'AZURE_ACCOUNT_NAME'): raise CommandError('AZURE_ACCOUNT_NAME setting is missing') if not hasattr(settings, 'AZURE_ACCOUNT_KEY'): raise CommandError('AZURE_ACCOUNT_KEY setting is missing') if self.container is None and not hasattr(settings, 'AZURE_DEFAULT_CONTAINER'): raise CommandError('AZURE_DEFAULT_CONTAINER setting is missing') self.log('Starting uploading from "%s" to Azure Storage ' 'container "%s"' % (self.source, self.container)) storage = AzureStorage(container=self.container) uploaded_files = [] for root, dirs, files in os.walk(self.source): #@UnusedVariable for f in files: if matches_patterns(f, self.ignore_patterns): continue path = os.path.join(root, f) blob_name = os.path.relpath(path, self.source).replace('\\', '/') if self.dir: blob_name = os.path.join(self.dir, blob_name) self.log('uploading %s...' % blob_name) try: with open(path, 'rb') as source_file: storage.save(blob_name, source_file) except Exception as e: self.log('upload aborted...') self.log(str(e), 3) return else: uploaded_files.append(blob_name) self.stdout.write('%s files uploaded.' % len(uploaded_files))
def list(self, ignore_patterns): # While ``StaticFileStorage`` itself is smart enough not to stumble # over this finder returning the full contents of STATIC_ROOT via # ``AssetsFileStorage``, ``CachedAssetsFileStorage`` is not. It would # create hashed versions of already hashed files. # # Since the development ``serve`` view will not use this ``list()`` # method, but the ``collectstatic`` command does, we can customize # it to deal with ``CachedAssetsFileStorage``. # # We restrict the files returned to known bundle output files. Those # will then be post-processed by ``CachedAssetsFileStorage`` and # properly hashed and rewritten. # # See also this discussion: # https://github.com/miracle2k/webassets/issues/114 env = get_env() if env.directory == getattr(settings, "STATIC_ROOT"): for bundle in env: try: output = bundle.resolve_output(env) except BundleError: # We don't have a version for this bundle continue if not matches_patterns(output, ignore_patterns) and self.storage.exists(output): yield output, self.storage else: # When ASSETS_ROOT is a separate directory independent of # STATIC_ROOT, we're good just letting all files be collected. for output in super(AssetsFinder, self).list(ignore_patterns): yield output
def post_process(self, paths, dry_run=False, **options): super_class = super(GZIPMixin, self) if hasattr(super_class, 'post_process'): for name, hashed_name, processed in super_class.post_process(paths.copy(), dry_run, **options): if hashed_name != name: paths[hashed_name] = (self, hashed_name) yield name, hashed_name, processed if dry_run: return for path in paths: if path: if not matches_patterns(path, self.gzip_patterns): continue original_file = self.open(path) if path.endswith('.css'): gzipped_path = '.'.join((path[:-4], 'gz', 'css')) elif path.endswith('.js'): gzipped_path = '.'.join((path[:-3], 'gz', 'js')) if self.exists(gzipped_path): self.delete(gzipped_path) gzipped_file = self._compress(original_file) gzipped_path = self.save(gzipped_path, gzipped_file) yield gzipped_path, gzipped_path, True
def list(self, ignore_patterns): """ Delegate the work of this method to the wrapped finder, but filter its results. """ for path, storage in self.wrapped.list(ignore_patterns): if utils.matches_patterns(path, self.include_patterns): yield path, storage
def should_save_gzipped_copy(path): """ returns True if the file specified by path should also have a copy gzipped and saved as get_gzipped_name(path) """ return matches_patterns(path, GZIP_PATTERNS)
def list(self, ignore_patterns): for package, data in ASSETS.items(): if "cache" in data: for src, dest in data["cache"]["paths"].items(): dest = self.to_cache_path(package, dest) if matches_patterns(dest, ignore_patterns): continue dest_dir = os.path.dirname(dest) if not os.path.exists(self.storage.path(dest_dir)): os.makedirs(self.storage.path(dest_dir)) if not os.path.exists(self.storage.path(dest)): with open(self.storage.path(dest), 'wb') as fp: fp.write(urlopen(self.to_url(src)).read()) yield dest, self.storage
def get_files(storage, match_patterns='*', ignore_patterns=None, location=''): if ignore_patterns is None: ignore_patterns = [] if match_patterns is None: match_patterns = [] directories, files = storage.listdir(location) for fn in files: if django_utils.matches_patterns(fn, ignore_patterns): continue if location: fn = os.path.join(location, fn) if not django_utils.matches_patterns(fn, match_patterns): continue yield fn for dir in directories: if django_utils.matches_patterns(dir, ignore_patterns): continue if location: dir = os.path.join(location, dir) if may_contain_match(dir, match_patterns) or django_utils.matches_patterns(dir, match_patterns): for fn in get_files(storage, match_patterns, ignore_patterns, dir): yield fn
def post_process(self, paths, dry_run=False, **options): """ Post process the given OrderedDict of files (called from collectstatic). Processing is actually two separate operations: 1. renaming files to include a hash of their content for cache-busting, and copying those files to the target storage. 2. adjusting files which contain references to other files so they refer to the cache-busting filenames. If either of these are performed on a file, then that file is considered post-processed. """ # don't even dare to process the files if we're in dry run mode if dry_run: return # where to store the new paths hashed_files = OrderedDict() # build a list of adjustable files adjustable_paths = [ path for path in paths if matches_patterns(path, self._patterns.keys()) ] # Do a single pass first. Post-process all files once, then repeat for # adjustable files. for name, hashed_name, processed, _ in self._post_process(paths, adjustable_paths, hashed_files): yield name, hashed_name, processed paths = {path: paths[path] for path in adjustable_paths} for i in range(self.max_post_process_passes): substitutions = False for name, hashed_name, processed, subst in self._post_process(paths, adjustable_paths, hashed_files): yield name, hashed_name, processed substitutions = substitutions or subst if not substitutions: break if substitutions: yield 'All', None, RuntimeError('Max post-process passes exceeded.') # Store the processed paths self.hashed_files.update(hashed_files)
def post_process(self, paths, dry_run=False, **options): """ Post process the given list of files (called from collectstatic). """ processed_files = [] # don't even dare to process the files if we're in dry run mode if dry_run: return processed_files # delete cache of all handled paths self.cache.delete_many([self.cache_key(path) for path in paths]) # only try processing the files we have patterns for matches = lambda path: matches_patterns(path, self._patterns.keys()) processing_paths = [path for path in paths if matches(path)] # then sort the files by the directory level path_level = lambda name: len(name.split(os.sep)) for name in sorted(paths, key=path_level, reverse=True): # first get a hashed name for the given file hashed_name = self.hashed_name(name) with self.open(name) as original_file: # then get the original's file content content = original_file.read() # to apply each replacement pattern on the content if name in processing_paths: converter = self.url_converter(name) for patterns in self._patterns.values(): for pattern in patterns: content = pattern.sub(converter, content) # then save the processed result if self.exists(hashed_name): self.delete(hashed_name) content_file = ContentFile(smart_str(content)) saved_name = self._save(hashed_name, content_file) hashed_name = force_unicode(saved_name.replace('\\', '/')) processed_files.append(hashed_name) # and then set the cache accordingly self.cache.set(self.cache_key(name), hashed_name) return processed_files
def _get_files(self, storage, ignore_patterns=None, location=''): """ Recursively walk the storage directories yielding the paths of all files that should be copied. """ if ignore_patterns is None: ignore_patterns = [] directories, files = storage.listdir(location) for fn in files: if utils.matches_patterns(fn, ignore_patterns): continue if location: fn = os.path.join(location, fn) yield fn for dir in directories: if self.match_ignored_dir('/'.join((storage.path(location), dir)), ignore_patterns): continue if location: dir = os.path.join(location, dir) for fn in self._get_files(storage, ignore_patterns, dir): yield fn
def _post_process(self, paths, adjustable_paths, hashed_files): # Sort the files by directory level def path_level(name): return len(name.split(os.sep)) for name in sorted(paths.keys(), key=path_level, reverse=True): substitutions = True # use the original, local file, not the copied-but-unprocessed # file, which might be somewhere far away, like S3 storage, path = paths[name] with storage.open(path) as original_file: cleaned_name = self.clean_name(name) hash_key = self.hash_key(cleaned_name) # generate the hash with the original content, even for # adjustable files. if hash_key not in hashed_files: hashed_name = self.hashed_name(name, original_file) else: hashed_name = hashed_files[hash_key] # then get the original's file content.. if hasattr(original_file, 'seek'): original_file.seek(0) hashed_file_exists = self.exists(hashed_name) processed = False # ..to apply each replacement pattern to the content if name in adjustable_paths: old_hashed_name = hashed_name content = original_file.read().decode(settings.FILE_CHARSET) for extension, patterns in self._patterns.items(): if matches_patterns(path, (extension,)): for pattern, template in patterns: converter = self.url_converter(name, hashed_files, template) try: content = pattern.sub(converter, content) except ValueError as exc: yield name, None, exc, False if hashed_file_exists: self.delete(hashed_name) # then save the processed result content_file = ContentFile(force_bytes(content)) # Save intermediate file for reference saved_name = self._save(hashed_name, content_file) hashed_name = self.hashed_name(name, content_file) if self.exists(hashed_name): self.delete(hashed_name) saved_name = self._save(hashed_name, content_file) hashed_name = self.clean_name(saved_name) # If the file hash stayed the same, this file didn't change if old_hashed_name == hashed_name: substitutions = False processed = True if not processed: # or handle the case in which neither processing nor # a change to the original file happened if not hashed_file_exists: processed = True saved_name = self._save(hashed_name, original_file) hashed_name = self.clean_name(saved_name) # and then set the cache accordingly hashed_files[hash_key] = hashed_name yield name, hashed_name, processed, substitutions
def post_process(self, paths, dry_run=False, **options): """ Copy from CachedFilesMixin https://code.djangoproject.com/ticket/19670 """ # don't even dare to process the files if we're in dry run mode if dry_run: return # where to store the new paths hashed_files = OrderedDict() # build a list of adjustable files matches = lambda path: matches_patterns(path, self._patterns.keys()) adjustable_paths = [path for path in paths if matches(path)] # then sort the files by the directory level path_level = lambda name: len(name.split(os.sep)) for name in sorted(paths.keys(), key=path_level, reverse=True): # use the original, local file, not the copied-but-unprocessed # file, which might be somewhere far away, like S3 storage, path = paths[name] with storage.open(path) as original_file: # generate the hash with the original content, even for # adjustable files. hashed_name = self.hashed_name(name, original_file) # then get the original's file content.. if hasattr(original_file, 'seek'): original_file.seek(0) hashed_file_exists = self.exists(hashed_name) processed = False # ..to apply each replacement pattern to the content if name in adjustable_paths: content = original_file.read().decode(settings.FILE_CHARSET) for extension, patterns in self._patterns.items(): if matches_patterns(path, (extension,)): for pattern, template in patterns: converter = self.url_converter(name, template) try: content = pattern.sub(converter, content) except ValueError as exc: yield name, None, exc if hashed_file_exists: self.delete(hashed_name) # then save the processed result content_file = ContentFile(force_bytes(content)) saved_name = self._save(hashed_name, content_file) hashed_name = force_text(self.clean_name(saved_name)) processed = True else: # or handle the case in which neither processing nor # a change to the original file happened if not hashed_file_exists: processed = True saved_name = self._save(hashed_name, original_file) hashed_name = force_text(self.clean_name(saved_name)) # and then set the cache accordingly hashed_files[self.hash_key(name)] = hashed_name yield name, hashed_name, processed # Finally store the processed paths self.hashed_files.update(hashed_files)
def list(self, ignore_patterns): for path, resource_info in self.resources.items(): if matches_patterns(path, ignore_patterns): continue self.fetch(path, resource_info) yield path, self.storage
def _post_process(self, paths, adjustable_paths, hashed_files): # Sort the files by directory level def path_level(name): return len(name.split(os.sep)) for name in sorted(paths.keys(), key=path_level, reverse=True): # Added by Rockallite: check whether hashing should be ignored cleaned_name = self.clean_name(name) # Ignore hashing by short-circuited the logic if cleaned_name in self.hashing_ignored_files: yield name, cleaned_name, False, False continue hash_key = self.hash_key(cleaned_name) if self.re_ignore_hashing is not None and self.re_ignore_hashing.search( cleaned_name): hashed_files[hash_key] = cleaned_name self.hashing_ignored_files.add(cleaned_name) yield name, cleaned_name, False, False continue substitutions = True # Commented out by Rockallite: old code always use original file # # use the original, local file, not the copied-but-unprocessed # # file, which might be somewhere far away, like S3 # storage, path = paths[name] # with storage.open(path) as original_file: # Added by Rockallite: new code which checks whether we should use # cached minified content storage, path = paths[name] cached_content = self.get_minified_content_file(name, paths=paths) if cached_content is None: # use the original, local file, not the copied-but-unprocessed # file, which might be somewhere far away, like S3 open_original_file = lambda: storage.open(path) else: # Use the cached and minified content open_original_file = lambda: cached_content with open_original_file() as original_file: # Added by Rockallite: end of new code # Commited out by Rockallite: cleaned name and hash key already # generated # cleaned_name = self.clean_name(name) # hash_key = self.hash_key(cleaned_name) # generate the hash with the original content, even for # adjustable files. if hash_key not in hashed_files: # Modified by Rockallite: use cleaned name (with backslashes # replaced by slashes) instead of "raw" name for hashing. # hashed_name = self.hashed_name(name, original_file) hashed_name = self.hashed_name(cleaned_name, original_file) else: hashed_name = hashed_files[hash_key] # then get the original's file content.. if hasattr(original_file, 'seek'): original_file.seek(0) hashed_file_exists = self.exists(hashed_name) processed = False # ..to apply each replacement pattern to the content if name in adjustable_paths: old_hashed_name = hashed_name content = original_file.read().decode( settings.FILE_CHARSET) # Added by Rockallite: flag indicating content substitution content_sub = False for extension, patterns in iteritems(self._patterns): if matches_patterns(path, (extension, )): for pattern, template in patterns: converter = self.url_converter( name, hashed_files, template) try: # Modified by Rockallite: get number of sub # content = pattern.sub(converter, content) content, num_sub = pattern.subn( converter, content) except ValueError as exc: yield name, None, exc, False # Added by Rockallite: check content subsitution else: if num_sub > 0: content_sub = True # Commented out by Rockallite: original code is a bit messy # if hashed_file_exists: # self.delete(hashed_name) # # then save the processed result # content_file = ContentFile(force_bytes(content)) # # Save intermediate file for reference # saved_name = self._save(hashed_name, content_file) # hashed_name = self.hashed_name(name, content_file) # # if self.exists(hashed_name): # self.delete(hashed_name) # # saved_name = self._save(hashed_name, content_file) # hashed_name = force_text(self.clean_name(saved_name)) # # If the file hash stayed the same, this file didn't change # if old_hashed_name == hashed_name: # substitutions = False # # processed = True # Added by Rockallite: new code begins here if content_sub: # Content is substituted. Re-calculate file hash content_file = ContentFile(force_bytes(content)) hashed_name = self.hashed_name(cleaned_name, content_file) if hashed_name == old_hashed_name: # The file didn't change substitutions = False else: # The file changed if not self.exists(hashed_name): # Save the file only if it doesn't exist saved_name = self._save( hashed_name, content_file, disable_minified_cache=True) hashed_name = force_text( self.clean_name(saved_name)) processed = True else: # The file didn't get substituted, thus didn't change. # Avoid unnecessary hashing calculation. substitutions = False # Comment by Rockallite: end of new code if not processed: # or handle the case in which neither processing nor # a change to the original file happened if not hashed_file_exists: processed = True saved_name = self._save(hashed_name, original_file, disable_minified_cache=True) hashed_name = force_text(self.clean_name(saved_name)) # Added by Rockallite: remember intermediate file if hash_key in hashed_files: old_hashed_name = hashed_files[hash_key] if old_hashed_name != hashed_name: self.intermediate_files.add(old_hashed_name) # and then set the cache accordingly hashed_files[hash_key] = hashed_name yield name, hashed_name, processed, substitutions
def post_process(self, paths, dry_run=False, **options): """ Post process the given OrderedDict of files (called from collectstatic). Processing is actually two separate operations: 1. renaming files to include a hash of their content for cache-busting, and copying those files to the target storage. 2. adjusting files which contain references to other files so they refer to the cache-busting filenames. If either of these are performed on a file, then that file is considered post-processed. """ # don't even dare to process the files if we're in dry run mode if dry_run: return # where to store the new paths hashed_files = OrderedDict() # build a list of adjustable files adjustable_paths = [ path for path in paths if matches_patterns(path, self._patterns.keys()) ] # then sort the files by the directory level def path_level(name): return len(name.split(os.sep)) for name in sorted(paths.keys(), key=path_level, reverse=True): # use the original, local file, not the copied-but-unprocessed # file, which might be somewhere far away, like S3 storage, path = paths[name] with storage.open(path) as original_file: # generate the hash with the original content, even for # adjustable files. hashed_name = self.hashed_name(name, original_file) # then get the original's file content.. if hasattr(original_file, 'seek'): original_file.seek(0) hashed_file_exists = self.exists(hashed_name) processed = False # ..to apply each replacement pattern to the content if name in adjustable_paths: content = original_file.read().decode(settings.FILE_CHARSET) for extension, patterns in iteritems(self._patterns): if matches_patterns(path, (extension,)): for pattern, template in patterns: converter = self.url_converter(name, template) try: content = pattern.sub(converter, content) except ValueError as exc: yield name, None, exc if hashed_file_exists: self.delete(hashed_name) # then save the processed result content_file = ContentFile(force_bytes(content)) saved_name = self._save(hashed_name, content_file) hashed_name = force_text(self.clean_name(saved_name)) processed = True else: # or handle the case in which neither processing nor # a change to the original file happened if not hashed_file_exists: processed = True saved_name = self._save(hashed_name, original_file) hashed_name = force_text(self.clean_name(saved_name)) # and then set the cache accordingly hashed_files[self.hash_key(name)] = hashed_name yield name, hashed_name, processed # Finally store the processed paths self.hashed_files.update(hashed_files)
def url(self, name, force=False): url = super(GZIPMixin, self).url(name, force) if matches_patterns(name, self.gzip_patterns): return "{0}.gz".format(url) return url
def ignore(self, path): for segment in self.split_path(path): if matches_patterns(segment, self.ignore_patterns): return True return False
def post_process(self, paths, dry_run=False, **options): """ Post process the given list of files (called from collectstatic). Processing is actually two separate operations: 1. renaming files to include a hash of their content for cache-busting, and copying those files to the target storage. 2. adjusting files which contain references to other files so they refer to the cache-busting filenames. If either of these are performed on a file, then that file is considered post-processed. """ # don't even dare to process the files if we're in dry run mode if dry_run: return # delete cache of all handled paths self.cache.delete_many([self.cache_key(path) for path in paths]) # build a list of adjustable files matches = lambda path: matches_patterns(path, self._patterns.keys()) adjustable_paths = [path for path in paths if matches(path)] # then sort the files by the directory level path_level = lambda name: len(name.split(os.sep)) for name in sorted(paths.keys(), key=path_level, reverse=True): # use the original, local file, not the copied-but-unprocessed # file, which might be somewhere far away, like S3 storage, path = paths[name] with storage.open(path) as original_file: # generate the hash with the original content, even for # adjustable files. hashed_name = self.hashed_name(name, original_file) # then get the original's file content.. if hasattr(original_file, 'seek'): original_file.seek(0) hashed_file_exists = self.exists(hashed_name) processed = False # ..to apply each replacement pattern to the content if name in adjustable_paths: content = original_file.read() converter = self.url_converter(name) for patterns in self._patterns.values(): for pattern in patterns: content = pattern.sub(converter, content) if hashed_file_exists: self.delete(hashed_name) # then save the processed result content_file = ContentFile(smart_str(content)) saved_name = self._save(hashed_name, content_file) hashed_name = force_unicode(saved_name.replace('\\', '/')) processed = True else: # or handle the case in which neither processing nor # a change to the original file happened if not hashed_file_exists: processed = True saved_name = self._save(hashed_name, original_file) hashed_name = force_unicode(saved_name.replace('\\', '/')) # and then set the cache accordingly self.cache.set(self.cache_key(name), hashed_name) yield name, hashed_name, processed
def list(self, ignore_patterns): for filename in settings.COMPRESS_SETS.keys(): if not matches_patterns(filename, ignore_patterns): yield filename, self.storage
def find(self, path, all=False): relpath = os.path.relpath(path, self.destination) if not django_utils.matches_patterns(relpath, self.match_patterns): return [] return super(NpmFinder, self).find(path, all=all)
def find(self, path, all=False): patterns = flatten_patterns(getattr(settings, 'NPM_FILE_PATTERNS', None)) relpath = os.path.relpath(path, getattr(settings, 'NPM_DESTINATION_PREFIX', '')) if not django_utils.matches_patterns(patterns, relpath): return [] return super(NpmFinder, self).find(path, all=all)
def _post_process(self, paths, adjustable_paths, hashed_files): # Sort the files by directory level def path_level(name): return len(name.split(os.sep)) for name in sorted(paths, key=path_level, reverse=True): substitutions = True # use the original, local file, not the copied-but-unprocessed # file, which might be somewhere far away, like S3 storage, path = paths[name] with storage.open(path) as original_file: cleaned_name = self.clean_name(name) hash_key = self.hash_key(cleaned_name) # generate the hash with the original content, even for # adjustable files. if hash_key not in hashed_files: hashed_name = self.hashed_name(name, original_file) else: hashed_name = hashed_files[hash_key] # then get the original's file content.. if hasattr(original_file, 'seek'): original_file.seek(0) hashed_file_exists = self.exists(hashed_name) processed = False # ..to apply each replacement pattern to the content if name in adjustable_paths: old_hashed_name = hashed_name content = original_file.read().decode(settings.FILE_CHARSET) for extension, patterns in self._patterns.items(): if matches_patterns(path, (extension,)): for pattern, template in patterns: converter = self.url_converter(name, hashed_files, template) try: content = pattern.sub(converter, content) except ValueError as exc: yield name, None, exc, False if hashed_file_exists: self.delete(hashed_name) # then save the processed result content_file = ContentFile(content.encode()) if self.keep_intermediate_files: # Save intermediate file for reference self._save(hashed_name, content_file) hashed_name = self.hashed_name(name, content_file) if self.exists(hashed_name): self.delete(hashed_name) saved_name = self._save(hashed_name, content_file) hashed_name = self.clean_name(saved_name) # If the file hash stayed the same, this file didn't change if old_hashed_name == hashed_name: substitutions = False processed = True if not processed: # or handle the case in which neither processing nor # a change to the original file happened if not hashed_file_exists: processed = True saved_name = self._save(hashed_name, original_file) hashed_name = self.clean_name(saved_name) # and then set the cache accordingly hashed_files[hash_key] = hashed_name yield name, hashed_name, processed, substitutions
def post_process(self, paths, dry_run=False, **options): super_class = super() processed_hash_names = [] if hasattr(super_class, "post_process"): for name, hashed_name, processed in super_class.post_process( paths.copy(), dry_run, **options ): if hashed_name != name: paths[hashed_name] = (self, hashed_name) processed_hash_names.append(hashed_name) yield name, hashed_name, processed if dry_run: return for path in processed_hash_names: if not matches_patterns(path, self.name_patterns): continue original_file = self.open(path) if original_file.size < self.minimum_size_bytes: continue gzipped_path = "{0}.gz".format(path) if not self.exists(gzipped_path): # This is the beauty of using django_pipeline. # If the .gz file exists, it means the source of it hasn't changed # even though it was re-written to disk, because we only bother # with files with hashed in the name. compressed_file = self._zopfli_compress(original_file) gzipped_path = self.save(gzipped_path, compressed_file) abs_path = os.path.join(settings.STATIC_ROOT, gzipped_path) if os.getenv("CI") or os.stat(abs_path).st_size > 1: yield gzipped_path, gzipped_path, True else: # Something went very wrong! size_before = os.stat(abs_path).st_size os.remove(abs_path) print( "The file {} was too small! ({} bytes)".format( abs_path, size_before ) ) brotli_path = "{0}.br".format(path) if not self.exists(brotli_path): # This is the beauty of using django_pipeline. # If the .gz file exists, it means the source of it hasn't changed # even though it was re-written to disk, because we only bother # with files with hashed in the name. compressed_file = self._brotli_compress(original_file) brotli_path = self.save(brotli_path, compressed_file) abs_path = os.path.join(settings.STATIC_ROOT, brotli_path) if os.getenv("CI") or os.stat(abs_path).st_size > 1: yield brotli_path, brotli_path, True else: # Something went very wrong! size_before = os.stat(abs_path).st_size os.remove(abs_path) print( "The file {} was too small! ({} bytes)".format( abs_path, size_before ) )
from __future__ import unicode_literals
def post_process(self, paths, dry_run=False, **options): """ Post process the given dictionary of files (called from collectstatic). Processing is actually two separate operations: 1. renaming files to include a hash of their content for cache-busting, and copying those files to the target storage. 2. adjusting files which contain references to other files so they refer to the cache-busting filenames. If either of these are performed on a file, then that file is considered post-processed. """ # don't even dare to process the files if we're in dry run mode if dry_run: return # where to store the new paths hashed_files = {} # build a list of adjustable files adjustable_paths = [ path for path in paths if matches_patterns(path, self._patterns) ] # Adjustable files to yield at end, keyed by the original path. processed_adjustable_paths = {} # Do a single pass first. Post-process all files once, yielding not # adjustable files and exceptions, and collecting adjustable files. for name, hashed_name, processed, _ in self._post_process( paths, adjustable_paths, hashed_files): if name not in adjustable_paths or isinstance( processed, Exception): yield name, hashed_name, processed else: processed_adjustable_paths[name] = (name, hashed_name, processed) paths = {path: paths[path] for path in adjustable_paths} substitutions = False for i in range(self.max_post_process_passes): substitutions = False for name, hashed_name, processed, subst in self._post_process( paths, adjustable_paths, hashed_files): # Overwrite since hashed_name may be newer. processed_adjustable_paths[name] = (name, hashed_name, processed) substitutions = substitutions or subst if not substitutions: break if substitutions: yield 'All', None, RuntimeError( 'Max post-process passes exceeded.') # Store the processed paths self.hashed_files.update(hashed_files) # Yield adjustable files with final, hashed name. yield from processed_adjustable_paths.values()
def get_minified_content_file(self, name, content=None, paths=None): if settings.DEBUG: # Return no cached minifiable file when debug mode is on return cleaned_name = self.clean_name(name) if cleaned_name in self.minified_files: # There is cached minified file. Return it return File(open(self.minified_files[cleaned_name], 'rb')) else: # No cached minified content. Check whether we should minify the # file content. pm_name = self.get_pre_minified_name(cleaned_name) if pm_name is None: # File already minified return min_func = None min_func_kwargs = None if (self.css_min_enabled or self.js_min_enabled) and \ not (self.re_ignore_min and self.re_ignore_min.search(cleaned_name)): # Minification mode is on and file isn't ignored if self.css_min_enabled and \ matches_patterns(cleaned_name, self.css_file_patterns): # Minify CSS min_func = self.css_min_func min_func_kwargs = self.css_min_func_kwargs elif self.js_min_enabled and \ matches_patterns(cleaned_name, self.js_file_patterns): # Minify JavaScript min_func = self.js_min_func min_func_kwargs = self.js_min_func_kwargs if min_func: # File content needs to be minified assert content is not None or paths is not None, \ '"content" and "paths" argument can\'t be both None' opened = False if content is None: storage, path = paths[name] content = storage.open(path) opened = True try: # content_text = content.read().decode(settings.FILE_CHARSET) content_text = content.read() finally: if opened: content.close() # Minify the content if min_func_kwargs is None: min_func_kwargs = {} content_text = min_func(content_text, **min_func_kwargs) # Convert to bytes and save it to a temporary file with NamedTemporaryFile(delete=False) as temp_file: temp_file.write(force_bytes(content_text)) # Cache the temp file path temp_file_path = temp_file.name self.minified_files[cleaned_name] = temp_file_path # Return minified file return File(open(temp_file_path, 'rb'))
def include(self, path): return matches_patterns(path, self.include_patterns)
def exclude(self, path): return matches_patterns(path, self.exclude_patterns)
def post_process(self, paths, dry_run=False, **options): """ Post process the given SortedDict of files (called from collectstatic). Processing is actually two separate operations: 1. renaming files to include a hash of their content for cache-busting, and copying those files to the target storage. 2. adjusting files which contain references to other files so they refer to the cache-busting filenames. If either of these are performed on a file, then that file is considered post-processed. """ # don't even dare to process the files if we're in dry run mode if dry_run: return # where to store the new paths hashed_paths = {} # build a list of adjustable files matches = lambda path: matches_patterns(path, self._patterns.keys()) adjustable_paths = [path for path in paths if matches(path)] # then sort the files by the directory level path_level = lambda name: len(name.split(os.sep)) for name in sorted(paths.keys(), key=path_level, reverse=True): # use the original, local file, not the copied-but-unprocessed # file, which might be somewhere far away, like S3 storage, path = paths[name] with storage.open(path) as original_file: # generate the hash with the original content, even for # adjustable files. hashed_name = self.hashed_name(name, original_file) # then get the original's file content.. if hasattr(original_file, 'seek'): original_file.seek(0) hashed_file_exists = self.exists(hashed_name) processed = False try: print(path) except: pass try: print(entry) except: pass # ..to apply each replacement pattern to the content if name in adjustable_paths: content = original_file.read().decode( settings.FILE_CHARSET) for patterns in self._patterns.values(): for pattern, template in patterns: converter = self.url_converter(name, template) try: content = pattern.sub(converter, content) except ValueError as exc: yield name, None, exc if hashed_file_exists: self.delete(hashed_name) # then save the processed result content_file = ContentFile(force_bytes(content)) saved_name = self._save(hashed_name, content_file) hashed_name = force_text(saved_name.replace('\\', '/')) processed = True else: # or handle the case in which neither processing nor # a change to the original file happened if not hashed_file_exists: processed = True saved_name = self._save(hashed_name, original_file) hashed_name = force_text(saved_name.replace('\\', '/')) # and then set the cache accordingly hashed_paths[self.cache_key(name.replace('\\', '/'))] = hashed_name yield name, hashed_name, processed # Finally set the cache self.cache.set_many(hashed_paths)
def find(self, path, all=False): relpath = os.path.relpath(path, self.destination) if not django_utils.matches_patterns(relpath, self.match_patterns): return [] return super(YarnFinder, self).find(path, all=all)
def post_process(self, paths, dry_run=False, **options): print "min enter super", len(paths) super_class = super(MinifyMixin, self) if hasattr(super_class, 'post_process'): for name, hashed_name, processed in super_class.post_process( paths.copy(), dry_run, **options): if hashed_name != name: if paths.has_key(name): del paths[name] paths[hashed_name] = (self, hashed_name) yield name, hashed_name, processed print "min leave super", len(paths) if not self._should_minify: return if dry_run: return hashed_files = OrderedDict() print "starting minify step" for path in paths: if path: if not matches_patterns(path, self.min_patterns): continue if self.min_anti_pattern in str(path): continue original_file = self.open(path) convoy_split_path = path.split(".") convoy_split_path.insert(-1, "cmin") convoy_min_path = ".".join(convoy_split_path) min_contents = False if CONVOY_USE_EXISTING_MIN_FILES: #This works best if minification is FIRST OR SECOND in the pipeline # if a minified file exists from the distribution, use it # we want all bugs in minified code to match the distributed bugs 1 to 1 split_path = path.split(".") # Kludge for if there is a hash in the filename # Tolerable because we falback to minifying it ourselves # TODO: break this out into a has_hash or has_fingerprint method # that looks at the filesystem for the un-hashed file # TODO: write a test that fails if django increases the hash length if len(split_path[-2]) == 12 and len(split_path) > 2: split_path.pop(-2) split_path.insert(-1, "min") dist_min_path = ".".join(split_path) if self.exists(dist_min_path): print "Using existing minified file %s" % dist_min_path #Copy the existing minified file into our name scheme f = self.open(dist_min_path) min_contents = f.read() f.close() if not min_contents: min_contents = self._min_compress(original_file, convoy_split_path[-1]) if self.exists(convoy_min_path): self.delete(convoy_min_path) saved_name = self.save(convoy_min_path, ContentFile(min_contents)) hashed_files[self.hash_key(path)] = convoy_min_path yield path, convoy_min_path, True self.hashed_files.update(hashed_files)