def get_files(storage, ignore_patterns=None, location=''):
"""
Recursively walk the storage directories yielding the paths
of all files that should be copied.
"""
if ignore_patterns is None:
ignore_patterns = []
directories, files = storage.listdir(location)
for fn in files:
if matches_patterns(fn, ignore_patterns) or (
location and matches_patterns(os.path.join(location, fn),
ignore_patterns)):
continue
if location:
fn = os.path.join(location, fn)
yield fn
for dir in directories:
if matches_patterns(dir, ignore_patterns) or (
location and matches_patterns(os.path.join(location, dir),
ignore_patterns)):
continue
if location:
dir = os.path.join(location, dir)
for fn in get_files(storage, ignore_patterns, dir):
yield fn
def collect(self) -> Dict[str, Any]:
result = super().collect()
try:
destination_dir = os.path.join(settings.STATIC_ROOT, 'js')
except IndexError:
# If the user does not want do have staticfiles, he should not get
# the webclient files either.
pass
else:
if self.dry_run:
self.log('Pretending to write WebclientJavaScriptView for all realms.', level=1)
else:
if not os.path.exists(destination_dir):
os.makedirs(destination_dir)
for realm in self.realms:
filename = self.js_filename.format(realm)
# Matches only the basename.
if matches_patterns(filename, self.ignore_patterns):
continue
path = os.path.join(destination_dir, filename)
if matches_patterns(path, self.ignore_patterns):
continue
content = self.view.get(realm=realm).content
with open(path, 'wb+') as f:
f.write(content)
message = "Written WebclientJavaScriptView for realm {} to '{}'".format(
realm,
path)
self.log(message, level=1)
result['modified'].append(path)
return result
def glob(cls, files=None):
'''
Glob a pattern or a list of pattern static storage relative(s).
'''
files = files or []
if isinstance(files, str):
matches = lambda path: matches_patterns(path, [files])
elif isinstance(files, (list, tuple)):
matches = lambda path: matches_patterns(path, files)
return [path for path in cls.get_static_files() if matches(path)]
def glob(cls, files=None):
'''
Glob a pattern or a list of pattern static storage relative(s).
'''
files = files or []
if isinstance(files, str):
matches = lambda path: matches_patterns(path, [files])
return [path for path in cls.get_static_files() if matches(path)]
elif isinstance(files, (list, tuple)):
all_files = cls.get_static_files()
sorted_result = []
for pattern in files:
sorted_result.extend([f for f in all_files if matches_patterns(f, [pattern])])
return sorted_result
def handle_noargs(self, **options):
self.set_options(**options)
if not os.path.isdir(self.source):
raise CommandError('Non existing local path: %s' % self.source)
if not hasattr(settings, 'AZURE_ACCOUNT_NAME'):
raise CommandError('AZURE_ACCOUNT_NAME setting is missing')
if not hasattr(settings, 'AZURE_ACCOUNT_KEY'):
raise CommandError('AZURE_ACCOUNT_KEY setting is missing')
if self.container is None and not hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
raise CommandError('AZURE_DEFAULT_CONTAINER setting is missing')
self.log('Starting uploading from "%s" to Azure Storage '
'container "%s"' % (self.source, self.container))
storage = AzureStorage(container=self.container)
uploaded_files = []
for root, dirs, files in os.walk(self.source): #@UnusedVariable
for f in files:
if matches_patterns(f, self.ignore_patterns):
continue
path = os.path.join(root, f)
blob_name = os.path.relpath(path, self.source).replace('\\', '/')
if self.dir:
blob_name = os.path.join(self.dir, blob_name)
self.log('uploading %s...' % blob_name)
try:
with open(path, 'rb') as source_file:
storage.save(blob_name, source_file)
except Exception as e:
self.log('upload aborted...')
self.log(str(e), 3)
return
else:
uploaded_files.append(blob_name)
self.stdout.write('%s files uploaded.' % len(uploaded_files))
def list(self, ignore_patterns):
# While ``StaticFileStorage`` itself is smart enough not to stumble
# over this finder returning the full contents of STATIC_ROOT via
# ``AssetsFileStorage``, ``CachedAssetsFileStorage`` is not. It would
# create hashed versions of already hashed files.
#
# Since the development ``serve`` view will not use this ``list()``
# method, but the ``collectstatic`` command does, we can customize
# it to deal with ``CachedAssetsFileStorage``.
#
# We restrict the files returned to known bundle output files. Those
# will then be post-processed by ``CachedAssetsFileStorage`` and
# properly hashed and rewritten.
#
# See also this discussion:
# https://github.com/miracle2k/webassets/issues/114
env = get_env()
if env.directory == getattr(settings, "STATIC_ROOT"):
for bundle in env:
try:
output = bundle.resolve_output(env)
except BundleError:
# We don't have a version for this bundle
continue
if not matches_patterns(output, ignore_patterns) and self.storage.exists(output):
yield output, self.storage
else:
# When ASSETS_ROOT is a separate directory independent of
# STATIC_ROOT, we're good just letting all files be collected.
for output in super(AssetsFinder, self).list(ignore_patterns):
yield output
def post_process(self, paths, dry_run=False, **options):
super_class = super(GZIPMixin, self)
if hasattr(super_class, 'post_process'):
for name, hashed_name, processed in super_class.post_process(paths.copy(), dry_run, **options):
if hashed_name != name:
paths[hashed_name] = (self, hashed_name)
yield name, hashed_name, processed
if dry_run:
return
for path in paths:
if path:
if not matches_patterns(path, self.gzip_patterns):
continue
original_file = self.open(path)
if path.endswith('.css'):
gzipped_path = '.'.join((path[:-4], 'gz', 'css'))
elif path.endswith('.js'):
gzipped_path = '.'.join((path[:-3], 'gz', 'js'))
if self.exists(gzipped_path):
self.delete(gzipped_path)
gzipped_file = self._compress(original_file)
gzipped_path = self.save(gzipped_path, gzipped_file)
yield gzipped_path, gzipped_path, True
def list(self, ignore_patterns):
"""
Delegate the work of this method to the wrapped finder, but filter its
results.
"""
for path, storage in self.wrapped.list(ignore_patterns):
if utils.matches_patterns(path, self.include_patterns):
yield path, storage
def should_save_gzipped_copy(path):
"""
returns True if the file specified by path should
also have a copy gzipped and saved as get_gzipped_name(path)
"""
return matches_patterns(path, GZIP_PATTERNS)
def list(self, ignore_patterns): for package, data in ASSETS.items(): if "cache" in data: for src, dest in data["cache"]["paths"].items(): dest = self.to_cache_path(package, dest) if matches_patterns(dest, ignore_patterns): continue dest_dir = os.path.dirname(dest) if not os.path.exists(self.storage.path(dest_dir)): os.makedirs(self.storage.path(dest_dir)) if not os.path.exists(self.storage.path(dest)): with open(self.storage.path(dest), 'wb') as fp: fp.write(urlopen(self.to_url(src)).read()) yield dest, self.storage
def get_files(storage, match_patterns='*', ignore_patterns=None, location=''):
if ignore_patterns is None:
ignore_patterns = []
if match_patterns is None:
match_patterns = []
directories, files = storage.listdir(location)
for fn in files:
if django_utils.matches_patterns(fn, ignore_patterns):
continue
if location:
fn = os.path.join(location, fn)
if not django_utils.matches_patterns(fn, match_patterns):
continue
yield fn
for dir in directories:
if django_utils.matches_patterns(dir, ignore_patterns):
continue
if location:
dir = os.path.join(location, dir)
if may_contain_match(dir, match_patterns) or django_utils.matches_patterns(dir, match_patterns):
for fn in get_files(storage, match_patterns, ignore_patterns, dir):
yield fn
def post_process(self, paths, dry_run=False, **options):
"""
Post process the given OrderedDict of files (called from collectstatic).
Processing is actually two separate operations:
1. renaming files to include a hash of their content for cache-busting,
and copying those files to the target storage.
2. adjusting files which contain references to other files so they
refer to the cache-busting filenames.
If either of these are performed on a file, then that file is considered
post-processed.
"""
# don't even dare to process the files if we're in dry run mode
if dry_run:
return
# where to store the new paths
hashed_files = OrderedDict()
# build a list of adjustable files
adjustable_paths = [
path for path in paths
if matches_patterns(path, self._patterns.keys())
]
# Do a single pass first. Post-process all files once, then repeat for
# adjustable files.
for name, hashed_name, processed, _ in self._post_process(paths, adjustable_paths, hashed_files):
yield name, hashed_name, processed
paths = {path: paths[path] for path in adjustable_paths}
for i in range(self.max_post_process_passes):
substitutions = False
for name, hashed_name, processed, subst in self._post_process(paths, adjustable_paths, hashed_files):
yield name, hashed_name, processed
substitutions = substitutions or subst
if not substitutions:
break
if substitutions:
yield 'All', None, RuntimeError('Max post-process passes exceeded.')
# Store the processed paths
self.hashed_files.update(hashed_files)
def post_process(self, paths, dry_run=False, **options):
"""
Post process the given list of files (called from collectstatic).
"""
processed_files = []
# don't even dare to process the files if we're in dry run mode
if dry_run:
return processed_files
# delete cache of all handled paths
self.cache.delete_many([self.cache_key(path) for path in paths])
# only try processing the files we have patterns for
matches = lambda path: matches_patterns(path, self._patterns.keys())
processing_paths = [path for path in paths if matches(path)]
# then sort the files by the directory level
path_level = lambda name: len(name.split(os.sep))
for name in sorted(paths, key=path_level, reverse=True):
# first get a hashed name for the given file
hashed_name = self.hashed_name(name)
with self.open(name) as original_file:
# then get the original's file content
content = original_file.read()
# to apply each replacement pattern on the content
if name in processing_paths:
converter = self.url_converter(name)
for patterns in self._patterns.values():
for pattern in patterns:
content = pattern.sub(converter, content)
# then save the processed result
if self.exists(hashed_name):
self.delete(hashed_name)
content_file = ContentFile(smart_str(content))
saved_name = self._save(hashed_name, content_file)
hashed_name = force_unicode(saved_name.replace('\\', '/'))
processed_files.append(hashed_name)
# and then set the cache accordingly
self.cache.set(self.cache_key(name), hashed_name)
return processed_files
def _get_files(self, storage, ignore_patterns=None, location=''):
"""
Recursively walk the storage directories yielding the paths
of all files that should be copied.
"""
if ignore_patterns is None:
ignore_patterns = []
directories, files = storage.listdir(location)
for fn in files:
if utils.matches_patterns(fn, ignore_patterns):
continue
if location:
fn = os.path.join(location, fn)
yield fn
for dir in directories:
if self.match_ignored_dir('/'.join((storage.path(location), dir)), ignore_patterns):
continue
if location:
dir = os.path.join(location, dir)
for fn in self._get_files(storage, ignore_patterns, dir):
yield fn
def _post_process(self, paths, adjustable_paths, hashed_files):
# Sort the files by directory level
def path_level(name):
return len(name.split(os.sep))
for name in sorted(paths.keys(), key=path_level, reverse=True):
substitutions = True
# use the original, local file, not the copied-but-unprocessed
# file, which might be somewhere far away, like S3
storage, path = paths[name]
with storage.open(path) as original_file:
cleaned_name = self.clean_name(name)
hash_key = self.hash_key(cleaned_name)
# generate the hash with the original content, even for
# adjustable files.
if hash_key not in hashed_files:
hashed_name = self.hashed_name(name, original_file)
else:
hashed_name = hashed_files[hash_key]
# then get the original's file content..
if hasattr(original_file, 'seek'):
original_file.seek(0)
hashed_file_exists = self.exists(hashed_name)
processed = False
# ..to apply each replacement pattern to the content
if name in adjustable_paths:
old_hashed_name = hashed_name
content = original_file.read().decode(settings.FILE_CHARSET)
for extension, patterns in self._patterns.items():
if matches_patterns(path, (extension,)):
for pattern, template in patterns:
converter = self.url_converter(name, hashed_files, template)
try:
content = pattern.sub(converter, content)
except ValueError as exc:
yield name, None, exc, False
if hashed_file_exists:
self.delete(hashed_name)
# then save the processed result
content_file = ContentFile(force_bytes(content))
# Save intermediate file for reference
saved_name = self._save(hashed_name, content_file)
hashed_name = self.hashed_name(name, content_file)
if self.exists(hashed_name):
self.delete(hashed_name)
saved_name = self._save(hashed_name, content_file)
hashed_name = self.clean_name(saved_name)
# If the file hash stayed the same, this file didn't change
if old_hashed_name == hashed_name:
substitutions = False
processed = True
if not processed:
# or handle the case in which neither processing nor
# a change to the original file happened
if not hashed_file_exists:
processed = True
saved_name = self._save(hashed_name, original_file)
hashed_name = self.clean_name(saved_name)
# and then set the cache accordingly
hashed_files[hash_key] = hashed_name
yield name, hashed_name, processed, substitutions
def post_process(self, paths, dry_run=False, **options):
""" Copy from CachedFilesMixin
https://code.djangoproject.com/ticket/19670
"""
# don't even dare to process the files if we're in dry run mode
if dry_run:
return
# where to store the new paths
hashed_files = OrderedDict()
# build a list of adjustable files
matches = lambda path: matches_patterns(path, self._patterns.keys())
adjustable_paths = [path for path in paths if matches(path)]
# then sort the files by the directory level
path_level = lambda name: len(name.split(os.sep))
for name in sorted(paths.keys(), key=path_level, reverse=True):
# use the original, local file, not the copied-but-unprocessed
# file, which might be somewhere far away, like S3
storage, path = paths[name]
with storage.open(path) as original_file:
# generate the hash with the original content, even for
# adjustable files.
hashed_name = self.hashed_name(name, original_file)
# then get the original's file content..
if hasattr(original_file, 'seek'):
original_file.seek(0)
hashed_file_exists = self.exists(hashed_name)
processed = False
# ..to apply each replacement pattern to the content
if name in adjustable_paths:
content = original_file.read().decode(settings.FILE_CHARSET)
for extension, patterns in self._patterns.items():
if matches_patterns(path, (extension,)):
for pattern, template in patterns:
converter = self.url_converter(name, template)
try:
content = pattern.sub(converter, content)
except ValueError as exc:
yield name, None, exc
if hashed_file_exists:
self.delete(hashed_name)
# then save the processed result
content_file = ContentFile(force_bytes(content))
saved_name = self._save(hashed_name, content_file)
hashed_name = force_text(self.clean_name(saved_name))
processed = True
else:
# or handle the case in which neither processing nor
# a change to the original file happened
if not hashed_file_exists:
processed = True
saved_name = self._save(hashed_name, original_file)
hashed_name = force_text(self.clean_name(saved_name))
# and then set the cache accordingly
hashed_files[self.hash_key(name)] = hashed_name
yield name, hashed_name, processed
# Finally store the processed paths
self.hashed_files.update(hashed_files)
def list(self, ignore_patterns):
for path, resource_info in self.resources.items():
if matches_patterns(path, ignore_patterns):
continue
self.fetch(path, resource_info)
yield path, self.storage
def _post_process(self, paths, adjustable_paths, hashed_files):
# Sort the files by directory level
def path_level(name):
return len(name.split(os.sep))
for name in sorted(paths.keys(), key=path_level, reverse=True):
# Added by Rockallite: check whether hashing should be ignored
cleaned_name = self.clean_name(name)
# Ignore hashing by short-circuited the logic
if cleaned_name in self.hashing_ignored_files:
yield name, cleaned_name, False, False
continue
hash_key = self.hash_key(cleaned_name)
if self.re_ignore_hashing is not None and self.re_ignore_hashing.search(
cleaned_name):
hashed_files[hash_key] = cleaned_name
self.hashing_ignored_files.add(cleaned_name)
yield name, cleaned_name, False, False
continue
substitutions = True
# Commented out by Rockallite: old code always use original file
# # use the original, local file, not the copied-but-unprocessed
# # file, which might be somewhere far away, like S3
# storage, path = paths[name]
# with storage.open(path) as original_file:
# Added by Rockallite: new code which checks whether we should use
# cached minified content
storage, path = paths[name]
cached_content = self.get_minified_content_file(name, paths=paths)
if cached_content is None:
# use the original, local file, not the copied-but-unprocessed
# file, which might be somewhere far away, like S3
open_original_file = lambda: storage.open(path)
else:
# Use the cached and minified content
open_original_file = lambda: cached_content
with open_original_file() as original_file:
# Added by Rockallite: end of new code
# Commited out by Rockallite: cleaned name and hash key already
# generated
# cleaned_name = self.clean_name(name)
# hash_key = self.hash_key(cleaned_name)
# generate the hash with the original content, even for
# adjustable files.
if hash_key not in hashed_files:
# Modified by Rockallite: use cleaned name (with backslashes
# replaced by slashes) instead of "raw" name for hashing.
# hashed_name = self.hashed_name(name, original_file)
hashed_name = self.hashed_name(cleaned_name, original_file)
else:
hashed_name = hashed_files[hash_key]
# then get the original's file content..
if hasattr(original_file, 'seek'):
original_file.seek(0)
hashed_file_exists = self.exists(hashed_name)
processed = False
# ..to apply each replacement pattern to the content
if name in adjustable_paths:
old_hashed_name = hashed_name
content = original_file.read().decode(
settings.FILE_CHARSET)
# Added by Rockallite: flag indicating content substitution
content_sub = False
for extension, patterns in iteritems(self._patterns):
if matches_patterns(path, (extension, )):
for pattern, template in patterns:
converter = self.url_converter(
name, hashed_files, template)
try:
# Modified by Rockallite: get number of sub
# content = pattern.sub(converter, content)
content, num_sub = pattern.subn(
converter, content)
except ValueError as exc:
yield name, None, exc, False
# Added by Rockallite: check content subsitution
else:
if num_sub > 0:
content_sub = True
# Commented out by Rockallite: original code is a bit messy
# if hashed_file_exists:
# self.delete(hashed_name)
# # then save the processed result
# content_file = ContentFile(force_bytes(content))
# # Save intermediate file for reference
# saved_name = self._save(hashed_name, content_file)
# hashed_name = self.hashed_name(name, content_file)
#
# if self.exists(hashed_name):
# self.delete(hashed_name)
#
# saved_name = self._save(hashed_name, content_file)
# hashed_name = force_text(self.clean_name(saved_name))
# # If the file hash stayed the same, this file didn't change
# if old_hashed_name == hashed_name:
# substitutions = False
#
# processed = True
# Added by Rockallite: new code begins here
if content_sub:
# Content is substituted. Re-calculate file hash
content_file = ContentFile(force_bytes(content))
hashed_name = self.hashed_name(cleaned_name,
content_file)
if hashed_name == old_hashed_name:
# The file didn't change
substitutions = False
else:
# The file changed
if not self.exists(hashed_name):
# Save the file only if it doesn't exist
saved_name = self._save(
hashed_name,
content_file,
disable_minified_cache=True)
hashed_name = force_text(
self.clean_name(saved_name))
processed = True
else:
# The file didn't get substituted, thus didn't change.
# Avoid unnecessary hashing calculation.
substitutions = False
# Comment by Rockallite: end of new code
if not processed:
# or handle the case in which neither processing nor
# a change to the original file happened
if not hashed_file_exists:
processed = True
saved_name = self._save(hashed_name,
original_file,
disable_minified_cache=True)
hashed_name = force_text(self.clean_name(saved_name))
# Added by Rockallite: remember intermediate file
if hash_key in hashed_files:
old_hashed_name = hashed_files[hash_key]
if old_hashed_name != hashed_name:
self.intermediate_files.add(old_hashed_name)
# and then set the cache accordingly
hashed_files[hash_key] = hashed_name
yield name, hashed_name, processed, substitutions
def post_process(self, paths, dry_run=False, **options):
"""
Post process the given OrderedDict of files (called from collectstatic).
Processing is actually two separate operations:
1. renaming files to include a hash of their content for cache-busting,
and copying those files to the target storage.
2. adjusting files which contain references to other files so they
refer to the cache-busting filenames.
If either of these are performed on a file, then that file is considered
post-processed.
"""
# don't even dare to process the files if we're in dry run mode
if dry_run:
return
# where to store the new paths
hashed_files = OrderedDict()
# build a list of adjustable files
adjustable_paths = [
path for path in paths
if matches_patterns(path, self._patterns.keys())
]
# then sort the files by the directory level
def path_level(name):
return len(name.split(os.sep))
for name in sorted(paths.keys(), key=path_level, reverse=True):
# use the original, local file, not the copied-but-unprocessed
# file, which might be somewhere far away, like S3
storage, path = paths[name]
with storage.open(path) as original_file:
# generate the hash with the original content, even for
# adjustable files.
hashed_name = self.hashed_name(name, original_file)
# then get the original's file content..
if hasattr(original_file, 'seek'):
original_file.seek(0)
hashed_file_exists = self.exists(hashed_name)
processed = False
# ..to apply each replacement pattern to the content
if name in adjustable_paths:
content = original_file.read().decode(settings.FILE_CHARSET)
for extension, patterns in iteritems(self._patterns):
if matches_patterns(path, (extension,)):
for pattern, template in patterns:
converter = self.url_converter(name, template)
try:
content = pattern.sub(converter, content)
except ValueError as exc:
yield name, None, exc
if hashed_file_exists:
self.delete(hashed_name)
# then save the processed result
content_file = ContentFile(force_bytes(content))
saved_name = self._save(hashed_name, content_file)
hashed_name = force_text(self.clean_name(saved_name))
processed = True
else:
# or handle the case in which neither processing nor
# a change to the original file happened
if not hashed_file_exists:
processed = True
saved_name = self._save(hashed_name, original_file)
hashed_name = force_text(self.clean_name(saved_name))
# and then set the cache accordingly
hashed_files[self.hash_key(name)] = hashed_name
yield name, hashed_name, processed
# Finally store the processed paths
self.hashed_files.update(hashed_files)
def url(self, name, force=False):
url = super(GZIPMixin, self).url(name, force)
if matches_patterns(name, self.gzip_patterns):
return "{0}.gz".format(url)
return url
def ignore(self, path):
for segment in self.split_path(path):
if matches_patterns(segment, self.ignore_patterns):
return True
return False
def post_process(self, paths, dry_run=False, **options):
"""
Post process the given list of files (called from collectstatic).
Processing is actually two separate operations:
1. renaming files to include a hash of their content for cache-busting,
and copying those files to the target storage.
2. adjusting files which contain references to other files so they
refer to the cache-busting filenames.
If either of these are performed on a file, then that file is considered
post-processed.
"""
# don't even dare to process the files if we're in dry run mode
if dry_run:
return
# delete cache of all handled paths
self.cache.delete_many([self.cache_key(path) for path in paths])
# build a list of adjustable files
matches = lambda path: matches_patterns(path, self._patterns.keys())
adjustable_paths = [path for path in paths if matches(path)]
# then sort the files by the directory level
path_level = lambda name: len(name.split(os.sep))
for name in sorted(paths.keys(), key=path_level, reverse=True):
# use the original, local file, not the copied-but-unprocessed
# file, which might be somewhere far away, like S3
storage, path = paths[name]
with storage.open(path) as original_file:
# generate the hash with the original content, even for
# adjustable files.
hashed_name = self.hashed_name(name, original_file)
# then get the original's file content..
if hasattr(original_file, 'seek'):
original_file.seek(0)
hashed_file_exists = self.exists(hashed_name)
processed = False
# ..to apply each replacement pattern to the content
if name in adjustable_paths:
content = original_file.read()
converter = self.url_converter(name)
for patterns in self._patterns.values():
for pattern in patterns:
content = pattern.sub(converter, content)
if hashed_file_exists:
self.delete(hashed_name)
# then save the processed result
content_file = ContentFile(smart_str(content))
saved_name = self._save(hashed_name, content_file)
hashed_name = force_unicode(saved_name.replace('\\', '/'))
processed = True
else:
# or handle the case in which neither processing nor
# a change to the original file happened
if not hashed_file_exists:
processed = True
saved_name = self._save(hashed_name, original_file)
hashed_name = force_unicode(saved_name.replace('\\', '/'))
# and then set the cache accordingly
self.cache.set(self.cache_key(name), hashed_name)
yield name, hashed_name, processed
def list(self, ignore_patterns):
for filename in settings.COMPRESS_SETS.keys():
if not matches_patterns(filename, ignore_patterns):
yield filename, self.storage
def find(self, path, all=False):
relpath = os.path.relpath(path, self.destination)
if not django_utils.matches_patterns(relpath, self.match_patterns):
return []
return super(NpmFinder, self).find(path, all=all)
def find(self, path, all=False):
patterns = flatten_patterns(getattr(settings, 'NPM_FILE_PATTERNS', None))
relpath = os.path.relpath(path, getattr(settings, 'NPM_DESTINATION_PREFIX', ''))
if not django_utils.matches_patterns(patterns, relpath):
return []
return super(NpmFinder, self).find(path, all=all)
def _post_process(self, paths, adjustable_paths, hashed_files):
# Sort the files by directory level
def path_level(name):
return len(name.split(os.sep))
for name in sorted(paths, key=path_level, reverse=True):
substitutions = True
# use the original, local file, not the copied-but-unprocessed
# file, which might be somewhere far away, like S3
storage, path = paths[name]
with storage.open(path) as original_file:
cleaned_name = self.clean_name(name)
hash_key = self.hash_key(cleaned_name)
# generate the hash with the original content, even for
# adjustable files.
if hash_key not in hashed_files:
hashed_name = self.hashed_name(name, original_file)
else:
hashed_name = hashed_files[hash_key]
# then get the original's file content..
if hasattr(original_file, 'seek'):
original_file.seek(0)
hashed_file_exists = self.exists(hashed_name)
processed = False
# ..to apply each replacement pattern to the content
if name in adjustable_paths:
old_hashed_name = hashed_name
content = original_file.read().decode(settings.FILE_CHARSET)
for extension, patterns in self._patterns.items():
if matches_patterns(path, (extension,)):
for pattern, template in patterns:
converter = self.url_converter(name, hashed_files, template)
try:
content = pattern.sub(converter, content)
except ValueError as exc:
yield name, None, exc, False
if hashed_file_exists:
self.delete(hashed_name)
# then save the processed result
content_file = ContentFile(content.encode())
if self.keep_intermediate_files:
# Save intermediate file for reference
self._save(hashed_name, content_file)
hashed_name = self.hashed_name(name, content_file)
if self.exists(hashed_name):
self.delete(hashed_name)
saved_name = self._save(hashed_name, content_file)
hashed_name = self.clean_name(saved_name)
# If the file hash stayed the same, this file didn't change
if old_hashed_name == hashed_name:
substitutions = False
processed = True
if not processed:
# or handle the case in which neither processing nor
# a change to the original file happened
if not hashed_file_exists:
processed = True
saved_name = self._save(hashed_name, original_file)
hashed_name = self.clean_name(saved_name)
# and then set the cache accordingly
hashed_files[hash_key] = hashed_name
yield name, hashed_name, processed, substitutions
def post_process(self, paths, dry_run=False, **options):
super_class = super()
processed_hash_names = []
if hasattr(super_class, "post_process"):
for name, hashed_name, processed in super_class.post_process(
paths.copy(), dry_run, **options
):
if hashed_name != name:
paths[hashed_name] = (self, hashed_name)
processed_hash_names.append(hashed_name)
yield name, hashed_name, processed
if dry_run:
return
for path in processed_hash_names:
if not matches_patterns(path, self.name_patterns):
continue
original_file = self.open(path)
if original_file.size < self.minimum_size_bytes:
continue
gzipped_path = "{0}.gz".format(path)
if not self.exists(gzipped_path):
# This is the beauty of using django_pipeline.
# If the .gz file exists, it means the source of it hasn't changed
# even though it was re-written to disk, because we only bother
# with files with hashed in the name.
compressed_file = self._zopfli_compress(original_file)
gzipped_path = self.save(gzipped_path, compressed_file)
abs_path = os.path.join(settings.STATIC_ROOT, gzipped_path)
if os.getenv("CI") or os.stat(abs_path).st_size > 1:
yield gzipped_path, gzipped_path, True
else:
# Something went very wrong!
size_before = os.stat(abs_path).st_size
os.remove(abs_path)
print(
"The file {} was too small! ({} bytes)".format(
abs_path, size_before
)
)
brotli_path = "{0}.br".format(path)
if not self.exists(brotli_path):
# This is the beauty of using django_pipeline.
# If the .gz file exists, it means the source of it hasn't changed
# even though it was re-written to disk, because we only bother
# with files with hashed in the name.
compressed_file = self._brotli_compress(original_file)
brotli_path = self.save(brotli_path, compressed_file)
abs_path = os.path.join(settings.STATIC_ROOT, brotli_path)
if os.getenv("CI") or os.stat(abs_path).st_size > 1:
yield brotli_path, brotli_path, True
else:
# Something went very wrong!
size_before = os.stat(abs_path).st_size
os.remove(abs_path)
print(
"The file {} was too small! ({} bytes)".format(
abs_path, size_before
)
)
from __future__ import unicode_literals
def post_process(self, paths, dry_run=False, **options):
"""
Post process the given dictionary of files (called from collectstatic).
Processing is actually two separate operations:
1. renaming files to include a hash of their content for cache-busting,
and copying those files to the target storage.
2. adjusting files which contain references to other files so they
refer to the cache-busting filenames.
If either of these are performed on a file, then that file is considered
post-processed.
"""
# don't even dare to process the files if we're in dry run mode
if dry_run:
return
# where to store the new paths
hashed_files = {}
# build a list of adjustable files
adjustable_paths = [
path for path in paths if matches_patterns(path, self._patterns)
]
# Adjustable files to yield at end, keyed by the original path.
processed_adjustable_paths = {}
# Do a single pass first. Post-process all files once, yielding not
# adjustable files and exceptions, and collecting adjustable files.
for name, hashed_name, processed, _ in self._post_process(
paths, adjustable_paths, hashed_files):
if name not in adjustable_paths or isinstance(
processed, Exception):
yield name, hashed_name, processed
else:
processed_adjustable_paths[name] = (name, hashed_name,
processed)
paths = {path: paths[path] for path in adjustable_paths}
substitutions = False
for i in range(self.max_post_process_passes):
substitutions = False
for name, hashed_name, processed, subst in self._post_process(
paths, adjustable_paths, hashed_files):
# Overwrite since hashed_name may be newer.
processed_adjustable_paths[name] = (name, hashed_name,
processed)
substitutions = substitutions or subst
if not substitutions:
break
if substitutions:
yield 'All', None, RuntimeError(
'Max post-process passes exceeded.')
# Store the processed paths
self.hashed_files.update(hashed_files)
# Yield adjustable files with final, hashed name.
yield from processed_adjustable_paths.values()
def get_minified_content_file(self, name, content=None, paths=None):
if settings.DEBUG:
# Return no cached minifiable file when debug mode is on
return
cleaned_name = self.clean_name(name)
if cleaned_name in self.minified_files:
# There is cached minified file. Return it
return File(open(self.minified_files[cleaned_name], 'rb'))
else:
# No cached minified content. Check whether we should minify the
# file content.
pm_name = self.get_pre_minified_name(cleaned_name)
if pm_name is None:
# File already minified
return
min_func = None
min_func_kwargs = None
if (self.css_min_enabled or self.js_min_enabled) and \
not (self.re_ignore_min and
self.re_ignore_min.search(cleaned_name)):
# Minification mode is on and file isn't ignored
if self.css_min_enabled and \
matches_patterns(cleaned_name, self.css_file_patterns):
# Minify CSS
min_func = self.css_min_func
min_func_kwargs = self.css_min_func_kwargs
elif self.js_min_enabled and \
matches_patterns(cleaned_name, self.js_file_patterns):
# Minify JavaScript
min_func = self.js_min_func
min_func_kwargs = self.js_min_func_kwargs
if min_func:
# File content needs to be minified
assert content is not None or paths is not None, \
'"content" and "paths" argument can\'t be both None'
opened = False
if content is None:
storage, path = paths[name]
content = storage.open(path)
opened = True
try:
# content_text = content.read().decode(settings.FILE_CHARSET)
content_text = content.read()
finally:
if opened:
content.close()
# Minify the content
if min_func_kwargs is None:
min_func_kwargs = {}
content_text = min_func(content_text, **min_func_kwargs)
# Convert to bytes and save it to a temporary file
with NamedTemporaryFile(delete=False) as temp_file:
temp_file.write(force_bytes(content_text))
# Cache the temp file path
temp_file_path = temp_file.name
self.minified_files[cleaned_name] = temp_file_path
# Return minified file
return File(open(temp_file_path, 'rb'))
def include(self, path):
return matches_patterns(path, self.include_patterns)
def url(self, name, force=False):
url = super(GZIPMixin, self).url(name, force)
if matches_patterns(name, self.gzip_patterns):
return "{0}.gz".format(url)
return url
def exclude(self, path):
return matches_patterns(path, self.exclude_patterns)
def post_process(self, paths, dry_run=False, **options):
"""
Post process the given SortedDict of files (called from collectstatic).
Processing is actually two separate operations:
1. renaming files to include a hash of their content for cache-busting,
and copying those files to the target storage.
2. adjusting files which contain references to other files so they
refer to the cache-busting filenames.
If either of these are performed on a file, then that file is considered
post-processed.
"""
# don't even dare to process the files if we're in dry run mode
if dry_run:
return
# where to store the new paths
hashed_paths = {}
# build a list of adjustable files
matches = lambda path: matches_patterns(path, self._patterns.keys())
adjustable_paths = [path for path in paths if matches(path)]
# then sort the files by the directory level
path_level = lambda name: len(name.split(os.sep))
for name in sorted(paths.keys(), key=path_level, reverse=True):
# use the original, local file, not the copied-but-unprocessed
# file, which might be somewhere far away, like S3
storage, path = paths[name]
with storage.open(path) as original_file:
# generate the hash with the original content, even for
# adjustable files.
hashed_name = self.hashed_name(name, original_file)
# then get the original's file content..
if hasattr(original_file, 'seek'):
original_file.seek(0)
hashed_file_exists = self.exists(hashed_name)
processed = False
try:
print(path)
except:
pass
try:
print(entry)
except:
pass
# ..to apply each replacement pattern to the content
if name in adjustable_paths:
content = original_file.read().decode(
settings.FILE_CHARSET)
for patterns in self._patterns.values():
for pattern, template in patterns:
converter = self.url_converter(name, template)
try:
content = pattern.sub(converter, content)
except ValueError as exc:
yield name, None, exc
if hashed_file_exists:
self.delete(hashed_name)
# then save the processed result
content_file = ContentFile(force_bytes(content))
saved_name = self._save(hashed_name, content_file)
hashed_name = force_text(saved_name.replace('\\', '/'))
processed = True
else:
# or handle the case in which neither processing nor
# a change to the original file happened
if not hashed_file_exists:
processed = True
saved_name = self._save(hashed_name, original_file)
hashed_name = force_text(saved_name.replace('\\', '/'))
# and then set the cache accordingly
hashed_paths[self.cache_key(name.replace('\\',
'/'))] = hashed_name
yield name, hashed_name, processed
# Finally set the cache
self.cache.set_many(hashed_paths)
def find(self, path, all=False):
relpath = os.path.relpath(path, self.destination)
if not django_utils.matches_patterns(relpath, self.match_patterns):
return []
return super(YarnFinder, self).find(path, all=all)
def post_process(self, paths, dry_run=False, **options):
print "min enter super", len(paths)
super_class = super(MinifyMixin, self)
if hasattr(super_class, 'post_process'):
for name, hashed_name, processed in super_class.post_process(
paths.copy(), dry_run, **options):
if hashed_name != name:
if paths.has_key(name):
del paths[name]
paths[hashed_name] = (self, hashed_name)
yield name, hashed_name, processed
print "min leave super", len(paths)
if not self._should_minify:
return
if dry_run:
return
hashed_files = OrderedDict()
print "starting minify step"
for path in paths:
if path:
if not matches_patterns(path, self.min_patterns):
continue
if self.min_anti_pattern in str(path):
continue
original_file = self.open(path)
convoy_split_path = path.split(".")
convoy_split_path.insert(-1, "cmin")
convoy_min_path = ".".join(convoy_split_path)
min_contents = False
if CONVOY_USE_EXISTING_MIN_FILES:
#This works best if minification is FIRST OR SECOND in the pipeline
# if a minified file exists from the distribution, use it
# we want all bugs in minified code to match the distributed bugs 1 to 1
split_path = path.split(".")
# Kludge for if there is a hash in the filename
# Tolerable because we falback to minifying it ourselves
# TODO: break this out into a has_hash or has_fingerprint method
# that looks at the filesystem for the un-hashed file
# TODO: write a test that fails if django increases the hash length
if len(split_path[-2]) == 12 and len(split_path) > 2:
split_path.pop(-2)
split_path.insert(-1, "min")
dist_min_path = ".".join(split_path)
if self.exists(dist_min_path):
print "Using existing minified file %s" % dist_min_path
#Copy the existing minified file into our name scheme
f = self.open(dist_min_path)
min_contents = f.read()
f.close()
if not min_contents:
min_contents = self._min_compress(original_file,
convoy_split_path[-1])
if self.exists(convoy_min_path):
self.delete(convoy_min_path)
saved_name = self.save(convoy_min_path,
ContentFile(min_contents))
hashed_files[self.hash_key(path)] = convoy_min_path
yield path, convoy_min_path, True
self.hashed_files.update(hashed_files)