def test_lang_file_minify_with_missing_file(self):
'''
Generates, removes one of the files, and subsequently tries again.
Used for testing failures half way through the script
'''
simple, lang = self.get_files()
lang_minify = minify.MinifyJs(lang)
combined_filename = lang_minify.get_combined_filename(force_generation=True)
self.assertLang(combined_filename)
minified_filename = lang_minify.get_minified_filename(force_generation=True)
pt_filename = utils.replace_lang(minified_filename, 'pt-br')
self.assertLang(minified_filename)
self.assertValidLangPath(minified_filename)
#remove one of the files and see if things will still work :)
assert os.path.isfile(pt_filename)
os.remove(pt_filename)
#try with from cache disabled
minified_filename = lang_minify.get_minified_filename(force_generation=True)
pt_filename = utils.replace_lang(minified_filename, 'pt-br')
#this shouldnt have been regenerated
#TODO: Maybe it would be desired behaviour to rebuild
assert not os.path.isfile(pt_filename)
#try with from cache enabled
OLD_FROM_CACHE, OLD_DEBUG = settings.FROM_CACHE, settings.DEBUG
settings.FROM_CACHE = True
settings.DEBUG = False
try:
minified_filename = lang_minify.get_minified_filename(force_generation=True)
pt_filename = utils.replace_lang(minified_filename, 'pt-br')
#this shouldnt have been regenerated
assert not os.path.isfile(pt_filename)
finally:
settings.FROM_CACHE = OLD_FROM_CACHE
settings.DEBUG = OLD_DEBUG
def test_lang_file_minify_with_missing_file(self):
'''
Generates, removes one of the files, and subsequently tries again.
Used for testing failures half way through the script
'''
simple, lang = self.get_files()
lang_minify = minify.MinifyJs(lang)
combined_filename = lang_minify.get_combined_filename(force_generation=True)
self.assertLang(combined_filename)
minified_filename = lang_minify.get_minified_filename(force_generation=True)
pt_filename = utils.replace_lang(minified_filename, 'pt-br')
self.assertLang(minified_filename)
self.assertValidLangPath(minified_filename)
#remove one of the files and see if things will still work :)
assert os.path.isfile(pt_filename)
os.remove(pt_filename)
#try with from cache disabled
minified_filename = lang_minify.get_minified_filename(force_generation=True)
pt_filename = utils.replace_lang(minified_filename, 'pt-br')
#this should have been regenerated
assert os.path.isfile(pt_filename)
#try with from cache enabled
OLD_FROM_CACHE, OLD_DEBUG = settings.FROM_CACHE, settings.DEBUG
settings.FROM_CACHE = True
settings.DEBUG = False
os.remove(pt_filename)
try:
minified_filename = lang_minify.get_minified_filename(force_generation=False)
pt_filename = utils.replace_lang(minified_filename, 'pt-br')
#this shouldnt have been regenerated
assert not os.path.isfile(pt_filename)
finally:
settings.FROM_CACHE = OLD_FROM_CACHE
settings.DEBUG = OLD_DEBUG
def get_minified_filename(self, force_generation=False):
'''
Returns the minified filename, for language specific files it will return
filename_<lang>.js
'''
input_filename = self.get_combined_filename()
output_filename = input_filename.replace('_debug_', '_mini_')
if output_filename in self.cache and not force_generation:
#if the output is cached, immediatly return it without checking the filesystem
return output_filename
else:
if not settings.DEBUG and not force_generation:
#while running on debug this isn't an error because a dummy cache backend is used
error_message = 'There is no file cache available, but we arent allowed to build the files. Searching for %s in %s' % (
output_filename, self.cache)
assert not settings.FROM_CACHE, error_message
#see if all the files we need are actually there
compiled_files_available = True
output_filenames = expand_on_lang(output_filename)
for lang_specific_filename in output_filenames:
if not os.path.isfile(lang_specific_filename):
compiled_files_available = False
# Keep a minified version of each file
# flip the minification of a combination of files in the combination of each minified file
# minify(combine(a,b,c)) = combine(minify(a), minify(a), minify(a))
non_localized_files = [
f for f in self.files if expand_on_lang(f) == [f]
]
non_localized_minified_filenames = [
self.minimize_file_to_cache(f) for f in non_localized_files
]
non_localized_filename = self._get_combined_filename(
non_localized_minified_filenames, force_generation)
# minify each localized file
minified_localized = defaultdict(list)
localized_files = [
f for f in self.files if expand_on_lang(f) != [f]
]
for f in localized_files:
for locale in get_languages_list(True):
loc_f = replace_lang(f, locale)
minified_localized[locale].append(
self.minimize_file_to_cache(loc_f))
# loop over locales and attach localized content to the non localized content
if not compiled_files_available or force_generation:
with open(non_localized_filename, "r") as fh:
not_localized_content = fh.read()
for locale in get_languages_list(bool(localized_files)):
filename = replace_lang(input_filename, locale)
lang_specific_output_path = filename.replace(
'_debug_', '_mini_')
tmp_filename = lang_specific_output_path + '.tmp'
if minified_localized[locale]:
localized_filename = self._get_combined_filename(
minified_localized[locale], force_generation)
else:
localized_filename = None
with open(tmp_filename, "w") as fh:
if localized_filename is not None:
with open(localized_filename) as loc_fh:
fh.write(loc_fh.read())
fh.write(not_localized_content)
#raise an error if the file exist, or remove it if rebuilding
if os.path.isfile(lang_specific_output_path):
os.remove(lang_specific_output_path)
if not force_generation:
logger.warn('%r already exists',
lang_specific_output_path)
os.rename(tmp_filename, lang_specific_output_path)
assert os.path.isfile(lang_specific_output_path)
self.cache[output_filename] = True
return output_filename
def _generate_combined_file(self, filename, files):
'''
Generate the combined file
If there is a <lang> param used we support is by generating multiple versions
And returning a combined filename with <lang> in it
filename = the file we are writing to
files = the list of files we are compiling
Generates a stripped version of each file. Expanding language_specific files where needed
Subsequently gets a combined file per locale
Finally it writes a file per locale with this output
'''
name = os.path.splitext(os.path.split(filename)[1])[0]
language_specific = has_lang(files)
#combined output per locale
combined_per_locale = dict()
#store the stripped file per path
stripped_files_dict = dict()
#loop through all files and combine them, expand if there is a <lang>
for file_path in files:
localized_paths = expand_on_lang(file_path)
for localized_path in localized_paths:
read_fh = open(localized_path)
# Add the spaceless version to the output
if SpacelessMiddleware:
data = SpacelessMiddleware.strip_content_safe(
read_fh.read())
else:
data = read_fh.read()
stripped_files_dict[localized_path] = data
read_fh.close()
#generate the combined file for each locale
locales = get_languages_list(language_specific)
for locale in locales:
#get the language_specific versions of the files and combine them
combined_output = ''
for file_path in files:
file_path = replace_lang(file_path, locale)
content = stripped_files_dict[file_path]
combined_output += content
combined_output += '\n'
#postfix some debug info to ensure we can check for the file's validity
if self.extension == 'js':
js = 'var file_%s = true;\n' % name
combined_output += js
js = 'var file_%s = true;\n' % name.replace('debug', 'mini')
combined_output += js
elif self.extension == 'css':
css = '#file_%s{color: #FF00CC;}\n' % name
combined_output += css
css = '#file_%s{color: #FF00CC;}\n' % name.replace(
'debug', 'mini')
combined_output += css
else:
raise TypeError('Extension %r is not supported' %
self.extension)
combined_per_locale[locale] = combined_output
#write the combined version per locale to temporary files and then move them
#to their locale specific file
for locale in locales:
combined_output = combined_per_locale[locale]
postfix = '%s.tmp' % locale
temp_file_path = filename + postfix
path, ext = os.path.splitext(filename)
final_file_path = filename.replace(
ext, '%s%s' % (locale, ext)) if locale else filename
#be atomic!
with portalocker.Lock(temp_file_path, timeout=MAX_WAIT) as fh:
fh.write(combined_output)
if os.path.isfile(final_file_path):
os.remove(final_file_path)
os.rename(filename + postfix, final_file_path)
if language_specific:
filename = append_lang(filename)
return filename
def get_minified_filename(self, force_generation=False):
'''
Returns the minified filename, for language specific files it will return
filename_<lang>.js
'''
input_filename = self.get_combined_filename()
output_filename = input_filename.replace('_debug_', '_mini_')
if output_filename in self.cache and not force_generation:
#if the output is cached, immediatly return it without checking the filesystem
return output_filename
else:
if not settings.DEBUG and not force_generation:
#while running on debug this isn't an error because a dummy cache backend is used
error_message = 'There is no file cache available, but we arent allowed to build the files. Searching for %s in %s' % (output_filename, self.cache)
assert not settings.FROM_CACHE, error_message
#see if all the files we need are actually there
compiled_files_available = True
output_filenames = expand_on_lang(output_filename)
for lang_specific_filename in output_filenames:
if not os.path.isfile(lang_specific_filename):
compiled_files_available = False
# Keep a minified version of each file
# flip the minification of a combination of files in the combination of each minified file
# minify(combine(a,b,c)) = combine(minify(a), minify(a), minify(a))
non_localized_files = [f for f in self.files if expand_on_lang(f) == [f]]
non_localized_minified_filenames = [self.minimize_file_to_cache(f) for f in non_localized_files]
non_localized_filename = self._get_combined_filename(non_localized_minified_filenames, force_generation)
# minify each localized file
minified_localized = defaultdict(list)
localized_files =[f for f in self.files if expand_on_lang(f) != [f]]
for f in localized_files:
for locale in get_languages_list(True):
loc_f = replace_lang(f, locale)
minified_localized[locale].append(self.minimize_file_to_cache(loc_f))
# loop over locales and attach localized content to the non localized content
if not compiled_files_available or force_generation:
with open(non_localized_filename, "r") as fh:
not_localized_content = fh.read()
for locale in get_languages_list(bool(localized_files)):
filename = replace_lang(input_filename, locale)
lang_specific_output_path = filename.replace('_debug_', '_mini_')
tmp_filename = lang_specific_output_path + '.tmp'
if minified_localized[locale]:
localized_filename = self._get_combined_filename(minified_localized[locale], force_generation)
else:
localized_filename = None
with open(tmp_filename, "w") as fh:
if localized_filename is not None:
with open(localized_filename) as loc_fh:
fh.write(loc_fh.read())
fh.write(not_localized_content)
#raise an error if the file exist, or remove it if rebuilding
if os.path.isfile(lang_specific_output_path):
os.remove(lang_specific_output_path)
if not force_generation:
logger.warn('%r already exists', lang_specific_output_path)
os.rename(tmp_filename, lang_specific_output_path)
assert os.path.isfile(lang_specific_output_path)
self.cache[output_filename] = True
return output_filename
def _generate_combined_file(self, filename, files):
'''
Generate the combined file
If there is a <lang> param used we support is by generating multiple versions
And returning a combined filename with <lang> in it
filename = the file we are writing to
files = the list of files we are compiling
Generates a stripped version of each file. Expanding language_specific files where needed
Subsequently gets a combined file per locale
Finally it writes a file per locale with this output
'''
name = os.path.splitext(os.path.split(filename)[1])[0]
language_specific = has_lang(files)
#combined output per locale
combined_per_locale = dict()
#store the stripped file per path
stripped_files_dict = dict()
#loop through all files and combine them, expand if there is a <lang>
for file_path in files:
localized_paths = expand_on_lang(file_path)
for localized_path in localized_paths:
read_fh = open(localized_path)
# Add the spaceless version to the output
if SpacelessMiddleware:
data = SpacelessMiddleware.strip_content_safe(read_fh.read())
else:
data = read_fh.read()
stripped_files_dict[localized_path] = data
read_fh.close()
#generate the combined file for each locale
locales = get_languages_list(language_specific)
for locale in locales:
#get the language_specific versions of the files and combine them
combined_output = ''
for file_path in files:
file_path = replace_lang(file_path, locale)
content = stripped_files_dict[file_path]
combined_output += content
combined_output += '\n'
#postfix some debug info to ensure we can check for the file's validity
if self.extension == 'js':
js = 'var file_%s = true;\n' % name
combined_output += js
js = 'var file_%s = true;\n' % name.replace('debug', 'mini')
combined_output += js
elif self.extension == 'css':
css = '#file_%s{color: #FF00CC;}\n' % name
combined_output += css
css = '#file_%s{color: #FF00CC;}\n' % name.replace('debug', 'mini')
combined_output += css
else:
raise TypeError('Extension %r is not supported'
% self.extension)
combined_per_locale[locale] = combined_output
#write the combined version per locale to temporary files and then move them
#to their locale specific file
for locale in locales:
combined_output = combined_per_locale[locale]
postfix = '%s.tmp' % locale
temp_file_path = filename + postfix
path, ext = os.path.splitext(filename)
final_file_path = filename.replace(ext, '%s%s' % (locale, ext)) if locale else filename
#be atomic!
with portalocker.Lock(temp_file_path, timeout=MAX_WAIT) as fh:
fh.write(combined_output)
if os.path.isfile(final_file_path):
os.remove(final_file_path)
os.rename(filename + postfix, final_file_path)
if language_specific:
filename = append_lang(filename)
return filename