Пример #1
0
 def test_lang_file_minify_with_missing_file(self):
     '''
     Generates, removes one of the files, and subsequently tries again.
     Used for testing failures half way through the script
     '''
     simple, lang = self.get_files()
     lang_minify = minify.MinifyJs(lang)
     combined_filename = lang_minify.get_combined_filename(force_generation=True)
     self.assertLang(combined_filename)
     minified_filename = lang_minify.get_minified_filename(force_generation=True)
     pt_filename = utils.replace_lang(minified_filename, 'pt-br')
     self.assertLang(minified_filename)
     self.assertValidLangPath(minified_filename)
     #remove one of the files and see if things will still work :)
     assert os.path.isfile(pt_filename)
     os.remove(pt_filename)
     
     #try with from cache disabled
     minified_filename = lang_minify.get_minified_filename(force_generation=True)
     pt_filename = utils.replace_lang(minified_filename, 'pt-br')
     #this shouldnt have been regenerated
     #TODO: Maybe it would be desired behaviour to rebuild
     assert not os.path.isfile(pt_filename)
     
     #try with from cache enabled
     OLD_FROM_CACHE, OLD_DEBUG = settings.FROM_CACHE, settings.DEBUG
     settings.FROM_CACHE = True
     settings.DEBUG = False
     try:
         minified_filename = lang_minify.get_minified_filename(force_generation=True)
         pt_filename = utils.replace_lang(minified_filename, 'pt-br')
         #this shouldnt have been regenerated
         assert not os.path.isfile(pt_filename)
     finally:
         settings.FROM_CACHE = OLD_FROM_CACHE
         settings.DEBUG = OLD_DEBUG
Пример #2
0
 def test_lang_file_minify_with_missing_file(self):
     '''
     Generates, removes one of the files, and subsequently tries again.
     Used for testing failures half way through the script
     '''
     simple, lang = self.get_files()
     lang_minify = minify.MinifyJs(lang)
     combined_filename = lang_minify.get_combined_filename(force_generation=True)
     self.assertLang(combined_filename)
     minified_filename = lang_minify.get_minified_filename(force_generation=True)
     pt_filename = utils.replace_lang(minified_filename, 'pt-br')
     self.assertLang(minified_filename)
     self.assertValidLangPath(minified_filename)
     #remove one of the files and see if things will still work :)
     assert os.path.isfile(pt_filename)
     os.remove(pt_filename)
     
     #try with from cache disabled
     minified_filename = lang_minify.get_minified_filename(force_generation=True)
     pt_filename = utils.replace_lang(minified_filename, 'pt-br')
     #this should have been regenerated
     assert os.path.isfile(pt_filename)
     
     #try with from cache enabled
     OLD_FROM_CACHE, OLD_DEBUG = settings.FROM_CACHE, settings.DEBUG
     settings.FROM_CACHE = True
     settings.DEBUG = False
     os.remove(pt_filename)
     try:
         minified_filename = lang_minify.get_minified_filename(force_generation=False)
         pt_filename = utils.replace_lang(minified_filename, 'pt-br')
         #this shouldnt have been regenerated
         assert not os.path.isfile(pt_filename)
     finally:
         settings.FROM_CACHE = OLD_FROM_CACHE
         settings.DEBUG = OLD_DEBUG
Пример #3
0
    def get_minified_filename(self, force_generation=False):
        '''
        Returns the minified filename, for language specific files it will return
        filename_<lang>.js
        '''
        input_filename = self.get_combined_filename()
        output_filename = input_filename.replace('_debug_', '_mini_')
        if output_filename in self.cache and not force_generation:
            #if the output is cached, immediatly return it without checking the filesystem
            return output_filename
        else:
            if not settings.DEBUG and not force_generation:
                #while running on debug this isn't an error because a dummy cache backend is used
                error_message = 'There is no file cache available, but we arent allowed to build the files. Searching for %s in %s' % (
                    output_filename, self.cache)
                assert not settings.FROM_CACHE, error_message
            #see if all the files we need are actually there
            compiled_files_available = True
            output_filenames = expand_on_lang(output_filename)
            for lang_specific_filename in output_filenames:
                if not os.path.isfile(lang_specific_filename):
                    compiled_files_available = False

            # Keep a minified version of each file
            # flip the minification of a combination of files in the combination of each minified file
            # minify(combine(a,b,c)) = combine(minify(a), minify(a), minify(a))
            non_localized_files = [
                f for f in self.files if expand_on_lang(f) == [f]
            ]
            non_localized_minified_filenames = [
                self.minimize_file_to_cache(f) for f in non_localized_files
            ]
            non_localized_filename = self._get_combined_filename(
                non_localized_minified_filenames, force_generation)

            # minify each localized file
            minified_localized = defaultdict(list)
            localized_files = [
                f for f in self.files if expand_on_lang(f) != [f]
            ]
            for f in localized_files:
                for locale in get_languages_list(True):
                    loc_f = replace_lang(f, locale)
                    minified_localized[locale].append(
                        self.minimize_file_to_cache(loc_f))

            # loop over locales and attach localized content to the non localized content
            if not compiled_files_available or force_generation:
                with open(non_localized_filename, "r") as fh:
                    not_localized_content = fh.read()
                for locale in get_languages_list(bool(localized_files)):
                    filename = replace_lang(input_filename, locale)
                    lang_specific_output_path = filename.replace(
                        '_debug_', '_mini_')
                    tmp_filename = lang_specific_output_path + '.tmp'

                    if minified_localized[locale]:
                        localized_filename = self._get_combined_filename(
                            minified_localized[locale], force_generation)
                    else:
                        localized_filename = None

                    with open(tmp_filename, "w") as fh:
                        if localized_filename is not None:
                            with open(localized_filename) as loc_fh:
                                fh.write(loc_fh.read())
                        fh.write(not_localized_content)

                    #raise an error if the file exist, or remove it if rebuilding
                    if os.path.isfile(lang_specific_output_path):
                        os.remove(lang_specific_output_path)
                        if not force_generation:
                            logger.warn('%r already exists',
                                        lang_specific_output_path)
                    os.rename(tmp_filename, lang_specific_output_path)
                    assert os.path.isfile(lang_specific_output_path)

            self.cache[output_filename] = True

        return output_filename
Пример #4
0
    def _generate_combined_file(self, filename, files):
        '''
        Generate the combined file
        If there is a <lang> param used we support is by generating multiple versions
        And returning a combined filename with <lang> in it
        
        filename = the file we are writing to
        files = the list of files we are compiling
        
        Generates a stripped version of each file. Expanding language_specific files where needed
        Subsequently gets a combined file per locale
        Finally it writes a file per locale with this output 
        '''
        name = os.path.splitext(os.path.split(filename)[1])[0]
        language_specific = has_lang(files)

        #combined output per locale
        combined_per_locale = dict()
        #store the stripped file per path
        stripped_files_dict = dict()

        #loop through all files and combine them, expand if there is a <lang>
        for file_path in files:
            localized_paths = expand_on_lang(file_path)
            for localized_path in localized_paths:
                read_fh = open(localized_path)
                # Add the spaceless version to the output
                if SpacelessMiddleware:
                    data = SpacelessMiddleware.strip_content_safe(
                        read_fh.read())
                else:
                    data = read_fh.read()
                stripped_files_dict[localized_path] = data
                read_fh.close()

        #generate the combined file for each locale
        locales = get_languages_list(language_specific)
        for locale in locales:
            #get the language_specific versions of the files and combine them
            combined_output = ''
            for file_path in files:
                file_path = replace_lang(file_path, locale)
                content = stripped_files_dict[file_path]
                combined_output += content
                combined_output += '\n'

            #postfix some debug info to ensure we can check for the file's validity
            if self.extension == 'js':
                js = 'var file_%s = true;\n' % name
                combined_output += js
                js = 'var file_%s = true;\n' % name.replace('debug', 'mini')
                combined_output += js
            elif self.extension == 'css':
                css = '#file_%s{color: #FF00CC;}\n' % name
                combined_output += css
                css = '#file_%s{color: #FF00CC;}\n' % name.replace(
                    'debug', 'mini')
                combined_output += css
            else:
                raise TypeError('Extension %r is not supported' %
                                self.extension)

            combined_per_locale[locale] = combined_output

        #write the combined version per locale to temporary files and then move them
        #to their locale specific file
        for locale in locales:
            combined_output = combined_per_locale[locale]
            postfix = '%s.tmp' % locale
            temp_file_path = filename + postfix

            path, ext = os.path.splitext(filename)
            final_file_path = filename.replace(
                ext, '%s%s' % (locale, ext)) if locale else filename

            #be atomic!
            with portalocker.Lock(temp_file_path, timeout=MAX_WAIT) as fh:
                fh.write(combined_output)

            if os.path.isfile(final_file_path):
                os.remove(final_file_path)
            os.rename(filename + postfix, final_file_path)

        if language_specific:
            filename = append_lang(filename)

        return filename
Пример #5
0
    def get_minified_filename(self, force_generation=False):
        '''
        Returns the minified filename, for language specific files it will return
        filename_<lang>.js
        '''
        input_filename = self.get_combined_filename()
        output_filename = input_filename.replace('_debug_', '_mini_')
        if output_filename in self.cache and not force_generation:
            #if the output is cached, immediatly return it without checking the filesystem
            return output_filename
        else:
            if not settings.DEBUG and not force_generation:
                #while running on debug this isn't an error because a dummy cache backend is used
                error_message = 'There is no file cache available, but we arent allowed to build the files. Searching for %s in %s' % (output_filename, self.cache)
                assert not settings.FROM_CACHE, error_message
            #see if all the files we need are actually there
            compiled_files_available = True
            output_filenames = expand_on_lang(output_filename)
            for lang_specific_filename in output_filenames:
                if not os.path.isfile(lang_specific_filename):
                    compiled_files_available = False

            # Keep a minified version of each file
            # flip the minification of a combination of files in the combination of each minified file
            # minify(combine(a,b,c)) = combine(minify(a), minify(a), minify(a))
            non_localized_files = [f for f in self.files if expand_on_lang(f) == [f]]
            non_localized_minified_filenames = [self.minimize_file_to_cache(f) for f in non_localized_files]
            non_localized_filename = self._get_combined_filename(non_localized_minified_filenames, force_generation)

            # minify each localized file
            minified_localized = defaultdict(list)
            localized_files =[f for f in self.files if expand_on_lang(f) != [f]]
            for f in localized_files:
                for locale in get_languages_list(True):
                    loc_f = replace_lang(f, locale)
                    minified_localized[locale].append(self.minimize_file_to_cache(loc_f))

            # loop over locales and attach localized content to the non localized content
            if not compiled_files_available or force_generation:
                with open(non_localized_filename, "r") as fh:
                    not_localized_content = fh.read()
                for locale in get_languages_list(bool(localized_files)):
                    filename = replace_lang(input_filename, locale)
                    lang_specific_output_path = filename.replace('_debug_', '_mini_')
                    tmp_filename = lang_specific_output_path + '.tmp'

                    if minified_localized[locale]:
                        localized_filename = self._get_combined_filename(minified_localized[locale], force_generation)
                    else:
                        localized_filename = None

                    with open(tmp_filename, "w") as fh:
                        if localized_filename is not None:
                            with open(localized_filename) as loc_fh:
                                fh.write(loc_fh.read())
                        fh.write(not_localized_content)

                    #raise an error if the file exist, or remove it if rebuilding
                    if os.path.isfile(lang_specific_output_path):
                        os.remove(lang_specific_output_path)
                        if not force_generation:
                            logger.warn('%r already exists', lang_specific_output_path)
                    os.rename(tmp_filename, lang_specific_output_path)
                    assert os.path.isfile(lang_specific_output_path)

            self.cache[output_filename] = True
            
        return output_filename
Пример #6
0
    def _generate_combined_file(self, filename, files):
        '''
        Generate the combined file
        If there is a <lang> param used we support is by generating multiple versions
        And returning a combined filename with <lang> in it
        
        filename = the file we are writing to
        files = the list of files we are compiling
        
        Generates a stripped version of each file. Expanding language_specific files where needed
        Subsequently gets a combined file per locale
        Finally it writes a file per locale with this output 
        '''
        name = os.path.splitext(os.path.split(filename)[1])[0]
        language_specific = has_lang(files)
        
        #combined output per locale
        combined_per_locale = dict()
        #store the stripped file per path
        stripped_files_dict = dict()

        #loop through all files and combine them, expand if there is a <lang>
        for file_path in files:
            localized_paths = expand_on_lang(file_path)
            for localized_path in localized_paths:
                read_fh = open(localized_path)
                # Add the spaceless version to the output
                if SpacelessMiddleware:
                    data = SpacelessMiddleware.strip_content_safe(read_fh.read())
                else:
                    data = read_fh.read()
                stripped_files_dict[localized_path] = data
                read_fh.close()
                

        #generate the combined file for each locale
        locales = get_languages_list(language_specific)
        for locale in locales:
            #get the language_specific versions of the files and combine them
            combined_output = ''
            for file_path in files:
                file_path = replace_lang(file_path, locale)
                content = stripped_files_dict[file_path]
                combined_output += content
                combined_output += '\n'
            
            #postfix some debug info to ensure we can check for the file's validity
            if self.extension == 'js':
                js = 'var file_%s = true;\n' % name
                combined_output += js
                js = 'var file_%s = true;\n' % name.replace('debug', 'mini')
                combined_output += js
            elif self.extension == 'css':
                css = '#file_%s{color: #FF00CC;}\n' % name
                combined_output += css
                css = '#file_%s{color: #FF00CC;}\n' % name.replace('debug', 'mini')
                combined_output += css
            else:
                raise TypeError('Extension %r is not supported'
                    % self.extension)
                
            combined_per_locale[locale] = combined_output
            
        #write the combined version per locale to temporary files and then move them
        #to their locale specific file
        for locale in locales:
            combined_output = combined_per_locale[locale]
            postfix = '%s.tmp' % locale
            temp_file_path = filename + postfix
            
            path, ext = os.path.splitext(filename)
            final_file_path = filename.replace(ext, '%s%s' % (locale, ext)) if locale else filename
            
            #be atomic!
            with portalocker.Lock(temp_file_path, timeout=MAX_WAIT) as fh:
                fh.write(combined_output)
    
            
            if os.path.isfile(final_file_path):
                os.remove(final_file_path)
            os.rename(filename + postfix, final_file_path)
        
        if language_specific:
            filename = append_lang(filename)

            
        return filename