def GetDataPackPair(self, lang, encoding): """Returns a (id, string) pair that represents the resource id and raw bytes of the data. This is used to generate the data pack data file. """ # TODO(benrg/joi): Move this and other implementations of GetDataPackPair # to grit.format.data_pack? from grit.format import rc_header id_map = rc_header.GetIds(self.GetRoot()) id = id_map[self.GetTextualIds()[0]] filename = self.ToRealPath(self.GetInputPath()) if self.attrs['flattenhtml'] == 'true': allow_external_script = self.attrs['allowexternalscript'] == 'true' data = self._GetFlattenedData( allow_external_script=allow_external_script) else: data = util.ReadFile(filename, util.BINARY) # Note that the minifier will only do anything if a minifier command # has been set in the command line. data = minifier.Minify(data, filename) use_gzip = self.attrs.get('compress', '') == 'gzip' if use_gzip and self.GetRoot().target_platform != 'ios': # We only use rsyncable compression on Linux. # We exclude ChromeOS since ChromeOS bots are Linux based but do not have # the --rsyncable option built in for gzip. See crbug.com/617950. if sys.platform == 'linux2' and 'chromeos' not in self.GetRoot( ).defines: data = grit.format.gzip_string.GzipStringRsyncable(data) else: data = grit.format.gzip_string.GzipString(data) # Include does not care about the encoding, because it only returns binary # data. return id, data
def GetDataPackValue(self, lang, encoding): '''Returns a str represenation for a data_pack entry.''' filename = self.ToRealPath(self.GetInputPath()) if self.attrs['flattenhtml'] == 'true': allow_external_script = self.attrs['allowexternalscript'] == 'true' data = self._GetFlattenedData( allow_external_script=allow_external_script) else: data = util.ReadFile(filename, util.BINARY) # Note that the minifier will only do anything if a minifier command # has been set in the command line. data = minifier.Minify(data, filename) # Include does not care about the encoding, because it only returns binary # data. return self.CompressDataIfNeeded(data)
def GetDataPackPair(self, lang, encoding): """Returns a (id, string) pair that represents the resource id and raw bytes of the data. This is used to generate the data pack data file. """ # TODO(benrg/joi): Move this and other implementations of GetDataPackPair # to grit.format.data_pack? from grit.format import rc_header id_map = rc_header.GetIds(self.GetRoot()) id = id_map[self.GetTextualIds()[0]] filename = self.ToRealPath(self.GetInputPath()) if self.attrs['flattenhtml'] == 'true': allow_external_script = self.attrs['allowexternalscript'] == 'true' data = self._GetFlattenedData(allow_external_script=allow_external_script) else: data = util.ReadFile(filename, util.BINARY) # Note that the minifier will only do anything if a minifier command # has been set in the command line. data = minifier.Minify(data, os.path.splitext(filename)[1]) if 'compress' in self.attrs and self.attrs['compress'] == 'gzip': # We only use rsyncable compression on Linux. # We exclude ChromeOS since ChromeOS bots are Linux based but do not have # the --rsyncable option built in for gzip. See crbug.com/617950. if sys.platform == 'linux2' and 'chromeos' not in self.GetRoot().defines: data = grit.format.gzip_string.GzipStringRsyncable(data) else: data = grit.format.gzip_string.GzipString(data) data = self.RESERVED_HEADER[0] + data elif data[:3] == self.RESERVED_HEADER: # We are reserving these 3 bytes as the header for gzipped files in the # data pack. 1f:8b is the first two bytes of a gzipped header, and ff is # a custom byte we throw in front of the gzip header so that we prevent # accidentally throwing this error on a resource we gzipped beforehand and # don't wish to compress again. If this exception is hit, change the first # byte of RESERVED_HEADER, and then mirror that update in # ui/base/resource/resource_bundle.h raise exception.ReservedHeaderCollision() # Include does not care about the encoding, because it only returns binary # data. return id, data
def GetDataPackPair(self, lang, encoding): """Returns a (id, string) pair that represents the resource id and raw bytes of the data. This is used to generate the data pack data file. """ # TODO(benrg/joi): Move this and other implementations of GetDataPackPair # to grit.format.data_pack? from grit.format import rc_header id_map = rc_header.GetIds(self.GetRoot()) id = id_map[self.GetTextualIds()[0]] filename = self.ToRealPath(self.GetInputPath()) if self.attrs['flattenhtml'] == 'true': allow_external_script = self.attrs['allowexternalscript'] == 'true' data = self._GetFlattenedData( allow_external_script=allow_external_script) else: data = util.ReadFile(filename, util.BINARY) # Note that the minifier will only do anything if a minifier command # has been set in the command line. data = minifier.Minify(data, filename) # Include does not care about the encoding, because it only returns binary # data. return id, self.CompressDataIfNeeded(data)
def DoInline(input_filename, grd_node, allow_external_script=False, preprocess_only=False, names_only=False, strip_whitespace=False, rewrite_function=None, filename_expansion_function=None): """Helper function that inlines the resources in a specified file. Reads input_filename, finds all the src attributes and attempts to inline the files they are referring to, then returns the result and the set of inlined files. Args: input_filename: name of file to read in grd_node: html node from the grd file for this include tag preprocess_only: Skip all HTML processing, only handle <if> and <include>. names_only: |nil| will be returned for the inlined contents (faster). strip_whitespace: remove whitespace and comments in the input files. rewrite_function: function(filepath, text, distribution) which will be called to rewrite html content before inlining images. filename_expansion_function: function(filename) which will be called to rewrite filenames before attempting to read them. Returns: a tuple of the inlined data as a string and the set of filenames of all the inlined files """ if filename_expansion_function: input_filename = filename_expansion_function(input_filename) input_filepath = os.path.dirname(input_filename) distribution = GetDistribution() # Keep track of all the files we inline. inlined_files = set() def SrcReplace(src_match, filepath=input_filepath, inlined_files=inlined_files): """Helper function to provide SrcInlineAsDataURL with the base file path""" return SrcInlineAsDataURL( src_match, filepath, distribution, inlined_files, names_only=names_only, filename_expansion_function=filename_expansion_function) def SrcsetReplace(srcset_match, filepath=input_filepath, inlined_files=inlined_files): """Helper function to provide SrcsetInlineAsDataURL with the base file path. """ return SrcsetInlineAsDataURL( srcset_match, filepath, distribution, inlined_files, names_only=names_only, filename_expansion_function=filename_expansion_function) def GetFilepath(src_match, base_path=input_filepath): matches = src_match.groupdict().iteritems() filename = [v for k, v in matches if k.startswith('file') and v][0] if filename.find(':') != -1: # filename is probably a URL, which we don't want to bother inlining return None filename = filename.replace('%DISTRIBUTION%', distribution) if filename_expansion_function: filename = filename_expansion_function(filename) return os.path.normpath(os.path.join(base_path, filename)) def IsConditionSatisfied(src_match): expr1 = src_match.group('expr1') or '' expr2 = src_match.group('expr2') or '' return grd_node is None or grd_node.EvaluateCondition(expr1 + expr2) def CheckConditionalElements(str): """Helper function to conditionally inline inner elements""" while True: begin_if = _BEGIN_IF_BLOCK.search(str) if begin_if is None: if _END_IF_BLOCK.search(str) is not None: raise Exception('Unmatched </if>') return str condition_satisfied = IsConditionSatisfied(begin_if) leading = str[0:begin_if.start()] content_start = begin_if.end() # Find matching "if" block end. count = 1 pos = begin_if.end() while True: end_if = _END_IF_BLOCK.search(str, pos) if end_if is None: raise Exception('Unmatched <if>') next_if = _BEGIN_IF_BLOCK.search(str, pos) if next_if is None or next_if.start() >= end_if.end(): count = count - 1 if count == 0: break pos = end_if.end() else: count = count + 1 pos = next_if.end() content = str[content_start:end_if.start()] trailing = str[end_if.end():] if condition_satisfied: str = leading + CheckConditionalElements(content) + trailing else: str = leading + trailing def InlineFileContents(src_match, pattern, inlined_files=inlined_files, strip_whitespace=False): """Helper function to inline external files of various types""" filepath = GetFilepath(src_match) if filepath is None: return src_match.group(0) inlined_files.add(filepath) if names_only: inlined_files.update( GetResourceFilenames( filepath, grd_node, allow_external_script, rewrite_function, filename_expansion_function=filename_expansion_function)) return "" # To recursively save inlined files, we need InlinedData instance returned # by DoInline. inlined_data_inst = DoInline( filepath, grd_node, allow_external_script=allow_external_script, strip_whitespace=strip_whitespace, filename_expansion_function=filename_expansion_function) inlined_files.update(inlined_data_inst.inlined_files) return pattern % inlined_data_inst.inlined_data def InlineIncludeFiles(src_match): """Helper function to directly inline generic external files (without wrapping them with any kind of tags). """ return InlineFileContents(src_match, '%s') def InlineScript(match): """Helper function to inline external script files""" attrs = (match.group('attrs1') + match.group('attrs2')).strip() if attrs: attrs = ' ' + attrs return InlineFileContents(match, '<script' + attrs + '>%s</script>', strip_whitespace=True) def InlineCSSText(text, css_filepath): """Helper function that inlines external resources in CSS text""" filepath = os.path.dirname(css_filepath) # Allow custom modifications before inlining images. if rewrite_function: text = rewrite_function(filepath, text, distribution) text = InlineCSSImages(text, filepath) return InlineCSSImports(text, filepath) def InlineCSSFile(src_match, pattern, base_path=input_filepath): """Helper function to inline external CSS files. Args: src_match: A regular expression match with a named group named "filename". pattern: The pattern to replace with the contents of the CSS file. base_path: The base path to use for resolving the CSS file. Returns: The text that should replace the reference to the CSS file. """ filepath = GetFilepath(src_match, base_path) if filepath is None: return src_match.group(0) # Even if names_only is set, the CSS file needs to be opened, because it # can link to images that need to be added to the file set. inlined_files.add(filepath) # Inline stylesheets included in this css file. text = _INCLUDE_RE.sub(InlineIncludeFiles, util.ReadFile(filepath, util.BINARY)) # When resolving CSS files we need to pass in the path so that relative URLs # can be resolved. return pattern % InlineCSSText(text, filepath) def InlineCSSImages(text, filepath=input_filepath): """Helper function that inlines external images in CSS backgrounds.""" # Replace contents of url() for css attributes: content, background, # or *-image. property_re = '(content|background|[\w-]*-image):[^;]*' url_value_re = 'url\((?!\[\[|{{)(?P<quote1>"|\'|)[^"\'()]*(?P=quote1)\)' image_set_value_re = 'image-set\(([ ]*url\((?!\[\[|{{)' + \ '(?P<quote2>"|\'|)[^"\'()]*(?P=quote2)\)' + \ '[ ]*[0-9.]*x[ ]*(,[ ]*)?)+\)' value_re = '(%s|%s)' % (url_value_re, image_set_value_re) css_re = property_re + value_re return re.sub(css_re, lambda m: InlineCSSUrls(m, filepath), text) def InlineCSSUrls(src_match, filepath=input_filepath): """Helper function that inlines each url on a CSS image rule match.""" # Replace contents of url() references in matches. return re.sub( 'url\((?P<quote>"|\'|)(?P<filename>[^"\'()]*)(?P=quote)\)', lambda m: SrcReplace(m, filepath), src_match.group(0)) def InlineCSSImports(text, filepath=input_filepath): """Helper function that inlines CSS files included via the @import directive. """ return re.sub( '@import\s+url\((?P<quote>"|\'|)(?P<filename>[^"\'()]*)' + '(?P=quote)\);', lambda m: InlineCSSFile(m, '%s', filepath), text) flat_text = util.ReadFile(input_filename, util.BINARY) # Check conditional elements, remove unsatisfied ones from the file. We do # this twice. The first pass is so that we don't even bother calling # InlineScript, InlineCSSFile and InlineIncludeFiles on text we're eventually # going to throw out anyway. flat_text = CheckConditionalElements(flat_text) flat_text = _INCLUDE_RE.sub(InlineIncludeFiles, flat_text) if not preprocess_only: if strip_whitespace: flat_text = minifier.Minify(flat_text, input_filename) if not allow_external_script: # We need to inline css and js before we inline images so that image # references gets inlined in the css and js flat_text = re.sub( '<script (?P<attrs1>.*?)src="(?P<filename>[^"\']*)"' + '(?P<attrs2>.*?)></script>', InlineScript, flat_text) flat_text = _STYLESHEET_RE.sub( lambda m: InlineCSSFile(m, '<style>%s</style>'), flat_text) # Check conditional elements, second pass. This catches conditionals in any # of the text we just inlined. flat_text = CheckConditionalElements(flat_text) if not preprocess_only: # Allow custom modifications before inlining images. if rewrite_function: flat_text = rewrite_function(input_filepath, flat_text, distribution) flat_text = _SRC_RE.sub(SrcReplace, flat_text) flat_text = _SRCSET_RE.sub(SrcsetReplace, flat_text) # TODO(arv): Only do this inside <style> tags. flat_text = InlineCSSImages(flat_text) flat_text = _ICON_RE.sub(SrcReplace, flat_text) if names_only: flat_text = None # Will contains garbage if the flag is set anyway. return InlinedData(flat_text, inlined_files)
def DoInline(input_filename, grd_node, allow_external_script=False, preprocess_only=False, names_only=False, strip_whitespace=False, rewrite_function=None, filename_expansion_function=None): """Helper function that inlines the resources in a specified file. Reads input_filename, finds all the src attributes and attempts to inline the files they are referring to, then returns the result and the set of inlined files. Args: input_filename: name of file to read in grd_node: html node from the grd file for this include tag preprocess_only: Skip all HTML processing, only handle <if> and <include>. names_only: |nil| will be returned for the inlined contents (faster). strip_whitespace: remove whitespace and comments in the input files. rewrite_function: function(filepath, text, distribution) which will be called to rewrite html content before inlining images. filename_expansion_function: function(filename) which will be called to rewrite filenames before attempting to read them. Returns: a tuple of the inlined data as a string and the set of filenames of all the inlined files """ if filename_expansion_function: input_filename = filename_expansion_function(input_filename) input_filepath = os.path.dirname(input_filename) distribution = GetDistribution() # Keep track of all the files we inline. inlined_files = set() def SrcReplace(src_match, filepath=input_filepath, inlined_files=inlined_files): """Helper function to provide SrcInlineAsDataURL with the base file path""" return SrcInlineAsDataURL( src_match, filepath, distribution, inlined_files, names_only=names_only, filename_expansion_function=filename_expansion_function) def SrcsetReplace(srcset_match, filepath=input_filepath, inlined_files=inlined_files): """Helper function to provide SrcsetInlineAsDataURL with the base file path. """ return SrcsetInlineAsDataURL( srcset_match, filepath, distribution, inlined_files, names_only=names_only, filename_expansion_function=filename_expansion_function) def GetFilepath(src_match, base_path=input_filepath): filename = [ v for k, v in src_match.groupdict().items() if k.startswith('file') and v ][0] if filename.find(':') != -1: # filename is probably a URL, which we don't want to bother inlining return None filename = filename.replace('%DISTRIBUTION%', distribution) if filename_expansion_function: filename = filename_expansion_function(filename) return os.path.normpath(os.path.join(base_path, filename)) def InlineFileContents(src_match, pattern, inlined_files=inlined_files, strip_whitespace=False): """Helper function to inline external files of various types""" filepath = GetFilepath(src_match) if filepath is None: return src_match.group(0) inlined_files.add(filepath) if names_only: inlined_files.update( GetResourceFilenames( filepath, grd_node, allow_external_script, rewrite_function, filename_expansion_function=filename_expansion_function)) return "" # To recursively save inlined files, we need InlinedData instance returned # by DoInline. inlined_data_inst = DoInline( filepath, grd_node, allow_external_script=allow_external_script, preprocess_only=preprocess_only, strip_whitespace=strip_whitespace, filename_expansion_function=filename_expansion_function) inlined_files.update(inlined_data_inst.inlined_files) return pattern % inlined_data_inst.inlined_data def InlineIncludeFiles(src_match): """Helper function to directly inline generic external files (without wrapping them with any kind of tags). """ return InlineFileContents(src_match, '%s') def InlineScript(match): """Helper function to inline external script files""" attrs = (match.group('attrs1') + match.group('attrs2')).strip() if attrs: attrs = ' ' + attrs return InlineFileContents(match, '<script' + attrs + '>%s</script>', strip_whitespace=True) def InlineCSSText(text, css_filepath): """Helper function that inlines external resources in CSS text""" filepath = os.path.dirname(css_filepath) # Allow custom modifications before inlining images. if rewrite_function: text = rewrite_function(filepath, text, distribution) text = InlineCSSImages(text, filepath) return InlineCSSImports(text, filepath) def InlineCSSFile(src_match, pattern, base_path=input_filepath): """Helper function to inline external CSS files. Args: src_match: A regular expression match with a named group named "filename". pattern: The pattern to replace with the contents of the CSS file. base_path: The base path to use for resolving the CSS file. Returns: The text that should replace the reference to the CSS file. """ filepath = GetFilepath(src_match, base_path) if filepath is None: return src_match.group(0) # Even if names_only is set, the CSS file needs to be opened, because it # can link to images that need to be added to the file set. inlined_files.add(filepath) # Inline stylesheets included in this css file. text = _INCLUDE_RE.sub(InlineIncludeFiles, util.ReadFile(filepath, 'utf-8')) # When resolving CSS files we need to pass in the path so that relative URLs # can be resolved. return pattern % InlineCSSText(text, filepath) def GetUrlRegexString(postfix=''): """Helper function that returns a string for a regex that matches url('') but not url([[ ]]) or url({{ }}). Appends |postfix| to group names. """ url_re = (r'url\((?!\[\[|{{)(?P<q%s>"|\'|)(?P<filename%s>[^"\'()]*)' r'(?P=q%s)\)') return url_re % (postfix, postfix, postfix) def InlineCSSImages(text, filepath=input_filepath): """Helper function that inlines external images in CSS backgrounds.""" # Replace contents of url() for css attributes: content, background, # or *-image. property_re = r'(content|background|[\w-]*-image):[^;]*' # Replace group names to prevent duplicates when forming value_re. image_set_value_re = (r'image-set\(([ ]*' + GetUrlRegexString('2') + r'[ ]*[0-9.]*x[ ]*(,[ ]*)?)+\)') value_re = '(%s|%s)' % (GetUrlRegexString(), image_set_value_re) css_re = property_re + value_re return re.sub(css_re, lambda m: InlineCSSUrls(m, filepath), text) def InlineCSSUrls(src_match, filepath=input_filepath): """Helper function that inlines each url on a CSS image rule match.""" # Replace contents of url() references in matches. return re.sub(GetUrlRegexString(), lambda m: SrcReplace(m, filepath), src_match.group(0)) def InlineCSSImports(text, filepath=input_filepath): """Helper function that inlines CSS files included via the @import directive. """ return re.sub(r'@import\s+' + GetUrlRegexString() + r';', lambda m: InlineCSSFile(m, '%s', filepath), text) flat_text = util.ReadFile(input_filename, 'utf-8') # Check conditional elements, remove unsatisfied ones from the file. We do # this twice. The first pass is so that we don't even bother calling # InlineScript, InlineCSSFile and InlineIncludeFiles on text we're eventually # going to throw out anyway. flat_text = CheckConditionalElements(grd_node, flat_text) flat_text = _INCLUDE_RE.sub(InlineIncludeFiles, flat_text) if not preprocess_only: if strip_whitespace: flat_text = minifier.Minify(flat_text.encode('utf-8'), input_filename).decode('utf-8') if not allow_external_script: # We need to inline css and js before we inline images so that image # references gets inlined in the css and js flat_text = re.sub( r'<script (?P<attrs1>.*?)src="(?P<filename>[^"\']*)"' r'(?P<attrs2>.*?)></script>', InlineScript, flat_text) flat_text = _STYLESHEET_RE.sub( lambda m: InlineCSSFile(m, '<style>%s</style>'), flat_text) # Check conditional elements, second pass. This catches conditionals in any # of the text we just inlined. flat_text = CheckConditionalElements(grd_node, flat_text) # Allow custom modifications before inlining images. if rewrite_function: flat_text = rewrite_function(input_filepath, flat_text, distribution) if not preprocess_only: flat_text = _SRC_RE.sub(SrcReplace, flat_text) flat_text = _SRCSET_RE.sub(SrcsetReplace, flat_text) # TODO(arv): Only do this inside <style> tags. flat_text = InlineCSSImages(flat_text) flat_text = _ICON_RE.sub(SrcReplace, flat_text) if names_only: flat_text = None # Will contains garbage if the flag is set anyway. return InlinedData(flat_text, inlined_files)