def GetDataPackPair(self, lang, encoding):
"""Returns a (id, string) pair that represents the resource id and raw
bytes of the data. This is used to generate the data pack data file.
"""
# TODO(benrg/joi): Move this and other implementations of GetDataPackPair
# to grit.format.data_pack?
from grit.format import rc_header
id_map = rc_header.GetIds(self.GetRoot())
id = id_map[self.GetTextualIds()[0]]
filename = self.ToRealPath(self.GetInputPath())
if self.attrs['flattenhtml'] == 'true':
allow_external_script = self.attrs['allowexternalscript'] == 'true'
data = self._GetFlattenedData(
allow_external_script=allow_external_script)
else:
data = util.ReadFile(filename, util.BINARY)
# Note that the minifier will only do anything if a minifier command
# has been set in the command line.
data = minifier.Minify(data, filename)
use_gzip = self.attrs.get('compress', '') == 'gzip'
if use_gzip and self.GetRoot().target_platform != 'ios':
# We only use rsyncable compression on Linux.
# We exclude ChromeOS since ChromeOS bots are Linux based but do not have
# the --rsyncable option built in for gzip. See crbug.com/617950.
if sys.platform == 'linux2' and 'chromeos' not in self.GetRoot(
).defines:
data = grit.format.gzip_string.GzipStringRsyncable(data)
else:
data = grit.format.gzip_string.GzipString(data)
# Include does not care about the encoding, because it only returns binary
# data.
return id, data
def GetDataPackValue(self, lang, encoding):
'''Returns a str represenation for a data_pack entry.'''
filename = self.ToRealPath(self.GetInputPath())
if self.attrs['flattenhtml'] == 'true':
allow_external_script = self.attrs['allowexternalscript'] == 'true'
data = self._GetFlattenedData(
allow_external_script=allow_external_script)
else:
data = util.ReadFile(filename, util.BINARY)
# Note that the minifier will only do anything if a minifier command
# has been set in the command line.
data = minifier.Minify(data, filename)
# Include does not care about the encoding, because it only returns binary
# data.
return self.CompressDataIfNeeded(data)
def GetDataPackPair(self, lang, encoding):
"""Returns a (id, string) pair that represents the resource id and raw
bytes of the data. This is used to generate the data pack data file.
"""
# TODO(benrg/joi): Move this and other implementations of GetDataPackPair
# to grit.format.data_pack?
from grit.format import rc_header
id_map = rc_header.GetIds(self.GetRoot())
id = id_map[self.GetTextualIds()[0]]
filename = self.ToRealPath(self.GetInputPath())
if self.attrs['flattenhtml'] == 'true':
allow_external_script = self.attrs['allowexternalscript'] == 'true'
data = self._GetFlattenedData(allow_external_script=allow_external_script)
else:
data = util.ReadFile(filename, util.BINARY)
# Note that the minifier will only do anything if a minifier command
# has been set in the command line.
data = minifier.Minify(data, os.path.splitext(filename)[1])
if 'compress' in self.attrs and self.attrs['compress'] == 'gzip':
# We only use rsyncable compression on Linux.
# We exclude ChromeOS since ChromeOS bots are Linux based but do not have
# the --rsyncable option built in for gzip. See crbug.com/617950.
if sys.platform == 'linux2' and 'chromeos' not in self.GetRoot().defines:
data = grit.format.gzip_string.GzipStringRsyncable(data)
else:
data = grit.format.gzip_string.GzipString(data)
data = self.RESERVED_HEADER[0] + data
elif data[:3] == self.RESERVED_HEADER:
# We are reserving these 3 bytes as the header for gzipped files in the
# data pack. 1f:8b is the first two bytes of a gzipped header, and ff is
# a custom byte we throw in front of the gzip header so that we prevent
# accidentally throwing this error on a resource we gzipped beforehand and
# don't wish to compress again. If this exception is hit, change the first
# byte of RESERVED_HEADER, and then mirror that update in
# ui/base/resource/resource_bundle.h
raise exception.ReservedHeaderCollision()
# Include does not care about the encoding, because it only returns binary
# data.
return id, data
def GetDataPackPair(self, lang, encoding):
"""Returns a (id, string) pair that represents the resource id and raw
bytes of the data. This is used to generate the data pack data file.
"""
# TODO(benrg/joi): Move this and other implementations of GetDataPackPair
# to grit.format.data_pack?
from grit.format import rc_header
id_map = rc_header.GetIds(self.GetRoot())
id = id_map[self.GetTextualIds()[0]]
filename = self.ToRealPath(self.GetInputPath())
if self.attrs['flattenhtml'] == 'true':
allow_external_script = self.attrs['allowexternalscript'] == 'true'
data = self._GetFlattenedData(
allow_external_script=allow_external_script)
else:
data = util.ReadFile(filename, util.BINARY)
# Note that the minifier will only do anything if a minifier command
# has been set in the command line.
data = minifier.Minify(data, filename)
# Include does not care about the encoding, because it only returns binary
# data.
return id, self.CompressDataIfNeeded(data)
def DoInline(input_filename,
grd_node,
allow_external_script=False,
preprocess_only=False,
names_only=False,
strip_whitespace=False,
rewrite_function=None,
filename_expansion_function=None):
"""Helper function that inlines the resources in a specified file.
Reads input_filename, finds all the src attributes and attempts to
inline the files they are referring to, then returns the result and
the set of inlined files.
Args:
input_filename: name of file to read in
grd_node: html node from the grd file for this include tag
preprocess_only: Skip all HTML processing, only handle <if> and <include>.
names_only: |nil| will be returned for the inlined contents (faster).
strip_whitespace: remove whitespace and comments in the input files.
rewrite_function: function(filepath, text, distribution) which will be
called to rewrite html content before inlining images.
filename_expansion_function: function(filename) which will be called to
rewrite filenames before attempting to read them.
Returns:
a tuple of the inlined data as a string and the set of filenames
of all the inlined files
"""
if filename_expansion_function:
input_filename = filename_expansion_function(input_filename)
input_filepath = os.path.dirname(input_filename)
distribution = GetDistribution()
# Keep track of all the files we inline.
inlined_files = set()
def SrcReplace(src_match,
filepath=input_filepath,
inlined_files=inlined_files):
"""Helper function to provide SrcInlineAsDataURL with the base file path"""
return SrcInlineAsDataURL(
src_match,
filepath,
distribution,
inlined_files,
names_only=names_only,
filename_expansion_function=filename_expansion_function)
def SrcsetReplace(srcset_match,
filepath=input_filepath,
inlined_files=inlined_files):
"""Helper function to provide SrcsetInlineAsDataURL with the base file
path.
"""
return SrcsetInlineAsDataURL(
srcset_match,
filepath,
distribution,
inlined_files,
names_only=names_only,
filename_expansion_function=filename_expansion_function)
def GetFilepath(src_match, base_path=input_filepath):
matches = src_match.groupdict().iteritems()
filename = [v for k, v in matches if k.startswith('file') and v][0]
if filename.find(':') != -1:
# filename is probably a URL, which we don't want to bother inlining
return None
filename = filename.replace('%DISTRIBUTION%', distribution)
if filename_expansion_function:
filename = filename_expansion_function(filename)
return os.path.normpath(os.path.join(base_path, filename))
def IsConditionSatisfied(src_match):
expr1 = src_match.group('expr1') or ''
expr2 = src_match.group('expr2') or ''
return grd_node is None or grd_node.EvaluateCondition(expr1 + expr2)
def CheckConditionalElements(str):
"""Helper function to conditionally inline inner elements"""
while True:
begin_if = _BEGIN_IF_BLOCK.search(str)
if begin_if is None:
if _END_IF_BLOCK.search(str) is not None:
raise Exception('Unmatched </if>')
return str
condition_satisfied = IsConditionSatisfied(begin_if)
leading = str[0:begin_if.start()]
content_start = begin_if.end()
# Find matching "if" block end.
count = 1
pos = begin_if.end()
while True:
end_if = _END_IF_BLOCK.search(str, pos)
if end_if is None:
raise Exception('Unmatched <if>')
next_if = _BEGIN_IF_BLOCK.search(str, pos)
if next_if is None or next_if.start() >= end_if.end():
count = count - 1
if count == 0:
break
pos = end_if.end()
else:
count = count + 1
pos = next_if.end()
content = str[content_start:end_if.start()]
trailing = str[end_if.end():]
if condition_satisfied:
str = leading + CheckConditionalElements(content) + trailing
else:
str = leading + trailing
def InlineFileContents(src_match,
pattern,
inlined_files=inlined_files,
strip_whitespace=False):
"""Helper function to inline external files of various types"""
filepath = GetFilepath(src_match)
if filepath is None:
return src_match.group(0)
inlined_files.add(filepath)
if names_only:
inlined_files.update(
GetResourceFilenames(
filepath,
grd_node,
allow_external_script,
rewrite_function,
filename_expansion_function=filename_expansion_function))
return ""
# To recursively save inlined files, we need InlinedData instance returned
# by DoInline.
inlined_data_inst = DoInline(
filepath,
grd_node,
allow_external_script=allow_external_script,
strip_whitespace=strip_whitespace,
filename_expansion_function=filename_expansion_function)
inlined_files.update(inlined_data_inst.inlined_files)
return pattern % inlined_data_inst.inlined_data
def InlineIncludeFiles(src_match):
"""Helper function to directly inline generic external files (without
wrapping them with any kind of tags).
"""
return InlineFileContents(src_match, '%s')
def InlineScript(match):
"""Helper function to inline external script files"""
attrs = (match.group('attrs1') + match.group('attrs2')).strip()
if attrs:
attrs = ' ' + attrs
return InlineFileContents(match,
'<script' + attrs + '>%s</script>',
strip_whitespace=True)
def InlineCSSText(text, css_filepath):
"""Helper function that inlines external resources in CSS text"""
filepath = os.path.dirname(css_filepath)
# Allow custom modifications before inlining images.
if rewrite_function:
text = rewrite_function(filepath, text, distribution)
text = InlineCSSImages(text, filepath)
return InlineCSSImports(text, filepath)
def InlineCSSFile(src_match, pattern, base_path=input_filepath):
"""Helper function to inline external CSS files.
Args:
src_match: A regular expression match with a named group named "filename".
pattern: The pattern to replace with the contents of the CSS file.
base_path: The base path to use for resolving the CSS file.
Returns:
The text that should replace the reference to the CSS file.
"""
filepath = GetFilepath(src_match, base_path)
if filepath is None:
return src_match.group(0)
# Even if names_only is set, the CSS file needs to be opened, because it
# can link to images that need to be added to the file set.
inlined_files.add(filepath)
# Inline stylesheets included in this css file.
text = _INCLUDE_RE.sub(InlineIncludeFiles,
util.ReadFile(filepath, util.BINARY))
# When resolving CSS files we need to pass in the path so that relative URLs
# can be resolved.
return pattern % InlineCSSText(text, filepath)
def InlineCSSImages(text, filepath=input_filepath):
"""Helper function that inlines external images in CSS backgrounds."""
# Replace contents of url() for css attributes: content, background,
# or *-image.
property_re = '(content|background|[\w-]*-image):[^;]*'
url_value_re = 'url\((?!\[\[|{{)(?P<quote1>"|\'|)[^"\'()]*(?P=quote1)\)'
image_set_value_re = 'image-set\(([ ]*url\((?!\[\[|{{)' + \
'(?P<quote2>"|\'|)[^"\'()]*(?P=quote2)\)' + \
'[ ]*[0-9.]*x[ ]*(,[ ]*)?)+\)'
value_re = '(%s|%s)' % (url_value_re, image_set_value_re)
css_re = property_re + value_re
return re.sub(css_re, lambda m: InlineCSSUrls(m, filepath), text)
def InlineCSSUrls(src_match, filepath=input_filepath):
"""Helper function that inlines each url on a CSS image rule match."""
# Replace contents of url() references in matches.
return re.sub(
'url\((?P<quote>"|\'|)(?P<filename>[^"\'()]*)(?P=quote)\)',
lambda m: SrcReplace(m, filepath), src_match.group(0))
def InlineCSSImports(text, filepath=input_filepath):
"""Helper function that inlines CSS files included via the @import
directive.
"""
return re.sub(
'@import\s+url\((?P<quote>"|\'|)(?P<filename>[^"\'()]*)' +
'(?P=quote)\);', lambda m: InlineCSSFile(m, '%s', filepath), text)
flat_text = util.ReadFile(input_filename, util.BINARY)
# Check conditional elements, remove unsatisfied ones from the file. We do
# this twice. The first pass is so that we don't even bother calling
# InlineScript, InlineCSSFile and InlineIncludeFiles on text we're eventually
# going to throw out anyway.
flat_text = CheckConditionalElements(flat_text)
flat_text = _INCLUDE_RE.sub(InlineIncludeFiles, flat_text)
if not preprocess_only:
if strip_whitespace:
flat_text = minifier.Minify(flat_text, input_filename)
if not allow_external_script:
# We need to inline css and js before we inline images so that image
# references gets inlined in the css and js
flat_text = re.sub(
'<script (?P<attrs1>.*?)src="(?P<filename>[^"\']*)"' +
'(?P<attrs2>.*?)></script>', InlineScript, flat_text)
flat_text = _STYLESHEET_RE.sub(
lambda m: InlineCSSFile(m, '<style>%s</style>'), flat_text)
# Check conditional elements, second pass. This catches conditionals in any
# of the text we just inlined.
flat_text = CheckConditionalElements(flat_text)
if not preprocess_only:
# Allow custom modifications before inlining images.
if rewrite_function:
flat_text = rewrite_function(input_filepath, flat_text,
distribution)
flat_text = _SRC_RE.sub(SrcReplace, flat_text)
flat_text = _SRCSET_RE.sub(SrcsetReplace, flat_text)
# TODO(arv): Only do this inside <style> tags.
flat_text = InlineCSSImages(flat_text)
flat_text = _ICON_RE.sub(SrcReplace, flat_text)
if names_only:
flat_text = None # Will contains garbage if the flag is set anyway.
return InlinedData(flat_text, inlined_files)
def DoInline(input_filename,
grd_node,
allow_external_script=False,
preprocess_only=False,
names_only=False,
strip_whitespace=False,
rewrite_function=None,
filename_expansion_function=None):
"""Helper function that inlines the resources in a specified file.
Reads input_filename, finds all the src attributes and attempts to
inline the files they are referring to, then returns the result and
the set of inlined files.
Args:
input_filename: name of file to read in
grd_node: html node from the grd file for this include tag
preprocess_only: Skip all HTML processing, only handle <if> and <include>.
names_only: |nil| will be returned for the inlined contents (faster).
strip_whitespace: remove whitespace and comments in the input files.
rewrite_function: function(filepath, text, distribution) which will be
called to rewrite html content before inlining images.
filename_expansion_function: function(filename) which will be called to
rewrite filenames before attempting to read them.
Returns:
a tuple of the inlined data as a string and the set of filenames
of all the inlined files
"""
if filename_expansion_function:
input_filename = filename_expansion_function(input_filename)
input_filepath = os.path.dirname(input_filename)
distribution = GetDistribution()
# Keep track of all the files we inline.
inlined_files = set()
def SrcReplace(src_match,
filepath=input_filepath,
inlined_files=inlined_files):
"""Helper function to provide SrcInlineAsDataURL with the base file path"""
return SrcInlineAsDataURL(
src_match,
filepath,
distribution,
inlined_files,
names_only=names_only,
filename_expansion_function=filename_expansion_function)
def SrcsetReplace(srcset_match,
filepath=input_filepath,
inlined_files=inlined_files):
"""Helper function to provide SrcsetInlineAsDataURL with the base file
path.
"""
return SrcsetInlineAsDataURL(
srcset_match,
filepath,
distribution,
inlined_files,
names_only=names_only,
filename_expansion_function=filename_expansion_function)
def GetFilepath(src_match, base_path=input_filepath):
filename = [
v for k, v in src_match.groupdict().items()
if k.startswith('file') and v
][0]
if filename.find(':') != -1:
# filename is probably a URL, which we don't want to bother inlining
return None
filename = filename.replace('%DISTRIBUTION%', distribution)
if filename_expansion_function:
filename = filename_expansion_function(filename)
return os.path.normpath(os.path.join(base_path, filename))
def InlineFileContents(src_match,
pattern,
inlined_files=inlined_files,
strip_whitespace=False):
"""Helper function to inline external files of various types"""
filepath = GetFilepath(src_match)
if filepath is None:
return src_match.group(0)
inlined_files.add(filepath)
if names_only:
inlined_files.update(
GetResourceFilenames(
filepath,
grd_node,
allow_external_script,
rewrite_function,
filename_expansion_function=filename_expansion_function))
return ""
# To recursively save inlined files, we need InlinedData instance returned
# by DoInline.
inlined_data_inst = DoInline(
filepath,
grd_node,
allow_external_script=allow_external_script,
preprocess_only=preprocess_only,
strip_whitespace=strip_whitespace,
filename_expansion_function=filename_expansion_function)
inlined_files.update(inlined_data_inst.inlined_files)
return pattern % inlined_data_inst.inlined_data
def InlineIncludeFiles(src_match):
"""Helper function to directly inline generic external files (without
wrapping them with any kind of tags).
"""
return InlineFileContents(src_match, '%s')
def InlineScript(match):
"""Helper function to inline external script files"""
attrs = (match.group('attrs1') + match.group('attrs2')).strip()
if attrs:
attrs = ' ' + attrs
return InlineFileContents(match,
'<script' + attrs + '>%s</script>',
strip_whitespace=True)
def InlineCSSText(text, css_filepath):
"""Helper function that inlines external resources in CSS text"""
filepath = os.path.dirname(css_filepath)
# Allow custom modifications before inlining images.
if rewrite_function:
text = rewrite_function(filepath, text, distribution)
text = InlineCSSImages(text, filepath)
return InlineCSSImports(text, filepath)
def InlineCSSFile(src_match, pattern, base_path=input_filepath):
"""Helper function to inline external CSS files.
Args:
src_match: A regular expression match with a named group named "filename".
pattern: The pattern to replace with the contents of the CSS file.
base_path: The base path to use for resolving the CSS file.
Returns:
The text that should replace the reference to the CSS file.
"""
filepath = GetFilepath(src_match, base_path)
if filepath is None:
return src_match.group(0)
# Even if names_only is set, the CSS file needs to be opened, because it
# can link to images that need to be added to the file set.
inlined_files.add(filepath)
# Inline stylesheets included in this css file.
text = _INCLUDE_RE.sub(InlineIncludeFiles,
util.ReadFile(filepath, 'utf-8'))
# When resolving CSS files we need to pass in the path so that relative URLs
# can be resolved.
return pattern % InlineCSSText(text, filepath)
def GetUrlRegexString(postfix=''):
"""Helper function that returns a string for a regex that matches url('')
but not url([[ ]]) or url({{ }}). Appends |postfix| to group names.
"""
url_re = (r'url\((?!\[\[|{{)(?P<q%s>"|\'|)(?P<filename%s>[^"\'()]*)'
r'(?P=q%s)\)')
return url_re % (postfix, postfix, postfix)
def InlineCSSImages(text, filepath=input_filepath):
"""Helper function that inlines external images in CSS backgrounds."""
# Replace contents of url() for css attributes: content, background,
# or *-image.
property_re = r'(content|background|[\w-]*-image):[^;]*'
# Replace group names to prevent duplicates when forming value_re.
image_set_value_re = (r'image-set\(([ ]*' + GetUrlRegexString('2') +
r'[ ]*[0-9.]*x[ ]*(,[ ]*)?)+\)')
value_re = '(%s|%s)' % (GetUrlRegexString(), image_set_value_re)
css_re = property_re + value_re
return re.sub(css_re, lambda m: InlineCSSUrls(m, filepath), text)
def InlineCSSUrls(src_match, filepath=input_filepath):
"""Helper function that inlines each url on a CSS image rule match."""
# Replace contents of url() references in matches.
return re.sub(GetUrlRegexString(), lambda m: SrcReplace(m, filepath),
src_match.group(0))
def InlineCSSImports(text, filepath=input_filepath):
"""Helper function that inlines CSS files included via the @import
directive.
"""
return re.sub(r'@import\s+' + GetUrlRegexString() + r';',
lambda m: InlineCSSFile(m, '%s', filepath), text)
flat_text = util.ReadFile(input_filename, 'utf-8')
# Check conditional elements, remove unsatisfied ones from the file. We do
# this twice. The first pass is so that we don't even bother calling
# InlineScript, InlineCSSFile and InlineIncludeFiles on text we're eventually
# going to throw out anyway.
flat_text = CheckConditionalElements(grd_node, flat_text)
flat_text = _INCLUDE_RE.sub(InlineIncludeFiles, flat_text)
if not preprocess_only:
if strip_whitespace:
flat_text = minifier.Minify(flat_text.encode('utf-8'),
input_filename).decode('utf-8')
if not allow_external_script:
# We need to inline css and js before we inline images so that image
# references gets inlined in the css and js
flat_text = re.sub(
r'<script (?P<attrs1>.*?)src="(?P<filename>[^"\']*)"'
r'(?P<attrs2>.*?)></script>', InlineScript, flat_text)
flat_text = _STYLESHEET_RE.sub(
lambda m: InlineCSSFile(m, '<style>%s</style>'), flat_text)
# Check conditional elements, second pass. This catches conditionals in any
# of the text we just inlined.
flat_text = CheckConditionalElements(grd_node, flat_text)
# Allow custom modifications before inlining images.
if rewrite_function:
flat_text = rewrite_function(input_filepath, flat_text, distribution)
if not preprocess_only:
flat_text = _SRC_RE.sub(SrcReplace, flat_text)
flat_text = _SRCSET_RE.sub(SrcsetReplace, flat_text)
# TODO(arv): Only do this inside <style> tags.
flat_text = InlineCSSImages(flat_text)
flat_text = _ICON_RE.sub(SrcReplace, flat_text)
if names_only:
flat_text = None # Will contains garbage if the flag is set anyway.
return InlinedData(flat_text, inlined_files)
