def Run(self, opts, args):
args = self.ParseOptions(args)
if len(args) != 1:
print(
'This tool takes a single tool-specific argument, the path to the\n'
'.rc file to process.')
return 2
self.SetOptions(opts)
path = args[0]
out_path = os.path.join(
util.dirname(path),
os.path.splitext(os.path.basename(path))[0] + '.grd')
rctext = util.ReadFile(path, self.input_encoding)
grd_text = unicode(self.Process(rctext, path))
with util.WrapOutputStream(file(out_path, 'w'), 'utf-8') as outfile:
outfile.write(grd_text)
print(
'Wrote output file %s.\nPlease check for TODO items in the file.' %
(out_path, ))
def InlineCSSFile(src_match, pattern, base_path=input_filepath):
"""Helper function to inline external CSS files.
Args:
src_match: A regular expression match with a named group named "filename".
pattern: The pattern to replace with the contents of the CSS file.
base_path: The base path to use for resolving the CSS file.
Returns:
The text that should replace the reference to the CSS file.
"""
filepath = GetFilepath(src_match, base_path)
if filepath is None:
return src_match.group(0)
# Even if names_only is set, the CSS file needs to be opened, because it
# can link to images that need to be added to the file set.
inlined_files.add(filepath)
# When resolving CSS files we need to pass in the path so that relative URLs
# can be resolved.
return pattern % InlineCSSText(util.ReadFile(filepath, util.BINARY),
filepath)
def RePack(output_file,
input_files,
whitelist_file=None,
suppress_removed_key_output=False,
output_info_filepath=None):
"""Write a new data pack file by combining input pack files.
Args:
output_file: path to the new data pack file.
input_files: a list of paths to the data pack files to combine.
whitelist_file: path to the file that contains the list of resource IDs
that should be kept in the output file or None to include
all resources.
suppress_removed_key_output: allows the caller to suppress the output from
RePackFromDataPackStrings.
output_info_file: If not None, specify the output .info filepath.
Raises:
KeyError: if there are duplicate keys or resource encoding is
inconsistent.
"""
input_data_packs = [ReadDataPack(filename) for filename in input_files]
input_info_files = [filename + '.info' for filename in input_files]
whitelist = None
if whitelist_file:
lines = util.ReadFile(whitelist_file, 'utf-8').strip().splitlines()
if not lines:
raise Exception('Whitelist file should not be empty')
whitelist = set(int(x) for x in lines)
inputs = [(p.resources, p.encoding) for p in input_data_packs]
resources, encoding = RePackFromDataPackStrings(
inputs, whitelist, suppress_removed_key_output)
WriteDataPack(resources, output_file, encoding)
if output_info_filepath is None:
output_info_filepath = output_file + '.info'
with open(output_info_filepath, 'w') as output_info_file:
for filename in input_info_files:
with open(filename, 'r') as info_file:
output_info_file.writelines(info_file.readlines())
def testRcIncludeFlattenedHtmlFile(self):
input_file = util.PathFromRoot('grit/testdata/include_test.html')
output_file = '%s/HTML_FILE1_include_test.html' % tempfile.gettempdir()
root = util.ParseGrdForUnittest('''
<includes>
<include name="HTML_FILE1" flattenhtml="true" file="%s" type="BINDATA" />
</includes>''' % input_file)
buf = StringIO.StringIO()
build.RcBuilder.ProcessNode(root,
DummyOutput('rc_all', 'en', output_file),
buf)
output = util.StripBlankLinesAndComments(buf.getvalue())
expected = (
_PREAMBLE +
u'HTML_FILE1 BINDATA "HTML_FILE1_include_test.html"'
)
# hackety hack to work on win32&lin
output = re.sub('"[c-zC-Z]:', '"', output)
self.assertEqual(expected, output)
file_contents = util.ReadFile(output_file, util.RAW_TEXT)
# Check for the content added by the <include> tag.
self.failUnless(file_contents.find('Hello Include!') != -1)
# Check for the content that was removed by if tag.
self.failUnless(file_contents.find('should be removed') == -1)
# Check for the content that was kept in place by if.
self.failUnless(file_contents.find('should be kept') != -1)
self.failUnless(file_contents.find('in the middle...') != -1)
self.failUnless(file_contents.find('at the end...') != -1)
# Check for nested content that was kept
self.failUnless(file_contents.find('nested true should be kept') != -1)
self.failUnless(
file_contents.find('silbing true should be kept') != -1)
# Check for removed "<if>" and "</if>" tags.
self.failUnless(file_contents.find('<if expr=') == -1)
self.failUnless(file_contents.find('</if>') == -1)
def ConvertFileToDataURL(filename, base_path, distribution, inlined_files,
names_only):
"""Convert filename to inlined data URI.
Takes a filename from ether "src" or "srcset", and attempts to read the file
at 'filename'. Returns data URI as string with given file inlined.
If it finds DIST_SUBSTR string in file name, replaces it with distribution.
If filename contains ':', it is considered URL and not translated.
Args:
filename: filename string from ether src or srcset attributes.
base_path: path that to look for files in
distribution: string that should replace DIST_SUBSTR
inlined_files: The name of the opened file is appended to this list.
names_only: If true, the function will not read the file but just return "".
It will still add the filename to |inlined_files|.
Returns:
string
"""
if filename.find(':') != -1:
# filename is probably a URL, which we don't want to bother inlining
return filename
filename = filename.replace(DIST_SUBSTR, distribution)
filepath = os.path.normpath(os.path.join(base_path, filename))
inlined_files.add(filepath)
if names_only:
return ""
mimetype = mimetypes.guess_type(filename)[0]
if mimetype is None:
raise Exception('%s is of an an unknown type and '
'cannot be stored in a data url.' % filename)
inline_data = base64.standard_b64encode(
util.ReadFile(filepath, util.BINARY))
return 'data:%s;base64,%s' % (mimetype, inline_data)
def testStructureNodeOutputfile(self):
input_file = util.PathFromRoot('grit/testdata/simple.html')
root = util.ParseGrdForUnittest('''\
<structures>
<structure type="tr_html" name="IDR_HTML" file="%s" />
</structures>''' % input_file)
struct, = root.GetChildrenOfType(structure.StructureNode)
# We must run the gatherer since we'll be wanting the translation of the
# file. The file exists in the location pointed to.
root.SetOutputLanguage('en')
root.RunGatherers()
output_dir = tempfile.gettempdir()
en_file = struct.FileForLanguage('en', output_dir)
self.failUnless(en_file == input_file)
fr_file = struct.FileForLanguage('fr', output_dir)
self.failUnless(fr_file == os.path.join(output_dir, 'fr_simple.html'))
contents = util.ReadFile(fr_file, util.RAW_TEXT)
self.failUnless(
contents.find('<p>') != -1) # should contain the markup
self.failUnless(contents.find('Hello!') == -1) # should be translated
def RePack(output_file, input_files, whitelist_file=None):
"""Write a new data pack file by combining input pack files.
Args:
output_file: path to the new data pack file.
input_files: a list of paths to the data pack files to combine.
whitelist_file: path to the file that contains the list of resource IDs
that should be kept in the output file or None to include
all resources.
Raises:
KeyError: if there are duplicate keys or resource encoding is
inconsistent.
"""
input_data_packs = [ReadDataPack(filename) for filename in input_files]
whitelist = None
if whitelist_file:
whitelist = util.ReadFile(whitelist_file,
util.RAW_TEXT).strip().split('\n')
whitelist = set(map(int, whitelist))
resources, encoding = RePackFromDataPackStrings(input_data_packs,
whitelist)
WriteDataPack(resources, output_file, encoding)
def GetDataPackPair(self, lang, encoding):
"""Returns a (id, string) pair that represents the resource id and raw
bytes of the data. This is used to generate the data pack data file.
"""
# TODO(benrg/joi): Move this and other implementations of GetDataPackPair
# to grit.format.data_pack?
from grit.format import rc_header
id_map = rc_header.GetIds(self.GetRoot())
id = id_map[self.GetTextualIds()[0]]
filename = self.ToRealPath(self.GetInputPath())
if self.attrs['flattenhtml'] == 'true':
allow_external_script = self.attrs['allowexternalscript'] == 'true'
data = self._GetFlattenedData(
allow_external_script=allow_external_script)
else:
data = util.ReadFile(filename, util.BINARY)
# Note that the minifier will only do anything if a minifier command
# has been set in the command line.
data = minifier.Minify(data, filename)
# Include does not care about the encoding, because it only returns binary
# data.
return id, self.CompressDataIfNeeded(data)
def _ReadFirstIdsFromFile(filename, defines):
"""Read the starting resource id values from |filename|. We also
expand variables of the form <(FOO) based on defines passed in on
the command line.
Returns a tuple, the absolute path of SRCDIR followed by the
first_ids dictionary.
"""
first_ids_dict = eval(util.ReadFile(filename, util.RAW_TEXT))
src_root_dir = os.path.abspath(
os.path.join(os.path.dirname(filename), first_ids_dict['SRCDIR']))
def ReplaceVariable(matchobj):
for key, value in defines.items():
if matchobj.group(1) == key:
value = os.path.abspath(value)
return value
return ''
renames = []
for grd_filename in first_ids_dict:
new_grd_filename = re.sub(r'<\(([A-Za-z_]+)\)', ReplaceVariable,
grd_filename)
if new_grd_filename != grd_filename:
abs_grd_filename = os.path.abspath(new_grd_filename)
if abs_grd_filename[:len(src_root_dir)] != src_root_dir:
new_grd_filename = os.path.basename(abs_grd_filename)
else:
new_grd_filename = abs_grd_filename[len(src_root_dir) + 1:]
new_grd_filename = new_grd_filename.replace('\\', '/')
renames.append((grd_filename, new_grd_filename))
for grd_filename, new_grd_filename in renames:
first_ids_dict[new_grd_filename] = first_ids_dict[grd_filename]
del (first_ids_dict[grd_filename])
return (src_root_dir, first_ids_dict)
def Parse(self):
"""Parses and inlines the represented file."""
filename = self.GetInputPath()
# If there is a grd_node, prefer its GetInputPath(), as that may do more
# processing to make the call to ToRealPath() below work correctly.
if self.grd_node:
filename = self.grd_node.GetInputPath()
if self.filename_expansion_function:
filename = self.filename_expansion_function(filename)
# Hack: some unit tests supply an absolute path and no root node.
if not os.path.isabs(filename):
filename = self.grd_node.ToRealPath(filename)
if self.flatten_html_:
self.inlined_text_ = html_inline.InlineToString(
filename,
self.grd_node,
allow_external_script=self.allow_external_script_,
strip_whitespace=True,
preprocess_only=self.preprocess_only_,
rewrite_function=lambda fp, t, d: ProcessImageSets(
fp,
t,
self.scale_factors_,
d,
filename_expansion_function=self.
filename_expansion_function),
filename_expansion_function=self.filename_expansion_function)
else:
distribution = html_inline.GetDistribution()
self.inlined_text_ = ProcessImageSets(
os.path.dirname(filename),
util.ReadFile(filename, 'utf-8'),
self.scale_factors_,
distribution,
filename_expansion_function=self.filename_expansion_function)
def Run(self, opts, args):
self.output_directory = '.'
first_ids_file = None
whitelist_filenames = []
assert_output_files = []
target_platform = None
depfile = None
depdir = None
rc_header_format = None
(own_opts, args) = getopt.getopt(args, 'a:o:D:E:f:w:t:h:',
('depdir=','depfile=','assert-file-list='))
for (key, val) in own_opts:
if key == '-a':
assert_output_files.append(val)
elif key == '--assert-file-list':
with open(val) as f:
assert_output_files += f.read().splitlines()
elif key == '-o':
self.output_directory = val
elif key == '-D':
name, val = util.ParseDefine(val)
self.defines[name] = val
elif key == '-E':
(env_name, env_value) = val.split('=', 1)
os.environ[env_name] = env_value
elif key == '-f':
# TODO([email protected]): Remove this override once change
# lands in WebKit.grd to specify the first_ids_file in the
# .grd itself.
first_ids_file = val
elif key == '-w':
whitelist_filenames.append(val)
elif key == '-t':
target_platform = val
elif key == '-h':
rc_header_format = val
elif key == '--depdir':
depdir = val
elif key == '--depfile':
depfile = val
if len(args):
print 'This tool takes no tool-specific arguments.'
return 2
self.SetOptions(opts)
if self.scons_targets:
self.VerboseOut('Using SCons targets to identify files to output.\n')
else:
self.VerboseOut('Output directory: %s (absolute path: %s)\n' %
(self.output_directory,
os.path.abspath(self.output_directory)))
if whitelist_filenames:
self.whitelist_names = set()
for whitelist_filename in whitelist_filenames:
self.VerboseOut('Using whitelist: %s\n' % whitelist_filename);
whitelist_contents = util.ReadFile(whitelist_filename, util.RAW_TEXT)
self.whitelist_names.update(whitelist_contents.strip().split('\n'))
self.res = grd_reader.Parse(opts.input,
debug=opts.extra_verbose,
first_ids_file=first_ids_file,
defines=self.defines,
target_platform=target_platform)
# Set an output context so that conditionals can use defines during the
# gathering stage; we use a dummy language here since we are not outputting
# a specific language.
self.res.SetOutputLanguage('en')
if rc_header_format:
self.res.AssignRcHeaderFormat(rc_header_format)
self.res.RunGatherers()
self.Process()
if assert_output_files:
if not self.CheckAssertedOutputFiles(assert_output_files):
return 2
if depfile and depdir:
self.GenerateDepfile(depfile, depdir)
return 0
def ReadDataPack(input_file): return ReadDataPackFromString(util.ReadFile(input_file, util.BINARY))
def DoInline(input_filename,
grd_node,
allow_external_script=False,
names_only=False,
rewrite_function=None):
"""Helper function that inlines the resources in a specified file.
Reads input_filename, finds all the src attributes and attempts to
inline the files they are referring to, then returns the result and
the set of inlined files.
Args:
input_filename: name of file to read in
grd_node: html node from the grd file for this include tag
names_only: |nil| will be returned for the inlined contents (faster).
rewrite_function: function(filepath, text, distribution) which will be
called to rewrite html content before inlining images.
Returns:
a tuple of the inlined data as a string and the set of filenames
of all the inlined files
"""
input_filepath = os.path.dirname(input_filename)
distribution = GetDistribution()
# Keep track of all the files we inline.
inlined_files = set()
def SrcReplace(src_match,
filepath=input_filepath,
inlined_files=inlined_files):
"""Helper function to provide SrcInlineAsDataURL with the base file path"""
return SrcInlineAsDataURL(src_match,
filepath,
distribution,
inlined_files,
names_only=names_only)
def GetFilepath(src_match):
filename = src_match.group('filename')
if filename.find(':') != -1:
# filename is probably a URL, which we don't want to bother inlining
return None
filename = filename.replace('%DISTRIBUTION%', distribution)
return os.path.join(input_filepath, filename)
def IsConditionSatisfied(src_match):
expression = src_match.group('expression')
return grd_node is None or grd_node.EvaluateCondition(expression)
def CheckConditionalElements(str):
"""Helper function to conditionally inline inner elements"""
while True:
begin_if = _BEGIN_IF_BLOCK.search(str)
if begin_if is None:
return str
condition_satisfied = IsConditionSatisfied(begin_if)
leading = str[0:begin_if.start()]
content_start = begin_if.end()
# Find matching "if" block end.
count = 1
pos = begin_if.end()
while True:
end_if = _END_IF_BLOCK.search(str, pos)
if end_if is None:
raise Exception('Unmatched <if>')
next_if = _BEGIN_IF_BLOCK.search(str, pos)
if next_if is None or next_if.start() >= end_if.end():
count = count - 1
if count == 0:
break
pos = end_if.end()
else:
count = count + 1
pos = next_if.end()
content = str[content_start:end_if.start()]
trailing = str[end_if.end():]
if condition_satisfied:
str = leading + CheckConditionalElements(content) + trailing
else:
str = leading + trailing
def InlineFileContents(src_match, pattern, inlined_files=inlined_files):
"""Helper function to inline external files of various types"""
filepath = GetFilepath(src_match)
if filepath is None:
return src_match.group(0)
inlined_files.add(filepath)
# Even if names_only is set, html files needs to be opened, because it
# can link to images that need to be added to the file set.
if names_only and not filepath.endswith('.html'):
return ""
return pattern % InlineToString(filepath, grd_node,
allow_external_script)
def InlineIncludeFiles(src_match):
"""Helper function to directly inline generic external files (without
wrapping them with any kind of tags).
"""
return InlineFileContents(src_match, '%s')
def InlineScript(match):
"""Helper function to inline external script files"""
attrs = (match.group('attrs1') + match.group('attrs2')).strip()
if attrs:
attrs = ' ' + attrs
return InlineFileContents(match, '<script' + attrs + '>%s</script>')
def InlineCSSText(text, css_filepath):
"""Helper function that inlines external resources in CSS text"""
filepath = os.path.dirname(css_filepath)
# Allow custom modifications before inlining images.
if rewrite_function:
text = rewrite_function(filepath, text, distribution)
return InlineCSSImages(text, filepath)
def InlineCSSFile(src_match, inlined_files=inlined_files):
"""Helper function to inline external css files.
Args:
src_match: A regular expression match with a named group named "filename".
Returns:
The text that should replace the reference to the CSS file.
"""
filepath = GetFilepath(src_match)
if filepath is None:
return src_match.group(0)
# Even if names_only is set, the CSS file needs to be opened, because it
# can link to images that need to be added to the file set.
inlined_files.add(filepath)
# When resolving CSS files we need to pass in the path so that relative URLs
# can be resolved.
return '<style>%s</style>' % InlineCSSText(
util.ReadFile(filepath, util.BINARY), filepath)
def InlineCSSImages(text, filepath=input_filepath):
"""Helper function that inlines external images in CSS backgrounds."""
# Replace contents of url() for css attributes: content, background,
# or *-image.
return re.sub(
'(?:content|background|[\w-]*-image):[^;]*' +
'(?:url\((?:\'|\")([^"\'\)\(]*)(?:\'|\")\)|' + 'image-set\(' +
'([ ]*url\((?:\'|\")([^"\'\)\(]*)(?:\'|\")\)' +
'[ ]*[0-9.]*x[ ]*(,[ ]*)?)*\))',
lambda m: InlineCSSUrls(m, filepath), text)
def InlineCSSUrls(src_match, filepath=input_filepath):
"""Helper function that inlines each url on a CSS image rule match."""
# Replace contents of url() references in matches.
return re.sub('url\((?:\'|\")(?P<filename>[^"\'\)\(]*)(?:\'|\")',
lambda m: SrcReplace(m, filepath), src_match.group(0))
flat_text = util.ReadFile(input_filename, util.BINARY)
if not allow_external_script:
# We need to inline css and js before we inline images so that image
# references gets inlined in the css and js
flat_text = re.sub(
'<script (?P<attrs1>.*?)src="(?P<filename>[^"\']*)"' +
'(?P<attrs2>.*?)></script>', InlineScript, flat_text)
flat_text = re.sub(
'<link rel="stylesheet".+?href="(?P<filename>[^"]*)".*?>',
InlineCSSFile, flat_text)
flat_text = re.sub('<include\s+src="(?P<filename>[^"\']*)".*>',
InlineIncludeFiles, flat_text)
# Check conditional elements, remove unsatisfied ones from the file.
flat_text = CheckConditionalElements(flat_text)
flat_text = re.sub('<(?!script)[^>]+?src="(?P<filename>[^"\']*)"',
SrcReplace, flat_text)
# Allow custom modifications before inlining images.
if rewrite_function:
flat_text = rewrite_function(input_filepath, flat_text, distribution)
# TODO(arv): Only do this inside <style> tags.
flat_text = InlineCSSImages(flat_text)
flat_text = re.sub('<link rel="icon".+?href="(?P<filename>[^"\']*)"',
SrcReplace, flat_text)
if names_only:
flat_text = None # Will contains garbage if the flag is set anyway.
return InlinedData(flat_text, inlined_files)
def GetData(self, *args): path, scale, req_scale = self._FindInputFile() data = util.ReadFile(self.grd_node.ToRealPath(path), util.BINARY) return _RescaleImage(data, scale, req_scale)
def _GetFilesInPak(pakname):
'''Get a list of the files that were actually included in the .pak output.
'''
data = util.ReadFile(pakname, util.BINARY)
# Hackity hack...
return [m.group(1) for m in re.finditer(r'CONTENTS_OF\((.*?)\)', data)]
def testExtractTranslations(self):
path = util.PathFromRoot('grit/testdata')
current_grd = grd_reader.Parse(
StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
<release seq="3">
<messages>
<message name="IDS_SIMPLE">
One
</message>
<message name="IDS_PLACEHOLDER">
<ph name="NUMBIRDS">%s<ex>3</ex></ph> birds
</message>
<message name="IDS_PLACEHOLDERS">
<ph name="ITEM">%d<ex>1</ex></ph> of <ph name="COUNT">%d<ex>3</ex></ph>
</message>
<message name="IDS_REORDERED_PLACEHOLDERS">
<ph name="ITEM">$1<ex>1</ex></ph> of <ph name="COUNT">$2<ex>3</ex></ph>
</message>
<message name="IDS_CHANGED">
This is the new version
</message>
<message name="IDS_TWIN_1">Hello</message>
<message name="IDS_TWIN_2">Hello</message>
<message name="IDS_NOT_TRANSLATEABLE" translateable="false">:</message>
<message name="IDS_LONGER_TRANSLATED">
Removed document <ph name="FILENAME">$1<ex>c:\temp</ex></ph>
</message>
<message name="IDS_DIFFERENT_TWIN_1">Howdie</message>
<message name="IDS_DIFFERENT_TWIN_2">Howdie</message>
</messages>
<structures>
<structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="klonk.rc" />
<structure type="menu" name="IDC_KLONKMENU" encoding="utf-16" file="klonk.rc" />
</structures>
</release>
</grit>'''), path)
current_grd.SetOutputLanguage('en')
current_grd.RunGatherers()
source_rc_path = util.PathFromRoot('grit/testdata/source.rc')
source_rc = util.ReadFile(source_rc_path, util.RAW_TEXT)
transl_rc_path = util.PathFromRoot('grit/testdata/transl.rc')
transl_rc = util.ReadFile(transl_rc_path, util.RAW_TEXT)
tool = transl2tc.TranslationToTc()
output_buf = StringIO.StringIO()
globopts = MakeOptions()
globopts.verbose = True
globopts.output_stream = output_buf
tool.Setup(globopts, [])
translations = tool.ExtractTranslations(current_grd, source_rc,
source_rc_path, transl_rc,
transl_rc_path)
values = translations.values()
output = output_buf.getvalue()
self.failUnless('Ein' in values)
self.failUnless('NUMBIRDS Vogeln' in values)
self.failUnless('ITEM von COUNT' in values)
self.failUnless(values.count('Hallo') == 1)
self.failIf('Dass war die alte Version' in values)
self.failIf(':' in values)
self.failIf('Dokument FILENAME ist entfernt worden' in values)
self.failIf('Nicht verwendet' in values)
self.failUnless(
('Howdie' in values or 'Hallo sagt man' in values)
and not ('Howdie' in values and 'Hallo sagt man' in values))
self.failUnless(
'XX01XX&SkraXX02XX&HaettaXX03XXThetta er "Klonk" sem eg fylaXX04XXgonkurinnXX05XXKlonk && er [good]XX06XX&HjalpXX07XX&Um...XX08XX'
in values)
self.failUnless('I lagi' in values)
self.failUnless(
output.count(
'Structure of message IDS_REORDERED_PLACEHOLDERS has changed'))
self.failUnless(output.count('Message IDS_CHANGED has changed'))
self.failUnless(
output.count(
'Structure of message IDS_LONGER_TRANSLATED has changed'))
self.failUnless(
output.count('Two different translations for "Howdie"'))
self.failUnless(
output.count(
'IDD_DIFFERENT_LENGTH_IN_TRANSL has wrong # of cliques'))
def Run(self, opts, args):
os.environ['cwd'] = os.getcwd()
self.output_directory = '.'
first_ids_file = None
predetermined_ids_file = None
whitelist_filenames = []
assert_output_files = []
target_platform = None
depfile = None
depdir = None
whitelist_support = False
write_only_new = False
depend_on_stamp = False
js_minifier = None
replace_ellipsis = True
(own_opts, args) = getopt.getopt(args, 'a:p:o:D:E:f:w:t:',
('depdir=','depfile=','assert-file-list=',
'help',
'output-all-resource-defines',
'no-output-all-resource-defines',
'no-replace-ellipsis',
'depend-on-stamp',
'js-minifier=',
'write-only-new=',
'whitelist-support'))
for (key, val) in own_opts:
if key == '-a':
assert_output_files.append(val)
elif key == '--assert-file-list':
with open(val) as f:
assert_output_files += f.read().splitlines()
elif key == '-o':
self.output_directory = val
elif key == '-D':
name, val = util.ParseDefine(val)
self.defines[name] = val
elif key == '-E':
(env_name, env_value) = val.split('=', 1)
os.environ[env_name] = env_value
elif key == '-f':
# TODO([email protected]): Remove this override once change
# lands in WebKit.grd to specify the first_ids_file in the
# .grd itself.
first_ids_file = val
elif key == '-w':
whitelist_filenames.append(val)
elif key == '--no-replace-ellipsis':
replace_ellipsis = False
elif key == '-p':
predetermined_ids_file = val
elif key == '-t':
target_platform = val
elif key == '--depdir':
depdir = val
elif key == '--depfile':
depfile = val
elif key == '--write-only-new':
write_only_new = val != '0'
elif key == '--depend-on-stamp':
depend_on_stamp = True
elif key == '--js-minifier':
js_minifier = val
elif key == '--whitelist-support':
whitelist_support = True
elif key == '--help':
self.ShowUsage()
sys.exit(0)
if len(args):
print 'This tool takes no tool-specific arguments.'
return 2
self.SetOptions(opts)
if self.scons_targets:
self.VerboseOut('Using SCons targets to identify files to output.\n')
else:
self.VerboseOut('Output directory: %s (absolute path: %s)\n' %
(self.output_directory,
os.path.abspath(self.output_directory)))
if whitelist_filenames:
self.whitelist_names = set()
for whitelist_filename in whitelist_filenames:
self.VerboseOut('Using whitelist: %s\n' % whitelist_filename);
whitelist_contents = util.ReadFile(whitelist_filename, util.RAW_TEXT)
self.whitelist_names.update(whitelist_contents.strip().split('\n'))
if js_minifier:
minifier.SetJsMinifier(js_minifier)
self.write_only_new = write_only_new
self.res = grd_reader.Parse(opts.input,
debug=opts.extra_verbose,
first_ids_file=first_ids_file,
predetermined_ids_file=predetermined_ids_file,
defines=self.defines,
target_platform=target_platform)
# Set an output context so that conditionals can use defines during the
# gathering stage; we use a dummy language here since we are not outputting
# a specific language.
self.res.SetOutputLanguage('en')
self.res.SetWhitelistSupportEnabled(whitelist_support)
self.res.RunGatherers()
# Replace ... with the single-character version. http://crbug.com/621772
if replace_ellipsis:
for node in self.res:
if isinstance(node, message.MessageNode):
node.SetReplaceEllipsis(True)
self.Process()
if assert_output_files:
if not self.CheckAssertedOutputFiles(assert_output_files):
return 2
if depfile and depdir:
self.GenerateDepfile(depfile, depdir, first_ids_file, depend_on_stamp)
return 0
def Test(data, encoding, expected_result):
with open('testfile', 'wb') as f:
f.write(data)
self.assertEqual(util.ReadFile('testfile', encoding),
expected_result)
def DoInline(input_filename,
grd_node,
allow_external_script=False,
preprocess_only=False,
names_only=False,
rewrite_function=None,
filename_expansion_function=None):
"""Helper function that inlines the resources in a specified file.
Reads input_filename, finds all the src attributes and attempts to
inline the files they are referring to, then returns the result and
the set of inlined files.
Args:
input_filename: name of file to read in
grd_node: html node from the grd file for this include tag
preprocess_only: Skip all HTML processing, only handle <if> and <include>.
names_only: |nil| will be returned for the inlined contents (faster).
rewrite_function: function(filepath, text, distribution) which will be
called to rewrite html content before inlining images.
filename_expansion_function: function(filename) which will be called to
rewrite filenames before attempting to read them.
Returns:
a tuple of the inlined data as a string and the set of filenames
of all the inlined files
"""
if filename_expansion_function:
input_filename = filename_expansion_function(input_filename)
input_filepath = os.path.dirname(input_filename)
distribution = GetDistribution()
# Keep track of all the files we inline.
inlined_files = set()
def SrcReplace(src_match,
filepath=input_filepath,
inlined_files=inlined_files):
"""Helper function to provide SrcInlineAsDataURL with the base file path"""
return SrcInlineAsDataURL(
src_match,
filepath,
distribution,
inlined_files,
names_only=names_only,
filename_expansion_function=filename_expansion_function)
def GetFilepath(src_match, base_path=input_filepath):
filename = src_match.group('filename')
if filename.find(':') != -1:
# filename is probably a URL, which we don't want to bother inlining
return None
filename = filename.replace('%DISTRIBUTION%', distribution)
if filename_expansion_function:
filename = filename_expansion_function(filename)
return os.path.normpath(os.path.join(base_path, filename))
def IsConditionSatisfied(src_match):
expression = src_match.group('expression')
return grd_node is None or grd_node.EvaluateCondition(expression)
def CheckConditionalElements(str):
"""Helper function to conditionally inline inner elements"""
while True:
begin_if = _BEGIN_IF_BLOCK.search(str)
if begin_if is None:
if _END_IF_BLOCK.search(str) is not None:
raise Exception('Unmatched </if>')
return str
condition_satisfied = IsConditionSatisfied(begin_if)
leading = str[0:begin_if.start()]
content_start = begin_if.end()
# Find matching "if" block end.
count = 1
pos = begin_if.end()
while True:
end_if = _END_IF_BLOCK.search(str, pos)
if end_if is None:
raise Exception('Unmatched <if>')
next_if = _BEGIN_IF_BLOCK.search(str, pos)
if next_if is None or next_if.start() >= end_if.end():
count = count - 1
if count == 0:
break
pos = end_if.end()
else:
count = count + 1
pos = next_if.end()
content = str[content_start:end_if.start()]
trailing = str[end_if.end():]
if condition_satisfied:
str = leading + CheckConditionalElements(content) + trailing
else:
str = leading + trailing
def InlineFileContents(src_match, pattern, inlined_files=inlined_files):
"""Helper function to inline external files of various types"""
filepath = GetFilepath(src_match)
if filepath is None:
return src_match.group(0)
inlined_files.add(filepath)
if names_only:
inlined_files.update(
GetResourceFilenames(
filepath,
allow_external_script,
rewrite_function,
filename_expansion_function=filename_expansion_function))
return ""
return pattern % InlineToString(
filepath,
grd_node,
allow_external_script=allow_external_script,
filename_expansion_function=filename_expansion_function)
def InlineIncludeFiles(src_match):
"""Helper function to directly inline generic external files (without
wrapping them with any kind of tags).
"""
return InlineFileContents(src_match, '%s')
def InlineScript(match):
"""Helper function to inline external script files"""
attrs = (match.group('attrs1') + match.group('attrs2')).strip()
if attrs:
attrs = ' ' + attrs
return InlineFileContents(match, '<script' + attrs + '>%s</script>')
def InlineCSSText(text, css_filepath):
"""Helper function that inlines external resources in CSS text"""
filepath = os.path.dirname(css_filepath)
# Allow custom modifications before inlining images.
if rewrite_function:
text = rewrite_function(filepath, text, distribution)
text = InlineCSSImages(text, filepath)
return InlineCSSImports(text, filepath)
def InlineCSSFile(src_match, pattern, base_path=input_filepath):
"""Helper function to inline external CSS files.
Args:
src_match: A regular expression match with a named group named "filename".
pattern: The pattern to replace with the contents of the CSS file.
base_path: The base path to use for resolving the CSS file.
Returns:
The text that should replace the reference to the CSS file.
"""
filepath = GetFilepath(src_match, base_path)
if filepath is None:
return src_match.group(0)
# Even if names_only is set, the CSS file needs to be opened, because it
# can link to images that need to be added to the file set.
inlined_files.add(filepath)
# When resolving CSS files we need to pass in the path so that relative URLs
# can be resolved.
return pattern % InlineCSSText(util.ReadFile(filepath, util.BINARY),
filepath)
def InlineCSSImages(text, filepath=input_filepath):
"""Helper function that inlines external images in CSS backgrounds."""
# Replace contents of url() for css attributes: content, background,
# or *-image.
return re.sub(
'(content|background|[\w-]*-image):[^;]*' +
'(url\((?P<quote1>"|\'|)[^"\'()]*(?P=quote1)\)|' + 'image-set\(' +
'([ ]*url\((?P<quote2>"|\'|)[^"\'()]*(?P=quote2)\)' +
'[ ]*[0-9.]*x[ ]*(,[ ]*)?)+\))',
lambda m: InlineCSSUrls(m, filepath), text)
def InlineCSSUrls(src_match, filepath=input_filepath):
"""Helper function that inlines each url on a CSS image rule match."""
# Replace contents of url() references in matches.
return re.sub(
'url\((?P<quote>"|\'|)(?P<filename>[^"\'()]*)(?P=quote)\)',
lambda m: SrcReplace(m, filepath), src_match.group(0))
def InlineCSSImports(text, filepath=input_filepath):
"""Helper function that inlines CSS files included via the @import
directive.
"""
return re.sub(
'@import\s+url\((?P<quote>"|\'|)(?P<filename>[^"\'()]*)' +
'(?P=quote)\);', lambda m: InlineCSSFile(m, '%s', filepath), text)
flat_text = util.ReadFile(input_filename, util.BINARY)
# Check conditional elements, remove unsatisfied ones from the file. We do
# this twice. The first pass is so that we don't even bother calling
# InlineScript, InlineCSSFile and InlineIncludeFiles on text we're eventually
# going to throw out anyway.
flat_text = CheckConditionalElements(flat_text)
if not preprocess_only:
if not allow_external_script:
# We need to inline css and js before we inline images so that image
# references gets inlined in the css and js
flat_text = re.sub(
'<script (?P<attrs1>.*?)src="(?P<filename>[^"\']*)"' +
'(?P<attrs2>.*?)></script>', InlineScript, flat_text)
flat_text = _STYLESHEET_RE.sub(
lambda m: InlineCSSFile(m, '<style>%s</style>'), flat_text)
flat_text = _INCLUDE_RE.sub(InlineIncludeFiles, flat_text)
# Check conditional elements, second pass. This catches conditionals in any
# of the text we just inlined.
flat_text = CheckConditionalElements(flat_text)
if not preprocess_only:
# Allow custom modifications before inlining images.
if rewrite_function:
flat_text = rewrite_function(input_filepath, flat_text,
distribution)
flat_text = _SRC_RE.sub(SrcReplace, flat_text)
# TODO(arv): Only do this inside <style> tags.
flat_text = InlineCSSImages(flat_text)
flat_text = _ICON_RE.sub(SrcReplace, flat_text)
if names_only:
flat_text = None # Will contains garbage if the flag is set anyway.
return InlinedData(flat_text, inlined_files)
def DoInline(input_filename,
grd_node,
allow_external_script=False,
preprocess_only=False,
names_only=False,
strip_whitespace=False,
rewrite_function=None,
filename_expansion_function=None):
"""Helper function that inlines the resources in a specified file.
Reads input_filename, finds all the src attributes and attempts to
inline the files they are referring to, then returns the result and
the set of inlined files.
Args:
input_filename: name of file to read in
grd_node: html node from the grd file for this include tag
preprocess_only: Skip all HTML processing, only handle <if> and <include>.
names_only: |nil| will be returned for the inlined contents (faster).
strip_whitespace: remove whitespace and comments in the input files.
rewrite_function: function(filepath, text, distribution) which will be
called to rewrite html content before inlining images.
filename_expansion_function: function(filename) which will be called to
rewrite filenames before attempting to read them.
Returns:
a tuple of the inlined data as a string and the set of filenames
of all the inlined files
"""
if filename_expansion_function:
input_filename = filename_expansion_function(input_filename)
input_filepath = os.path.dirname(input_filename)
distribution = GetDistribution()
# Keep track of all the files we inline.
inlined_files = set()
def SrcReplace(src_match,
filepath=input_filepath,
inlined_files=inlined_files):
"""Helper function to provide SrcInlineAsDataURL with the base file path"""
return SrcInlineAsDataURL(
src_match,
filepath,
distribution,
inlined_files,
names_only=names_only,
filename_expansion_function=filename_expansion_function)
def SrcsetReplace(srcset_match,
filepath=input_filepath,
inlined_files=inlined_files):
"""Helper function to provide SrcsetInlineAsDataURL with the base file
path.
"""
return SrcsetInlineAsDataURL(
srcset_match,
filepath,
distribution,
inlined_files,
names_only=names_only,
filename_expansion_function=filename_expansion_function)
def GetFilepath(src_match, base_path=input_filepath):
filename = [
v for k, v in src_match.groupdict().items()
if k.startswith('file') and v
][0]
if filename.find(':') != -1:
# filename is probably a URL, which we don't want to bother inlining
return None
filename = filename.replace('%DISTRIBUTION%', distribution)
if filename_expansion_function:
filename = filename_expansion_function(filename)
return os.path.normpath(os.path.join(base_path, filename))
def InlineFileContents(src_match,
pattern,
inlined_files=inlined_files,
strip_whitespace=False):
"""Helper function to inline external files of various types"""
filepath = GetFilepath(src_match)
if filepath is None:
return src_match.group(0)
inlined_files.add(filepath)
if names_only:
inlined_files.update(
GetResourceFilenames(
filepath,
grd_node,
allow_external_script,
rewrite_function,
filename_expansion_function=filename_expansion_function))
return ""
# To recursively save inlined files, we need InlinedData instance returned
# by DoInline.
inlined_data_inst = DoInline(
filepath,
grd_node,
allow_external_script=allow_external_script,
preprocess_only=preprocess_only,
strip_whitespace=strip_whitespace,
filename_expansion_function=filename_expansion_function)
inlined_files.update(inlined_data_inst.inlined_files)
return pattern % inlined_data_inst.inlined_data
def InlineIncludeFiles(src_match):
"""Helper function to directly inline generic external files (without
wrapping them with any kind of tags).
"""
return InlineFileContents(src_match, '%s')
def InlineScript(match):
"""Helper function to inline external script files"""
attrs = (match.group('attrs1') + match.group('attrs2')).strip()
if attrs:
attrs = ' ' + attrs
return InlineFileContents(match,
'<script' + attrs + '>%s</script>',
strip_whitespace=True)
def InlineCSSText(text, css_filepath):
"""Helper function that inlines external resources in CSS text"""
filepath = os.path.dirname(css_filepath)
# Allow custom modifications before inlining images.
if rewrite_function:
text = rewrite_function(filepath, text, distribution)
text = InlineCSSImages(text, filepath)
return InlineCSSImports(text, filepath)
def InlineCSSFile(src_match, pattern, base_path=input_filepath):
"""Helper function to inline external CSS files.
Args:
src_match: A regular expression match with a named group named "filename".
pattern: The pattern to replace with the contents of the CSS file.
base_path: The base path to use for resolving the CSS file.
Returns:
The text that should replace the reference to the CSS file.
"""
filepath = GetFilepath(src_match, base_path)
if filepath is None:
return src_match.group(0)
# Even if names_only is set, the CSS file needs to be opened, because it
# can link to images that need to be added to the file set.
inlined_files.add(filepath)
# Inline stylesheets included in this css file.
text = _INCLUDE_RE.sub(InlineIncludeFiles,
util.ReadFile(filepath, 'utf-8'))
# When resolving CSS files we need to pass in the path so that relative URLs
# can be resolved.
return pattern % InlineCSSText(text, filepath)
def GetUrlRegexString(postfix=''):
"""Helper function that returns a string for a regex that matches url('')
but not url([[ ]]) or url({{ }}). Appends |postfix| to group names.
"""
url_re = (r'url\((?!\[\[|{{)(?P<q%s>"|\'|)(?P<filename%s>[^"\'()]*)'
r'(?P=q%s)\)')
return url_re % (postfix, postfix, postfix)
def InlineCSSImages(text, filepath=input_filepath):
"""Helper function that inlines external images in CSS backgrounds."""
# Replace contents of url() for css attributes: content, background,
# or *-image.
property_re = r'(content|background|[\w-]*-image):[^;]*'
# Replace group names to prevent duplicates when forming value_re.
image_set_value_re = (r'image-set\(([ ]*' + GetUrlRegexString('2') +
r'[ ]*[0-9.]*x[ ]*(,[ ]*)?)+\)')
value_re = '(%s|%s)' % (GetUrlRegexString(), image_set_value_re)
css_re = property_re + value_re
return re.sub(css_re, lambda m: InlineCSSUrls(m, filepath), text)
def InlineCSSUrls(src_match, filepath=input_filepath):
"""Helper function that inlines each url on a CSS image rule match."""
# Replace contents of url() references in matches.
return re.sub(GetUrlRegexString(), lambda m: SrcReplace(m, filepath),
src_match.group(0))
def InlineCSSImports(text, filepath=input_filepath):
"""Helper function that inlines CSS files included via the @import
directive.
"""
return re.sub(r'@import\s+' + GetUrlRegexString() + r';',
lambda m: InlineCSSFile(m, '%s', filepath), text)
flat_text = util.ReadFile(input_filename, 'utf-8')
# Check conditional elements, remove unsatisfied ones from the file. We do
# this twice. The first pass is so that we don't even bother calling
# InlineScript, InlineCSSFile and InlineIncludeFiles on text we're eventually
# going to throw out anyway.
flat_text = CheckConditionalElements(grd_node, flat_text)
flat_text = _INCLUDE_RE.sub(InlineIncludeFiles, flat_text)
if not preprocess_only:
if strip_whitespace:
flat_text = minifier.Minify(flat_text.encode('utf-8'),
input_filename).decode('utf-8')
if not allow_external_script:
# We need to inline css and js before we inline images so that image
# references gets inlined in the css and js
flat_text = re.sub(
r'<script (?P<attrs1>.*?)src="(?P<filename>[^"\']*)"'
r'(?P<attrs2>.*?)></script>', InlineScript, flat_text)
flat_text = _STYLESHEET_RE.sub(
lambda m: InlineCSSFile(m, '<style>%s</style>'), flat_text)
# Check conditional elements, second pass. This catches conditionals in any
# of the text we just inlined.
flat_text = CheckConditionalElements(grd_node, flat_text)
# Allow custom modifications before inlining images.
if rewrite_function:
flat_text = rewrite_function(input_filepath, flat_text, distribution)
if not preprocess_only:
flat_text = _SRC_RE.sub(SrcReplace, flat_text)
flat_text = _SRCSET_RE.sub(SrcsetReplace, flat_text)
# TODO(arv): Only do this inside <style> tags.
flat_text = InlineCSSImages(flat_text)
flat_text = _ICON_RE.sub(SrcReplace, flat_text)
if names_only:
flat_text = None # Will contains garbage if the flag is set anyway.
return InlinedData(flat_text, inlined_files)
def GetData(self, *args):
path, scale, req_scale = self._FindInputFile()
data = util.ReadFile(self.grd_node.ToRealPath(path), util.BINARY)
data = _RescaleImage(data, scale, req_scale)
data = _MoveSpecialChunksToFront(data)
return data
def main():
parser = argparse.ArgumentParser(description='Replace the Google Chrome incognito image with Pedobear')
parser.add_argument('--restore', action='store_true', help='Restore from backup')
args = parser.parse_args()
version = check_output(["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "--version"]).replace("Google Chrome", "").strip()
resource_path = "/Applications/Google Chrome.app/Contents/Versions/" + version + "/Google Chrome Framework.framework/Resources/resources.pak"
resource_backup_path = "/Applications/Google Chrome.app/Contents/Versions/" + version + "/Google Chrome Framework.framework/Resources/resources.bak"
if os.path.isfile(resource_path):
if args.restore:
if os.path.isfile(resource_backup_path):
print "Restoring from backup..."
shutil.copy2(resource_backup_path, resource_path)
print "Resource file restored."
else:
print "Tried to restore, but backup file not found at " + resource_backup_path
return
print "Resource file found."
data = ReadDataPack(resource_path)
original_1x = util.ReadFile('1x_base64png_original.txt', UTF8)
original_2x = util.ReadFile('2x_base64png_original.txt', UTF8)
pedobear_1x = util.ReadFile('1x_base64png_pedobear.txt', UTF8)
pedobear_2x = util.ReadFile('2x_base64png_pedobear.txt', UTF8)
found = False
already_patched = False
data2 = {}
for (resource_id, text) in data.resources.iteritems():
if original_1x in text:
found = True
text = text.replace(original_1x, pedobear_1x)
elif pedobear_1x in text:
already_patched = True
if original_2x in text:
found = True
text = text.replace(original_2x, pedobear_2x)
elif pedobear_2x in text:
already_patched = True
data2[resource_id] = text
if found:
print "Incognito images found."
print "Creating backup in " + resource_backup_path
shutil.copy2(resource_path, resource_backup_path)
print "Patching " + resource_path
WriteDataPack(data2, resource_path, UTF8)
print "Done! Quit and reopen Chrome."
elif already_patched:
print "Resource file is already patched."
else:
print "Could not find incognito image in resource file, aborting..."
else:
print "Could not find resource.pak to patch, aborting..."