def _AddMissingLocalesInGnOutputs(gn_lines, wanted_locales):
intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
# NOTE: Since this may insert new lines to each interval, process the
# list in reverse order to maintain valid (start,end) positions during
# the iteration.
for start, end in reversed(intervals):
if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
continue
locales = set()
for pos in xrange(start, end):
lang = _GetAndroidGnOutputLocale(gn_lines[pos])
locale = resource_utils.ToChromiumLocaleName(lang)
locales.add(locale)
missing_locales = wanted_locales.difference(locales)
if not missing_locales:
continue
src_locale = 'bg'
src_values = 'values-%s/' % resource_utils.ToAndroidLocaleName(
src_locale)
src_line = None
for pos in xrange(start, end):
if src_values in gn_lines[pos]:
src_line = gn_lines[pos]
break
if not src_line:
raise Exception('Cannot find output list item with "%s" locale' %
src_locale)
line_count = end - 1
for locale in missing_locales:
if locale == _DEFAULT_LOCALE:
dst_line = src_line.replace('values-%s/' % src_locale,
'values/')
else:
dst_line = src_line.replace(
'values-%s/' % src_locale,
'values-%s/' % resource_utils.ToAndroidLocaleName(locale))
gn_lines.insert(line_count, dst_line)
line_count += 1
gn_lines = _SortListSubRange(
gn_lines, start, line_count,
lambda line: _RE_GN_VALUES_LIST_LINE.match(line).group(1))
return gn_lines
def _ToAndroidLocales(locale_whitelist, support_zh_hk):
"""Converts the list of Chrome locales to Android config locale qualifiers.
Args:
locale_whitelist: A list of Chromium locale names.
support_zh_hk: True if we need to support zh-HK by duplicating
the zh-TW strings.
Returns:
A set of matching Android config locale qualifier names.
"""
ret = set()
for locale in locale_whitelist:
locale = resource_utils.ToAndroidLocaleName(locale)
if locale is None or ('-' in locale and '-r' not in locale):
raise Exception('Unsupported Chromium locale name: %s' % locale)
ret.add(locale)
# Always keep non-regional fall-backs.
language = locale.split('-')[0]
ret.add(language)
# We don't actually support zh-HK in Chrome on Android, but we mimic the
# native side behavior where we use zh-TW resources when the locale is set to
# zh-HK. See https://crbug.com/780847.
if support_zh_hk:
assert not any('HK' in l for l in locale_whitelist), (
'Remove special logic if zh-HK is now supported (crbug.com/780847).'
)
ret.add('zh-rHK')
return set(ret)
def _RenameLocaleResourceDirs(resource_dirs):
"""Rename locale resource directories into standard names when necessary.
This is necessary to deal with the fact that older Android releases only
support ISO 639-1 two-letter codes, and sometimes even obsolete versions
of them.
In practice it means:
* 3-letter ISO 639-2 qualifiers are renamed under a corresponding
2-letter one. E.g. for Filipino, strings under values-fil/ will be moved
to a new corresponding values-tl/ sub-directory.
* Modern ISO 639-1 codes will be renamed to their obsolete variant
for Indonesian, Hebrew and Yiddish (e.g. 'values-in/ -> values-id/).
* Norwegian macrolanguage strings will be renamed to Bokmål (main
Norway language). See http://crbug.com/920960. In practice this
means that 'values-no/ -> values-nb/' unless 'values-nb/' already
exists.
* BCP 47 langauge tags will be renamed to an equivalent ISO 639-1
locale qualifier if possible (e.g. 'values-b+en+US/ -> values-en-rUS').
Though this is not necessary at the moment, because no third-party
package that Chromium links against uses these for the current list of
supported locales, this may change when the list is extended in the
future).
Args:
resource_dirs: list of top-level resource directories.
Returns:
A dictionary mapping renamed paths to their original location
(e.g. '.../values-tl/strings.xml' -> ' .../values-fil/strings.xml').
"""
renamed_paths = dict()
for resource_dir in resource_dirs:
for path in _IterFiles(resource_dir):
locale = resource_utils.FindLocaleInStringResourceFilePath(path)
if not locale:
continue
cr_locale = resource_utils.ToChromiumLocaleName(locale)
if not cr_locale:
continue # Unsupported Android locale qualifier!?
locale2 = resource_utils.ToAndroidLocaleName(cr_locale)
if locale != locale2:
path2 = path.replace('/values-%s/' % locale,
'/values-%s/' % locale2)
if path == path2:
raise Exception(
'Could not substitute locale %s for %s in %s' %
(locale, locale2, path))
if os.path.exists(path2):
# This happens sometimes, e.g. some libraries provide both
# values-nb/ and values-no/ with the same content.
continue
build_utils.MakeDirectory(os.path.dirname(path2))
shutil.move(path, path2)
renamed_paths[os.path.relpath(path2,
resource_dir)] = os.path.relpath(
path, resource_dir)
return renamed_paths
def _RewriteLanguageAssetPath(src_path):
"""Rewrite the destination path of a locale asset for language-based splits.
Should only be used when generating bundles with language-based splits.
This will rewrite paths that look like locales/<locale>.pak into
locales#<language>/<locale>.pak, where <language> is the language code
from the locale.
Returns new path.
"""
if not src_path.startswith(_LOCALES_SUBDIR) or not src_path.endswith(
'.pak'):
return [src_path]
locale = src_path[len(_LOCALES_SUBDIR):-4]
android_locale = resource_utils.ToAndroidLocaleName(locale)
# The locale format is <lang>-<region> or <lang>. Extract the language.
pos = android_locale.find('-')
if pos >= 0:
android_language = android_locale[:pos]
else:
android_language = android_locale
if android_language == _FALLBACK_LANGUAGE:
# Fallback language .pak files must be placed in a different directory
# to ensure they are always stored in the base module.
result_path = 'assets/fallback-locales/%s.pak' % locale
else:
# Other language .pak files go into a language-specific asset directory
# that bundletool will store in separate split APKs.
result_path = 'assets/locales#lang_%s/%s.pak' % (android_language,
locale)
return result_path
def main():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
build_utils.AddDepfileOption(parser)
parser.add_argument('--locale-list',
required=True,
help='GN-list of Chrome-specific locale names.')
parser.add_argument('--output-zip',
required=True,
help='Output zip archive path.')
args = parser.parse_args()
locale_list = build_utils.ParseGnList(args.locale_list)
if not locale_list:
raise Exception('Locale list cannot be empty!')
with build_utils.AtomicOutput(args.output_zip) as tmp_file:
with zipfile.ZipFile(tmp_file, 'w') as out_zip:
# First, write the default value, since aapt requires one.
_AddLocaleResourceFileToZip(out_zip, '', _DEFAULT_CHROME_LOCALE)
for locale in locale_list:
android_locale = resource_utils.ToAndroidLocaleName(locale)
_AddLocaleResourceFileToZip(out_zip, android_locale, locale)
if args.depfile:
build_utils.WriteDepfile(args.depfile, args.output_zip)
def _GetAndroidGnOutputLocale(line):
"""Check a GN list, and return its Android locale if it is an output .xml"""
m = _RE_GN_VALUES_LIST_LINE.match(line)
if not m:
return None
if m.group(1): # First group is optional and contains group 2.
return m.group(2)
return resource_utils.ToAndroidLocaleName(_DEFAULT_LOCALE)
def _AddMissingLocalesInGrdAndroidOutputs(grd_file, grd_lines, wanted_locales):
"""Fix an input .grd line by adding missing Android outputs.
Args:
grd_file: Input .grd file path.
grd_lines: Input .grd line list.
wanted_locales: set of Chromium locale names.
Returns:
A new list of .grd lines, containing new <output> elements when needed
for locales from |wanted_locales| that were not part of the input.
"""
intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
for start, end in reversed(intervals):
locales = set()
for pos in xrange(start, end):
lang = _RE_LANG_ATTRIBUTE.search(grd_lines[pos]).group(1)
locale = _FixChromiumLangAttribute(lang)
locales.add(locale)
missing_locales = wanted_locales.difference(locales)
if not missing_locales:
continue
src_locale = 'bg'
src_lang_attribute = 'lang="%s"' % src_locale
src_line = None
for pos in xrange(start, end):
if src_lang_attribute in grd_lines[pos]:
src_line = grd_lines[pos]
break
if not src_line:
raise Exception(
'Cannot find <output> element with "%s" lang attribute' %
src_locale)
line_count = end - 1
for locale in missing_locales:
android_locale = resource_utils.ToAndroidLocaleName(locale)
dst_line = src_line.replace('lang="%s"' % src_locale,
'lang="%s"' % locale).replace(
'values-%s/' % src_locale,
'values-%s/' % android_locale)
grd_lines.insert(line_count, dst_line)
line_count += 1
# Sort the new <output> elements.
return _SortGrdElementsRanges(grd_lines, _IsGrdAndroidOutputLine)
def _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
wanted_locales):
"""Check all <output> elements in specific input .grd lines range.
This really checks the following:
- Filenames exist for each listed locale.
- Filenames are well-formed.
Args:
grd_lines: Input .grd lines.
start: Sub-range start position in input line list.
end: Sub-range limit position in input line list.
wanted_locales: Set of wanted Chromium locale names.
Returns:
List of error message strings for this input. Empty on success.
"""
errors = []
for pos in xrange(start, end):
line = grd_lines[pos]
m = _RE_LANG_ATTRIBUTE.search(line)
if not m:
continue
lang = m.group(1)
cr_locale = _FixChromiumLangAttribute(lang)
m = _RE_FILENAME_ATTRIBUTE.search(line)
if not m:
errors.append(
'%d: Missing filename attribute in <output> element' % pos + 1)
else:
filename = m.group(1)
if not filename.endswith('.xml'):
errors.append('%d: Filename should end with ".xml": %s' %
(pos + 1, filename))
dirname = os.path.basename(os.path.dirname(filename))
prefix = ('values-%s' %
resource_utils.ToAndroidLocaleName(cr_locale)
if cr_locale != _DEFAULT_LOCALE else 'values')
if dirname != prefix:
errors.append('%s: Directory name should be %s: %s' %
(pos + 1, prefix, filename))
return errors
def _ToAndroidLocales(locale_allowlist):
"""Converts the list of Chrome locales to Android config locale qualifiers.
Args:
locale_allowlist: A list of Chromium locale names.
Returns:
A set of matching Android config locale qualifier names.
"""
ret = set()
for locale in locale_allowlist:
locale = resource_utils.ToAndroidLocaleName(locale)
if locale is None or ('-' in locale and '-r' not in locale):
raise Exception('Unsupported Chromium locale name: %s' % locale)
ret.add(locale)
# Always keep non-regional fall-backs.
language = locale.split('-')[0]
ret.add(language)
return ret
def _RenameLocaleResourceDirs(resource_dirs, path_info):
"""Rename locale resource directories into standard names when necessary.
This is necessary to deal with the fact that older Android releases only
support ISO 639-1 two-letter codes, and sometimes even obsolete versions
of them.
In practice it means:
* 3-letter ISO 639-2 qualifiers are renamed under a corresponding
2-letter one. E.g. for Filipino, strings under values-fil/ will be moved
to a new corresponding values-tl/ sub-directory.
* Modern ISO 639-1 codes will be renamed to their obsolete variant
for Indonesian, Hebrew and Yiddish (e.g. 'values-in/ -> values-id/).
* Norwegian macrolanguage strings will be renamed to Bokmal (main
Norway language). See http://crbug.com/920960. In practice this
means that 'values-no/ -> values-nb/' unless 'values-nb/' already
exists.
* BCP 47 langauge tags will be renamed to an equivalent ISO 639-1
locale qualifier if possible (e.g. 'values-b+en+US/ -> values-en-rUS').
Args:
resource_dirs: list of top-level resource directories.
"""
for resource_dir in resource_dirs:
ignore_dirs = {}
for path in _IterFiles(resource_dir):
locale = resource_utils.FindLocaleInStringResourceFilePath(path)
if not locale:
continue
cr_locale = resource_utils.ToChromiumLocaleName(locale)
if not cr_locale:
continue # Unsupported Android locale qualifier!?
locale2 = resource_utils.ToAndroidLocaleName(cr_locale)
if locale != locale2:
path2 = path.replace('/values-%s/' % locale,
'/values-%s/' % locale2)
if path == path2:
raise Exception(
'Could not substitute locale %s for %s in %s' %
(locale, locale2, path))
# Ignore rather than rename when the destination resources config
# already exists.
# e.g. some libraries provide both values-nb/ and values-no/.
# e.g. material design provides:
# * res/values-rUS/values-rUS.xml
# * res/values-b+es+419/values-b+es+419.xml
config_dir = os.path.dirname(path2)
already_has_renamed_config = ignore_dirs.get(config_dir)
if already_has_renamed_config is None:
# Cache the result of the first time the directory is encountered
# since subsequent encounters will find the directory already exists
# (due to the rename).
already_has_renamed_config = os.path.exists(config_dir)
ignore_dirs[config_dir] = already_has_renamed_config
if already_has_renamed_config:
continue
build_utils.MakeDirectory(os.path.dirname(path2))
shutil.move(path, path2)
path_info.RegisterRename(os.path.relpath(path, resource_dir),
os.path.relpath(path2, resource_dir))
def GenerateBundleApks(bundle_path,
bundle_apks_path,
aapt2_path,
keystore_path,
keystore_password,
keystore_alias,
mode=None,
local_testing=False,
minimal=False,
minimal_sdk_version=None,
check_for_noop=True,
system_image_locales=None,
optimize_for=None):
"""Generate an .apks archive from a an app bundle if needed.
Args:
bundle_path: Input bundle file path.
bundle_apks_path: Output bundle .apks archive path. Name must end with
'.apks' or this operation will fail.
aapt2_path: Path to aapt2 build tool.
keystore_path: Path to keystore.
keystore_password: Keystore password, as a string.
keystore_alias: Keystore signing key alias.
mode: Build mode, which must be either None or one of BUILD_APKS_MODES.
minimal: Create the minimal set of apks possible (english-only).
minimal_sdk_version: Use this sdkVersion when |minimal| or
|system_image_locales| args are present.
check_for_noop: Use md5_check to short-circuit when inputs have not changed.
system_image_locales: Locales to package in the APK when mode is "system"
or "system_compressed".
optimize_for: Overrides split configuration, which must be None or
one of OPTIMIZE_FOR_OPTIONS.
"""
device_spec = None
if minimal_sdk_version:
assert minimal or system_image_locales, (
'minimal_sdk_version is only used when minimal or system_image_locales '
'is specified')
if minimal:
# Measure with one language split installed. Use Hindi because it is
# popular. resource_size.py looks for splits/base-hi.apk.
# Note: English is always included since it's in base-master.apk.
device_spec = _CreateDeviceSpec(bundle_path, minimal_sdk_version,
['hi'])
elif mode in _SYSTEM_MODES:
if not system_image_locales:
raise Exception('system modes require system_image_locales')
# Bundletool doesn't seem to understand device specs with locales in the
# form of "<lang>-r<region>", so just provide the language code instead.
locales = [
resource_utils.ToAndroidLocaleName(l).split('-')[0]
for l in system_image_locales
]
device_spec = _CreateDeviceSpec(bundle_path, minimal_sdk_version,
locales)
def rebuild():
logging.info('Building %s', bundle_apks_path)
with build_utils.TempDir() as tmp_dir:
tmp_apks_file = os.path.join(tmp_dir, 'output.apks')
cmd_args = [
'build-apks',
'--aapt2=%s' % aapt2_path,
'--output=%s' % tmp_apks_file,
'--ks=%s' % keystore_path,
'--ks-pass=pass:%s' % keystore_password,
'--ks-key-alias=%s' % keystore_alias,
'--overwrite',
]
input_bundle_path = bundle_path
# Work around bundletool not respecting uncompressDexFiles setting.
# b/176198991
if mode not in _SYSTEM_MODES and _BundleMinSdkVersion(
bundle_path) >= 27:
input_bundle_path = os.path.join(tmp_dir, 'system.aab')
_FixBundleDexCompressionGlob(bundle_path, input_bundle_path)
cmd_args += ['--bundle=%s' % input_bundle_path]
if local_testing:
cmd_args += ['--local-testing']
if mode is not None:
if mode not in BUILD_APKS_MODES:
raise Exception(
'Invalid mode parameter %s (should be in %s)' %
(mode, BUILD_APKS_MODES))
cmd_args += ['--mode=' + mode]
if optimize_for:
if optimize_for not in OPTIMIZE_FOR_OPTIONS:
raise Exception('Invalid optimize_for parameter %s '
'(should be in %s)' %
(mode, OPTIMIZE_FOR_OPTIONS))
cmd_args += ['--optimize-for=' + optimize_for]
if device_spec:
spec_file = os.path.join(tmp_dir, 'device.json')
with open(spec_file, 'w') as f:
json.dump(device_spec, f)
cmd_args += ['--device-spec=' + spec_file]
bundletool.RunBundleTool(cmd_args)
shutil.move(tmp_apks_file, bundle_apks_path)
if check_for_noop:
input_paths = [
bundle_path,
bundletool.BUNDLETOOL_JAR_PATH,
aapt2_path,
keystore_path,
]
input_strings = [
keystore_password,
keystore_alias,
device_spec,
]
if mode is not None:
input_strings.append(mode)
# Avoid rebuilding (saves ~20s) when the input files have not changed. This
# is essential when calling the apk_operations.py script multiple times with
# the same bundle (e.g. out/Debug/bin/monochrome_public_bundle run).
md5_check.CallAndRecordIfStale(rebuild,
input_paths=input_paths,
input_strings=input_strings,
output_paths=[bundle_apks_path])
else:
rebuild()
def GenerateBundleApks(bundle_path,
bundle_apks_path,
aapt2_path,
keystore_path,
keystore_password,
keystore_alias,
mode=None,
minimal=False,
minimal_sdk_version=None,
check_for_noop=True,
system_image_locales=None,
optimize_for=None):
"""Generate an .apks archive from a an app bundle if needed.
Args:
bundle_path: Input bundle file path.
bundle_apks_path: Output bundle .apks archive path. Name must end with
'.apks' or this operation will fail.
aapt2_path: Path to aapt2 build tool.
keystore_path: Path to keystore.
keystore_password: Keystore password, as a string.
keystore_alias: Keystore signing key alias.
mode: Build mode, which must be either None or one of BUILD_APKS_MODES.
minimal: Create the minimal set of apks possible (english-only).
minimal_sdk_version: Use this sdkVersion when |minimal| or
|system_image_locales| args are present.
check_for_noop: Use md5_check to short-circuit when inputs have not changed.
system_image_locales: Locales to package in the APK when mode is "system"
or "system_compressed".
optimize_for: Overrides split configuration, which must be None or
one of OPTIMIZE_FOR_OPTIONS.
"""
device_spec = None
if minimal_sdk_version:
assert minimal or system_image_locales, (
'minimal_sdk_version is only used when minimal or system_image_locales '
'is specified')
if minimal:
# Measure with one language split installed. Use Hindi because it is
# popular. resource_size.py looks for splits/base-hi.apk.
# Note: English is always included since it's in base-master.apk.
device_spec = _CreateDeviceSpec(bundle_path, minimal_sdk_version, ['hi'])
elif mode in _SYSTEM_MODES:
if not system_image_locales:
raise Exception('system modes require system_image_locales')
# Bundletool doesn't seem to understand device specs with locales in the
# form of "<lang>-r<region>", so just provide the language code instead.
locales = [
resource_utils.ToAndroidLocaleName(l).split('-')[0]
for l in system_image_locales
]
device_spec = _CreateDeviceSpec(bundle_path, minimal_sdk_version, locales)
def rebuild():
logging.info('Building %s', bundle_apks_path)
with tempfile.NamedTemporaryFile(suffix='.apks') as tmp_apks_file:
cmd_args = [
'build-apks',
'--aapt2=%s' % aapt2_path,
'--output=%s' % tmp_apks_file.name,
'--bundle=%s' % bundle_path,
'--ks=%s' % keystore_path,
'--ks-pass=pass:%s' % keystore_password,
'--ks-key-alias=%s' % keystore_alias,
'--overwrite',
]
if mode is not None:
if mode not in BUILD_APKS_MODES:
raise Exception('Invalid mode parameter %s (should be in %s)' %
(mode, BUILD_APKS_MODES))
cmd_args += ['--mode=' + mode]
if optimize_for:
if optimize_for not in OPTIMIZE_FOR_OPTIONS:
raise Exception('Invalid optimize_for parameter %s '
'(should be in %s)' %
(mode, OPTIMIZE_FOR_OPTIONS))
cmd_args += ['--optimize-for=' + optimize_for]
with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as spec_file:
if device_spec:
json.dump(device_spec, spec_file)
spec_file.flush()
cmd_args += ['--device-spec=' + spec_file.name]
bundletool.RunBundleTool(cmd_args)
# Make the resulting .apks file hermetic.
with build_utils.TempDir() as temp_dir, \
build_utils.AtomicOutput(bundle_apks_path, only_if_changed=False) as f:
files = build_utils.ExtractAll(tmp_apks_file.name, temp_dir)
build_utils.DoZip(files, f, base_dir=temp_dir)
if check_for_noop:
# NOTE: BUNDLETOOL_JAR_PATH is added to input_strings, rather than
# input_paths, to speed up MD5 computations by about 400ms (the .jar file
# contains thousands of class files which are checked independently,
# resulting in an .md5.stamp of more than 60000 lines!).
input_paths = [bundle_path, aapt2_path, keystore_path]
input_strings = [
keystore_password,
keystore_alias,
bundletool.BUNDLETOOL_JAR_PATH,
# NOTE: BUNDLETOOL_VERSION is already part of BUNDLETOOL_JAR_PATH, but
# it's simpler to assume that this may not be the case in the future.
bundletool.BUNDLETOOL_VERSION,
device_spec,
]
if mode is not None:
input_strings.append(mode)
# Avoid rebuilding (saves ~20s) when the input files have not changed. This
# is essential when calling the apk_operations.py script multiple times with
# the same bundle (e.g. out/Debug/bin/monochrome_public_bundle run).
md5_check.CallAndRecordIfStale(
rebuild,
input_paths=input_paths,
input_strings=input_strings,
output_paths=[bundle_apks_path])
else:
rebuild()
def _CheckGrdOutputElementRangeForAndroid(grd_lines, start, end,
wanted_locales):
"""Check all <output> elements in specific input .grd lines range.
This really checks the following:
- Each item has a correct 'lang' attribute.
- There are no duplicated lines for the same 'lang' attribute.
- Filenames are well-formed.
- That there are no extra locales that Chromium doesn't want.
- That no wanted locale is missing.
Args:
grd_lines: Input .grd lines.
start: Sub-range start position in input line list.
end: Sub-range limit position in input line list.
wanted_locales: Set of wanted Chromium locale names.
Returns:
List of error message strings for this input. Empty on success.
"""
errors = []
locales = set()
for pos in xrange(start, end):
line = grd_lines[pos]
m = _RE_LANG_ATTRIBUTE.search(line)
if not m:
errors.append('%d: Missing "lang" attribute in <output> element' %
pos + 1)
continue
lang = m.group(1)
cr_locale = _FixChromiumLangAttribute(lang)
if cr_locale in locales:
errors.append('%d: Redefinition of <output> for "%s" locale' %
(pos + 1, lang))
locales.add(cr_locale)
m = _RE_FILENAME_ATTRIBUTE.search(line)
if not m:
errors.append(
'%d: Missing filename attribute in <output> element' % pos + 1)
else:
filename = m.group(1)
if not filename.endswith('.xml'):
errors.append('%d: Filename should end with ".xml": %s' %
(pos + 1, filename))
dirname = os.path.basename(os.path.dirname(filename))
prefix = ('values-%s' %
resource_utils.ToAndroidLocaleName(cr_locale)
if cr_locale != _DEFAULT_LOCALE else 'values')
if dirname != prefix:
errors.append('%s: Directory name should be %s: %s' %
(pos + 1, prefix, filename))
extra_locales = locales.difference(wanted_locales)
if extra_locales:
errors.append('%d-%d: Extra locales found: %s' %
(start + 1, end + 1, sorted(extra_locales)))
missing_locales = wanted_locales.difference(locales)
if missing_locales:
errors.append('%d-%d: Missing locales: %s' %
(start + 1, end + 1, sorted(missing_locales)))
return errors