def main():
p = argparse.ArgumentParser(
description="Remove obsolete strings and reformat files in localized repositories"
)
p.add_argument("--noupdates", help="Do not pull from remote", action="store_true")
p.add_argument(
"--wetrun",
help="Commit local changes and push them to remote",
action="store_true",
)
p.add_argument(
"--locale", help="Run on a specific locale", action="store", default=""
)
args = p.parse_args()
# Read paths from config file
[l10n_clones_path, quarantine_path] = local_config.read_config(
["l10n_clones_path", "quarantine_path"]
)
if args.locale:
locales = [args.locale]
else:
locales = [x for x in os.listdir(l10n_clones_path) if not x.startswith(".")]
# Exclude locales still working on Mercurial directly
excluded_locales = [
"ja",
"ja-JP-mac",
]
locales = sorted([x for x in locales if x not in excluded_locales])
# Get a list of supported files in the source repository
source_file_list = extractFileList(quarantine_path)
for locale in locales:
print("Locale: {}".format(locale))
locale_path = os.path.join(l10n_clones_path, locale)
# Update locale repository, unless --noupdates was called explicitly
if not args.noupdates:
print("Updating repository...")
subprocess.run(["hg", "-R", locale_path, "pull", "-u"])
# Create list of target files
target_file_list = extractFileList(locale_path)
# Ignore files in locale that are not available in the source
target_file_list = [f for f in target_file_list if f in source_file_list]
# Read source and target and write the output overwriting the existing
# localized file
for filename in target_file_list:
source_filename = os.path.join(quarantine_path, filename)
target_filename = os.path.join(locale_path, filename)
with open(source_filename) as f:
source_content = f.read()
source_parser = getParser(filename)
source_parser.readUnicode(source_content)
reference = list(source_parser.walk())
with open(target_filename) as f:
target_content = f.read()
target_parser = getParser(filename)
target_parser.readUnicode(target_content)
target = list(target_parser.walk())
output = serialize(filename, reference, target, {})
# zh-CN has an extra string that needs to be kept (start page)
if (
locale == "zh-CN"
and filename == "browser/chrome/browser/browser.properties"
):
output += b"\n# DO NOT REMOVE: this string is used to set up the home page for zh-CN\n"
output += b"browser.startup.homepage = https://start.firefoxchina.cn\n"
with open(target_filename, "wb") as f:
f.write(output)
if args.wetrun:
# Commit changes
subprocess.run(["hg", "-R", locale_path, "addremove"])
subprocess.run(
[
"hg",
"-R",
locale_path,
"commit",
"-m",
"Remove obsolete strings and reformat files",
]
)
subprocess.run(["hg", "-R", locale_path, "push"])
else:
print("(dry run)")
def main():
p = argparse.ArgumentParser(
description="Add missing FTL files in localized repositories")
p.add_argument("--noupdates",
help="Do not pull from remote",
action="store_true")
p.add_argument(
"--wetrun",
help="Commit local changes and push them to remote",
action="store_true",
)
p.add_argument("--locale",
help="Run on a specific locale",
action="store",
default="")
p.add_argument(
"--threshold",
help=
"Minimum percentage of completion in completion under which locales are ignored",
action="store",
default="70",
)
args = p.parse_args()
# Read paths from config file
[l10n_clones_path, quarantine_path
] = local_config.read_config(["l10n_clones_path", "quarantine_path"])
# Get the list of locales
if args.locale:
locales = [args.locale]
else:
locales = sorted(
[x for x in os.listdir(l10n_clones_path) if not x.startswith(".")])
# Get a list of FTL files in the source repository
source_files = extractFileList(quarantine_path)
# Get completion stats for locales from Pontoon
query = """
{
firefox: project(slug: "thunderbird") {
localizations {
locale {
code
},
missingStrings,
totalStrings
}
}
}
"""
pontoon_stats = {}
try:
print("Reading Pontoon stats...")
url = "https://pontoon.mozilla.org/graphql?query={}".format(
urlquote(query))
response = urlopen(url)
json_data = json.load(response)
for project, project_data in json_data["data"].items():
for element in project_data["localizations"]:
locale = element["locale"]["code"]
pontoon_stats[locale] = round(
(float(element["totalStrings"] -
element["missingStrings"])) /
element["totalStrings"] * 100,
2,
)
except Exception as e:
print(e)
ignored_locales = {
"missing": [],
"incomplete": [],
}
threshold = int(args.threshold)
files_total = 0
out_log = []
# If a locale is not available in Pontoon, but it's been requested
# explicitly, fake stats to run the commands anyway.
requested_locale = args.locale
if requested_locale and requested_locale not in pontoon_stats:
print(
"Locale {} is not available in Pontoon but was requested explicitly. Assuming completion at 100%"
.format(requested_locale))
pontoon_stats[requested_locale] = 100
for locale in locales:
# Ignore a locale if it's not in Pontoon or is below 60%
if locale not in pontoon_stats and locale not in ["ja-JP-mac"]:
ignored_locales["missing"].append(locale)
continue
if locale not in ["ja-JP-mac"] and pontoon_stats[locale] < threshold:
ignored_locales["incomplete"].append(locale)
continue
l10n_repo = os.path.join(l10n_clones_path, locale)
# Update locale repository, unless --noupdates was called explicitly
if not args.noupdates:
subprocess.run(["hg", "-R", l10n_repo, "pull", "-u"])
# Create list of files
locale_files = extractFileList(l10n_repo)
added_files = 0
for file_name in source_files:
if file_name not in locale_files:
full_file_name = os.path.join(l10n_repo, file_name)
file_path = os.path.dirname(full_file_name)
if not os.path.isdir(file_path):
# Create missing folder
print("Creating missing folder: {}".format(
os.path.relpath(file_path, l10n_repo)))
os.makedirs(file_path)
with open(full_file_name, "w") as f:
f.write(
"# This Source Code Form is subject to the terms of the Mozilla Public\n"
"# License, v. 2.0. If a copy of the MPL was not distributed with this\n"
"# file, You can obtain one at http://mozilla.org/MPL/2.0/.\n"
)
added_files += 1
if added_files > 0:
out_log.append("{}: added {} files".format(locale, added_files))
files_total += added_files
if args.wetrun:
subprocess.run(["hg", "-R", l10n_repo, "addremove"])
subprocess.run([
"hg",
"-R",
l10n_repo,
"commit",
"-m",
"Bug 1586984 - Add empty FTL files to repository to avoid English fallback",
])
subprocess.run(["hg", "-R", l10n_repo, "push"])
if not args.wetrun:
print("*** DRY RUN ***")
print("Total files added: {}".format(files_total))
print("\n".join(out_log))
if ignored_locales["missing"]:
print("Locales not available in Pontoon ({}):".format(
len(ignored_locales["missing"])))
print(", ".join(ignored_locales["missing"]))
if ignored_locales["incomplete"]:
print("Locales below {}% ({}):".format(
threshold, len(ignored_locales["incomplete"])))
print(", ".join(ignored_locales["incomplete"]))
def main():
p = argparse.ArgumentParser(
description="Remove obsolete and empty files in localized repositories"
)
p.add_argument("--noupdates",
help="Do not pull from remote",
action="store_true")
p.add_argument(
"--wetrun",
help="Commit local changes and push them to remote",
action="store_true",
)
p.add_argument("--locale",
help="Run on a specific locale",
action="store",
default="")
args = p.parse_args()
# Read paths from config file
[l10n_clones_path, quarantine_path
] = local_config.read_config(["l10n_clones_path", "quarantine_path"])
# Get the list of locales
if args.locale:
locales = [args.locale]
else:
locales = sorted(
[x for x in os.listdir(l10n_clones_path) if not x.startswith(".")])
# Store the list of files in quarantine
source_file_list = extractFileList(quarantine_path)
for locale in locales:
print("Locale: {}".format(locale))
need_commit = False
locale_path = os.path.join(l10n_clones_path, locale)
# Update locale repository, unless --noupdates was called explicitly
if not args.noupdates:
print("Updating repository...")
subprocess.run(["hg", "-R", locale_path, "pull", "-u"])
target_file_list = extractFileList(locale_path)
# Remove obsolete files (file in the locale, but not available in quarantine)
obsolete_files_list = []
for filename in target_file_list:
if filename not in source_file_list:
obsolete_files_list.append(filename)
if args.wetrun:
os.remove(os.path.join(locale_path, filename))
need_commit = True
obsolete_files_list.sort()
if obsolete_files_list:
print("Obsolete files:")
print("\n".join(obsolete_files_list))
# Find files containing no strings
empty_files_list = []
empty_files_list = findEmptyFiles(locale_path)
if empty_files_list:
if args.wetrun:
for filename in empty_files_list:
os.remove(os.path.join(locale_path, filename))
need_commit = True
print("Empty files:")
print("\n".join(empty_files_list))
if args.wetrun:
if need_commit:
# Commit changes
subprocess.run(["hg", "-R", locale_path, "addremove"])
subprocess.run([
"hg",
"-R",
locale_path,
"commit",
"-m",
"Bug 1443175 - Remove obsolete and empty files",
])
subprocess.run(["hg", "-R", locale_path, "push"])
else:
print("(dry run)")
def main():
p = argparse.ArgumentParser(
description="Remove Thunderbird folder from non Thunderbird locales")
p.add_argument("--noupdates",
help="Do not pull from remote",
action="store_true")
p.add_argument(
"--wetrun",
help="Commit local changes and push them to remote",
action="store_true",
)
p.add_argument("--locale",
help="Run on a specific locale",
action="store",
default="")
args = p.parse_args()
# Read paths from config file
[l10n_clones_path, quarantine_path
] = local_config.read_config(["l10n_clones_path", "quarantine_path"])
# Get the list of locales
if args.locale:
locales = [args.locale]
else:
locales = sorted(
[x for x in os.listdir(l10n_clones_path) if not x.startswith(".")])
for locale in locales:
print("Locale: {}".format(locale))
need_commit = False
locale_path = os.path.join(l10n_clones_path, locale)
# Update locale repository, unless --noupdates was called explicitly
if not args.noupdates:
print("Updating repository...")
subprocess.run(["hg", "-R", locale_path, "pull", "-u"])
# Only look at the "mail" subfolder
locale_mail_path = os.path.join(locale_path, "mail")
target_file_list = extractFileList(locale_mail_path)
filename = "messenger/aboutRights.ftl"
if len(target_file_list) == 1 and target_file_list == [filename]:
# The only file available is aboutRights.ftl, remove the folder
if args.wetrun:
os.remove(os.path.join(locale_mail_path, filename))
need_commit = True
if args.wetrun:
if need_commit:
# Commit changes
subprocess.run(["hg", "-R", locale_path, "addremove"])
subprocess.run([
"hg",
"-R",
locale_path,
"commit",
"-m",
"Remove migrated aboutRights.ftl from non Thunderbird locale",
])
subprocess.run(["hg", "-R", locale_path, "push"])
else:
print("(dry run)")