def _upload_translation(branch, lang_object):
if no_crowdin_branch(branch, get_crowdin_details()):
logging.error("Branch '{}' not found.".format(branch))
sys.exit(1)
logging.info(
"Crowdin: uploading translation files for '{}' to '{}'...".format(
lang_object[utils.KEY_CROWDIN_CODE], branch))
url = UPLOAD_TRANSLATION_URL.format(
branch=branch, language=lang_object[utils.KEY_CROWDIN_CODE])
file_names = []
for name in os.listdir(utils.local_locale_path(lang_object)):
if is_string_file(name):
file_names.append(name)
for name in os.listdir(utils.local_perseus_locale_path(lang_object)):
if is_string_file(name):
file_names.append(name)
for chunk in _chunks(file_names):
references = [_translation_upload_ref(f, lang_object) for f in chunk]
r = requests.post(url, files=references)
r.raise_for_status()
for ref in references:
ref[1].close()
logging.info("Crowdin: upload succeeded!")
def main(title, message, link_text):
"""
Generate JSON suitable for sending in nutrition facts notifications
"""
supported_languages = utils.supported_languages(include_in_context=False,
include_english=True)
output = {}
for lang_object in supported_languages:
file_path = os.path.join(utils.local_locale_path(lang_object),
FILE_NAME)
i18n = {}
with open(file_path) as f:
input_data = json.load(f)
if title in input_data:
i18n[I18N_TITLE] = input_data[title]
if message in input_data:
i18n[I18N_MESSAGE] = input_data[message]
if link_text in input_data:
i18n[I18N_LINK_TEXT] = input_data[link_text]
output[lang_object[utils.KEY_INTL_CODE]] = i18n
# output JSON
print(
json.dumps(output,
sort_keys=True,
indent=2,
separators=(",", ": "),
ensure_ascii=False))
def _translation_upload_ref(file_name, lang_object):
if file_name == PERSEUS_CSV: # hack for perseus, assumes the same file name
source_path = utils.local_perseus_locale_path(lang_object)
else:
source_path = utils.local_locale_path(lang_object)
file_pointer = open(os.path.join(source_path, file_name), "rb")
return ("files[{0}]".format(file_name), file_pointer)
def _process_downloaded_files():
"""
Convert all CSV json files to JSON and ensure consistent diffs with ordered keys.
Also copy over django.po files
"""
for lang_object in utils.available_languages(include_in_context=True):
locale_path = utils.local_locale_path(lang_object)
perseus_path = utils.local_perseus_locale_path(lang_object)
csv_locale_dir_path = os.path.join(utils.local_locale_csv_path(),
lang_object["crowdin_code"])
perseus_locale_dir_path = os.path.join(
utils.local_perseus_locale_csv_path(), lang_object["crowdin_code"])
# Make sure that the Perseus directory for CSV_FILES/{lang_code} exists
if not os.path.exists(perseus_locale_dir_path):
os.makedirs(perseus_locale_dir_path)
files = os.listdir(csv_locale_dir_path) + os.listdir(
perseus_locale_dir_path)
for file_name in files:
if file_name == PERSEUS_CSV:
file_path = os.path.join(perseus_locale_dir_path, file_name)
else:
file_path = os.path.join(csv_locale_dir_path, file_name)
if file_name == DJANGO_PO:
shutil.move(file_path, locale_path)
continue
if "csv" not in file_name:
continue
# Account for csv reading differences in Pythons 2 and 3
try:
newline = None if sys.version_info[0] < 3 else ""
mode = "r+b" if sys.version_info[0] < 3 else "r"
encoding = None if sys.version_info[0] < 3 else "utf-8"
csv_file = io.open(file_path,
mode=mode,
encoding=encoding,
newline=newline)
except EnvironmentError:
logging.info(
"Failed to find CSV file in: {}".format(file_path))
continue
with csv_file as f:
csv_data = list(row for row in csv.DictReader(f))
data = _locale_data_from_csv(csv_data)
if file_name == PERSEUS_CSV:
utils.json_dump_formatted(data, perseus_path,
file_name.replace("csv", "json"))
else:
utils.json_dump_formatted(data, locale_path,
file_name.replace("csv", "json"))
def command_download(branch):
"""
Downloads and updates the local translation files from the given branch on Crowdin
"""
logging.info("Crowdin: downloading '{}'...".format(branch))
# delete previous files
_wipe_translations(utils.LOCALE_PATH)
_wipe_translations(utils.PERSEUS_LOCALE_PATH)
for lang_object in utils.supported_languages(include_in_context=True):
code = lang_object[utils.KEY_CROWDIN_CODE]
url = DOWNLOAD_URL.format(language=code, branch=branch)
r = requests.get(url)
r.raise_for_status()
z = zipfile.ZipFile(io.BytesIO(r.content))
target = utils.local_locale_path(lang_object)
logging.info("\tExtracting {} to {}".format(code, target))
z.extractall(target)
# hack for perseus
perseus_target = utils.local_perseus_locale_path(lang_object)
if not os.path.exists(perseus_target):
os.makedirs(perseus_target)
shutil.move(
os.path.join(target, PERSEUS_FILE),
os.path.join(perseus_target, PERSEUS_FILE),
)
_format_json_files() # clean them up to make git diffs more meaningful
logging.info("Crowdin: download succeeded!")
def command_gen_subset_fonts():
"""
Creates custom fonts that attempt to contain all the glyphs and other font features
that are used in user-facing text for the translation in each language.
We make a separate subset font for common strings, which generally overaps somewhat
with the individual language subsets. This slightly increases how much the client
needs to download on first request, but reduces Kolibri's distribution size by a
couple megabytes.
"""
logging.info("generating subset fonts...")
_clean_up(SCOPE_COMMON)
_clean_up(SCOPE_SUBSET)
_subset_and_merge_fonts(
text=" ".join(_get_common_strings()),
default_font=NOTO_SANS_LATIN,
subset_reg_path=_woff_font_path(SCOPE_COMMON, is_bold=False),
subset_bold_path=_woff_font_path(SCOPE_COMMON, is_bold=True),
)
languages = utils.supported_languages(include_in_context=True,
include_english=True)
for lang_info in languages:
logging.info("gen subset for {}".format(lang_info[utils.KEY_ENG_NAME]))
strings = []
strings.extend(_get_lang_strings(utils.local_locale_path(lang_info)))
strings.extend(
_get_lang_strings(utils.local_perseus_locale_path(lang_info)))
name = lang_info[utils.KEY_INTL_CODE]
_subset_and_merge_fonts(
text=" ".join(strings),
default_font=lang_info[utils.KEY_DEFAULT_FONT],
subset_reg_path=_woff_font_path(_scoped(SCOPE_SUBSET, name),
is_bold=False),
subset_bold_path=_woff_font_path(_scoped(SCOPE_SUBSET, name),
is_bold=True),
)
# generate common subset file
_generate_inline_font_css(name=SCOPE_COMMON, font_family=SCOPE_COMMON)
# generate language-specific subset font files
languages = utils.supported_languages(include_in_context=True,
include_english=True)
for lang in languages:
_generate_inline_font_css(
name=_scoped(SCOPE_SUBSET, lang[utils.KEY_INTL_CODE]),
font_family=SCOPE_SUBSET,
)
logging.info("subsets created")
def _format_json_files():
"""
re-print all json files to ensure consistent diffs with ordered keys
"""
locale_paths = []
for lang_object in utils.supported_languages(include_in_context=True):
locale_paths.append(utils.local_locale_path(lang_object))
locale_paths.append(utils.local_perseus_locale_path(lang_object))
for locale_path in locale_paths:
for file_name in os.listdir(locale_path):
if not file_name.endswith(".json"):
continue
file_path = os.path.join(locale_path, file_name)
with io.open(file_path, mode="r", encoding="utf-8") as f:
data = json.load(f)
utils.json_dump_formatted(data, file_path)
def main(title, message, link_text):
"""
Generate JSON suitable for sending in nutrition facts notifications
"""
available_languages = utils.available_languages(include_in_context=False,
include_english=True)
output = {}
for lang_object in available_languages:
file_path = os.path.join(utils.local_locale_path(lang_object),
FILE_NAME)
i18n = {}
# If the language code is "en", parse csv file instead of json file.
# Note that `make i18n-extract-frontend` should have been run to generate the csv file.
if lang_object[utils.KEY_INTL_CODE] == "en":
file_path = file_path.replace("json", "csv")
with open(file_path) as f:
csv_reader = csv.DictReader(f)
for row in csv_reader:
identifier = row["Identifier"]
if title == identifier:
i18n[I18N_TITLE] = row["Source String"]
if message == identifier:
i18n[I18N_MESSAGE] = row["Source String"]
if link_text == identifier:
i18n[I18N_LINK_TEXT] = row["Source String"]
else:
with open(file_path) as f:
input_data = json.load(f)
if title in input_data:
i18n[I18N_TITLE] = input_data[title]
if message in input_data:
i18n[I18N_MESSAGE] = input_data[message]
if link_text in input_data:
i18n[I18N_LINK_TEXT] = input_data[link_text]
output[lang_object[utils.KEY_INTL_CODE]] = i18n
# output JSON
print(
json.dumps(output,
sort_keys=True,
indent=2,
separators=(",", ": "),
ensure_ascii=False))
def _format_json_files():
"""
re-print all json files to ensure consistent diffs with ordered keys
"""
for lang_object in utils.supported_languages(include_in_context=True):
locale_path = utils.local_locale_path(lang_object)
perseus_path = utils.local_perseus_locale_path(lang_object)
csv_locale_dir_path = os.path.join(utils.local_locale_csv_path(),
lang_object["crowdin_code"])
for file_name in os.listdir(csv_locale_dir_path):
if file_name.endswith("json"):
# Then it is a Perseus JSON file - just copy it.
source = os.path.join(csv_locale_dir_path, file_name)
target = os.path.join(perseus_path, file_name)
try:
os.makedirs(perseus_path)
except:
pass
shutil.copyfile(source, target)
continue
elif not file_name.endswith("csv"):
continue
csv_path = os.path.join(csv_locale_dir_path, file_name)
# Account for csv reading differences in Pythons 2 and 3
if sys.version_info[0] < 3:
csv_file = open(csv_path, "rb")
else:
csv_file = open(csv_path, "r", newline="")
with csv_file as f:
csv_data = list(row for row in csv.DictReader(f))
data = _locale_data_from_csv(csv_data)
utils.json_dump_formatted(data, locale_path,
file_name.replace("csv", "json"))
