def _process_downloaded_files():
"""
Convert all CSV json files to JSON and ensure consistent diffs with ordered keys.
Also copy over django.po files
"""
for lang_object in utils.available_languages(include_in_context=True):
locale_path = utils.local_locale_path(lang_object)
perseus_path = utils.local_perseus_locale_path(lang_object)
csv_locale_dir_path = os.path.join(utils.local_locale_csv_path(),
lang_object["crowdin_code"])
perseus_locale_dir_path = os.path.join(
utils.local_perseus_locale_csv_path(), lang_object["crowdin_code"])
# Make sure that the Perseus directory for CSV_FILES/{lang_code} exists
if not os.path.exists(perseus_locale_dir_path):
os.makedirs(perseus_locale_dir_path)
files = os.listdir(csv_locale_dir_path) + os.listdir(
perseus_locale_dir_path)
for file_name in files:
if file_name == PERSEUS_CSV:
file_path = os.path.join(perseus_locale_dir_path, file_name)
else:
file_path = os.path.join(csv_locale_dir_path, file_name)
if file_name == DJANGO_PO:
shutil.move(file_path, locale_path)
continue
if "csv" not in file_name:
continue
# Account for csv reading differences in Pythons 2 and 3
try:
newline = None if sys.version_info[0] < 3 else ""
mode = "r+b" if sys.version_info[0] < 3 else "r"
encoding = None if sys.version_info[0] < 3 else "utf-8"
csv_file = io.open(file_path,
mode=mode,
encoding=encoding,
newline=newline)
except EnvironmentError:
logging.info(
"Failed to find CSV file in: {}".format(file_path))
continue
with csv_file as f:
csv_data = list(row for row in csv.DictReader(f))
data = _locale_data_from_csv(csv_data)
if file_name == PERSEUS_CSV:
utils.json_dump_formatted(data, perseus_path,
file_name.replace("csv", "json"))
else:
utils.json_dump_formatted(data, locale_path,
file_name.replace("csv", "json"))
def download_translations(branch):
"""
Download translations from the given branch
"""
checkPerseus()
checkApiKey()
logging.info("Crowdin: downloading '{}'...".format(branch))
# delete previous files
_wipe_translations(utils.LOCALE_PATH)
if utils.PERSEUS_LOCALE_PATH:
_wipe_translations(utils.PERSEUS_LOCALE_PATH)
for lang_object in utils.available_languages(include_in_context=True):
code = lang_object[utils.KEY_CROWDIN_CODE]
url = DOWNLOAD_URL.format(language=code, branch=branch)
r = requests.get(url)
r.raise_for_status()
z = zipfile.ZipFile(io.BytesIO(r.content))
target = utils.local_locale_csv_path()
logging.info("\tExtracting {} to {}".format(code, target))
z.extractall(target)
if utils.PERSEUS_LOCALE_PATH:
# hack for perseus
perseus_target = os.path.join(
utils.local_perseus_locale_csv_path(),
lang_object["crowdin_code"])
# TODO - Update this to work with perseus properly - likely to need to update
# the kolibri-exercise-perseus-plugin repo directly to produce a CSV for its
# translations.
if not os.path.exists(perseus_target):
os.makedirs(perseus_target)
try:
shutil.move(
os.path.join(target, lang_object["crowdin_code"],
PERSEUS_CSV),
os.path.join(perseus_target, PERSEUS_CSV),
)
except Exception as e:
logging.error("Ignoring an exception")
logging.error(e)
# TODO Don't need to format here... going to do this in the new command.
_process_downloaded_files(
) # clean them up to make git diffs more meaningful
logging.info("Crowdin: download succeeded!")
def command_download(branch):
"""
Downloads and updates the local translation files from the given branch on Crowdin
"""
logging.info("Crowdin: downloading '{}'...".format(branch))
# delete previous files
_wipe_translations(utils.LOCALE_PATH)
_wipe_translations(utils.PERSEUS_LOCALE_PATH)
for lang_object in utils.supported_languages(include_in_context=True):
code = lang_object[utils.KEY_CROWDIN_CODE]
url = DOWNLOAD_URL.format(language=code, branch=branch)
r = requests.get(url)
r.raise_for_status()
z = zipfile.ZipFile(io.BytesIO(r.content))
target = utils.local_locale_csv_path()
logging.info("\tExtracting {} to {}".format(code, target))
z.extractall(target)
# hack for perseus
perseus_target = utils.local_perseus_locale_csv_path()
## TODO - Update this to work with perseus properly - likely to need to update
## the kolibri-exercise-perseus-plugin repo directly to produce a CSV for its
## translations.
if not os.path.exists(perseus_target):
os.makedirs(perseus_target)
try:
shutil.move(
os.path.join(target, PERSEUS_CSV),
os.path.join(perseus_target, PERSEUS_CSV),
)
except:
pass
## TODO Don't need to format here... going to do this in the new command.
_format_json_files() # clean them up to make git diffs more meaningful
logging.info("Crowdin: download succeeded!")