def _process_downloaded_files(): """ Convert all CSV json files to JSON and ensure consistent diffs with ordered keys. Also copy over django.po files """ for lang_object in utils.available_languages(include_in_context=True): locale_path = utils.local_locale_path(lang_object) perseus_path = utils.local_perseus_locale_path(lang_object) csv_locale_dir_path = os.path.join(utils.local_locale_csv_path(), lang_object["crowdin_code"]) perseus_locale_dir_path = os.path.join( utils.local_perseus_locale_csv_path(), lang_object["crowdin_code"]) # Make sure that the Perseus directory for CSV_FILES/{lang_code} exists if not os.path.exists(perseus_locale_dir_path): os.makedirs(perseus_locale_dir_path) files = os.listdir(csv_locale_dir_path) + os.listdir( perseus_locale_dir_path) for file_name in files: if file_name == PERSEUS_CSV: file_path = os.path.join(perseus_locale_dir_path, file_name) else: file_path = os.path.join(csv_locale_dir_path, file_name) if file_name == DJANGO_PO: shutil.move(file_path, locale_path) continue if "csv" not in file_name: continue # Account for csv reading differences in Pythons 2 and 3 try: newline = None if sys.version_info[0] < 3 else "" mode = "r+b" if sys.version_info[0] < 3 else "r" encoding = None if sys.version_info[0] < 3 else "utf-8" csv_file = io.open(file_path, mode=mode, encoding=encoding, newline=newline) except EnvironmentError: logging.info( "Failed to find CSV file in: {}".format(file_path)) continue with csv_file as f: csv_data = list(row for row in csv.DictReader(f)) data = _locale_data_from_csv(csv_data) if file_name == PERSEUS_CSV: utils.json_dump_formatted(data, perseus_path, file_name.replace("csv", "json")) else: utils.json_dump_formatted(data, locale_path, file_name.replace("csv", "json"))
def download_translations(branch): """ Download translations from the given branch """ checkPerseus() checkApiKey() logging.info("Crowdin: downloading '{}'...".format(branch)) # delete previous files _wipe_translations(utils.LOCALE_PATH) if utils.PERSEUS_LOCALE_PATH: _wipe_translations(utils.PERSEUS_LOCALE_PATH) for lang_object in utils.available_languages(include_in_context=True): code = lang_object[utils.KEY_CROWDIN_CODE] url = DOWNLOAD_URL.format(language=code, branch=branch) r = requests.get(url) r.raise_for_status() z = zipfile.ZipFile(io.BytesIO(r.content)) target = utils.local_locale_csv_path() logging.info("\tExtracting {} to {}".format(code, target)) z.extractall(target) if utils.PERSEUS_LOCALE_PATH: # hack for perseus perseus_target = os.path.join( utils.local_perseus_locale_csv_path(), lang_object["crowdin_code"]) # TODO - Update this to work with perseus properly - likely to need to update # the kolibri-exercise-perseus-plugin repo directly to produce a CSV for its # translations. if not os.path.exists(perseus_target): os.makedirs(perseus_target) try: shutil.move( os.path.join(target, lang_object["crowdin_code"], PERSEUS_CSV), os.path.join(perseus_target, PERSEUS_CSV), ) except Exception as e: logging.error("Ignoring an exception") logging.error(e) # TODO Don't need to format here... going to do this in the new command. _process_downloaded_files( ) # clean them up to make git diffs more meaningful logging.info("Crowdin: download succeeded!")
def command_download(branch): """ Downloads and updates the local translation files from the given branch on Crowdin """ logging.info("Crowdin: downloading '{}'...".format(branch)) # delete previous files _wipe_translations(utils.LOCALE_PATH) _wipe_translations(utils.PERSEUS_LOCALE_PATH) for lang_object in utils.supported_languages(include_in_context=True): code = lang_object[utils.KEY_CROWDIN_CODE] url = DOWNLOAD_URL.format(language=code, branch=branch) r = requests.get(url) r.raise_for_status() z = zipfile.ZipFile(io.BytesIO(r.content)) target = utils.local_locale_csv_path() logging.info("\tExtracting {} to {}".format(code, target)) z.extractall(target) # hack for perseus perseus_target = utils.local_perseus_locale_csv_path() ## TODO - Update this to work with perseus properly - likely to need to update ## the kolibri-exercise-perseus-plugin repo directly to produce a CSV for its ## translations. if not os.path.exists(perseus_target): os.makedirs(perseus_target) try: shutil.move( os.path.join(target, PERSEUS_CSV), os.path.join(perseus_target, PERSEUS_CSV), ) except: pass ## TODO Don't need to format here... going to do this in the new command. _format_json_files() # clean them up to make git diffs more meaningful logging.info("Crowdin: download succeeded!")