def _process_downloaded_files():
"""
Convert all CSV json files to JSON and ensure consistent diffs with ordered keys.
Also copy over django.po files
"""
for lang_object in utils.available_languages(include_in_context=True):
locale_path = utils.local_locale_path(lang_object)
perseus_path = utils.local_perseus_locale_path(lang_object)
csv_locale_dir_path = os.path.join(utils.local_locale_csv_path(),
lang_object["crowdin_code"])
perseus_locale_dir_path = os.path.join(
utils.local_perseus_locale_csv_path(), lang_object["crowdin_code"])
# Make sure that the Perseus directory for CSV_FILES/{lang_code} exists
if not os.path.exists(perseus_locale_dir_path):
os.makedirs(perseus_locale_dir_path)
files = os.listdir(csv_locale_dir_path) + os.listdir(
perseus_locale_dir_path)
for file_name in files:
if file_name == PERSEUS_CSV:
file_path = os.path.join(perseus_locale_dir_path, file_name)
else:
file_path = os.path.join(csv_locale_dir_path, file_name)
if file_name == DJANGO_PO:
shutil.move(file_path, locale_path)
continue
if "csv" not in file_name:
continue
# Account for csv reading differences in Pythons 2 and 3
try:
newline = None if sys.version_info[0] < 3 else ""
mode = "r+b" if sys.version_info[0] < 3 else "r"
encoding = None if sys.version_info[0] < 3 else "utf-8"
csv_file = io.open(file_path,
mode=mode,
encoding=encoding,
newline=newline)
except EnvironmentError:
logging.info(
"Failed to find CSV file in: {}".format(file_path))
continue
with csv_file as f:
csv_data = list(row for row in csv.DictReader(f))
data = _locale_data_from_csv(csv_data)
if file_name == PERSEUS_CSV:
utils.json_dump_formatted(data, perseus_path,
file_name.replace("csv", "json"))
else:
utils.json_dump_formatted(data, locale_path,
file_name.replace("csv", "json"))
def _format_json_files():
"""
re-print all json files to ensure consistent diffs with ordered keys
"""
locale_paths = []
for lang_object in utils.supported_languages(include_in_context=True):
locale_paths.append(utils.local_locale_path(lang_object))
locale_paths.append(utils.local_perseus_locale_path(lang_object))
for locale_path in locale_paths:
for file_name in os.listdir(locale_path):
if not file_name.endswith(".json"):
continue
file_path = os.path.join(locale_path, file_name)
with io.open(file_path, mode="r", encoding="utf-8") as f:
data = json.load(f)
utils.json_dump_formatted(data, file_path)
def _format_json_files():
"""
re-print all json files to ensure consistent diffs with ordered keys
"""
for lang_object in utils.supported_languages(include_in_context=True):
locale_path = utils.local_locale_path(lang_object)
perseus_path = utils.local_perseus_locale_path(lang_object)
csv_locale_dir_path = os.path.join(utils.local_locale_csv_path(),
lang_object["crowdin_code"])
for file_name in os.listdir(csv_locale_dir_path):
if file_name.endswith("json"):
# Then it is a Perseus JSON file - just copy it.
source = os.path.join(csv_locale_dir_path, file_name)
target = os.path.join(perseus_path, file_name)
try:
os.makedirs(perseus_path)
except:
pass
shutil.copyfile(source, target)
continue
elif not file_name.endswith("csv"):
continue
csv_path = os.path.join(csv_locale_dir_path, file_name)
# Account for csv reading differences in Pythons 2 and 3
if sys.version_info[0] < 3:
csv_file = open(csv_path, "rb")
else:
csv_file = open(csv_path, "r", newline="")
with csv_file as f:
csv_data = list(row for row in csv.DictReader(f))
data = _locale_data_from_csv(csv_data)
utils.json_dump_formatted(data, locale_path,
file_name.replace("csv", "json"))
def update_manifest(ref=None):
"""
Given a git reference in the Noto repo, such as a git commit hash or tag, extract
information about the fonts available for use and save that information to the
manifest file.
The Noto repo currently contains both an older style and the newer "Phase 3"
fonts. Phase 3 fonts have more consistent internal metrics which makes them amenable
to being merged together, which we make use of. The older fonts are still usable,
but cannot be merged together.
Noto also contains both standard and "UI" variants of many fonts. When a font has a
UI variant, it means that some of the glyphs in the standard variant are very tall
and might overflow a typical line of text; the UI variant has the glypsh redrawn
to fit.
When searching for fonts to include, we take all language fonts that have both a
regular and a bold variant, with preference given to Phase 3 and UI variants.
"""
# grab the head of master
if not ref:
logging.info("Using head of master")
ref = _request("git/refs/heads/master")["object"]["sha"]
logging.info("Generating new manifest for reference '{}'".format(ref))
git_tree = _request("git/trees/{}?recursive=1".format(ref))
# backups
font_info = _font_info(git_tree, ref, OLD_STYLE_PATH, _old_download_url)
# prefer phase 3, replacing old-styles when possible
font_info.update(_font_info(git_tree, ref, PHASE_3_PATH, _p3_download_url))
new_manifest = {KEY_REF: ref, KEY_FONTS: font_info}
utils.json_dump_formatted(new_manifest, FONTS_SOURCE, FONT_MANIFEST_NAME)