def delete_all_generated_redirecting_pages(extended):
"""
deletes all the generated pages (inlcuding status.md)
:param extended: if True then the deleted pages will be printed
"""
for language in get_all_languages():
deleted_pages = []
include_page_path_to_main = get_include_page_path_to_main(
language=language)
if os.path.isfile(include_page_path_to_main):
os.remove(include_page_path_to_main)
deleted_pages.append(include_page_path_to_main)
include_page_path_to_toc = get_include_page_path_to_toc(
language=language)
if os.path.isfile(include_page_path_to_toc):
os.remove(include_page_path_to_toc)
deleted_pages.append(include_page_path_to_toc)
if language != MAIN_LANGUAGE:
deleted_pages = delete_redirecting_help_pages(language=language)
index = get_local_file_path(language=language, page="index.md")
if os.path.isfile(index):
os.remove(index)
deleted_pages.append("index.md")
num_deleted_pages = len(deleted_pages)
logger.ok(
"Deleted {num_deleted_pages} page(s) for language '{lang}'".format(
lang=language, num_deleted_pages=num_deleted_pages))
if extended and num_deleted_pages != 0:
logger.neutral("\t{}".format(deleted_pages))
if os.path.isfile(FILE_STATUS):
os.remove(FILE_STATUS)
logger.ok("Delete the markdown status file")
def status_create_markdown():
"""
creates a markdown file of the current status and opens it
"""
def write_properties(property_files):
markdown.append("\n| Property file | Keys | Keys translated | Keys not translated | % translated |\n")
markdown.append("| ------------- | ---- | --------------- | ------------------- | ------------ |\n")
for file in property_files:
lines = read_file(file)
keys = get_translations_as_dict(lines=lines)
keys_missing_value = get_empty_keys(lines=lines)
num_keys = len(keys)
num_keys_missing_value = len(keys_missing_value)
num_keys_translated = num_keys - num_keys_missing_value
percent_translated = int((num_keys_translated / float(num_keys)) * 100) if num_keys != 0 else 0
markdown.append("| {file} | {num_keys} | {num_keys_translated} | {num_keys_missing} | {percent_translated} |\n"
.format(file=get_filename(filepath=file), num_keys=num_keys, num_keys_translated=num_keys_translated, num_keys_missing=num_keys_missing_value, percent_translated=percent_translated))
markdown = []
date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
markdown.append("### Localization files status ({date} - Branch `{branch}` `{hash}`)\n".format(date=date, branch=get_current_branch(), hash=get_current_hash_short()))
write_properties(property_files=get_all_jabref_properties())
write_properties(property_files=get_all_menu_properties())
write_file(STATUS_FILE, markdown)
logger.ok("Current status written to {}".format(STATUS_FILE))
open_file(STATUS_FILE)
def remove_all_help_suffixes(extended):
for language in get_all_languages():
renamed_pages = remove_help_suffix(language=language)
num_renamed_pages = len(renamed_pages)
logger.ok("Language: '{language}'".format(language=language))
logger.ok("\tremoved the 'Help' suffix from {} pages".format(num_renamed_pages))
if extended and num_renamed_pages != 0:
logger.neutral("\t\t{}".format(renamed_pages))
def check_language_status(language, extended):
"""
checks the status of this language and prints it on the console (maybe call `update` before?)
:param language: string
:param extended: boolean: if the specific pages should be printed
"""
outdated_pages = get_outdated_pages(language=language)
num_outdated_pages = len(outdated_pages)
if language == MAIN_LANGUAGE:
main_not_translated_pages = get_pages_not_in_main_language()
num_main_not_translated_pages = len(main_not_translated_pages)
log = logger.ok if num_main_not_translated_pages == 0 and num_outdated_pages == 0 else logger.error
log("Main Language: '{lang}'".format(lang=MAIN_LANGUAGE))
if num_main_not_translated_pages == 0:
logger.ok("\thas no conflicts")
else:
for key, value in main_not_translated_pages.iteritems():
logger.error(
"\tlanguage '{lang}' has {count} additional page(s)".
format(lang=key, count=len(value)))
if extended and len(value) != 0:
logger.neutral("\t\t{pages}".format(pages=value))
else:
not_translated_pages = get_not_translated_pages(
main_language=MAIN_LANGUAGE, secondary_language=language)
num_not_translated_pages = len(not_translated_pages)
not_redirected_pages = get_not_redirected_pages(
main_language=MAIN_LANGUAGE, secondary_language=language)
num_not_redirected_pages = len(not_redirected_pages)
log = logger.ok if num_not_redirected_pages == 0 and num_not_translated_pages == 0 and num_outdated_pages == 0 else logger.error
log("Language: '{lang}'".format(lang=language))
log = logger.ok if num_not_translated_pages == 0 else logger.error
log("\thas {num_not_translated_pages} not translated page(s)".format(
num_not_translated_pages=num_not_translated_pages))
if num_not_translated_pages != 0 and extended:
logger.neutral("\t\t{not_translated_pages}".format(
not_translated_pages=not_translated_pages))
log = logger.ok if num_not_redirected_pages == 0 else logger.error
log("\thas {num_not_redirected_pages} not redirected page(s)".format(
num_not_redirected_pages=num_not_redirected_pages))
if num_not_redirected_pages != 0 and extended:
logger.neutral("\t\t{not_redirected_pages}".format(
not_redirected_pages=not_redirected_pages))
log = logger.ok if num_outdated_pages == 0 else logger.error
log("\thas {num_outdated_pages} outdated page(s)".format(
num_outdated_pages=num_outdated_pages))
if num_outdated_pages != 0 and extended:
logger.neutral(
"\t\t{outdated_pages}".format(outdated_pages=outdated_pages))
def check_properties(main_property_file, property_files):
main_lines = read_file(filename=main_property_file)
main_keys = get_keys_from_lines(lines=main_lines)
# the main property file gets compared to itself, but that is OK
for file in property_files:
filename = get_filename(filepath=file)
lines = read_file(file)
keys = get_keys_from_lines(lines=lines)
keys_missing = get_missing_keys(main_keys, keys)
keys_obsolete = get_missing_keys(keys, main_keys)
keys_duplicate = get_duplicates(lines=lines)
keys_not_translated = get_empty_keys(lines=lines)
num_keys = len(keys)
num_keys_missing = len(keys_missing)
num_keys_not_translated = len(keys_not_translated)
num_keys_obsolete = len(keys_obsolete)
num_keys_duplicate = len(keys_duplicate)
num_keys_translated = num_keys - num_keys_not_translated
log = logger.error if num_keys_missing != 0 or num_keys_not_translated != 0 or num_keys_obsolete != 0 or num_keys_duplicate != 0 else logger.ok
log("Status of file '{file}' with {num_keys} Keys".format(
file=filename, num_keys=num_keys))
logger.ok("\t{} translated keys".format(num_keys_translated))
log = logger.error if num_keys_not_translated != 0 else logger.ok
log("\t{} not translated keys".format(num_keys_not_translated))
if extended and num_keys_not_translated != 0:
logger.neutral(u"\t\t{}".format(
", ".join(keys_not_translated)))
log = logger.error if num_keys_missing != 0 else logger.ok
log("\t{} missing keys".format(num_keys_missing))
if extended and num_keys_missing != 0:
logger.neutral(u"\t\t{}".format(", ".join(keys_missing)))
log = logger.error if num_keys_obsolete != 0 else logger.ok
log("\t{} obsolete keys".format(num_keys_obsolete))
if extended and num_keys_obsolete != 0:
logger.neutral(u"\t\t{}".format(", ".join(keys_obsolete)))
log = logger.error if num_keys_duplicate != 0 else logger.ok
log("\t{} duplicates".format(num_keys_duplicate))
if extended and num_keys_duplicate != 0:
logger.neutral(u"\t\t{}".format(", ".join(keys_duplicate)))
def check_properties(main_property_file, property_files):
main_lines = read_file(filename=main_property_file)
main_keys = get_keys_from_lines(lines=main_lines)
# the main property file gets compared to itself, but that is OK
for file in property_files:
filename = get_filename(filepath=file)
lines = read_file(file)
keys = get_keys_from_lines(lines=lines)
keys_missing = get_missing_keys(main_keys, keys)
keys_obsolete = get_missing_keys(keys, main_keys)
keys_duplicate = get_duplicates(lines=lines)
keys_not_translated = get_empty_keys(lines=lines)
num_keys = len(keys)
num_keys_missing = len(keys_missing)
num_keys_not_translated = len(keys_not_translated)
num_keys_obsolete = len(keys_obsolete)
num_keys_duplicate = len(keys_duplicate)
num_keys_translated = num_keys - num_keys_not_translated
log = logger.error if num_keys_missing != 0 or num_keys_not_translated != 0 or num_keys_obsolete != 0 or num_keys_duplicate != 0 else logger.ok
log("Status of file '{file}' with {num_keys} Keys".format(file=filename, num_keys=num_keys))
logger.ok("\t{} translated keys".format(num_keys_translated))
log = logger.error if num_keys_not_translated != 0 else logger.ok
log("\t{} not translated keys".format(num_keys_not_translated))
if extended and num_keys_not_translated != 0:
logger.neutral(u"\t\t{}".format(", ".join(keys_not_translated)))
log = logger.error if num_keys_missing != 0 else logger.ok
log("\t{} missing keys".format(num_keys_missing))
if extended and num_keys_missing != 0:
logger.neutral(u"\t\t{}".format(", ".join(keys_missing)))
log = logger.error if num_keys_obsolete != 0 else logger.ok
log("\t{} obsolete keys".format(num_keys_obsolete))
if extended and num_keys_obsolete != 0:
logger.neutral(u"\t\t{}".format(", ".join(keys_obsolete)))
log = logger.error if num_keys_duplicate != 0 else logger.ok
log("\t{} duplicates".format(num_keys_duplicate))
if extended and num_keys_duplicate != 0:
logger.neutral(u"\t\t{}".format(", ".join(keys_duplicate)))
def do_api(path, account_id):
if os.path.isfile(API_KEY_FILE):
with open(API_KEY_FILE) as f:
key = f.read().strip()
r = requests.get(API_ENDPOINT + path, auth=(account_id, key))
if r.status_code == 200:
log.ok("Success!")
return 0
else:
if r.status_code == 401:
log.err("Invalid account_id and/or api_key given")
elif r.status_code == 403:
log.err("You have no permission to use the that resource")
elif r.status_code == 404:
log.err("The endpoint could not be found")
return 1
else:
log.err("Could not find API key file [{key_file}]".format(key_file=API_KEY_FILE))
return 1
def update_index(extended):
"""
updates the index of all languages
:param extended: boolean: if the pages which could not be processed be printed
"""
def get_link_title_and_categories(language, page):
"""
:param language: string
:param page: string
:return: (string, string, string): file link, title, categories
"""
with open(get_local_file_path(language=MAIN_LANGUAGE,
page=page)) as yaml_main_page:
with open(get_local_file_path(language=language,
page=page)) as yaml_page:
main_post = frontmatter.load(yaml_main_page)
post = frontmatter.load(yaml_page)
file_link = get_relative_file_link(language=language,
page=page)
title = post[
FRONTMATTER_TITLE] if FRONTMATTER_TITLE in post.keys(
) else ""
if not title:
title = main_post[
FRONTMATTER_TITLE] if FRONTMATTER_TITLE in main_post.keys(
) else ""
if language != MAIN_LANGUAGE and FRONTMATTER_CATEGORIES in post.keys(
):
logger.warn(
u"categories are only needed to be declared in the {main} file, ignoring categories in following: '{lang}', '{file}'"
.format(main=MAIN_LANGUAGE, lang=language, file=page))
# getting the categories from the english file, prevents getting a translated category
categories = main_post[
FRONTMATTER_CATEGORIES] if FRONTMATTER_CATEGORIES in main_post.keys(
) else []
for key in categories:
if not does_category_exist(key):
logger.error(
u"Following category is not going to be considered '{lang}', '{file}', '{key}'"
.format(lang=language, file=page, key=key))
return file_link, title, categories
def create_index_file(order, index_file, index, indentation=2):
"""
creates the index file out of the category order and the presorted pages
:param order: list of strings: the order of the categories
:param index_file: list of strings
:param index: dict of strings and dicts: the presorted pages
:param indentation: int: how much the category is indented
"""
# 2 loops to write the link before subsections
for key, value in sorted(index.items(), key=lambda x: x[1]):
if type(value) is not dict:
index_file.append(u"- [{title}]({link})\n".format(title=value,
link=key))
last_category = ""
for category in order:
if type(category) is list:
create_index_file(order=category,
index_file=index_file,
index=index[last_category],
indentation=indentation + 1)
else:
last_category = category
if category not in index.keys():
logger.error(
u"\tFollowing category is non-existent: {category}".
format(category=category))
continue
translated_category = get_localization(language=language,
key=category)
index_file.append(u"\n{indentation} {title}\n".format(
indentation="#" * indentation, title=translated_category))
create_index_file(order=[],
index_file=index_file,
index=index[category],
indentation=indentation + 1)
considered = True
index_file.append("\n")
for language in get_all_languages():
renamed_pages = remove_help_suffix(language=language)
redirected_pages = generate_missing_redirects(language=language)
generate_inlcudes(language=language)
missing_frontmatter = []
num_pages_on_index = 0
index = {}
for page in get_help_pages_in(language=language):
if is_old_help_page_redirecting_to_new_one(language=language,
page=page):
continue
file_link, title, categories = get_link_title_and_categories(
language=language, page=page)
if not title or not categories:
missing_frontmatter.append(file_link)
continue
index_tmp = index
for category in categories:
if category not in index_tmp:
new_dict = {}
index_tmp[category] = new_dict
index_tmp = new_dict
else:
index_tmp = index_tmp[category]
index_tmp[file_link] = title
num_pages_on_index += 1
title = get_localization(language=language, key="Help contents")
more_questions = get_localization(
language=language,
key=
"You can't find a solution to your problem? You still have questions?"
)
forum_hint = get_localization(
language=language, key="Use the online forum to get more support!")
index_file = [
get_index_header(title=title,
more_questions=more_questions,
forum=forum_hint)
]
create_index_file(order=get_categories_order(),
index_file=index_file,
index=index)
write_file(language + "/index.md", index_file)
num_missing_frontmatter = len(missing_frontmatter)
num_redirected_pages = len(redirected_pages)
num_renamed_pages = len(renamed_pages)
logger.ok("Language: '{language}'".format(language=language))
logger.ok("\tgenerated the 'include' pages")
logger.ok("\tremoved the 'Help' suffix from {} pages".format(
num_renamed_pages))
if extended and num_renamed_pages != 0:
logger.neutral("\t\t{}".format(renamed_pages))
if language != MAIN_LANGUAGE:
logger.ok("\tredirected {} page(s)".format(num_redirected_pages))
if num_redirected_pages != 0 and extended:
logger.neutral("\t\t{}".format(redirected_pages))
if num_missing_frontmatter != 0:
logger.error("\t{} page(s) with missing frontmatter".format(
num_missing_frontmatter))
if extended:
logger.neutral("\t\t{}".format(missing_frontmatter))
logger.ok("\tcreated index with {} page(s)".format(num_pages_on_index))
def update_properties(main_property_file, other_property_files):
main_lines = read_file(filename=main_property_file)
# saved the stripped lines
write_file(main_property_file, main_lines)
main_keys = get_keys_from_lines(lines=main_lines)
main_duplicates = get_duplicates(lines=main_lines)
num_main_duplicates = len(main_duplicates)
if num_main_duplicates != 0:
logger.error(
"There are {num_duplicates} duplicates in {file}, please fix them manually"
.format(num_duplicates=num_main_duplicates,
file=get_filename(filepath=main_property_file)))
if extended:
logger.neutral(u"\t{}".format(", ".join(main_duplicates)))
return
for other_property_file in other_property_files:
filename = get_filename(filepath=other_property_file)
lines = read_file(filename=other_property_file)
keys, not_fixed, fixed = fix_duplicates(lines=lines)
num_keys = len(keys)
num_not_fixed = len(not_fixed)
num_fixed = len(fixed)
if num_not_fixed != 0:
logger.error(
"There are {num_not_fixed_duplicates} ambiguous duplicates in {file}, please fix them manually"
.format(num_not_fixed_duplicates=num_not_fixed,
file=filename))
if extended:
logger.error(u"\t{}".format(u", ".join(not_fixed)))
continue
keys_missing = get_missing_keys(main_keys, keys)
keys_obsolete = get_missing_keys(keys, main_keys)
num_keys_missing = len(keys_missing)
num_keys_obsolete = len(keys_obsolete)
for missing_key in keys_missing:
keys[missing_key] = ""
for obsolete_key in keys_obsolete:
del keys[obsolete_key]
other_lines_to_write = []
for line in main_lines:
key = get_key_from_line(line)
if key is not None:
other_lines_to_write.append(u"{key}={value}\r\n".format(
key=key, value=keys[key]))
else:
other_lines_to_write.append(line)
sorted = len(lines) != len(other_lines_to_write)
if not sorted:
for old_line, new_lines in zip(lines, other_lines_to_write):
if old_line != new_lines:
sorted = True
write_file(filename=other_property_file,
content=other_lines_to_write)
logger.ok("Processing file '{file}' with {num_keys} Keys".format(
file=filename, num_keys=num_keys))
if num_fixed != 0:
logger.ok(
"\tfixed {} unambiguous duplicates".format(num_fixed))
if extended:
logger.neutral(u"\t\t{}".format(", ".join(fixed)))
if num_keys_missing != 0:
logger.ok("\tadded {} missing keys".format(num_keys_missing))
if extended:
logger.neutral(u"\t\t{}".format(", ".join(keys_missing)))
if num_keys_obsolete != 0:
logger.ok(
"\tdeleted {} obsolete keys".format(num_keys_obsolete))
if extended:
logger.neutral(u"\t\t{}".format(", ".join(keys_obsolete)))
if sorted:
logger.ok("\thas been sorted successfully")
def update_properties(main_property_file, other_property_files):
main_lines = read_file(filename=main_property_file)
# saved the stripped lines
write_file(main_property_file, main_lines)
main_keys = get_keys_from_lines(lines=main_lines)
main_duplicates = get_duplicates(lines=main_lines)
num_main_duplicates = len(main_duplicates)
if num_main_duplicates != 0:
logger.error("There are {num_duplicates} duplicates in {file}, please fix them manually".format(num_duplicates=num_main_duplicates, file=get_filename(filepath=main_property_file)))
if extended:
logger.neutral(u"\t{}".format(", ".join(main_duplicates)))
return
for other_property_file in other_property_files:
filename = get_filename(filepath=other_property_file)
lines = read_file(filename=other_property_file)
keys, not_fixed, fixed = fix_duplicates(lines=lines)
num_keys = len(keys)
num_not_fixed = len(not_fixed)
num_fixed = len(fixed)
if num_not_fixed != 0:
logger.error("There are {num_not_fixed_duplicates} ambiguous duplicates in {file}, please fix them manually".format(num_not_fixed_duplicates=num_not_fixed, file=filename))
if extended:
logger.error(u"\t{}".format(u", ".join(not_fixed)))
continue
keys_missing = get_missing_keys(main_keys, keys)
keys_obsolete = get_missing_keys(keys, main_keys)
num_keys_missing = len(keys_missing)
num_keys_obsolete = len(keys_obsolete)
for missing_key in keys_missing:
keys[missing_key] = ""
for obsolete_key in keys_obsolete:
del keys[obsolete_key]
other_lines_to_write = []
for line in main_lines:
key = get_key_from_line(line)
if key is not None:
other_lines_to_write.append(u"{key}={value}\r\n".format(key=key, value=keys[key]))
else:
other_lines_to_write.append(line)
sorted = len(lines) != len(other_lines_to_write)
if not sorted:
for old_line, new_lines in zip(lines, other_lines_to_write):
if old_line != new_lines:
sorted = True
write_file(filename=other_property_file, content=other_lines_to_write)
logger.ok("Processing file '{file}' with {num_keys} Keys".format(file=filename, num_keys=num_keys))
if num_fixed != 0:
logger.ok("\tfixed {} unambiguous duplicates".format(num_fixed))
if extended:
logger.neutral(u"\t\t{}".format(", ".join(fixed)))
if num_keys_missing != 0:
logger.ok("\tadded {} missing keys".format(num_keys_missing))
if extended:
logger.neutral(u"\t\t{}".format(", ".join(keys_missing)))
if num_keys_obsolete != 0:
logger.ok("\tdeleted {} obsolete keys".format(num_keys_obsolete))
if extended:
logger.neutral(u"\t\t{}".format(", ".join(keys_obsolete)))
if sorted:
logger.ok("\thas been sorted successfully")