def prepare(language):
"""Prepares all necessary files for Mother Tongues dictionary web app and API
:param str language: path to either a txt file with paths to one or more MTD language configuration files **or** a directory containing MTD language configuration files
"""
language = os.path.abspath(language)
configs = return_configs_from_path(language)
ls = LanguageSuite(configs)
names = [l['config']['L1'] for l in ls.config_objects]
dictionaries = [Dictionary(l) for l in ls.config_objects]
write_static(dictionaries)
write_swagger(dictionaries)
set_active_dictionaries(ls.config_objects)
try:
if 40 in logger._cache and logger._cache[40]:
click.echo(
"Sorry, your build finished with some errors. Please look at your logs/messages above and try again."
)
else:
click.echo(
f"Successfully built static files for the following dictionaries: {names}. You may now run the app."
)
except AttributeError:
click.echo(
f"Successfully built static files for the following dictionaries: {names}. You may now run the app. *Warning* Mother Tongues uses logger caching to check if your build finished with errors. Because you are using a version of Python < 3.7 this feature is disabled and running your dictionary might not work."
)
def export(language, export_type, output):
"""Exports Mother Tongues Dictionary
:param str language: path to either a txt file with paths to one or more MTD language configuration files **or** a directory containing MTD language configuration files
:param str export_type: choose type of export: ["raw-json", "raw-xlsx", "raw-csv", "raw-psv", "raw-tsv", "raw-html", "js", "json", "web", "mobile", "github"]
:param str output: choose where output is exported to
"""
if export_type in ["mobile"]:
click.echo(f"this feature is coming soon")
else:
language = os.path.abspath(language)
configs = return_configs_from_path(language)
ls = LanguageSuite(configs)
dictionaries = [Dictionary(l) for l in ls.config_objects]
if output:
output = os.path.abspath(output)
if export_type.startswith('raw'):
for d in dictionaries:
ext = export_type.split('-')[1]
output_name = os.path.join(output, f"{d.name}.{ext}")
d.export_raw_data(output_name, export_type=ext)
elif export_type == "js" or export_type == "json":
for d in dictionaries:
config_output_name = os.path.join(
output, f"config-{d.name}.{export_type}")
data_output_name = os.path.join(
output, f"dict_cached-{d.name}.{export_type}")
with open(config_output_name, 'w', encoding='utf8') as f:
f.write(d.return_formatted_config(form=export_type))
with open(data_output_name, 'w', encoding='utf8') as f:
f.write(d.return_formatted_data(form=export_type))
elif export_type == 'github':
for d in dictionaries:
push_to_github(d)
elif export_type == "web":
freezer = Freezer(create_app())
@freezer.register_generator
def show_dictionary():
for l in ACTIVE:
language = slugify(l['config']['L1'])
yield {
'path': f"/dictionaries/{language}/",
'language': language
}
@freezer.register_generator
def show_stats():
for l in ACTIVE:
language = slugify(l['config']['L1'])
yield {
'path': f"/statistics/{language}/",
'language': language
}
freezer.freeze()
build_dir = os.path.join(os.path.dirname(parent_dir.__file__),
"build")
copy_tree(build_dir, os.path.join(output, "mtd-output"))
def create_suite(language_configs):
""" Given a list of LanguageConfigs (or a list of paths), create corresponding Dictionaries
Args:
language_configs (List[Union[LanguageConfig, str]]): A valid language config, or path to one
Returns:
LanguageSuite: The LanguageSuite of all dictionaries
"""
return LanguageSuite(language_configs)
def available(path):
""" Return names of all language configs at specified path
:param str path: path to either a txt file with paths to one or more MTD language configuration files **or** a directory containing MTD language configuration files
"""
try:
configs = return_configs_from_path(path)
ls = LanguageSuite(configs)
names = [l['config']['L1'] for l in ls.config_objects]
if names:
click.echo(f"The following languages are available at {path}: {names}")
else:
click.echo(UnfoundConfigError(path))
except UnfoundConfigError:
click.echo(UnfoundConfigError(path))
def check(language, alphabet):
''' Utility to check various aspects of your Mother Tongues Dictionary.
'''
base_template = '''
===========================================================
= MTD Report =
= =
= =
= =
= =
= =
===========================================================
Time: {time}
Dictionaries
++++++++++++
{dictionaries}
'''
dictionary_template = '''
{L1} to {L2} Dictionary
-----------------------
Config
------
Alphabet: {original_alphabet}
Tests
-----
Alphabet Test: {alphabet_test}
Characters Not in Alphabet: {letters_not_in_alphabet}
Null Items Test: {null_test}
Number of Null Items: {number_of_null_rows}
Duplicates Test: {duplicate_test}
Number of Duplicates: {number_of_duplicates}
Data
----
{duplicates}
'''
time = str(dt.now())
dictionaries = []
success = '✓'
fail = '✗'
language = os.path.abspath(language)
configs = return_configs_from_path(language)
ls = LanguageSuite(configs)
for i, lang in enumerate(ls.config_objects):
df = ls.dictionaries[i].df
dictionary = {'L1': lang.config['config']['L1'], 'L2': lang.config['config']['L2'], 'tests': {}}
# Check Alphabet
excess = check_alphabet(lang.config['config']['alphabet'], df)
if excess:
dictionary['tests']['alphabet_test'] = fail
else:
dictionary['tests']['alphabet_test'] = success
dictionary['letters_not_in_alphabet'] = json.dumps(excess)
# Check Null
null = return_null(df)
if null:
dictionary['tests']['null_test'] = fail
else:
dictionary['tests']['null_test'] = success
dictionary['number_of_null_rows'] = len(null)
# Check Duplicates
dupes = return_dupes(df)
if dupes:
dictionary['tests']['duplicate_test'] = fail
else:
dictionary['tests']['duplicate_test'] = success
dictionary['number_of_duplicates'] = len(dupes)
dictionary['duplicates'] = [x['value'] for x in dupes]
dictionary['original_alphabet'] = json.dumps(lang.config['config']['alphabet'])
dictionaries.append(dictionary)
base = base_template.format(time=time,
dictionaries='\n'.join([dictionary_template.format(L1=d['L1'],
L2=d['L2'],
alphabet_test=d['tests']['alphabet_test'],
letters_not_in_alphabet=d['letters_not_in_alphabet'],
null_test=d['tests']['null_test'],
number_of_null_rows=d['number_of_null_rows'],
duplicate_test=d['tests']['duplicate_test'],
number_of_duplicates=d['number_of_duplicates'],
duplicates=d['duplicates'],
original_alphabet=d['original_alphabet']) for d in dictionaries]))
with open(f'mtd-{time}.log', 'w') as f:
f.write(base)
def create_suite(
language_configs: List[Union[LanguageConfig, str]]) -> LanguageSuite:
""" Create a LanguageSuite from a list of LanguageConfigs or paths to language configs
"""
return LanguageSuite(language_configs)