def create_dictionary(
language_config: Union[LanguageConfig, str]) -> Dictionary:
""" Given a LanguageConfig (or a path to one), create a Dictionary
"""
if isinstance(language_config, LanguageConfig):
return Dictionary(language_config)
else:
language_config = LanguageConfig(language_config)
return Dictionary(language_config)
def push_to_github(dictionary: Dictionary):
'''Push Dictionary to GitHub
'''
if not 'github_credentials_path' in dictionary.config:
raise CredentialsMissingError(dictionary.name)
else:
base_url = "https://api.github.com/repos/"
with open(dictionary.config['github_credentials_path'], 'r', encoding='utf8') as f:
creds = json.load(f)
config_path = os.path.join(base_url, creds['user'], creds['repo'], 'contents', creds['path'], 'config.js')
dict_path = os.path.join(base_url, creds['user'], creds['repo'], 'contents', creds['path'], 'dict_cached.js')
config_r = requests.get(config_path)
dict_r = requests.get(dict_path)
config_rjson = config_r.json()
dict_rjson = dict_r.json()
try:
config_sha = config_rjson['sha']
dict_sha = dict_rjson['sha']
except KeyError:
config_sha = ""
dict_sha = ""
config_b64 = b64encode(str.encode(dictionary.return_formatted_config('js')))
config_json = json.dumps({
"message": "update",
"content": config_b64.decode('utf8'),
"sha": config_sha
})
dict_b64 = b64encode(str.encode(dictionary.return_formatted_data('js')))
dict_json = json.dumps({
"message": "update",
"content": dict_b64.decode('utf8'),
"sha": dict_sha
})
headers={'Authorization': 'token {tok}'.format(tok=creds['token'])}
if input(f"Overwrite file at {config_path}? ") in ['yes', 'y']:
code = requests.put(config_path, data=config_json, headers=headers)
logger.info(f"Tried pushing file to {config_path} and received code {code} in response.")
else:
print("Aborted. You must type 'yes' or 'y' to push to GitHub.")
if input(f"Overwrite file at {dict_path}? ") in ['yes', 'y']:
code = requests.put(dict_path, data=dict_json, headers=headers)
logger.info(f"Tried pushing file to {config_path} and received code {code} in response.")
else:
print("Aborted. You must type 'yes' or 'y' to push to GitHub.")
def create_dictionary(language_config):
""" Given a LanguageConfig (or a path to one), create a Dictionary
Args:
language_config (Union[LanguageConfig, str]): A valid language config, or path to one
Returns:
Dictionary: The Dictionary based on the provided configuration
"""
if isinstance(language_config, LanguageConfig):
return Dictionary(language_config)
else:
language_config = LanguageConfig(language_config)
return Dictionary(language_config)
def show_stats(language):
if not slugify(language) in active_names:
abort(404)
else:
stats = {}
# Add Handler to collect problems with Dictionary
problems = []
lh = ListHandler(problems)
lh.setLevel(logging.INFO)
logger.addHandler(lh)
dictionary = Dictionary(active_names[language])
logger.removeHandler(lh)
df = dictionary.df
stats['total_len'] = len(dictionary)
if 'audio' in df:
stats['audio_len'] = return_len_of_not_null(df, 'audio')
else:
stats['audio_len'] = 0
if 'img' in df:
stats['img_len'] = return_len_of_not_null(df, 'img')
else:
stats['img_len'] = 0
stats['source'] = return_unique_len(df, 'source')
problems = sorted(problems, key=lambda k: k['levelno'], reverse=True)
return render_template('stats.html',
name=language,
stats=stats,
alphabet=dictionary.config['alphabet'],
problems=problems)
def prepare(language):
"""Prepares all necessary files for Mother Tongues dictionary web app and API
:param str language: path to either a txt file with paths to one or more MTD language configuration files **or** a directory containing MTD language configuration files
"""
language = os.path.abspath(language)
configs = return_configs_from_path(language)
ls = LanguageSuite(configs)
names = [l['config']['L1'] for l in ls.config_objects]
dictionaries = [Dictionary(l) for l in ls.config_objects]
write_static(dictionaries)
write_swagger(dictionaries)
set_active_dictionaries(ls.config_objects)
try:
if 40 in logger._cache and logger._cache[40]:
click.echo(
"Sorry, your build finished with some errors. Please look at your logs/messages above and try again."
)
else:
click.echo(
f"Successfully built static files for the following dictionaries: {names}. You may now run the app."
)
except AttributeError:
click.echo(
f"Successfully built static files for the following dictionaries: {names}. You may now run the app. *Warning* Mother Tongues uses logger caching to check if your build finished with errors. Because you are using a version of Python < 3.7 this feature is disabled and running your dictionary might not work."
)
def export(language, export_type, output):
"""Exports Mother Tongues Dictionary
:param str language: path to either a txt file with paths to one or more MTD language configuration files **or** a directory containing MTD language configuration files
:param str export_type: choose type of export: ["raw-json", "raw-xlsx", "raw-csv", "raw-psv", "raw-tsv", "raw-html", "js", "json", "web", "mobile", "github"]
:param str output: choose where output is exported to
"""
if export_type in ["mobile"]:
click.echo(f"this feature is coming soon")
else:
language = os.path.abspath(language)
configs = return_configs_from_path(language)
ls = LanguageSuite(configs)
dictionaries = [Dictionary(l) for l in ls.config_objects]
if output:
output = os.path.abspath(output)
if export_type.startswith('raw'):
for d in dictionaries:
ext = export_type.split('-')[1]
output_name = os.path.join(output, f"{d.name}.{ext}")
d.export_raw_data(output_name, export_type=ext)
elif export_type == "js" or export_type == "json":
for d in dictionaries:
config_output_name = os.path.join(
output, f"config-{d.name}.{export_type}")
data_output_name = os.path.join(
output, f"dict_cached-{d.name}.{export_type}")
with open(config_output_name, 'w', encoding='utf8') as f:
f.write(d.return_formatted_config(form=export_type))
with open(data_output_name, 'w', encoding='utf8') as f:
f.write(d.return_formatted_data(form=export_type))
elif export_type == 'github':
for d in dictionaries:
push_to_github(d)
elif export_type == "web":
freezer = Freezer(create_app())
@freezer.register_generator
def show_dictionary():
for l in ACTIVE:
language = slugify(l['config']['L1'])
yield {
'path': f"/dictionaries/{language}/",
'language': language
}
@freezer.register_generator
def show_stats():
for l in ACTIVE:
language = slugify(l['config']['L1'])
yield {
'path': f"/statistics/{language}/",
'language': language
}
freezer.freeze()
build_dir = os.path.join(os.path.dirname(parent_dir.__file__),
"build")
copy_tree(build_dir, os.path.join(output, "mtd-output"))
def __init__(self, config_paths: List[Union[str, LanguageConfig]]):
self.languages_path = os.path.dirname(ldir.__file__)
self.config_objects = []
for cp in config_paths:
if isinstance(cp, LanguageConfig):
self.config_objects.append(cp)
else:
self.config_objects.append(LanguageConfig(cp))
self.dictionaries = [Dictionary(co) for co in self.config_objects]
def __init__(self):
self.available = [{
"plain": l['config']['L1'],
"slug": slugify(l['config']['L1'])
} for l in ACTIVE]
self.dictionaries = [Dictionary(d) for d in ACTIVE]
self.parser = reqparse.RequestParser()
self.parser.add_argument(
'only-config',
dest='only-config',
type=bool,
location='args',
default=False,
required=False,
help='Return only config',
)
self.parser.add_argument(
'only-data',
dest='only-data',
type=bool,
location='args',
default=False,
required=False,
help='Return only data',
)
self.parser.add_argument(
'available',
dest='available',
type=bool,
location='args',
default=False,
required=False,
help='Return all available languages',
)
self.parser.add_argument(
'name',
dest='name',
type=str,
location='args',
action='append',
required=False,
help='Specify languages',
)