def create_dictionary( language_config: Union[LanguageConfig, str]) -> Dictionary: """ Given a LanguageConfig (or a path to one), create a Dictionary """ if isinstance(language_config, LanguageConfig): return Dictionary(language_config) else: language_config = LanguageConfig(language_config) return Dictionary(language_config)
def push_to_github(dictionary: Dictionary): '''Push Dictionary to GitHub ''' if not 'github_credentials_path' in dictionary.config: raise CredentialsMissingError(dictionary.name) else: base_url = "https://api.github.com/repos/" with open(dictionary.config['github_credentials_path'], 'r', encoding='utf8') as f: creds = json.load(f) config_path = os.path.join(base_url, creds['user'], creds['repo'], 'contents', creds['path'], 'config.js') dict_path = os.path.join(base_url, creds['user'], creds['repo'], 'contents', creds['path'], 'dict_cached.js') config_r = requests.get(config_path) dict_r = requests.get(dict_path) config_rjson = config_r.json() dict_rjson = dict_r.json() try: config_sha = config_rjson['sha'] dict_sha = dict_rjson['sha'] except KeyError: config_sha = "" dict_sha = "" config_b64 = b64encode(str.encode(dictionary.return_formatted_config('js'))) config_json = json.dumps({ "message": "update", "content": config_b64.decode('utf8'), "sha": config_sha }) dict_b64 = b64encode(str.encode(dictionary.return_formatted_data('js'))) dict_json = json.dumps({ "message": "update", "content": dict_b64.decode('utf8'), "sha": dict_sha }) headers={'Authorization': 'token {tok}'.format(tok=creds['token'])} if input(f"Overwrite file at {config_path}? ") in ['yes', 'y']: code = requests.put(config_path, data=config_json, headers=headers) logger.info(f"Tried pushing file to {config_path} and received code {code} in response.") else: print("Aborted. You must type 'yes' or 'y' to push to GitHub.") if input(f"Overwrite file at {dict_path}? ") in ['yes', 'y']: code = requests.put(dict_path, data=dict_json, headers=headers) logger.info(f"Tried pushing file to {config_path} and received code {code} in response.") else: print("Aborted. You must type 'yes' or 'y' to push to GitHub.")
def create_dictionary(language_config): """ Given a LanguageConfig (or a path to one), create a Dictionary Args: language_config (Union[LanguageConfig, str]): A valid language config, or path to one Returns: Dictionary: The Dictionary based on the provided configuration """ if isinstance(language_config, LanguageConfig): return Dictionary(language_config) else: language_config = LanguageConfig(language_config) return Dictionary(language_config)
def show_stats(language): if not slugify(language) in active_names: abort(404) else: stats = {} # Add Handler to collect problems with Dictionary problems = [] lh = ListHandler(problems) lh.setLevel(logging.INFO) logger.addHandler(lh) dictionary = Dictionary(active_names[language]) logger.removeHandler(lh) df = dictionary.df stats['total_len'] = len(dictionary) if 'audio' in df: stats['audio_len'] = return_len_of_not_null(df, 'audio') else: stats['audio_len'] = 0 if 'img' in df: stats['img_len'] = return_len_of_not_null(df, 'img') else: stats['img_len'] = 0 stats['source'] = return_unique_len(df, 'source') problems = sorted(problems, key=lambda k: k['levelno'], reverse=True) return render_template('stats.html', name=language, stats=stats, alphabet=dictionary.config['alphabet'], problems=problems)
def prepare(language): """Prepares all necessary files for Mother Tongues dictionary web app and API :param str language: path to either a txt file with paths to one or more MTD language configuration files **or** a directory containing MTD language configuration files """ language = os.path.abspath(language) configs = return_configs_from_path(language) ls = LanguageSuite(configs) names = [l['config']['L1'] for l in ls.config_objects] dictionaries = [Dictionary(l) for l in ls.config_objects] write_static(dictionaries) write_swagger(dictionaries) set_active_dictionaries(ls.config_objects) try: if 40 in logger._cache and logger._cache[40]: click.echo( "Sorry, your build finished with some errors. Please look at your logs/messages above and try again." ) else: click.echo( f"Successfully built static files for the following dictionaries: {names}. You may now run the app." ) except AttributeError: click.echo( f"Successfully built static files for the following dictionaries: {names}. You may now run the app. *Warning* Mother Tongues uses logger caching to check if your build finished with errors. Because you are using a version of Python < 3.7 this feature is disabled and running your dictionary might not work." )
def export(language, export_type, output): """Exports Mother Tongues Dictionary :param str language: path to either a txt file with paths to one or more MTD language configuration files **or** a directory containing MTD language configuration files :param str export_type: choose type of export: ["raw-json", "raw-xlsx", "raw-csv", "raw-psv", "raw-tsv", "raw-html", "js", "json", "web", "mobile", "github"] :param str output: choose where output is exported to """ if export_type in ["mobile"]: click.echo(f"this feature is coming soon") else: language = os.path.abspath(language) configs = return_configs_from_path(language) ls = LanguageSuite(configs) dictionaries = [Dictionary(l) for l in ls.config_objects] if output: output = os.path.abspath(output) if export_type.startswith('raw'): for d in dictionaries: ext = export_type.split('-')[1] output_name = os.path.join(output, f"{d.name}.{ext}") d.export_raw_data(output_name, export_type=ext) elif export_type == "js" or export_type == "json": for d in dictionaries: config_output_name = os.path.join( output, f"config-{d.name}.{export_type}") data_output_name = os.path.join( output, f"dict_cached-{d.name}.{export_type}") with open(config_output_name, 'w', encoding='utf8') as f: f.write(d.return_formatted_config(form=export_type)) with open(data_output_name, 'w', encoding='utf8') as f: f.write(d.return_formatted_data(form=export_type)) elif export_type == 'github': for d in dictionaries: push_to_github(d) elif export_type == "web": freezer = Freezer(create_app()) @freezer.register_generator def show_dictionary(): for l in ACTIVE: language = slugify(l['config']['L1']) yield { 'path': f"/dictionaries/{language}/", 'language': language } @freezer.register_generator def show_stats(): for l in ACTIVE: language = slugify(l['config']['L1']) yield { 'path': f"/statistics/{language}/", 'language': language } freezer.freeze() build_dir = os.path.join(os.path.dirname(parent_dir.__file__), "build") copy_tree(build_dir, os.path.join(output, "mtd-output"))
def __init__(self, config_paths: List[Union[str, LanguageConfig]]): self.languages_path = os.path.dirname(ldir.__file__) self.config_objects = [] for cp in config_paths: if isinstance(cp, LanguageConfig): self.config_objects.append(cp) else: self.config_objects.append(LanguageConfig(cp)) self.dictionaries = [Dictionary(co) for co in self.config_objects]
def __init__(self): self.available = [{ "plain": l['config']['L1'], "slug": slugify(l['config']['L1']) } for l in ACTIVE] self.dictionaries = [Dictionary(d) for d in ACTIVE] self.parser = reqparse.RequestParser() self.parser.add_argument( 'only-config', dest='only-config', type=bool, location='args', default=False, required=False, help='Return only config', ) self.parser.add_argument( 'only-data', dest='only-data', type=bool, location='args', default=False, required=False, help='Return only data', ) self.parser.add_argument( 'available', dest='available', type=bool, location='args', default=False, required=False, help='Return all available languages', ) self.parser.add_argument( 'name', dest='name', type=str, location='args', action='append', required=False, help='Specify languages', )