def stage_descriptions(env): run_gen_tool(env.gen_tool, out=env.get_subprocess_out(), err=env.get_subprocess_out(), intermediate_data_path=env.intermediate_path, user_resource_path=env.user_resource_path, dump_wikipedia_urls=env.wiki_url_path, idToWikidata=env.id_to_wikidata_path) langs = ("en", "ru", "es") checker = check_and_get_checker(env.popularity_path) download_from_wikipedia_tags(env.wiki_url_path, env.descriptions_path, langs, checker) download_from_wikidata_tags(env.id_to_wikidata_path, env.descriptions_path, langs, checker) @country_stage_log def stage_write_descriptions(env, country, **kwargs): stages.run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.mwm_path, user_resource_path=env.user_resource_path, wikipedia_pages=env.descriptions_path, idToWikidata=env.id_to_wikidata_path, output=country, **kwargs) mwms = env.get_mwm_names() countries = filter(lambda x: x not in WORLDS_NAMES, mwms) with ThreadPool() as pool: pool.map(partial(stage_write_descriptions, env), countries)
def main(): log.setLevel(logging.WARNING) wikipediaapi.log.setLevel(logging.WARNING) args = parse_args() wikipedia_file = args.wikipedia wikidata_file = args.wikidata output_dir = args.output_dir popularity_file = args.popularity langs = list(itertools.chain.from_iterable(args.langs)) os.makedirs(output_dir, exist_ok=True) checker = check_and_get_checker(popularity_file) download_from_wikipedia_tags(wikipedia_file, output_dir, langs, checker) if wikidata_file is None: log.warning(f"Wikidata file not set.") elif os.path.exists(wikidata_file): download_from_wikidata_tags(wikidata_file, output_dir, langs, checker) else: log.warning(f"Wikidata ({wikidata_file}) file not set.")
def apply(self, env: Env): run_gen_tool( env.gen_tool, out=env.get_subprocess_out(), err=env.get_subprocess_out(), intermediate_data_path=env.paths.intermediate_data_path, user_resource_path=env.paths.user_resource_path, dump_wikipedia_urls=env.paths.wiki_url_path, idToWikidata=env.paths.id_to_wikidata_path, ) langs = ("en", "ru", "es", "fr", "de") checker = check_and_get_checker(env.paths.popularity_path) download_from_wikipedia_tags(env.paths.wiki_url_path, env.paths.descriptions_path, langs, checker) download_from_wikidata_tags(env.paths.id_to_wikidata_path, env.paths.descriptions_path, langs, checker)