def download_raw_file_to_cache(repo_data): url = repo_data["url"] url_filename = repo_data["url_filename"] cache_dir = repo_data["cache_dir"] filepath = repo_data["filepath"] cached_path(url, cache_dir=cache_dir) shutil.move(Path(UNDERTHESEA_FOLDER) / cache_dir / url_filename, Path(UNDERTHESEA_FOLDER) / cache_dir / filepath)
def download_zip_file_to_cache(repo_data): url = repo_data["url"] cache_dir = repo_data["cache_dir"] url_filename = repo_data["url_filename"] cached_path(url, cache_dir=cache_dir) filepath = Path(CACHE_ROOT) / cache_dir / url_filename cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(filepath) zip.extractall(cache_folder) os.remove(filepath)
def download(model_name): if model_name not in REPO: print(f"No matching distribution found for '{model_name}'") return model_path = REPO[model_name]["model_path"] cache_dir = REPO[model_name]["cache_dir"] model_path = Path(CACHE_ROOT) / cache_dir / model_path if Path(model_path).exists(): print(f"Model is already existed: '{model_name}' in {model_path}") return if model_name == "TC_GENERAL": url = "https://www.dropbox.com/s/866offu8wglrcej/tc_svm_vntc_20190607.zip?dl=1" cached_path(url, cache_dir=cache_dir) model_path = Path( CACHE_ROOT) / cache_dir / "tc_svm_vntc_20190607.zip?dl=1" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / "tc_svm_vntc_20190607", Path(CACHE_ROOT) / cache_dir / "TC_GENERAL", ) os.remove(model_path) if model_name == "TC_GENERAL_V131": url = "https://github.com/undertheseanlp/playground/releases/download/1.3.x/tc_svm_vntc_20201228.zip" cached_path(url, cache_dir=cache_dir) model_path = Path( CACHE_ROOT) / cache_dir / "tc_svm_vntc_20201228.zip" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / "tc_svm_vntc_20201228", Path(CACHE_ROOT) / cache_dir / "TC_GENERAL_V131", ) os.remove(model_path) if model_name == "TC_BANK_V131": model_id = "tc_svm_ubc-1_20210107" url = f"https://github.com/undertheseanlp/playground/releases/download/1.3.x/{model_id}.zip" cached_path(url, cache_dir=cache_dir) model_path = Path(CACHE_ROOT) / cache_dir / f"{model_id}.zip" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / model_id, Path(CACHE_ROOT) / cache_dir / model_name, ) os.remove(model_path) if model_name == "TC_BANK": url = "https://www.dropbox.com/s/prrjlypbrr6ze6p/tc_svm_uts2017_bank_20190607.zip?dl=1" cached_path(url, cache_dir=cache_dir) model_path = Path( CACHE_ROOT ) / cache_dir / "tc_svm_uts2017_bank_20190607.zip?dl=1" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / "tc_svm_uts2017_bank_20190607", Path(CACHE_ROOT) / cache_dir / "TC_BANK", ) os.remove(model_path) if model_name == "SA_GENERAL": url = "https://www.dropbox.com/s/xfj1ity3egabv77/sa_svm_aivivn2019_20190615.zip?dl=1" cached_path(url, cache_dir=cache_dir) model_path = Path( CACHE_ROOT) / cache_dir / "sa_svm_aivivn2019_20190615.zip?dl=1" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / "sa_svm_aivivn2019_20190615", Path(CACHE_ROOT) / cache_dir / "SA_GENERAL", ) os.remove(model_path) if model_name == "SA_GENERAL_V131": model_id = "sa_svm_vlsp2016-sa_20210107" url = f"https://github.com/undertheseanlp/playground/releases/download/1.3.x/{model_id}.zip" cached_path(url, cache_dir=cache_dir) model_path = Path(CACHE_ROOT) / cache_dir / f"{model_id}.zip" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / model_id, Path(CACHE_ROOT) / cache_dir / model_name, ) os.remove(model_path) if model_name == "SA_BANK": url = "https://www.dropbox.com/s/yo6sf6ofpdb3hlh/sa_svm_uts2017_bank_20190611.zip?dl=1" cached_path(url, cache_dir=cache_dir) model_path = Path( CACHE_ROOT ) / cache_dir / "sa_svm_uts2017_bank_20190611.zip?dl=1" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / "sa_svm_uts2017_bank_20190611", Path(CACHE_ROOT) / cache_dir / "SA_BANK", ) os.remove(model_path) if model_name == "SA_BANK_V131": model_id = "sa_svm_ubs-1_20210107" url = f"https://github.com/undertheseanlp/playground/releases/download/1.3.x/{model_id}.zip" cached_path(url, cache_dir=cache_dir) model_path = Path(CACHE_ROOT) / cache_dir / f"{model_id}.zip" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / model_id, Path(CACHE_ROOT) / cache_dir / model_name, ) os.remove(model_path)
def download(model_name): if model_name not in REPO: print(f"No matching distribution found for '{model_name}'") return model_path = REPO[model_name]["model_path"] cache_dir = REPO[model_name]["cache_dir"] model_path = Path(CACHE_ROOT) / cache_dir / model_path if Path(model_path).exists(): print(f"Model is already existed: '{model_name}' in {model_path}") return if model_name == "TC_GENERAL": url = "https://www.dropbox.com/s/866offu8wglrcej/tc_svm_vntc_20190607.zip?dl=1" cached_path(url, cache_dir=cache_dir) model_path = Path( CACHE_ROOT) / cache_dir / "tc_svm_vntc_20190607.zip?dl=1" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / "tc_svm_vntc_20190607", Path(CACHE_ROOT) / cache_dir / "TC_GENERAL", ) os.remove(model_path) if model_name == "TC_BANK": url = "https://www.dropbox.com/s/prrjlypbrr6ze6p/tc_svm_uts2017_bank_20190607.zip?dl=1" cached_path(url, cache_dir=cache_dir) model_path = Path( CACHE_ROOT ) / cache_dir / "tc_svm_uts2017_bank_20190607.zip?dl=1" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / "tc_svm_uts2017_bank_20190607", Path(CACHE_ROOT) / cache_dir / "TC_BANK", ) os.remove(model_path) if model_name == "SA_GENERAL": url = "https://www.dropbox.com/s/xfj1ity3egabv77/sa_svm_aivivn2019_20190615.zip?dl=1" cached_path(url, cache_dir=cache_dir) model_path = Path( CACHE_ROOT) / cache_dir / "sa_svm_aivivn2019_20190615.zip?dl=1" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / "sa_svm_aivivn2019_20190615", Path(CACHE_ROOT) / cache_dir / "SA_GENERAL", ) os.remove(model_path) if model_name == "SA_BANK": url = "https://www.dropbox.com/s/yo6sf6ofpdb3hlh/sa_svm_uts2017_bank_20190611.zip?dl=1" cached_path(url, cache_dir=cache_dir) model_path = Path( CACHE_ROOT ) / cache_dir / "sa_svm_uts2017_bank_20190611.zip?dl=1" cache_folder = Path(CACHE_ROOT) / cache_dir zip = zipfile.ZipFile(model_path) zip.extractall(cache_folder) os.rename( Path(CACHE_ROOT) / cache_dir / "sa_svm_uts2017_bank_20190611", Path(CACHE_ROOT) / cache_dir / "SA_BANK", ) os.remove(model_path)