def download_raw_file_to_cache(repo_data):
url = repo_data["url"]
url_filename = repo_data["url_filename"]
cache_dir = repo_data["cache_dir"]
filepath = repo_data["filepath"]
cached_path(url, cache_dir=cache_dir)
shutil.move(Path(UNDERTHESEA_FOLDER) / cache_dir / url_filename,
Path(UNDERTHESEA_FOLDER) / cache_dir / filepath)
def download_zip_file_to_cache(repo_data):
url = repo_data["url"]
cache_dir = repo_data["cache_dir"]
url_filename = repo_data["url_filename"]
cached_path(url, cache_dir=cache_dir)
filepath = Path(CACHE_ROOT) / cache_dir / url_filename
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(filepath)
zip.extractall(cache_folder)
os.remove(filepath)
def download(model_name):
if model_name not in REPO:
print(f"No matching distribution found for '{model_name}'")
return
model_path = REPO[model_name]["model_path"]
cache_dir = REPO[model_name]["cache_dir"]
model_path = Path(CACHE_ROOT) / cache_dir / model_path
if Path(model_path).exists():
print(f"Model is already existed: '{model_name}' in {model_path}")
return
if model_name == "TC_GENERAL":
url = "https://www.dropbox.com/s/866offu8wglrcej/tc_svm_vntc_20190607.zip?dl=1"
cached_path(url, cache_dir=cache_dir)
model_path = Path(
CACHE_ROOT) / cache_dir / "tc_svm_vntc_20190607.zip?dl=1"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / "tc_svm_vntc_20190607",
Path(CACHE_ROOT) / cache_dir / "TC_GENERAL",
)
os.remove(model_path)
if model_name == "TC_GENERAL_V131":
url = "https://github.com/undertheseanlp/playground/releases/download/1.3.x/tc_svm_vntc_20201228.zip"
cached_path(url, cache_dir=cache_dir)
model_path = Path(
CACHE_ROOT) / cache_dir / "tc_svm_vntc_20201228.zip"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / "tc_svm_vntc_20201228",
Path(CACHE_ROOT) / cache_dir / "TC_GENERAL_V131",
)
os.remove(model_path)
if model_name == "TC_BANK_V131":
model_id = "tc_svm_ubc-1_20210107"
url = f"https://github.com/undertheseanlp/playground/releases/download/1.3.x/{model_id}.zip"
cached_path(url, cache_dir=cache_dir)
model_path = Path(CACHE_ROOT) / cache_dir / f"{model_id}.zip"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / model_id,
Path(CACHE_ROOT) / cache_dir / model_name,
)
os.remove(model_path)
if model_name == "TC_BANK":
url = "https://www.dropbox.com/s/prrjlypbrr6ze6p/tc_svm_uts2017_bank_20190607.zip?dl=1"
cached_path(url, cache_dir=cache_dir)
model_path = Path(
CACHE_ROOT
) / cache_dir / "tc_svm_uts2017_bank_20190607.zip?dl=1"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / "tc_svm_uts2017_bank_20190607",
Path(CACHE_ROOT) / cache_dir / "TC_BANK",
)
os.remove(model_path)
if model_name == "SA_GENERAL":
url = "https://www.dropbox.com/s/xfj1ity3egabv77/sa_svm_aivivn2019_20190615.zip?dl=1"
cached_path(url, cache_dir=cache_dir)
model_path = Path(
CACHE_ROOT) / cache_dir / "sa_svm_aivivn2019_20190615.zip?dl=1"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / "sa_svm_aivivn2019_20190615",
Path(CACHE_ROOT) / cache_dir / "SA_GENERAL",
)
os.remove(model_path)
if model_name == "SA_GENERAL_V131":
model_id = "sa_svm_vlsp2016-sa_20210107"
url = f"https://github.com/undertheseanlp/playground/releases/download/1.3.x/{model_id}.zip"
cached_path(url, cache_dir=cache_dir)
model_path = Path(CACHE_ROOT) / cache_dir / f"{model_id}.zip"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / model_id,
Path(CACHE_ROOT) / cache_dir / model_name,
)
os.remove(model_path)
if model_name == "SA_BANK":
url = "https://www.dropbox.com/s/yo6sf6ofpdb3hlh/sa_svm_uts2017_bank_20190611.zip?dl=1"
cached_path(url, cache_dir=cache_dir)
model_path = Path(
CACHE_ROOT
) / cache_dir / "sa_svm_uts2017_bank_20190611.zip?dl=1"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / "sa_svm_uts2017_bank_20190611",
Path(CACHE_ROOT) / cache_dir / "SA_BANK",
)
os.remove(model_path)
if model_name == "SA_BANK_V131":
model_id = "sa_svm_ubs-1_20210107"
url = f"https://github.com/undertheseanlp/playground/releases/download/1.3.x/{model_id}.zip"
cached_path(url, cache_dir=cache_dir)
model_path = Path(CACHE_ROOT) / cache_dir / f"{model_id}.zip"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / model_id,
Path(CACHE_ROOT) / cache_dir / model_name,
)
os.remove(model_path)
def download(model_name):
if model_name not in REPO:
print(f"No matching distribution found for '{model_name}'")
return
model_path = REPO[model_name]["model_path"]
cache_dir = REPO[model_name]["cache_dir"]
model_path = Path(CACHE_ROOT) / cache_dir / model_path
if Path(model_path).exists():
print(f"Model is already existed: '{model_name}' in {model_path}")
return
if model_name == "TC_GENERAL":
url = "https://www.dropbox.com/s/866offu8wglrcej/tc_svm_vntc_20190607.zip?dl=1"
cached_path(url, cache_dir=cache_dir)
model_path = Path(
CACHE_ROOT) / cache_dir / "tc_svm_vntc_20190607.zip?dl=1"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / "tc_svm_vntc_20190607",
Path(CACHE_ROOT) / cache_dir / "TC_GENERAL",
)
os.remove(model_path)
if model_name == "TC_BANK":
url = "https://www.dropbox.com/s/prrjlypbrr6ze6p/tc_svm_uts2017_bank_20190607.zip?dl=1"
cached_path(url, cache_dir=cache_dir)
model_path = Path(
CACHE_ROOT
) / cache_dir / "tc_svm_uts2017_bank_20190607.zip?dl=1"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / "tc_svm_uts2017_bank_20190607",
Path(CACHE_ROOT) / cache_dir / "TC_BANK",
)
os.remove(model_path)
if model_name == "SA_GENERAL":
url = "https://www.dropbox.com/s/xfj1ity3egabv77/sa_svm_aivivn2019_20190615.zip?dl=1"
cached_path(url, cache_dir=cache_dir)
model_path = Path(
CACHE_ROOT) / cache_dir / "sa_svm_aivivn2019_20190615.zip?dl=1"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / "sa_svm_aivivn2019_20190615",
Path(CACHE_ROOT) / cache_dir / "SA_GENERAL",
)
os.remove(model_path)
if model_name == "SA_BANK":
url = "https://www.dropbox.com/s/yo6sf6ofpdb3hlh/sa_svm_uts2017_bank_20190611.zip?dl=1"
cached_path(url, cache_dir=cache_dir)
model_path = Path(
CACHE_ROOT
) / cache_dir / "sa_svm_uts2017_bank_20190611.zip?dl=1"
cache_folder = Path(CACHE_ROOT) / cache_dir
zip = zipfile.ZipFile(model_path)
zip.extractall(cache_folder)
os.rename(
Path(CACHE_ROOT) / cache_dir / "sa_svm_uts2017_bank_20190611",
Path(CACHE_ROOT) / cache_dir / "SA_BANK",
)
os.remove(model_path)
