def _update_all():
print("Update Corpus...")
with TinyDB(corpus_db_path()) as local_db:
item_all = local_db.all()
query = Query()
for item in item_all:
name = item["name"]
if "file_name" in item.keys():
local_db.update({"filename": item["file_name"]},
query.name == name)
elif "file" in item.keys():
local_db.update({"filename": item["file"]}, query.name == name)
local_db.close()
def get_corpus_db_detail(name: str) -> dict:
"""
Get details about a corpus, using information from local catalog.
:param str name: name corpus
:return: details about a corpus
:rtype: dict
"""
local_db = TinyDB(corpus_db_path())
query = Query()
res = local_db.search(query.name == name)
local_db.close()
if res:
return res[0]
return dict()
def remove(name: str) -> bool:
"""
Remove corpus
:param str name: corpus name
:return: **True** if the corpus is found and succesfully removed.
Otherwise, it returns **False**.
:rtype: bool
:Example:
::
from pythainlp.corpus import remove, get_corpus_path, get_corpus
print(remove('ttc'))
# output: True
print(get_corpus_path('ttc'))
# output: None
get_corpus('ttc')
# output:
# FileNotFoundError: [Errno 2] No such file or directory:
# '/usr/local/lib/python3.6/dist-packages/pythainlp/corpus/ttc'
"""
db = TinyDB(corpus_db_path())
query = Query()
data = db.search(query.name == name)
if data:
path = get_corpus_path(name)
os.remove(path)
db.remove(query.name == name)
db.close()
return True
db.close()
return False
def download(name: str,
force: bool = False,
url: str = None,
version: str = None) -> bool:
"""
Download corpus.
The available corpus names can be seen in this file:
https://github.com/PyThaiNLP/pythainlp-corpus/blob/master/db.json
:param str name: corpus name
:param bool force: force download
:param str url: URL of the corpus catalog
:param str version: Version of the corpus
:return: **True** if the corpus is found and succesfully downloaded.
Otherwise, it returns **False**.
:rtype: bool
:Example:
::
from pythainlp.corpus import download
download('wiki_lm_lstm', force=True)
# output:
# Corpus: wiki_lm_lstm
# - Downloading: wiki_lm_lstm 0.1
# thwiki_lm.pth: 26%|██▌ | 114k/434k [00:00<00:00, 690kB/s]
By default, downloaded corpus and model will be saved in
``$HOME/pythainlp-data/``
(e.g. ``/Users/bact/pythainlp-data/wiki_lm_lstm.pth``).
"""
if not url:
url = corpus_db_url()
corpus_db = get_corpus_db(url)
if not corpus_db:
print(f"Cannot download corpus catalog from: {url}")
return False
corpus_db = corpus_db.json()
# check if corpus is available
if name in list(corpus_db.keys()):
local_db = TinyDB(corpus_db_path())
query = Query()
corpus = corpus_db[name.lower()]
print("Corpus:", name)
if version is None:
version = corpus["latest_version"]
corpus_versions = corpus["versions"][version]
file_name = corpus_versions["filename"]
found = local_db.search((query.name == name)
& (query.version == version))
# If not found in local, download
if force or not found:
print(f"- Downloading: {name} {version}")
_download(
corpus_versions["download_url"],
file_name,
)
_check_hash(
file_name,
corpus_versions["md5"],
)
if found:
local_db.update({"version": version}, query.name == name)
else:
local_db.insert({
"name": name,
"version": version,
"filename": file_name
})
else:
if local_db.search(query.name == name
and query.version == version):
# Already has the same version
print("- Already up to date.")
else:
# Has the corpus but different version
current_ver = local_db.search(query.name == name)[0]["version"]
print(f"- Existing version: {current_ver}")
print(f"- New version available: {version}")
print("- Use download(data_name, force=True) to update")
local_db.close()
return True
print("Corpus not found:", name)
return False