def check_and_install_transliteration(force=False):
# 'en' is not a supported transliteration language
transliteration_languages = [l.code for l in languages if l.code != "en"]
# check installed
install_needed = []
if not force:
t_packages_path = Path(polyglot.polyglot_path) / "transliteration2"
for lang in transliteration_languages:
if not (t_packages_path / lang /
f"transliteration.{lang}.tar.bz2").exists():
install_needed.append(lang)
else:
install_needed = transliteration_languages
# install the needed transliteration packages
if install_needed:
print(
f"Installing transliteration models for the following languages: {', '.join(install_needed)}"
)
from polyglot.downloader import Downloader
downloader = Downloader()
for lang in install_needed:
downloader.download(f"transliteration2.{lang}")
def _download_polyglot_data():
downloader = Downloader()
# Download PT and ES embeddings for mention detection
downloader.download('embeddings2.pt')
downloader.download('embeddings2.es')
# Download NER models
downloader.download('ner2.pt')
downloader.download('ner2.es')
def download(args):
""" Download polyglot packages and models."""
downloader = Downloader(server_index_url = args.server_index_url)
if args.packages:
for pkg_id in args.packages:
rv = downloader.download(info_or_id=unicode(pkg_id), download_dir=args.dir,
quiet=args.quiet, force=args.force,
halt_on_error=args.halt_on_error)
if rv == False and args.halt_on_error:
break
else:
downloader.download(download_dir=args.dir, quiet=args.quiet, force=args.force,
halt_on_error=args.halt_on_error)
