def _download_and_link(resource_alias, resource_fullname, compatibility,
pip_args, verbose):
version = get_resources_version(resource_fullname, resource_alias,
compatibility)
url_tail = '{r}-{v}/{r}-{v}.tar.gz#egg={r}=={v}'.format(
r=resource_fullname, v=version)
download_url = __about__.__download_url__ + '/' + url_tail
exit_code = install_remote_package(download_url, pip_args)
if exit_code != 0:
sys.exit(exit_code)
try:
# Get package path here because link uses
# pip.get_installed_distributions() to check if the resource is a
# package, which fails if the resource was just installed via
# subprocess
package_path = get_package_path(resource_fullname)
link_path, resources_dir = link_resources(resource_fullname,
resource_alias,
force=True,
resources_path=package_path)
if verbose:
pretty_print("%s --> %s" % (str(resources_dir), str(link_path)),
"You can now load the resources via "
"snips_nlu.load_resources('%s')" % resource_alias,
title="Linking successful",
level=PrettyPrintLevel.SUCCESS)
except: # pylint:disable=bare-except
pretty_print(
"Creating a shortcut link for '{r}' didn't work.\nYou can "
"still load the resources using the full package name: "
"snips_nlu.load_resources('{n}')".format(r=resource_alias,
n=resource_fullname),
title="The language resources were successfully downloaded, "
"however linking failed.",
level=PrettyPrintLevel.WARNING)
def main():
commands = {
"train": train,
"parse": parse,
"download": download,
"download-all-languages": download_all_languages,
"link": link,
"generate-dataset": generate_dataset,
"cross-val-metrics": cross_val_metrics,
"train-test-metrics": train_test_metrics,
}
if len(sys.argv) == 1:
pretty_print(', '.join(commands),
title="Available commands",
exits=1,
level=PrettyPrintLevel.INFO)
command = sys.argv.pop(1)
sys.argv[0] = 'snips-nlu %s' % command
if command in commands:
plac.call(commands[command], sys.argv[1:])
else:
pretty_print("Available: %s" % ', '.join(commands),
title="Unknown command: %s" % command,
exits=1,
level=PrettyPrintLevel.INFO)
def link(origin, link_name, force=False, resources_path=None):
"""
Create a symlink for language resources within the snips_nlu/data
directory. Accepts either the name of a pip package, or the local path to
the resources data directory.
Linking resources allows loading them via
snips_nlu.load_resources(link_name).
"""
if is_package(origin):
resources_path = get_package_path(origin)
else:
resources_path = Path(origin) if resources_path is None \
else Path(resources_path)
if not resources_path.exists():
raise OSError("%s not found" % str(resources_path))
link_path = DATA_PATH / link_name
if link_path.is_symlink() and not force:
raise OSError("Symlink already exists: %s" % str(link_path))
elif link_path.is_symlink():
link_path.unlink()
elif link_path.exists():
raise OSError("Symlink cannot be overwritten: %s" % str(link_path))
resources_sub_dir = get_resources_sub_directory(resources_path)
create_symlink(link_path, resources_sub_dir)
pretty_print("%s --> %s" % (str(resources_sub_dir), str(link_path)),
"You can now load the resources via "
"snips_nlu.load_resources('%s')" % link_name,
title="Linking successful", level=PrettyPrintLevel.SUCCESS)
def _download_and_link_entity(long_resource_name, entity_name, language,
compatibility, pip_args):
full_resource_name = long_resource_name + "_" + language
version = get_resources_version(full_resource_name, entity_name,
compatibility)
entity_alias = get_builtin_entity_shortname(entity_name).lower()
entity_base_url = _get_entity_base_url(language, entity_alias, version)
latest = get_json(entity_base_url + "/latest",
"Latest entity resources version")
latest_url = "{b}/{n}#egg={r}=={v}".format(
b=entity_base_url, n=latest["filename"], r=full_resource_name,
v=latest["version"])
exit_code = install_remote_package(latest_url, pip_args)
if exit_code != 0:
sys.exit(exit_code)
try:
# Get package path here because link uses
# pip.get_installed_distributions() to check if the resource is a
# package, which fails if the resource was just installed via
# subprocess
package_path = get_package_path(full_resource_name)
link_alias = entity_alias + "_" + language
link_path, resources_dir = link_resources(
full_resource_name, link_alias, force=True,
resources_path=package_path)
pretty_print("%s --> %s" % (str(resources_dir), str(link_path)),
"You can now use the '%s' builtin entity" % entity_name,
title="Linking successful",
level=PrettyPrintLevel.SUCCESS)
except: # pylint:disable=bare-except
pretty_print(
"Creating a shortcut link for '%s' didn't work." % entity_name,
title="The builtin entity resources were successfully downloaded, "
"however linking failed.",
level=PrettyPrintLevel.ERROR)
def print_compatibility_error(language):
from snips_nlu.cli.utils import PrettyPrintLevel, pretty_print
pretty_print(
"Language resources for '{lang}' could not be loaded.\nYou may "
"have to download resources again using "
"'python -m snips_nlu download {lang}'".format(lang=language),
"This can happen when you update the snips-nlu library.",
title="Something went wrong while loading resources",
level=PrettyPrintLevel.ERROR)
def _get_compatibility():
version = __about__.__version__
table = get_json(__about__.__compatibility__, "Compatibility table")
compatibility = table["snips-nlu"]
if version not in compatibility:
pretty_print("No compatible resources found for version %s" % version,
title="Resources compatibility error", exits=1,
level=PrettyPrintLevel.ERROR)
return compatibility[version]
def link(origin, link_name, force=False, resources_path=None):
"""
Create a symlink for language resources within the snips_nlu/data
directory. Accepts either the name of a pip package, or the local path to
the resources data directory.
Linking resources allows loading them via
snips_nlu.load_resources(link_name).
"""
link_path, resources_dir = link_resources(origin, link_name, force,
resources_path)
pretty_print("%s --> %s" % (str(resources_dir), str(link_path)),
title="Linking successful", level=PrettyPrintLevel.SUCCESS)
def _download_and_link(resource_alias, resource_fullname, compatibility,
pip_args, verbose):
import sys
from snips_nlu import __about__
from snips_nlu.cli.link import link_resources
from snips_nlu.cli.utils import (
PrettyPrintLevel, get_resources_version, install_remote_package,
pretty_print)
from snips_nlu.common.utils import get_package_path
version = get_resources_version(resource_fullname, resource_alias,
compatibility)
url_tail = '{r}-{v}/{r}-{v}.tar.gz#egg={r}=={v}'.format(
r=resource_fullname, v=version)
download_url = __about__.__download_url__ + '/' + url_tail
exit_code = install_remote_package(download_url, pip_args)
if exit_code != 0:
sys.exit(exit_code)
try:
# Get package path here because link uses
# pip.get_installed_distributions() to check if the resource is a
# package, which fails if the resource was just installed via
# subprocess
package_path = get_package_path(resource_fullname)
link_path, resources_dir = link_resources(
resource_fullname, resource_alias, force=True,
resources_path=package_path)
if verbose:
pretty_print("%s --> %s" % (str(resources_dir), str(link_path)),
title="Linking successful",
level=PrettyPrintLevel.SUCCESS)
except OSError as e: # pylint:disable=bare-except
pretty_print(
"Creating a shortcut link for '%s' didn't work: %s"
% (resource_alias, repr(e)),
title="The language resources were successfully downloaded, "
"however linking failed.",
level=PrettyPrintLevel.ERROR)
def download(resource_name, direct=False,
*pip_args): # pylint:disable=keyword-arg-before-vararg
"""Download compatible resources for the specified language"""
if direct:
dl = _download_resources(
'{r}/{r}.tar.gz#egg={r}'.format(r=resource_name), pip_args)
if dl != 0:
sys.exit(dl)
else:
resource_name = resource_name.lower()
shortcuts = get_json(__about__.__shortcuts__, "Resource shortcuts")
full_resource_name = shortcuts.get(resource_name, resource_name)
compatibility = _get_compatibility()
version = _get_resources_version(full_resource_name, compatibility)
dl = _download_resources('{r}-{v}/{r}-{v}.tar.gz#egg={r}=={v}'
.format(r=full_resource_name, v=version),
pip_args)
if dl != 0:
sys.exit(dl)
try:
# Get package path here because link uses
# pip.get_installed_distributions() to check if the resource is a
# package, which fails if the resource was just installed via
# subprocess
package_path = get_package_path(full_resource_name)
link(full_resource_name, resource_name, force=True,
resources_path=package_path)
except: # pylint:disable=bare-except
pretty_print(
"Creating a shortcut link for '{r}' didn't work.\nYou can "
"still load the resources via its full package name: "
"snips_nlu.load_resources('{n}')".format(r=resource_name,
n=full_resource_name),
title="Language resources were successfully downloaded, "
"however linking failed.",
level=PrettyPrintLevel.WARNING)
def _get_resources_version(resource_name, compatibility):
if resource_name not in compatibility:
pretty_print("No resources found for '%s'" % resource_name,
title="Resources compatibility error", exits=1,
level=PrettyPrintLevel.ERROR)
return compatibility[resource_name][0]