def remove_model(args):
"""Remove installed model."""
# TODO: Remove .archive and .config for the model.
model = args.model
# Determine if remove all model pkgs or a certain model pkg.
cache = None
if model is None:
path = utils.get_init_dir()
if os.path.exists(path):
msg = "*Completely* remove all installed models in '{}'"
else:
msg = "The local model folder '{}' does not exist. Nothing to do."
msg = msg.format(path)
print(msg)
return
else:
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
path = utils.get_package_dir(model)
if os.path.exists(utils.get_package_cache_dir(model)):
cache = utils.get_package_cache_dir(model)
msg = "Remove '{}/'"
# Check that the model is installed.
utils.check_model_installed(model)
if utils.yes_or_no(msg, path, yes=True):
# Remove package installation dir
shutil.rmtree(path)
# Remove package config dir as well without ask
path = utils.get_package_config_dir(model)
if os.path.exists(path):
shutil.rmtree(path)
# Ask if remove cached files
if cache is not None and utils.yes_or_no(
"Remove cache '{}/' as well", cache, yes=False
):
shutil.rmtree(cache)
archive = utils.get_package_archive_dir(model)
if os.path.exists(archive):
shutil.rmtree(archive)
else:
if model is None and not args.quiet:
utils.print_next_step("remove")
def interact_get_match_photos(term, img_download_path, default_key_file,
key_file):
if term is not None:
urls, _ = check_key(default_key_file, key_file, term)
img_download_path = os.path.join(img_download_path,
'_'.join(term.split()))
os.makedirs(img_download_path, exist_ok=True)
else:
msg = "\n Please type in a path or URL of a photo for face recognition."
msg += "\n Or type in a search term to ask Bing to find a photo for you."
msg += "\n Or Ctrl-c to quit.\n"
print(msg)
yes = mlutils.yes_or_no(
"Would you like to use Bing to search a photo for you", yes=True)
if yes:
msg = (
"\nPlease type in a search term to ask Bing to find a photo for you:"
"\n(For example, Satya and Bill Gates)")
term = ask_for_input(msg)
urls, _ = check_key(default_key_file, key_file, term)
img_download_path = os.path.join(img_download_path,
'_'.join(term.split()))
os.makedirs(img_download_path, exist_ok=True)
else:
urls = get_url_path_list(
ask_for_input("Then please type in path or URL of the photo:"))
return urls, term, img_download_path
def remove_mlm(args):
"""Remove downloaded {} files.""".format(EXT_MLM)
mlm = glob.glob(os.path.join(utils.get_init_dir(), "*.mlm"))
mlm.sort()
for m in mlm:
if utils.yes_or_no("Remove model package archive '{}'", m, yes=True):
os.remove(m)
def interact_search_for(name, number, key_file, img_dir_path, img_urls=None):
"""Search photos for <name> interactively."""
msg = (
"\nNow photos of '{0}' will be searched by Bing on the Internet. Found Photos"
"\nwill be shown one-by-one, you may need to help choosing {1} photos in which"
"\n'{0}' is the only person in order to set up a face database of '{0}'."
"\n\nPlease close each window (Ctrl-w) to proceed.")
print(msg.format(name, number))
if img_urls is None:
img_urls, total_img_num = search_images(name, key_file)
count = 0
for url in img_urls:
try:
msg = "\n [{}/{}] Downloading the photo from\n {}"
print(msg.format(str(count + 1).zfill(2), number, url))
path, _ = download_img(url, img_dir_path, name)
except (ConnectionResetError, urllib.error.URLError):
print(" URL access failed!")
continue
show_image(path)
yes = mlutils.yes_or_no(
" Is '{}' the only face in the photo or do you want to use it",
name,
yes=True)
if yes:
print(" The photo is saved as\n {}".
format(path))
count += 1
if count == number:
break
else:
os.remove(path)
def dispatch(args):
"""Dispatch other commands to the appropriate model provided script."""
cmd = args.cmd
model = args.model
path = utils.get_package_dir(model)
param = " ".join(args.param)
# Get working dir if any.
if args.workding_dir is not None:
utils.update_working_dir(model, args.workding_dir)
if args.workding_dir == '':
args.workding_dir = None
else:
args.working_dir = utils.get_working_dir(model)
# Get conda environment name if any.
conda_env_name = utils.get_conda_env_name(model)
# Check that the model is installed and has commands.
utils.check_model_installed(model)
entry = utils.load_description(model)
if 'commands' not in entry or len(entry['commands']) == 0:
raise utils.CommandNotFoundException(cmd, model)
# Correct misspelled command if possible.
matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands']))
if matched_cmd is not None:
cmd = matched_cmd
# Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml.
meta = entry['meta']
if 'display' in meta and cmd in meta['display'] and os.environ.get(
'DISPLAY', '') == '':
msg = "Graphic display is required but not available for command '{}'. Continue"
yes = utils.yes_or_no(msg, cmd, yes=False)
if not yes:
msg = """
To enable DISPLAY be sure to connect to the server using 'ssh -X'
or else connect to the server's desktop using a local X server like X2Go.
"""
sys.stdout.write(msg)
sys.exit(1)
# Obtain the default/chosen language for the package.
lang = meta["languages"]
# Deal with malformed 'languages' field
lang_opts = {"python": "py", "R": "R"}
for k in list(lang_opts):
if lang in k:
lang = lang_opts[k]
break
# Obtain the specified script file.
script = cmd + "." + lang
logger = logging.getLogger(__name__)
logger.debug("Execute the script: " + os.path.join(path, script))
if cmd not in list(entry['commands']) or not os.path.exists(
os.path.join(path, script)):
raise utils.CommandNotFoundException(cmd, model)
# Determine the interpreter to use
#
# .R => Rscript; .py => python, etc.
interpreter = utils.interpreter(script)
# Change working dir if needed
if args.workding_dir is not None:
script = os.path.join(path, script)
path = args.workding_dir
# _MLHUB_CMD_CWD: a environment variable indicates current working
# directory where command `ml xxx` is invoked.
# _MLHUB_MODEL_NAME: env variable indicates the name of the model.
#
# The above two env vars can be obtained by helper function, such
# as utils.get_cmd_cwd(). And model package developer should be
# use the helper function instead of the env vars directly.
command = "export _MLHUB_CMD_CWD='{}'; export _MLHUB_MODEL_NAME='{}'; {} {} {}".format(
os.getcwd(), model, interpreter, script, param)
# Run script inside conda environment if specified
if conda_env_name is not None:
command = 'bash -c "source activate {}; {}"'.format(
conda_env_name, command)
logger.debug("(cd " + path + "; " + command + ")")
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
missing_r_dep = False
if proc.returncode != 0:
errors = errors.decode("utf-8")
# Check if it is Python dependency unsatisfied
dep_required = re.compile(
r"ModuleNotFoundError: No module named '(.*)'").search(errors)
# Check if R dependency unsatisified
if dep_required is None:
dep_required = re.compile(
r"there is no package called ‘(.*)’").search(errors)
if dep_required is not None:
missing_r_dep = True
# Check if required data resource not found
data_required = re.compile(
r"mlhub.utils.DataResourceNotFoundException").search(errors)
if dep_required is not None: # Dependency unsatisfied
dep_required = dep_required.group(1)
logger.error("Dependency unsatisfied: {}\n{}".format(
dep_required, errors))
raise utils.LackDependencyException(dep_required, missing_r_dep,
model)
elif data_required is not None: # Data not found
raise utils.DataResourceNotFoundException()
else: # Other unknown errors
print("An error was encountered:\n")
print(errors)
else:
# Suggest next step
if not args.quiet:
utils.print_next_step(cmd, description=entry, model=model)
def install_model(args):
"""Install a model.
Args:
args: Command line args parsed by argparse.
args.model (str): mlm/zip path, mlm/zip url, model name, GitHub repo,
like mlhubber/mlhub, or MLHUB.yaml on github repo,
like mlhubber/audit:doc/MLHUB.yaml.
"""
logger = logging.getLogger(__name__)
logger.info('Install a model.')
logger.debug('args: {}'.format(args))
model = args.model # model pkg name
location = args.model # pkg file path or URL
version = None # model pkg version
mlhubyaml = None # MLHUB.yaml path or URL
# Obtain the model URL if not a local file.
if not utils.is_archive(model) and not utils.is_url(
model) and '/' not in model:
# Model package name, which can be found in mlhub repo.
# Like:
# $ ml install audit
#
# We assume the URL got from mlhub repo is a link to a mlm/zip/tar file
# or a GitHub repo reference or MLHUB.yaml.
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
# Get model pkg meta data from mlhub repo.
location, version, meta_list = utils.get_model_info_from_repo(
model, args.mlhub)
# Update bash completion list.
utils.update_model_completion({e['meta']['name'] for e in meta_list})
if not utils.is_archive(location):
# Model from GitHub.
# Like:
# $ ml install mlhubber/audit
# $ ml install mlhubber/audit:doc/MLHUB.yaml
# $ ml install https://github.com/mlhubber/audit/...
mlhubyaml = utils.get_pkgyaml_github_url(location) # URL to MLHUB.yaml
location = utils.get_githubrepo_zip_url(location)
logger.debug("location: {}".format(location))
logger.debug("mlhubyaml: {}".format(mlhubyaml))
# Determine the path of downloaded/existing model package file
pkgfile = None
if utils.is_archive(location):
pkgfile = os.path.basename(location) # pkg file name
elif utils.is_url(location):
pkgfile = utils.get_url_filename(location)
# Query archive type if not available from file name per se.
while pkgfile is None or not utils.is_archive(pkgfile):
print(
"The file type cannot be determined.\n"
"Please give it a file name with explicit valid archive extension: ",
end='')
pkgfile = input()
uncompressdir = pkgfile[:pkgfile.rfind(
'.')] # Dir Where pkg file is extracted
# Installation.
entry = None # Meta info read from MLHUB.yaml
with tempfile.TemporaryDirectory() as mlhubtmpdir:
# Determine the local path of the model package
if utils.is_url(location):
local = os.path.join(mlhubtmpdir, pkgfile) # downloaded
else:
local = location # local file path
uncompressdir = os.path.join(mlhubtmpdir, uncompressdir)
# Obtain model version.
if version is None:
if utils.ends_with_mlm(
pkgfile): # Get version number from MLM file name.
model, version = utils.interpret_mlm_name(pkgfile)
elif not utils.is_github_url(
location): # Get MLHUB.yaml inside the archive file.
if utils.is_url(
location
): # Download the package file because it is not from GitHub.
utils.download_model_pkg(location, local, pkgfile,
args.quiet)
if not args.quiet:
print("Extracting '{}' ...\n".format(pkgfile))
utils.unpack_with_promote(local,
uncompressdir,
valid_name=pkgfile)
mlhubyaml = utils.get_available_pkgyaml(
uncompressdir) # Path to MLHUB.yaml
if mlhubyaml is not None: # Get version number from MLHUB.yaml
entry = utils.read_mlhubyaml(mlhubyaml)
meta = entry["meta"]
model = meta["name"]
version = meta["version"]
utils.update_model_completion({model
}) # Update bash completion list.
# Check if model is already installed.
install_path = utils.get_package_dir(model) # Installation path
if os.path.exists(install_path):
installed_version = utils.load_description(
model)['meta']['version']
# Ensure version number is string.
installed_version = str(installed_version)
version = str(version)
if StrictVersion(installed_version) > StrictVersion(version):
yes = utils.yes_or_no(
"Downgrade '{}' from version '{}' to version '{}'",
model,
installed_version,
version,
yes=True)
elif StrictVersion(installed_version) == StrictVersion(version):
yes = utils.yes_or_no(
"Replace '{}' version '{}' with version '{}'",
model,
installed_version,
version,
yes=True)
else:
yes = utils.yes_or_no(
"Upgrade '{}' from version '{}' to version '{}'",
model,
installed_version,
version,
yes=True)
if not yes:
sys.exit(0)
else:
print()
shutil.rmtree(install_path)
# Uncompress package file.
if not os.path.exists(
uncompressdir
): # Model pkg mlm or GitHub pkg has not unzipped yet.
if utils.is_url(location): # Download the package file if needed.
utils.download_model_pkg(location, local, pkgfile, args.quiet)
if not args.quiet:
print("Extracting '{}' ...\n".format(pkgfile))
utils.unpack_with_promote(local, uncompressdir, valid_name=pkgfile)
# Install package files.
#
# Because it is time-consuming to download all package files one-by-one , we
# download the whole zipball from the repo first, then re-arrange the files
# according to `dependencies` -> `files` in MLHUB.yaml if any.
# Find if any files specified in MLHUB.yaml
if mlhubyaml is None: # MLM file which can obtain version number from it name.
mlhubyaml = utils.get_available_pkgyaml(uncompressdir)
entry = utils.read_mlhubyaml(mlhubyaml)
depspec = None
if 'dependencies' in entry:
depspec = entry['dependencies']
elif 'dependencies' in entry['meta']:
depspec = entry['meta']['dependencies']
file_spec = None
if depspec is not None and 'files' in depspec:
file_spec = {'files': depspec['files']}
elif 'files' in entry:
file_spec = {'files': entry['files']}
if file_spec is not None: # install package files if they are specified in MLHUB.yaml
# MLHUB.yaml should always be at the package root.
os.mkdir(install_path)
if utils.is_url(
mlhubyaml
): # We currently only support MLHUB.yaml specified on GitHub.
if mlhubyaml.startswith("https://api"):
urllib.request.urlretrieve(
json.loads(urllib.request.urlopen(
mlhubyaml).read())['download_url'],
os.path.join(install_path, MLHUB_YAML))
else:
urllib.request.urlretrieve(
mlhubyaml, os.path.join(install_path, MLHUB_YAML))
else:
shutil.move(mlhubyaml, install_path)
# All package files except MLHUB.yaml should be specified in 'files' of MLHUB.yaml
try:
utils.install_file_deps(
utils.flatten_mlhubyaml_deps(file_spec)[0][1],
model,
downloadir=uncompressdir)
except utils.ModePkgInstallationFileNotFoundException as e:
if os.path.exists(install_path):
shutil.rmtree(install_path)
raise
else:
# Otherwise, put all files under package dir.
# **Note** Here we must make sure <instal_path> does not exist.
# Otherwise, <unzipdir> will be inside <install_path>
shutil.move(uncompressdir, install_path)
# Update bash completion list.
utils.update_command_completion(
set(utils.load_description(model)['commands']))
# Update working dir if any.
if args.workding_dir is not None:
utils.update_working_dir(model, args.workding_dir)
if not args.quiet:
# Informative message about the size of the installed model.
print("Installed '{}' into '{}' ({:,} bytes).".format(
model, install_path, utils.dir_size(install_path)))
# Suggest next step. README or DOWNLOAD
utils.print_next_step('install', model=model)
def generalkey(key_file, priv_info, verbose=True, ask=True):
"""Load key, location, etc from file or ask user and save.
The user is asked for an general key, location, etc.
The provided information is saved into a file
for future use. The contents of that file is the key.
{"key":"abcd1234abcda4f2f6e9f565df34ef24","location":"eastaustralia"}
"""
# Set up messages.
msg_request = """\
Private information is required to access this service.
See the README for more details.
"""
msg_found = f"""\
The following file has been found and is assumed to
contain the private information.
{key_file}"""
msg_saved = """
That information has been saved into the file:
{}""".format(key_file)
# Obtain the key/location/etc.
if os.path.isfile(key_file) and os.path.getsize(key_file) > 0:
if verbose:
print(msg_found, file=sys.stderr)
if ask:
yes = yes_or_no(
"\nUse this private information ('d' to "
"display, 'n' to update)",
third_choice=True)
else:
yes = True
if yes == "d":
with open(key_file, 'r') as handle:
parsed = json.load(handle)
print(f"\n{json.dumps(parsed, indent=2)}")
yes = yes_or_no("\nUse this private information "
"(type 'n' to update)")
if not yes:
print("\n" + msg_request, file=sys.stderr)
data = {}
if any(isinstance(el, list) for el in priv_info):
# private:
# Azure speech: key*, location
#
# In this case, priv_info = [[Azure speech, [key, location]]]
for item in priv_info:
service = item[0]
nested_dic = {}
for elem in item[1]:
key_or_other = ask_info(elem, service)
nested_dic[elem] = key_or_other
data[service] = nested_dic
# private:
# Azure speech:key*, location
# In this case, the priv_info = [key, location]
else:
for item in priv_info:
key_or_other = ask_info(item, "")
data[item] = key_or_other
# Write data into json file
with open(key_file, "w") as outfile:
json.dump(data, outfile)
outfile.close()
print(msg_saved, file=sys.stderr)
else:
print(msg_request, file=sys.stderr)
if ask:
data = {}
if any(isinstance(el, list) for el in priv_info):
# private:
# Azure speech:key*, location
#
# In this case, priv_info = [[Azure speech, [key, location]]]
for item in priv_info:
service = item[0]
nested_dic = {}
for elem in item[1]:
key_or_other = ask_info(elem, service)
nested_dic[elem] = key_or_other
data[service] = nested_dic
# private:
# Azure speech:key*, location
# In this case, the priv_info = [key, location]
else:
for item in priv_info:
key_or_other = ask_info(item, "")
data[item] = key_or_other
# Write data into json file
with open(key_file, "w") as outfile:
json.dump(data, outfile)
outfile.close()
print(msg_saved, file=sys.stderr)
def install_model(args):
"""Install a model.
Args:
args: Command line args parsed by argparse.
args.model (str): mlm/zip path, mlm/zip url, model name, GitHub repo,
like mlhubber/mlhub, or MLHUB.yaml on github repo,
like mlhubber/audit:doc/MLHUB.yaml.
"""
logger = logging.getLogger(__name__)
logger.info("Install a model.")
logger.debug(f"args: {args}")
# Avoid 403 errors which result when the header identifies itself
# as python urllib or is empty and thus the web site assumes it is
# a robot. We are not a robot but a user downloading a file. This
# will ensure gitlab is okay with retrieving from a URL by adding
# a header rather than no header. TODO move to using Requests.
opener = urllib.request.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
urllib.request.install_opener(opener)
model = args.model # model pkg name
location = args.model # pkg file path or URL
key = args.i # SSH key
version = None # model pkg version
mlhubyaml = None # MLHUB.yaml path or URL
repo_obj = None # RepoTypeURL object for related URL interpretation
maybe_private = False # Maybe private repo
# Obtain the model URL if not a local file.
if (
not utils.is_archive_file(model)
and not utils.is_url(model)
and "/" not in model
):
# Model package name, which can be found in mlhub repo.
# Like:
# $ ml install audit
#
# We assume the URL got from mlhub repo is a link to a mlm/zip/tar file
# or a GitHub repo reference or MLHUB.yaml.
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
# Get model pkg meta data from mlhub repo.
location, version, meta_list = utils.get_model_info_from_repo(
model, args.mlhub
)
# Update bash completion list.
utils.update_model_completion({e["meta"]["name"] for e in meta_list})
if not utils.is_archive_file(location):
# Model from a repo such as GitHub, GitLab, Bitbucket etc.
#
# Possible options are:
# $ ml install mlhubber/audit # latest commit on the master branch of GitHub repo mlhubber/audit
# $ ml install mlhubber/audit@dev # latest commit on the dev branch of GitHub repo mlhubber/audit
# $ ml install mlhubber/audit@0001ea4 # commit 0001ea4 of mlhubber/audit
# $ ml install mlhubber/audit:doc/MLHUB.yaml # latest commit on master, but a specified YAML file
# $ ml install https://github.com/mlhubber/audit/... # Arbitrary GitHub link address
#
# $ ml install github:mlhubber/audit # GitHub repo, the same as ml install mlhubber/audit
#
# $ ml install gitlab:mlhubber/audit@2fe89kh:doc/MLHUB.yaml # GitLab repo
# $ ml install https://https://gitlab.com/mlhubber/audit/... # GitLab repo
#
# $ ml install bitbucket:mlhubber/audit # BitBucket repo
# $ ml install https://bitbucket.org/mlhubber/audit/... # BitBucket repo
repo_obj = utils.RepoTypeURL.get_repo_obj(location)
try:
mlhubyaml = repo_obj.get_pkg_yaml_url()
location = repo_obj.compose_repo_zip_url()
logger.debug(f"location: {location}")
logger.debug(f"mlhubyaml: {mlhubyaml}")
except utils.DescriptionYAMLNotFoundException: # Maybe private repo
maybe_private = True
pass
# Determine the path of downloaded/existing model package file
pkgfile = None
if maybe_private: # Maybe private repo
pkgfile = repo_obj.repo
elif utils.is_archive_file(location):
pkgfile = os.path.basename(location) # pkg file name
elif utils.is_url(location):
pkgfile = utils.get_url_filename(location)
# Query archive type if not available from file name per se.
if not maybe_private:
while pkgfile is None or not utils.is_archive_file(pkgfile):
print(
"The file type cannot be determined.\n"
"Please give it a file name with explicit valid archive extension: ",
end="",
)
pkgfile = input()
if maybe_private:
uncompressdir = pkgfile
else:
uncompressdir = pkgfile[
: pkgfile.rfind(".")
] # Dir Where pkg file is extracted
# Installation.
entry = None # Meta info read from MLHUB.yaml
with tempfile.TemporaryDirectory() as mlhubtmpdir:
# Determine the local path of the model package
if maybe_private:
local = None
elif utils.is_url(location):
local = os.path.join(mlhubtmpdir, pkgfile) # downloaded
else:
local = location # local file path
uncompressdir = os.path.join(mlhubtmpdir, uncompressdir)
# Obtain model version.
if version is None:
if utils.ends_with_mlm(
pkgfile
): # Get version number from MLM file name.
model, version = utils.interpret_mlm_name(pkgfile)
elif not repo_obj:
# Get MLHUB.yaml inside the archive file.
if utils.is_url(
location
): # Download the package file because it is not from GitHub.
utils.download_model_pkg(
location, local, pkgfile, args.quiet
)
if not args.quiet:
print("Extracting '{}' ...\n".format(pkgfile))
utils.unpack_with_promote(
local, uncompressdir, valid_name=pkgfile
)
mlhubyaml = utils.get_available_pkgyaml(
uncompressdir
) # Path to MLHUB.yaml
elif maybe_private:
identity_env = (
"GIT_SSH_COMMAND='ssh -i {}' ".format(key) if key else ""
)
command = "cd {}; {}git clone {}; cd {}; git checkout {}".format(
mlhubtmpdir,
identity_env,
repo_obj.get_ssh_clone_url(),
repo_obj.repo,
repo_obj.ref,
)
proc = subprocess.Popen(
command, shell=True, stderr=subprocess.PIPE
)
output, errors = proc.communicate()
if proc.returncode != 0:
raise utils.InstallFailedException(errors.decode("utf-8"))
if repo_obj.path:
mlhubyaml = os.path.join(uncompressdir, repo_obj.path)
else:
mlhubyaml = utils.get_available_pkgyaml(
uncompressdir
) # Path to MLHUB.yaml
if mlhubyaml is not None: # Get version number from MLHUB.yaml
entry = utils.read_mlhubyaml(mlhubyaml)
meta = entry["meta"]
model = meta["name"]
version = meta["version"]
utils.update_model_completion(
{model}
) # Update bash completion list.
# Check if model is already installed.
install_path = utils.get_package_dir(model) # Installation path
if os.path.exists(install_path):
installed_version = utils.load_description(model)["meta"][
"version"
]
# Ensure version number is string.
installed_version = str(installed_version)
version = str(version)
if StrictVersion(installed_version) > StrictVersion(version):
yes = utils.yes_or_no(
"Downgrade '{}' from version '{}' to version '{}'",
model,
installed_version,
version,
yes=True,
)
elif StrictVersion(installed_version) == StrictVersion(version):
yes = utils.yes_or_no(
"Replace '{}' version '{}' with version '{}'",
model,
installed_version,
version,
yes=True,
)
else:
yes = utils.yes_or_no(
"Upgrade '{}' from version '{}' to version '{}'",
model,
installed_version,
version,
yes=True,
)
if not yes:
# Suggest next step before exiting, as if an install has just happened.
utils.print_next_step("install", model=model)
sys.exit(0)
else:
print()
shutil.rmtree(install_path)
# Uncompress package file.
if not os.path.exists(
uncompressdir
): # Model pkg mlm or GitHub pkg has not unzipped yet.
if utils.is_url(location): # Download the package file if needed.
utils.download_model_pkg(location, local, pkgfile, args.quiet)
if not args.quiet:
print("Extracting '{}' ...\n".format(pkgfile))
utils.unpack_with_promote(local, uncompressdir, valid_name=pkgfile)
# Install package files.
#
# Because it is time-consuming to download all package files one-by-one , we
# download the whole zipball from the repo first, then re-arrange the files
# according to `dependencies` -> `files` in MLHUB.yaml if any.
# Find if any files specified in MLHUB.yaml
if (
mlhubyaml is None
): # MLM file which can obtain version number from it name.
mlhubyaml = utils.get_available_pkgyaml(uncompressdir)
entry = utils.read_mlhubyaml(mlhubyaml)
depspec = None
if "dependencies" in entry:
depspec = entry["dependencies"]
elif "dependencies" in entry["meta"]:
depspec = entry["meta"]["dependencies"]
file_spec = None
if depspec is not None and "files" in depspec:
file_spec = {"files": depspec["files"]}
elif "files" in entry:
file_spec = {"files": entry["files"]}
if (
file_spec is not None
): # install package files if they are specified in MLHUB.yaml
# MLHUB.yaml should always be at the package root.
os.mkdir(install_path)
if utils.is_url(
mlhubyaml
): # We currently only support MLHUB.yaml specified on GitHub.
if mlhubyaml.startswith("https://api"):
urllib.request.urlretrieve(
json.loads(urllib.request.urlopen(mlhubyaml).read())[
"download_url"
],
os.path.join(install_path, MLHUB_YAML),
)
else:
urllib.request.urlretrieve(
mlhubyaml, os.path.join(install_path, MLHUB_YAML)
)
else:
shutil.move(mlhubyaml, install_path)
# All package files except MLHUB.yaml should be specified in 'files' of MLHUB.yaml
try:
utils.install_file_deps(
utils.flatten_mlhubyaml_deps(file_spec)[0][1],
model,
downloadir=uncompressdir,
yes=True,
)
except utils.ModelPkgInstallationFileNotFoundException:
if os.path.exists(install_path):
shutil.rmtree(install_path)
raise
else:
# Otherwise, put all files under package dir.
# **Note** Here we must make sure <install_path> does not exist.
# Otherwise, <unzipdir> will be inside <install_path>
shutil.move(uncompressdir, install_path)
# Update bash completion list.
utils.update_command_completion(
set(utils.load_description(model)["commands"])
)
# Update working dir if any.
if args.working_dir is not None:
utils.update_working_dir(model, args.working_dir)
if not args.quiet:
# Informative message about the size of the installed model.
msg = f"Found '{model}' version {version}.\n\nInstalled '{model}' "
msg += f"into '{install_path}/' ({utils.dir_size(install_path):,} bytes)."
print(msg)
# Suggest next step. README or DOWNLOAD
utils.print_next_step("install", model=model)
else:
name = ask_for_input("\nPlease give a person's name to recognise (For example, Satya)")
else:
name = ask_for_input("\nPlease give the person's name to capture")
# ----------------------------------------------------------------------
# Generate face database or load existing one
# ----------------------------------------------------------------------
if not use_database and name in data and args.data is None: # Face data is available for the name
if args.batch:
use_database = True
else: # Ask whether or not to use available face data if face database exists
msg = ("\nYou have searched '{0}' before! So there are face data for '{0}'."
"\nWould you like to use the data")
use_database = mlutils.yes_or_no(msg, name, yes=True)
key_file = None
img_download_path = None
hasdigestag = False
if not use_database: # Face data needs to be obtained from other source instead of existing database
if args.data is None: # Search for photos interactively
print("\n To recognise '{0}' in arbitrary photos, sample photos of '{0}'\n are needed.".format(name))
img_download_path = make_name_dir(SINGLE_FACE_IMG_PATH, name)
hasdigestag = True
if args.capture:
interact_capture_photos(name, args.count, img_download_path)
else:
img_urls, _ = check_key(KEY_FILE, args.key, name) # Get Bing search subscription API key
key_file = KEY_FILE
def dispatch(args):
"""Dispatch other commands to the appropriate model provided script."""
cmd = args.cmd
model = args.model
path = utils.get_package_dir(model)
param = " ".join(args.param)
# Get working dir if any.
if args.workding_dir is not None:
utils.update_working_dir(model, args.workding_dir)
if args.workding_dir == '':
args.workding_dir = None
else:
args.working_dir = utils.get_working_dir(model)
# Get conda environment name if any.
conda_env_name = utils.get_conda_env_name(model)
# Check that the model is installed and has commands.
utils.check_model_installed(model)
entry = utils.load_description(model)
if 'commands' not in entry or len(entry['commands']) == 0:
raise utils.CommandNotFoundException(cmd, model)
# Correct misspelled command if possible.
matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands']))
if matched_cmd is not None:
cmd = matched_cmd
# Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml.
meta = entry['meta']
if 'display' in meta and cmd in meta['display'] and os.environ.get(
'DISPLAY', '') == '':
msg = "Graphic display is required but not available for command '{}'. Continue"
yes = utils.yes_or_no(msg, cmd, yes=False)
if not yes:
msg = """
To enable DISPLAY be sure to connect to the server using 'ssh -X'
or else connect to the server's desktop using a local X server like X2Go.
"""
sys.stdout.write(msg)
sys.exit(1)
# Obtain the default/chosen language for the package.
lang = meta["languages"]
# Deal with malformed 'languages' field
lang_opts = {"python": "py", "R": "R"}
for k in list(lang_opts):
if lang in k:
lang = lang_opts[k]
break
# Obtain the specified script file.
script = cmd + "." + lang
logger = logging.getLogger(__name__)
logger.debug("Execute the script: " + os.path.join(path, script))
if cmd not in list(entry['commands']) or not os.path.exists(
os.path.join(path, script)):
raise utils.CommandNotFoundException(cmd, model)
# Determine the interpreter to use
#
# .R => Rscript; .py => python, etc.
interpreter = utils.interpreter(script)
# Change working dir if needed
if args.workding_dir is not None:
script = os.path.join(path, script)
path = args.workding_dir
# Handle python environment
python_pkg_bin = None
python_pkg_path = None
if script.endswith('py'):
python_pkg_base = os.path.sep.join(
[utils.get_package_dir(model), '.python'])
python_pkg_path = python_pkg_base + site.USER_SITE
python_pkg_bin = python_pkg_base + site.USER_BASE + '/bin'
# TODO: Make sure to document:
# $ sudo apt-get install -y python3-pip
# $ /usr/bin/pip3 install mlhub
# Since in DSVM, the default pip is conda's pip, so if we stick to
# use system's command, then the installation of MLHub itself should
# be completed via system's pip, otherwise, MLHub will not work.
if sys.executable != SYS_PYTHON_CMD:
python_pkg_path = python_pkg_base + site.getsitepackages()[0]
python_pkg_bin = python_pkg_base + site.PREFIXES[0] + '/bin'
if utils.get_sys_python_pkg_usage(model):
utils.print_on_stderr(MSG_INCOMPATIBLE_PYTHON_ENV, model)
# _MLHUB_CMD_CWD: a environment variable indicates current working
# directory where command `ml xxx` is invoked.
# _MLHUB_MODEL_NAME: env variable indicates the name of the model.
#
# The above two env vars can be obtained by helper function, such
# as utils.get_cmd_cwd(). And model package developer should be
# use the helper function instead of the env vars directly.
env_var = "export _MLHUB_CMD_CWD='{}'; ".format(os.getcwd())
env_var += "export _MLHUB_MODEL_NAME='{}'; ".format(model)
env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable)
env_var += "export PYTHONPATH='{}'; ".format(
python_pkg_path) if python_pkg_path else ""
env_var += "export PATH=\"{}:$PATH\"; ".format(
python_pkg_bin) if python_pkg_bin else ""
command = "{}{} {} {}".format(env_var, interpreter, script, param)
# Run script inside conda environment if specified
if conda_env_name is not None:
command = '{} -c "source activate {}; {}"'.format(
BASH_CMD, conda_env_name, command)
logger.debug("(cd " + path + "; " + command + ")")
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
missing_r_dep = False
if proc.returncode != 0:
errors = errors.decode("utf-8")
# Check if it is Python dependency unsatisfied
dep_required = re.compile(
r"ModuleNotFoundError: No module named '(.*)'").search(errors)
# Check if R dependency unsatisified
if dep_required is None:
dep_required = re.compile(
r"there is no package called ‘(.*)’").search(errors)
if dep_required is not None:
missing_r_dep = True
# Check if required data resource not found
data_required = re.compile(
r"mlhub.utils.DataResourceNotFoundException").search(errors)
if dep_required is not None: # Dependency unsatisfied
dep_required = dep_required.group(1)
logger.error("Dependency unsatisfied: {}\n{}".format(
dep_required, errors))
raise utils.LackDependencyException(dep_required, missing_r_dep,
model)
elif data_required is not None: # Data not found
raise utils.DataResourceNotFoundException()
else: # Other unknown errors
print("An error was encountered:\n")
print(errors)
else:
# Suggest next step
if not args.quiet:
utils.print_next_step(cmd, description=entry, model=model)
def check_key(default_key_file, key_file=None, search_term='cat'):
"""Check subscription key or ask valid key if possible."""
key = None
img_urls = None
total_img_num = 0
if key_file is None:
if not os.path.exists(default_key_file):
msg = """
To search photos, Bing image search API is used:
https://azure.microsoft.com/en-us/services/cognitive-services/bing-image-search-api/
And a Bing search API subscription key is needed.
A 7-days free trail Azure account can be created at:
https://azure.microsoft.com/en-us/try/cognitive-services/?api=search-api-v7
You can try it and obtain the key.
"""
print(msg)
else:
yes = mlutils.yes_or_no(
"A Bing subscription key is found locally! Would you like to use it",
yes=True)
if yes: # Load Bing search API key if available
key_file = default_key_file
else:
key_file = get_abspath(key_file)
if not os.path.exists(
key_file): # key_file is exactly the key, not a file
key = key_file
key_file = default_key_file
save_key(key, key_file)
if key is None and key_file is not None:
key = load_key(key_file)
if key is not None:
try:
img_urls, total_img_num = search_images(search_term, key)
save_key(key, default_key_file)
except urllib.error.URLError:
print("Maybe the key is wrong!")
key = None
if key is None:
msg = "Please paste the key below (Your key will be kept in\n'{}'):".format(
default_key_file)
print(msg)
msg = "> "
while key is None:
key = getpass(msg)
try:
img_urls, total_img_num = search_images(search_term, key)
save_key(key, default_key_file)
except urllib.error.URLError:
print("Maybe the key is wrong! Please input again.")
key = None
return img_urls, total_img_num
