def remove_model(args):
"""Remove installed model."""
# TODO: Remove .archive and .config for the model.
model = args.model
# Determine if remove all model pkgs or a certain model pkg.
cache = None
if model is None:
path = utils.get_init_dir()
if os.path.exists(path):
msg = "*Completely* remove all installed models in '{}'"
else:
msg = "The local model folder '{}' does not exist. Nothing to do."
msg = msg.format(path)
print(msg)
return
else:
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
path = utils.get_package_dir(model)
if os.path.exists(utils.get_package_cache_dir(model)):
cache = utils.get_package_cache_dir(model)
msg = "Remove '{}/'"
# Check that the model is installed.
utils.check_model_installed(model)
if utils.yes_or_no(msg, path, yes=True):
# Remove package installation dir
shutil.rmtree(path)
# Remove package config dir as well without ask
path = utils.get_package_config_dir(model)
if os.path.exists(path):
shutil.rmtree(path)
# Ask if remove cached files
if cache is not None and utils.yes_or_no(
"Remove cache '{}/' as well", cache, yes=False
):
shutil.rmtree(cache)
archive = utils.get_package_archive_dir(model)
if os.path.exists(archive):
shutil.rmtree(archive)
else:
if model is None and not args.quiet:
utils.print_next_step("remove")
def remove_mlm(args):
"""Remove downloaded {} files.""".format(EXT_MLM)
mlm = glob.glob(os.path.join(utils.get_init_dir(), "*.mlm"))
mlm.sort()
for m in mlm:
if utils.yes_or_no("Remove model package archive '{}'", m, yes=True):
os.remove(m)
def list_available(args):
"""List the name and title of the models in the Hub."""
# Setup.
logger = logging.getLogger(__name__)
logger.info("List available models.")
logger.debug(f"args: {args}")
meta, repo = utils.get_repo_meta_data(args.mlhub)
model_names = [entry["meta"]["name"] for entry in meta]
# Update bash completion list.
utils.update_model_completion(set(model_names))
# List model name only.
if args.name_only:
print("\n".join(model_names))
return
# Provide some context.
if not args.quiet:
msg = "The repository '{}' provides the following models:\n"
print(msg.format(repo))
# List the meta data.
for entry in meta:
utils.print_meta_line(entry)
# Suggest next step.
if not args.quiet:
utils.print_next_step("available")
if not os.path.exists(utils.get_init_dir()):
print(
"Why not give the 'rain' model a go...\n\n"
" $ ml install rain\n"
)
def list_installed(args):
"""List the installed models."""
logger = logging.getLogger(__name__)
logger.info('List installed models.')
# Find installed models, ignoring special folders like R.
init = utils.get_init_dir()
if os.path.exists(init):
msg = " in '{}'.".format(init)
models = [
f for f in os.listdir(init) if os.path.isdir(os.path.join(init, f))
and f != "R" and not f.startswith('.') and not f.startswith('_')
]
else:
msg = ". '{}' does not exist.".format(init)
models = []
models.sort()
# Only list model names
if args.name_only:
print('\n'.join(models))
return
# Report on how many models we found installed.
mcnt = len(models)
plural = "s" if mcnt != 1 else ""
print("Found {} model{} installed{}".format(mcnt, plural, msg))
# Report on each of the installed models.
if mcnt > 0:
print("")
invalid_models = []
for p in models:
try:
entry = utils.load_description(p)
utils.print_meta_line(entry)
except (utils.DescriptionYAMLNotFoundException,
utils.MalformedYAMLException, KeyError):
mcnt -= 1
invalid_models.append(p)
continue
# Update bash completion list.
if 'commands' in entry:
utils.update_command_completion(set(entry['commands']))
invalid_mcnt = len(invalid_models)
if invalid_mcnt > 0:
print("\nOf which {} model package{} {} broken:\n".format(
invalid_mcnt, 's' if invalid_mcnt > 1 else '',
'are' if invalid_mcnt > 1 else 'is'))
print(" ====> \033[31m" + ', '.join(invalid_models) + "\033[0m")
print(utils.get_command_suggestion('remove'))
# Suggest next step.
if not args.quiet:
if mcnt > 0:
utils.print_next_step('installed', scenario='exist')
else:
utils.print_next_step('installed', scenario='none')
def main():
"""Main program for the command line script."""
# --------------------------------------------------
# Global option parser. See mlhub.constants.OPTIONS
# --------------------------------------------------
logger.info("Create global option parser.")
global_option_parser = argparse.ArgumentParser(
add_help=False # Disable -h or --help. Use custom help msg instead.
)
utils.OptionAdder(global_option_parser, constants.OPTIONS).add_alloptions()
# --------------------------------------------------
# Parse version
# --------------------------------------------------
logger.info("Parse global options.")
args, extras = global_option_parser.parse_known_args(sys.argv[1:])
if args.debug: # Add console log handler to log debug message to console
logger.info('Enable printing out debug log on console.')
utils.add_log_handler(logger, logging.StreamHandler(), logging.DEBUG,
constants.LOG_CONSOLE_FORMAT)
logger.debug('args: {}, extra_args: {}'.format(args, extras))
# Get the first positional argument.
pos_args = [(i, arg) for i, arg in enumerate(sys.argv[1:])
if not arg.startswith('-')]
first_pos_arg_index, first_pos_arg = pos_args[0] if len(
pos_args) != 0 else (None, None)
logger.debug('First positional argument: {}'.format(first_pos_arg))
if args.version:
logger.info('Query version.')
# --------------------------------------------------
# Query the version of the model, for example
# $ ml rain -v
# $ ml -v rain
# Otherwise, output the version of ml
# $ ml -v
# --------------------------------------------------
if first_pos_arg is not None: # Print model version
print(first_pos_arg, "version", utils.get_version(first_pos_arg))
else: # Print mlhub version
print(constants.APP, "version", utils.get_version())
return 0
# --------------------------------------------------
# Parse command line args for basic commands or model specific commands
# --------------------------------------------------
# Correct misspelled command if possible.
if first_pos_arg is not None:
# Only match basic commands since model pkg commands are more specific which would be
# better to be checked after the model pkg name is known.
matched_cmd = utils.get_misspelled_command(first_pos_arg,
list(constants.COMMANDS))
if matched_cmd is not None:
sys.argv[first_pos_arg_index + 1] = matched_cmd
first_pos_arg = matched_cmd
# Dispatch commands.
if first_pos_arg is not None and first_pos_arg not in constants.COMMANDS:
# Model specific commands, such as demo, display.
logger.info("Parse model specific dommands.")
model_cmd_parser = argparse.ArgumentParser(
prog=constants.CMD,
parents=[global_option_parser],
add_help=False # Use custom help message
)
model_cmd_parser.add_argument('cmd', metavar='command')
model_cmd_parser.add_argument('model')
args, extras = model_cmd_parser.parse_known_args()
logger.debug("args: {}".format(args))
logger.debug("extra_args: {}".format(extras))
# Simple help message for the model-specific command
if '--help' in extras or '-h' in extras:
logger.debug("Help for command '{}' of '{}'".format(
args.cmd, args.model))
utils.print_model_cmd_help(utils.load_description(args.model),
args.cmd)
return 0
setattr(args, 'func', commands.dispatch)
setattr(args, 'param', extras)
else:
# Basic commands, such as install, readme. See mlhub.constants.COMMANDS
logger.info("Parse basic common commands.")
basic_cmd_parser = argparse.ArgumentParser(
prog=constants.CMD,
description="Access models from the ML Hub.",
parents=[global_option_parser])
subparsers = basic_cmd_parser.add_subparsers(title='subcommands',
dest="cmd")
utils.SubCmdAdder(subparsers, commands,
constants.COMMANDS).add_allsubcmds()
args = basic_cmd_parser.parse_args()
logger.debug("args: {}".format(args))
# Print usage for incorrect argument
if "func" not in args:
utils.print_usage()
return 0
# Ensure we have a trailing slash on the mlhub.
if args.mlhub is not None:
constants.MLHUB = os.path.join(args.mlhub, "")
if args.mlmetavar is not None:
constants.CMD = args.mlmetavar
# --------------------------------------------------
# Dispatch commands
# --------------------------------------------------
try:
args.func(args)
except utils.MLInitCreateException as e:
msg = "The below '{}' init folder cannot be created:\n {}"
utils.print_error_exit(msg, constants.APP, e.args[0])
except utils.MLTmpDirCreateException as e:
msg = "The below '{}' tmp folder cannot be created:\n {}"
utils.print_error_exit(msg, constants.APP, e.args[0])
except utils.MalformedMLMFileNameException as e:
msg = "Malformed {} file:\n {}"
utils.print_error_exit(msg, constants.EXT_MLM, e.args[0])
except utils.MalformedYAMLException as e:
name = e.args[0]
if os.path.sep in name or '/' in name:
msg = "Malformed YAML file:\n {}"
else:
msg = "Malformed description for model package '{}'!"
utils.print_error_exit(msg, e.args[0])
except utils.ModelURLAccessException as e:
msg = "URL access failed:\n {}"
utils.print_error_exit(msg, e.args[0])
except utils.YAMLFileAccessException as e:
msg = "YAML file access failed:\n {}"
utils.print_error_exit(msg, e.args[0])
except utils.RepoAccessException as e:
utils.print_error("Cannot access the ML Hub repository:\n {}",
e.args[0])
if not args.quiet: # Suggest check if any models installed, since mlhub repo not accessible
utils.print_commands_suggestions_on_stderr('installed')
sys.exit(1)
except utils.ModelNotFoundOnRepoException as e:
msg = "No model named '{}' was found on\n {}"
utils.print_error(msg, e.args[0], e.args[1])
if not args.quiet: # Suggest check if any models available, since specified model not found
utils.print_commands_suggestions_on_stderr('available')
sys.exit(1)
except utils.ModelDownloadHaltException as e:
msg = "URL - '{}' failed:\n {}".format(e.args[0], e.args[1])
utils.print_error_exit(msg)
except utils.DescriptionYAMLNotFoundException as e:
msg = "No MLHUB description file found: {}"
location = e.args[0]
if not utils.is_url(location):
msg += "\n The model package may be broken!"
utils.print_error(msg, location)
if not args.quiet: # Suggest remove broken package or install new model
utils.print_commands_suggestions_on_stderr('remove', 'install')
else:
msg += "\n The given location may be wrong!"
utils.print_error(msg, location)
sys.exit(1)
except utils.ModelNotInstalledException as e:
msg = "model '{}' is not installed ({})."
utils.print_error(msg, e.args[0], utils.get_init_dir())
if not args.quiet: # Suggest install model package or check if any available
utils.print_commands_suggestions_on_stderr('installed',
'available', 'install')
sys.exit(1)
except utils.ModelReadmeNotFoundException as e:
msg = "The '{}' model does not have a '{}' file:\n {}\n"
utils.print_error(msg, e.args[0], constants.README, e.args[1])
if not args.quiet: # Suggest remove broken package or install new model
utils.print_commands_suggestions_on_stderr('remove', 'install')
sys.exit(1)
except utils.UnsupportedScriptExtensionException as e:
msg = "Could not determine an interpreter for extension '{}'"
utils.print_error_exit(msg, e.args[0])
except utils.CommandNotFoundException as e:
msg = "The command '{}' was not found for this model '{}'."
utils.print_error(msg, e.args[0], e.args[1])
if not args.quiet: # Suggest check if any available commands
utils.print_commands_suggestions_on_stderr('commands')
sys.exit(1)
except utils.LackDependencyException as e:
msg = "Required {} dependencies are not installed for this model: \n ====> \033[31m{}\033[0m"
utils.print_error(msg, 'R' if e.args[1] else 'Python', e.args[0])
if not args.quiet: # Suggest install dependencies
utils.print_commands_suggestions_on_stderr('configure')
sys.exit(1)
except utils.LackPrerequisiteException as e:
msg = "Required pre-requisite not found: \n ====> \033[31m{}\033[0m"
utils.print_error(msg, e.args[0])
if not args.quiet: # Suggest install dependencies
msg = "\nTo install required pre-requisites:\n\n $ ml configure\n"
utils.print_on_stderr(msg)
sys.exit(1)
except utils.DataResourceNotFoundException:
msg = "Some data or model files required by the model package are missing!"
utils.print_error(msg)
if not args.quiet: # Suggest download data
utils.print_commands_suggestions_on_stderr('configure')
sys.exit(1)
except utils.MalformedPackagesDotYAMLException as e:
msg = (
"There is no '{}' available for the model package '{}' which may be under maintenance now.\n"
"Please try again later.")
utils.print_error(msg, e.args[0], e.args[1])
if not args.quiet: # Suggest check if any models available, since specified model not available
utils.print_commands_suggestions_on_stderr('available')
sys.exit(1)
except utils.ModePkgInstallationFileNotFoundException as e:
msg = "No such package file: {}\n The model package may be broken!"
utils.print_error_exit(msg, e.args[0])
except utils.ModelPkgDependencyFileNotFoundException as e:
msg = "Failed to get File dependency: {}\n"
utils.print_error_exit(msg, e.args[0])
except utils.ConfigureFailedException: # configure failed, then just quit
sys.exit(1)
except (KeyboardInterrupt, EOFError): # Catch Ctrl-C and Ctrl-D
print()
sys.exit(1)
def list_installed(args):
"""List the installed models."""
logger = logging.getLogger(__name__)
logger.info("List installed models.")
# Find installed models, ignoring special folders like R.
init = utils.get_init_dir()
if os.path.exists(init):
msg = f" in '{init}'."
models = [
f
for f in os.listdir(init)
if os.path.isdir(os.path.join(init, f))
and f != "R"
and not f.startswith(".")
and not f.startswith("_")
]
else:
msg = f". '{init}' does not exist."
models = []
models.sort()
# Only list model names
if args.name_only:
print("\n".join(models))
return
# Report on how many models we found installed.
mcnt = len(models)
plural = "s" if mcnt != 1 else ""
print(f"Found {mcnt} model{plural} installed{msg}")
# Report on each of the installed models.
if mcnt > 0:
print("")
invalid_models = []
for p in models:
try:
entry = utils.load_description(p)
utils.print_meta_line(entry)
except (
utils.DescriptionYAMLNotFoundException,
utils.MalformedYAMLException,
KeyError,
):
mcnt -= 1
invalid_models.append(p)
continue
# Update bash completion list.
if "commands" in entry:
utils.update_command_completion(set(entry["commands"]))
invalid_mcnt = len(invalid_models)
if invalid_mcnt > 0:
print(
"\nOf which {} model package{} {} broken:\n".format(
invalid_mcnt,
"s" if invalid_mcnt > 1 else "",
"are" if invalid_mcnt > 1 else "is",
)
)
print(" ====> \033[31m" + ", ".join(invalid_models) + "\033[0m")
print(utils.get_command_suggestion("remove"))
# Suggest next step.
if not args.quiet:
if mcnt > 0:
utils.print_next_step("installed", scenario="exist")
else:
utils.print_next_step("installed", scenario="none")
