def remove_model(args): """Remove installed model.""" # TODO: Remove .archive and .config for the model. model = args.model # Determine if remove all model pkgs or a certain model pkg. cache = None if model is None: path = utils.get_init_dir() if os.path.exists(path): msg = "*Completely* remove all installed models in '{}'" else: msg = "The local model folder '{}' does not exist. Nothing to do." msg = msg.format(path) print(msg) return else: # Correct model name if possible. matched_model = utils.get_misspelled_pkg(model) if matched_model is not None: model = matched_model path = utils.get_package_dir(model) if os.path.exists(utils.get_package_cache_dir(model)): cache = utils.get_package_cache_dir(model) msg = "Remove '{}/'" # Check that the model is installed. utils.check_model_installed(model) if utils.yes_or_no(msg, path, yes=True): # Remove package installation dir shutil.rmtree(path) # Remove package config dir as well without ask path = utils.get_package_config_dir(model) if os.path.exists(path): shutil.rmtree(path) # Ask if remove cached files if cache is not None and utils.yes_or_no( "Remove cache '{}/' as well", cache, yes=False ): shutil.rmtree(cache) archive = utils.get_package_archive_dir(model) if os.path.exists(archive): shutil.rmtree(archive) else: if model is None and not args.quiet: utils.print_next_step("remove")
def remove_mlm(args): """Remove downloaded {} files.""".format(EXT_MLM) mlm = glob.glob(os.path.join(utils.get_init_dir(), "*.mlm")) mlm.sort() for m in mlm: if utils.yes_or_no("Remove model package archive '{}'", m, yes=True): os.remove(m)
def list_available(args): """List the name and title of the models in the Hub.""" # Setup. logger = logging.getLogger(__name__) logger.info("List available models.") logger.debug(f"args: {args}") meta, repo = utils.get_repo_meta_data(args.mlhub) model_names = [entry["meta"]["name"] for entry in meta] # Update bash completion list. utils.update_model_completion(set(model_names)) # List model name only. if args.name_only: print("\n".join(model_names)) return # Provide some context. if not args.quiet: msg = "The repository '{}' provides the following models:\n" print(msg.format(repo)) # List the meta data. for entry in meta: utils.print_meta_line(entry) # Suggest next step. if not args.quiet: utils.print_next_step("available") if not os.path.exists(utils.get_init_dir()): print( "Why not give the 'rain' model a go...\n\n" " $ ml install rain\n" )
def list_installed(args): """List the installed models.""" logger = logging.getLogger(__name__) logger.info('List installed models.') # Find installed models, ignoring special folders like R. init = utils.get_init_dir() if os.path.exists(init): msg = " in '{}'.".format(init) models = [ f for f in os.listdir(init) if os.path.isdir(os.path.join(init, f)) and f != "R" and not f.startswith('.') and not f.startswith('_') ] else: msg = ". '{}' does not exist.".format(init) models = [] models.sort() # Only list model names if args.name_only: print('\n'.join(models)) return # Report on how many models we found installed. mcnt = len(models) plural = "s" if mcnt != 1 else "" print("Found {} model{} installed{}".format(mcnt, plural, msg)) # Report on each of the installed models. if mcnt > 0: print("") invalid_models = [] for p in models: try: entry = utils.load_description(p) utils.print_meta_line(entry) except (utils.DescriptionYAMLNotFoundException, utils.MalformedYAMLException, KeyError): mcnt -= 1 invalid_models.append(p) continue # Update bash completion list. if 'commands' in entry: utils.update_command_completion(set(entry['commands'])) invalid_mcnt = len(invalid_models) if invalid_mcnt > 0: print("\nOf which {} model package{} {} broken:\n".format( invalid_mcnt, 's' if invalid_mcnt > 1 else '', 'are' if invalid_mcnt > 1 else 'is')) print(" ====> \033[31m" + ', '.join(invalid_models) + "\033[0m") print(utils.get_command_suggestion('remove')) # Suggest next step. if not args.quiet: if mcnt > 0: utils.print_next_step('installed', scenario='exist') else: utils.print_next_step('installed', scenario='none')
def main(): """Main program for the command line script.""" # -------------------------------------------------- # Global option parser. See mlhub.constants.OPTIONS # -------------------------------------------------- logger.info("Create global option parser.") global_option_parser = argparse.ArgumentParser( add_help=False # Disable -h or --help. Use custom help msg instead. ) utils.OptionAdder(global_option_parser, constants.OPTIONS).add_alloptions() # -------------------------------------------------- # Parse version # -------------------------------------------------- logger.info("Parse global options.") args, extras = global_option_parser.parse_known_args(sys.argv[1:]) if args.debug: # Add console log handler to log debug message to console logger.info('Enable printing out debug log on console.') utils.add_log_handler(logger, logging.StreamHandler(), logging.DEBUG, constants.LOG_CONSOLE_FORMAT) logger.debug('args: {}, extra_args: {}'.format(args, extras)) # Get the first positional argument. pos_args = [(i, arg) for i, arg in enumerate(sys.argv[1:]) if not arg.startswith('-')] first_pos_arg_index, first_pos_arg = pos_args[0] if len( pos_args) != 0 else (None, None) logger.debug('First positional argument: {}'.format(first_pos_arg)) if args.version: logger.info('Query version.') # -------------------------------------------------- # Query the version of the model, for example # $ ml rain -v # $ ml -v rain # Otherwise, output the version of ml # $ ml -v # -------------------------------------------------- if first_pos_arg is not None: # Print model version print(first_pos_arg, "version", utils.get_version(first_pos_arg)) else: # Print mlhub version print(constants.APP, "version", utils.get_version()) return 0 # -------------------------------------------------- # Parse command line args for basic commands or model specific commands # -------------------------------------------------- # Correct misspelled command if possible. if first_pos_arg is not None: # Only match basic commands since model pkg commands are more specific which would be # better to be checked after the model pkg name is known. matched_cmd = utils.get_misspelled_command(first_pos_arg, list(constants.COMMANDS)) if matched_cmd is not None: sys.argv[first_pos_arg_index + 1] = matched_cmd first_pos_arg = matched_cmd # Dispatch commands. if first_pos_arg is not None and first_pos_arg not in constants.COMMANDS: # Model specific commands, such as demo, display. logger.info("Parse model specific dommands.") model_cmd_parser = argparse.ArgumentParser( prog=constants.CMD, parents=[global_option_parser], add_help=False # Use custom help message ) model_cmd_parser.add_argument('cmd', metavar='command') model_cmd_parser.add_argument('model') args, extras = model_cmd_parser.parse_known_args() logger.debug("args: {}".format(args)) logger.debug("extra_args: {}".format(extras)) # Simple help message for the model-specific command if '--help' in extras or '-h' in extras: logger.debug("Help for command '{}' of '{}'".format( args.cmd, args.model)) utils.print_model_cmd_help(utils.load_description(args.model), args.cmd) return 0 setattr(args, 'func', commands.dispatch) setattr(args, 'param', extras) else: # Basic commands, such as install, readme. See mlhub.constants.COMMANDS logger.info("Parse basic common commands.") basic_cmd_parser = argparse.ArgumentParser( prog=constants.CMD, description="Access models from the ML Hub.", parents=[global_option_parser]) subparsers = basic_cmd_parser.add_subparsers(title='subcommands', dest="cmd") utils.SubCmdAdder(subparsers, commands, constants.COMMANDS).add_allsubcmds() args = basic_cmd_parser.parse_args() logger.debug("args: {}".format(args)) # Print usage for incorrect argument if "func" not in args: utils.print_usage() return 0 # Ensure we have a trailing slash on the mlhub. if args.mlhub is not None: constants.MLHUB = os.path.join(args.mlhub, "") if args.mlmetavar is not None: constants.CMD = args.mlmetavar # -------------------------------------------------- # Dispatch commands # -------------------------------------------------- try: args.func(args) except utils.MLInitCreateException as e: msg = "The below '{}' init folder cannot be created:\n {}" utils.print_error_exit(msg, constants.APP, e.args[0]) except utils.MLTmpDirCreateException as e: msg = "The below '{}' tmp folder cannot be created:\n {}" utils.print_error_exit(msg, constants.APP, e.args[0]) except utils.MalformedMLMFileNameException as e: msg = "Malformed {} file:\n {}" utils.print_error_exit(msg, constants.EXT_MLM, e.args[0]) except utils.MalformedYAMLException as e: name = e.args[0] if os.path.sep in name or '/' in name: msg = "Malformed YAML file:\n {}" else: msg = "Malformed description for model package '{}'!" utils.print_error_exit(msg, e.args[0]) except utils.ModelURLAccessException as e: msg = "URL access failed:\n {}" utils.print_error_exit(msg, e.args[0]) except utils.YAMLFileAccessException as e: msg = "YAML file access failed:\n {}" utils.print_error_exit(msg, e.args[0]) except utils.RepoAccessException as e: utils.print_error("Cannot access the ML Hub repository:\n {}", e.args[0]) if not args.quiet: # Suggest check if any models installed, since mlhub repo not accessible utils.print_commands_suggestions_on_stderr('installed') sys.exit(1) except utils.ModelNotFoundOnRepoException as e: msg = "No model named '{}' was found on\n {}" utils.print_error(msg, e.args[0], e.args[1]) if not args.quiet: # Suggest check if any models available, since specified model not found utils.print_commands_suggestions_on_stderr('available') sys.exit(1) except utils.ModelDownloadHaltException as e: msg = "URL - '{}' failed:\n {}".format(e.args[0], e.args[1]) utils.print_error_exit(msg) except utils.DescriptionYAMLNotFoundException as e: msg = "No MLHUB description file found: {}" location = e.args[0] if not utils.is_url(location): msg += "\n The model package may be broken!" utils.print_error(msg, location) if not args.quiet: # Suggest remove broken package or install new model utils.print_commands_suggestions_on_stderr('remove', 'install') else: msg += "\n The given location may be wrong!" utils.print_error(msg, location) sys.exit(1) except utils.ModelNotInstalledException as e: msg = "model '{}' is not installed ({})." utils.print_error(msg, e.args[0], utils.get_init_dir()) if not args.quiet: # Suggest install model package or check if any available utils.print_commands_suggestions_on_stderr('installed', 'available', 'install') sys.exit(1) except utils.ModelReadmeNotFoundException as e: msg = "The '{}' model does not have a '{}' file:\n {}\n" utils.print_error(msg, e.args[0], constants.README, e.args[1]) if not args.quiet: # Suggest remove broken package or install new model utils.print_commands_suggestions_on_stderr('remove', 'install') sys.exit(1) except utils.UnsupportedScriptExtensionException as e: msg = "Could not determine an interpreter for extension '{}'" utils.print_error_exit(msg, e.args[0]) except utils.CommandNotFoundException as e: msg = "The command '{}' was not found for this model '{}'." utils.print_error(msg, e.args[0], e.args[1]) if not args.quiet: # Suggest check if any available commands utils.print_commands_suggestions_on_stderr('commands') sys.exit(1) except utils.LackDependencyException as e: msg = "Required {} dependencies are not installed for this model: \n ====> \033[31m{}\033[0m" utils.print_error(msg, 'R' if e.args[1] else 'Python', e.args[0]) if not args.quiet: # Suggest install dependencies utils.print_commands_suggestions_on_stderr('configure') sys.exit(1) except utils.LackPrerequisiteException as e: msg = "Required pre-requisite not found: \n ====> \033[31m{}\033[0m" utils.print_error(msg, e.args[0]) if not args.quiet: # Suggest install dependencies msg = "\nTo install required pre-requisites:\n\n $ ml configure\n" utils.print_on_stderr(msg) sys.exit(1) except utils.DataResourceNotFoundException: msg = "Some data or model files required by the model package are missing!" utils.print_error(msg) if not args.quiet: # Suggest download data utils.print_commands_suggestions_on_stderr('configure') sys.exit(1) except utils.MalformedPackagesDotYAMLException as e: msg = ( "There is no '{}' available for the model package '{}' which may be under maintenance now.\n" "Please try again later.") utils.print_error(msg, e.args[0], e.args[1]) if not args.quiet: # Suggest check if any models available, since specified model not available utils.print_commands_suggestions_on_stderr('available') sys.exit(1) except utils.ModePkgInstallationFileNotFoundException as e: msg = "No such package file: {}\n The model package may be broken!" utils.print_error_exit(msg, e.args[0]) except utils.ModelPkgDependencyFileNotFoundException as e: msg = "Failed to get File dependency: {}\n" utils.print_error_exit(msg, e.args[0]) except utils.ConfigureFailedException: # configure failed, then just quit sys.exit(1) except (KeyboardInterrupt, EOFError): # Catch Ctrl-C and Ctrl-D print() sys.exit(1)
def list_installed(args): """List the installed models.""" logger = logging.getLogger(__name__) logger.info("List installed models.") # Find installed models, ignoring special folders like R. init = utils.get_init_dir() if os.path.exists(init): msg = f" in '{init}'." models = [ f for f in os.listdir(init) if os.path.isdir(os.path.join(init, f)) and f != "R" and not f.startswith(".") and not f.startswith("_") ] else: msg = f". '{init}' does not exist." models = [] models.sort() # Only list model names if args.name_only: print("\n".join(models)) return # Report on how many models we found installed. mcnt = len(models) plural = "s" if mcnt != 1 else "" print(f"Found {mcnt} model{plural} installed{msg}") # Report on each of the installed models. if mcnt > 0: print("") invalid_models = [] for p in models: try: entry = utils.load_description(p) utils.print_meta_line(entry) except ( utils.DescriptionYAMLNotFoundException, utils.MalformedYAMLException, KeyError, ): mcnt -= 1 invalid_models.append(p) continue # Update bash completion list. if "commands" in entry: utils.update_command_completion(set(entry["commands"])) invalid_mcnt = len(invalid_models) if invalid_mcnt > 0: print( "\nOf which {} model package{} {} broken:\n".format( invalid_mcnt, "s" if invalid_mcnt > 1 else "", "are" if invalid_mcnt > 1 else "is", ) ) print(" ====> \033[31m" + ", ".join(invalid_models) + "\033[0m") print(utils.get_command_suggestion("remove")) # Suggest next step. if not args.quiet: if mcnt > 0: utils.print_next_step("installed", scenario="exist") else: utils.print_next_step("installed", scenario="none")