def main():
"""Main program for the command line script."""
# --------------------------------------------------
# Global option parser. See mlhub.constants.OPTIONS
# --------------------------------------------------
logger.info("Create global option parser.")
global_option_parser = argparse.ArgumentParser(
add_help=False # Disable -h or --help. Use custom help msg instead.
)
utils.OptionAdder(global_option_parser, constants.OPTIONS).add_alloptions()
# --------------------------------------------------
# Parse version
# --------------------------------------------------
logger.info("Parse global options.")
args, extras = global_option_parser.parse_known_args(sys.argv[1:])
if args.debug: # Add console log handler to log debug message to console
logger.info('Enable printing out debug log on console.')
utils.add_log_handler(logger, logging.StreamHandler(), logging.DEBUG,
constants.LOG_CONSOLE_FORMAT)
logger.debug('args: {}, extra_args: {}'.format(args, extras))
# Get the first positional argument.
pos_args = [(i, arg) for i, arg in enumerate(sys.argv[1:])
if not arg.startswith('-')]
first_pos_arg_index, first_pos_arg = pos_args[0] if len(
pos_args) != 0 else (None, None)
logger.debug('First positional argument: {}'.format(first_pos_arg))
if args.version:
logger.info('Query version.')
# --------------------------------------------------
# Query the version of the model, for example
# $ ml rain -v
# $ ml -v rain
# Otherwise, output the version of ml
# $ ml -v
# --------------------------------------------------
if first_pos_arg is not None: # Print model version
print(first_pos_arg, "version", utils.get_version(first_pos_arg))
else: # Print mlhub version
print(constants.APP, "version", utils.get_version())
return 0
# --------------------------------------------------
# Parse command line args for basic commands or model specific commands
# --------------------------------------------------
# Correct misspelled command if possible.
if first_pos_arg is not None:
# Only match basic commands since model pkg commands are more specific which would be
# better to be checked after the model pkg name is known.
matched_cmd = utils.get_misspelled_command(first_pos_arg,
list(constants.COMMANDS))
if matched_cmd is not None:
sys.argv[first_pos_arg_index + 1] = matched_cmd
first_pos_arg = matched_cmd
# Dispatch commands.
if first_pos_arg is not None and first_pos_arg not in constants.COMMANDS:
# Model specific commands, such as demo, display.
logger.info("Parse model specific dommands.")
model_cmd_parser = argparse.ArgumentParser(
prog=constants.CMD,
parents=[global_option_parser],
add_help=False # Use custom help message
)
model_cmd_parser.add_argument('cmd', metavar='command')
model_cmd_parser.add_argument('model')
args, extras = model_cmd_parser.parse_known_args()
logger.debug("args: {}".format(args))
logger.debug("extra_args: {}".format(extras))
# Simple help message for the model-specific command
if '--help' in extras or '-h' in extras:
logger.debug("Help for command '{}' of '{}'".format(
args.cmd, args.model))
utils.print_model_cmd_help(utils.load_description(args.model),
args.cmd)
return 0
setattr(args, 'func', commands.dispatch)
setattr(args, 'param', extras)
else:
# Basic commands, such as install, readme. See mlhub.constants.COMMANDS
logger.info("Parse basic common commands.")
basic_cmd_parser = argparse.ArgumentParser(
prog=constants.CMD,
description="Access models from the ML Hub.",
parents=[global_option_parser])
subparsers = basic_cmd_parser.add_subparsers(title='subcommands',
dest="cmd")
utils.SubCmdAdder(subparsers, commands,
constants.COMMANDS).add_allsubcmds()
args = basic_cmd_parser.parse_args()
logger.debug("args: {}".format(args))
# Print usage for incorrect argument
if "func" not in args:
utils.print_usage()
return 0
# Ensure we have a trailing slash on the mlhub.
if args.mlhub is not None:
constants.MLHUB = os.path.join(args.mlhub, "")
if args.mlmetavar is not None:
constants.CMD = args.mlmetavar
# --------------------------------------------------
# Dispatch commands
# --------------------------------------------------
try:
args.func(args)
except utils.MLInitCreateException as e:
msg = "The below '{}' init folder cannot be created:\n {}"
utils.print_error_exit(msg, constants.APP, e.args[0])
except utils.MLTmpDirCreateException as e:
msg = "The below '{}' tmp folder cannot be created:\n {}"
utils.print_error_exit(msg, constants.APP, e.args[0])
except utils.MalformedMLMFileNameException as e:
msg = "Malformed {} file:\n {}"
utils.print_error_exit(msg, constants.EXT_MLM, e.args[0])
except utils.MalformedYAMLException as e:
name = e.args[0]
if os.path.sep in name or '/' in name:
msg = "Malformed YAML file:\n {}"
else:
msg = "Malformed description for model package '{}'!"
utils.print_error_exit(msg, e.args[0])
except utils.ModelURLAccessException as e:
msg = "URL access failed:\n {}"
utils.print_error_exit(msg, e.args[0])
except utils.YAMLFileAccessException as e:
msg = "YAML file access failed:\n {}"
utils.print_error_exit(msg, e.args[0])
except utils.RepoAccessException as e:
utils.print_error("Cannot access the ML Hub repository:\n {}",
e.args[0])
if not args.quiet: # Suggest check if any models installed, since mlhub repo not accessible
utils.print_commands_suggestions_on_stderr('installed')
sys.exit(1)
except utils.ModelNotFoundOnRepoException as e:
msg = "No model named '{}' was found on\n {}"
utils.print_error(msg, e.args[0], e.args[1])
if not args.quiet: # Suggest check if any models available, since specified model not found
utils.print_commands_suggestions_on_stderr('available')
sys.exit(1)
except utils.ModelDownloadHaltException as e:
msg = "URL - '{}' failed:\n {}".format(e.args[0], e.args[1])
utils.print_error_exit(msg)
except utils.DescriptionYAMLNotFoundException as e:
msg = "No MLHUB description file found: {}"
location = e.args[0]
if not utils.is_url(location):
msg += "\n The model package may be broken!"
utils.print_error(msg, location)
if not args.quiet: # Suggest remove broken package or install new model
utils.print_commands_suggestions_on_stderr('remove', 'install')
else:
msg += "\n The given location may be wrong!"
utils.print_error(msg, location)
sys.exit(1)
except utils.ModelNotInstalledException as e:
msg = "model '{}' is not installed ({})."
utils.print_error(msg, e.args[0], utils.get_init_dir())
if not args.quiet: # Suggest install model package or check if any available
utils.print_commands_suggestions_on_stderr('installed',
'available', 'install')
sys.exit(1)
except utils.ModelReadmeNotFoundException as e:
msg = "The '{}' model does not have a '{}' file:\n {}\n"
utils.print_error(msg, e.args[0], constants.README, e.args[1])
if not args.quiet: # Suggest remove broken package or install new model
utils.print_commands_suggestions_on_stderr('remove', 'install')
sys.exit(1)
except utils.UnsupportedScriptExtensionException as e:
msg = "Could not determine an interpreter for extension '{}'"
utils.print_error_exit(msg, e.args[0])
except utils.CommandNotFoundException as e:
msg = "The command '{}' was not found for this model '{}'."
utils.print_error(msg, e.args[0], e.args[1])
if not args.quiet: # Suggest check if any available commands
utils.print_commands_suggestions_on_stderr('commands')
sys.exit(1)
except utils.LackDependencyException as e:
msg = "Required {} dependencies are not installed for this model: \n ====> \033[31m{}\033[0m"
utils.print_error(msg, 'R' if e.args[1] else 'Python', e.args[0])
if not args.quiet: # Suggest install dependencies
utils.print_commands_suggestions_on_stderr('configure')
sys.exit(1)
except utils.LackPrerequisiteException as e:
msg = "Required pre-requisite not found: \n ====> \033[31m{}\033[0m"
utils.print_error(msg, e.args[0])
if not args.quiet: # Suggest install dependencies
msg = "\nTo install required pre-requisites:\n\n $ ml configure\n"
utils.print_on_stderr(msg)
sys.exit(1)
except utils.DataResourceNotFoundException:
msg = "Some data or model files required by the model package are missing!"
utils.print_error(msg)
if not args.quiet: # Suggest download data
utils.print_commands_suggestions_on_stderr('configure')
sys.exit(1)
except utils.MalformedPackagesDotYAMLException as e:
msg = (
"There is no '{}' available for the model package '{}' which may be under maintenance now.\n"
"Please try again later.")
utils.print_error(msg, e.args[0], e.args[1])
if not args.quiet: # Suggest check if any models available, since specified model not available
utils.print_commands_suggestions_on_stderr('available')
sys.exit(1)
except utils.ModePkgInstallationFileNotFoundException as e:
msg = "No such package file: {}\n The model package may be broken!"
utils.print_error_exit(msg, e.args[0])
except utils.ModelPkgDependencyFileNotFoundException as e:
msg = "Failed to get File dependency: {}\n"
utils.print_error_exit(msg, e.args[0])
except utils.ConfigureFailedException: # configure failed, then just quit
sys.exit(1)
except (KeyboardInterrupt, EOFError): # Catch Ctrl-C and Ctrl-D
print()
sys.exit(1)
def main():
"""Main program for the command line script."""
#------------------------------------
# COMMAND LINE PARSER
#------------------------------------
parser = argparse.ArgumentParser(
prog=CMD,
description="Access models from the ML Hub.",
)
#------------------------------------
# --DEBUG
#------------------------------------
parser.add_argument('--debug',
action='store_true',
help="Display debug information.")
#------------------------------------
# --QUIET
parser.add_argument('--quiet', action='store_true', help="Reduce noise.")
#------------------------------------
# --INIT-DIR
#------------------------------------
parser.add_argument(
'--init-dir',
help="Use this as the init dir instead of '{}'.".format(MLINIT))
#------------------------------------
# --MLHUB
#------------------------------------
parser.add_argument('--mlhub',
help="Use this ML Hub instead of '{}'.".format(MLHUB))
#------------------------------------
# --CMD
#------------------------------------
parser.add_argument(
'--cmd', help="Command display name instead of '{}'.".format(CMD))
#------------------------------------
# We support a basic set of commands
# and then any model specific
# commands provided in the archive.
#------------------------------------
subparsers = parser.add_subparsers(
title='subcommands',
dest="cmd",
)
#------------------------------------
# AVAILABLE
#------------------------------------
parser_available = subparsers.add_parser(
"available",
aliases=['avail'],
description="List the models available from the ML Hub",
)
parser_available.set_defaults(func=commands.list_available)
#------------------------------------
# INSTALLED
#------------------------------------
parser_installed = subparsers.add_parser(
"installed",
description="List the locally installed models",
)
parser_installed.set_defaults(func=commands.list_installed)
#------------------------------------
# CLEAN
#------------------------------------
parser_clean = subparsers.add_parser(
"clean",
description="Remove downloaded .mlm archive files",
)
parser_clean.set_defaults(func=commands.remove_mlm)
#------------------------------------
# INSTALL
#------------------------------------
parser_install = subparsers.add_parser(
"install",
description="Locally install a model downloaded from a ML Hub",
)
parser_install.add_argument("model")
parser_install.set_defaults(func=commands.install_model)
#------------------------------------
# DOWNLOAD
#------------------------------------
parser_download = subparsers.add_parser(
"download",
description="Download the actual (large) pre-built model",
)
parser_download.add_argument("model")
parser_download.set_defaults(func=commands.download_model)
#------------------------------------
# README
#------------------------------------
parser_readme = subparsers.add_parser(
"readme",
description="Display the model's README information",
)
parser_readme.add_argument("model")
parser_readme.set_defaults(func=commands.readme)
#------------------------------------
# LICENSE
#------------------------------------
parser_license = subparsers.add_parser(
"license",
description="Display the model's LICENSE information",
)
parser_license.add_argument("model")
parser_license.set_defaults(func=commands.license)
#------------------------------------
# COMMANDS
#------------------------------------
parser_cmds = subparsers.add_parser(
"commands",
description="List all of the commands supported by the model",
)
parser_cmds.add_argument("model")
parser_cmds.set_defaults(func=commands.list_model_commands)
#------------------------------------
# CONFIGURE
#------------------------------------
parser_configure = subparsers.add_parser(
"configure",
description="Configure the dependencies required for the model",
)
parser_configure.add_argument("model")
parser_configure.set_defaults(func=commands.configure_model)
#------------------------------------
# MODEL SPECIFIC COMMANDS
#
# Need to make this general or dynamic
# based on the comannds available
# from the locally installed models?
#------------------------------------
parser_cmd = subparsers.add_parser(
"demo",
aliases=['print', 'display', 'score', 'rebuild'],
description="Model commands",
)
parser_cmd.add_argument("model")
parser_cmd.add_argument("param", nargs="*")
parser_cmd.set_defaults(func=commands.dispatch)
#------------------------------------
# DONATE
#------------------------------------
parser_donate = subparsers.add_parser(
"donate",
description="Consider a donation to the author",
)
parser_donate.add_argument("model")
parser_donate.set_defaults(func=commands.donate)
#------------------------------------
# ACTION
#------------------------------------
sys.argv.pop(0)
args = parser.parse_args(sys.argv)
# Ensure we have a trainling slash on the mlhub.
if args.mlhub is not None: mlhub = os.path.join(args.mlhub, "")
if args.cmd is not None: constants.CMD = args.cmd
if args.debug:
constants.debug = True
print(DEBUG + str(args))
if not "func" in args:
utils.print_usage()
return 0
args.func(args)