def main():
"""Main program for the command line script."""
# --------------------------------------------------
# Global option parser. See mlhub.constants.OPTIONS
# --------------------------------------------------
logger.info("Create global option parser.")
global_option_parser = argparse.ArgumentParser(
add_help=False # Disable -h or --help. Use custom help msg instead.
)
utils.OptionAdder(global_option_parser, constants.OPTIONS).add_alloptions()
# --------------------------------------------------
# Parse version
# --------------------------------------------------
logger.info("Parse global options.")
args, extras = global_option_parser.parse_known_args(sys.argv[1:])
if args.debug: # Add console log handler to log debug message to console
logger.info('Enable printing out debug log on console.')
utils.add_log_handler(logger, logging.StreamHandler(), logging.DEBUG,
constants.LOG_CONSOLE_FORMAT)
logger.debug('args: {}, extra_args: {}'.format(args, extras))
# Get the first positional argument.
pos_args = [(i, arg) for i, arg in enumerate(sys.argv[1:])
if not arg.startswith('-')]
first_pos_arg_index, first_pos_arg = pos_args[0] if len(
pos_args) != 0 else (None, None)
logger.debug('First positional argument: {}'.format(first_pos_arg))
if args.version:
logger.info('Query version.')
# --------------------------------------------------
# Query the version of the model, for example
# $ ml rain -v
# $ ml -v rain
# Otherwise, output the version of ml
# $ ml -v
# --------------------------------------------------
if first_pos_arg is not None: # Print model version
print(first_pos_arg, "version", utils.get_version(first_pos_arg))
else: # Print mlhub version
print(constants.APP, "version", utils.get_version())
return 0
# --------------------------------------------------
# Parse command line args for basic commands or model specific commands
# --------------------------------------------------
# Correct misspelled command if possible.
if first_pos_arg is not None:
# Only match basic commands since model pkg commands are more specific which would be
# better to be checked after the model pkg name is known.
matched_cmd = utils.get_misspelled_command(first_pos_arg,
list(constants.COMMANDS))
if matched_cmd is not None:
sys.argv[first_pos_arg_index + 1] = matched_cmd
first_pos_arg = matched_cmd
# Dispatch commands.
if first_pos_arg is not None and first_pos_arg not in constants.COMMANDS:
# Model specific commands, such as demo, display.
logger.info("Parse model specific dommands.")
model_cmd_parser = argparse.ArgumentParser(
prog=constants.CMD,
parents=[global_option_parser],
add_help=False # Use custom help message
)
model_cmd_parser.add_argument('cmd', metavar='command')
model_cmd_parser.add_argument('model')
args, extras = model_cmd_parser.parse_known_args()
logger.debug("args: {}".format(args))
logger.debug("extra_args: {}".format(extras))
# Simple help message for the model-specific command
if '--help' in extras or '-h' in extras:
logger.debug("Help for command '{}' of '{}'".format(
args.cmd, args.model))
utils.print_model_cmd_help(utils.load_description(args.model),
args.cmd)
return 0
setattr(args, 'func', commands.dispatch)
setattr(args, 'param', extras)
else:
# Basic commands, such as install, readme. See mlhub.constants.COMMANDS
logger.info("Parse basic common commands.")
basic_cmd_parser = argparse.ArgumentParser(
prog=constants.CMD,
description="Access models from the ML Hub.",
parents=[global_option_parser])
subparsers = basic_cmd_parser.add_subparsers(title='subcommands',
dest="cmd")
utils.SubCmdAdder(subparsers, commands,
constants.COMMANDS).add_allsubcmds()
args = basic_cmd_parser.parse_args()
logger.debug("args: {}".format(args))
# Print usage for incorrect argument
if "func" not in args:
utils.print_usage()
return 0
# Ensure we have a trailing slash on the mlhub.
if args.mlhub is not None:
constants.MLHUB = os.path.join(args.mlhub, "")
if args.mlmetavar is not None:
constants.CMD = args.mlmetavar
# --------------------------------------------------
# Dispatch commands
# --------------------------------------------------
try:
args.func(args)
except utils.MLInitCreateException as e:
msg = "The below '{}' init folder cannot be created:\n {}"
utils.print_error_exit(msg, constants.APP, e.args[0])
except utils.MLTmpDirCreateException as e:
msg = "The below '{}' tmp folder cannot be created:\n {}"
utils.print_error_exit(msg, constants.APP, e.args[0])
except utils.MalformedMLMFileNameException as e:
msg = "Malformed {} file:\n {}"
utils.print_error_exit(msg, constants.EXT_MLM, e.args[0])
except utils.MalformedYAMLException as e:
name = e.args[0]
if os.path.sep in name or '/' in name:
msg = "Malformed YAML file:\n {}"
else:
msg = "Malformed description for model package '{}'!"
utils.print_error_exit(msg, e.args[0])
except utils.ModelURLAccessException as e:
msg = "URL access failed:\n {}"
utils.print_error_exit(msg, e.args[0])
except utils.YAMLFileAccessException as e:
msg = "YAML file access failed:\n {}"
utils.print_error_exit(msg, e.args[0])
except utils.RepoAccessException as e:
utils.print_error("Cannot access the ML Hub repository:\n {}",
e.args[0])
if not args.quiet: # Suggest check if any models installed, since mlhub repo not accessible
utils.print_commands_suggestions_on_stderr('installed')
sys.exit(1)
except utils.ModelNotFoundOnRepoException as e:
msg = "No model named '{}' was found on\n {}"
utils.print_error(msg, e.args[0], e.args[1])
if not args.quiet: # Suggest check if any models available, since specified model not found
utils.print_commands_suggestions_on_stderr('available')
sys.exit(1)
except utils.ModelDownloadHaltException as e:
msg = "URL - '{}' failed:\n {}".format(e.args[0], e.args[1])
utils.print_error_exit(msg)
except utils.DescriptionYAMLNotFoundException as e:
msg = "No MLHUB description file found: {}"
location = e.args[0]
if not utils.is_url(location):
msg += "\n The model package may be broken!"
utils.print_error(msg, location)
if not args.quiet: # Suggest remove broken package or install new model
utils.print_commands_suggestions_on_stderr('remove', 'install')
else:
msg += "\n The given location may be wrong!"
utils.print_error(msg, location)
sys.exit(1)
except utils.ModelNotInstalledException as e:
msg = "model '{}' is not installed ({})."
utils.print_error(msg, e.args[0], utils.get_init_dir())
if not args.quiet: # Suggest install model package or check if any available
utils.print_commands_suggestions_on_stderr('installed',
'available', 'install')
sys.exit(1)
except utils.ModelReadmeNotFoundException as e:
msg = "The '{}' model does not have a '{}' file:\n {}\n"
utils.print_error(msg, e.args[0], constants.README, e.args[1])
if not args.quiet: # Suggest remove broken package or install new model
utils.print_commands_suggestions_on_stderr('remove', 'install')
sys.exit(1)
except utils.UnsupportedScriptExtensionException as e:
msg = "Could not determine an interpreter for extension '{}'"
utils.print_error_exit(msg, e.args[0])
except utils.CommandNotFoundException as e:
msg = "The command '{}' was not found for this model '{}'."
utils.print_error(msg, e.args[0], e.args[1])
if not args.quiet: # Suggest check if any available commands
utils.print_commands_suggestions_on_stderr('commands')
sys.exit(1)
except utils.LackDependencyException as e:
msg = "Required {} dependencies are not installed for this model: \n ====> \033[31m{}\033[0m"
utils.print_error(msg, 'R' if e.args[1] else 'Python', e.args[0])
if not args.quiet: # Suggest install dependencies
utils.print_commands_suggestions_on_stderr('configure')
sys.exit(1)
except utils.LackPrerequisiteException as e:
msg = "Required pre-requisite not found: \n ====> \033[31m{}\033[0m"
utils.print_error(msg, e.args[0])
if not args.quiet: # Suggest install dependencies
msg = "\nTo install required pre-requisites:\n\n $ ml configure\n"
utils.print_on_stderr(msg)
sys.exit(1)
except utils.DataResourceNotFoundException:
msg = "Some data or model files required by the model package are missing!"
utils.print_error(msg)
if not args.quiet: # Suggest download data
utils.print_commands_suggestions_on_stderr('configure')
sys.exit(1)
except utils.MalformedPackagesDotYAMLException as e:
msg = (
"There is no '{}' available for the model package '{}' which may be under maintenance now.\n"
"Please try again later.")
utils.print_error(msg, e.args[0], e.args[1])
if not args.quiet: # Suggest check if any models available, since specified model not available
utils.print_commands_suggestions_on_stderr('available')
sys.exit(1)
except utils.ModePkgInstallationFileNotFoundException as e:
msg = "No such package file: {}\n The model package may be broken!"
utils.print_error_exit(msg, e.args[0])
except utils.ModelPkgDependencyFileNotFoundException as e:
msg = "Failed to get File dependency: {}\n"
utils.print_error_exit(msg, e.args[0])
except utils.ConfigureFailedException: # configure failed, then just quit
sys.exit(1)
except (KeyboardInterrupt, EOFError): # Catch Ctrl-C and Ctrl-D
print()
sys.exit(1)
def dispatch(args):
"""Dispatch other commands to the appropriate model provided script."""
cmd = args.cmd
model = args.model
path = utils.get_package_dir(model)
param = " ".join(args.param)
# Get working dir if any.
if args.workding_dir is not None:
utils.update_working_dir(model, args.workding_dir)
if args.workding_dir == '':
args.workding_dir = None
else:
args.working_dir = utils.get_working_dir(model)
# Get conda environment name if any.
conda_env_name = utils.get_conda_env_name(model)
# Check that the model is installed and has commands.
utils.check_model_installed(model)
entry = utils.load_description(model)
if 'commands' not in entry or len(entry['commands']) == 0:
raise utils.CommandNotFoundException(cmd, model)
# Correct misspelled command if possible.
matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands']))
if matched_cmd is not None:
cmd = matched_cmd
# Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml.
meta = entry['meta']
if 'display' in meta and cmd in meta['display'] and os.environ.get(
'DISPLAY', '') == '':
msg = "Graphic display is required but not available for command '{}'. Continue"
yes = utils.yes_or_no(msg, cmd, yes=False)
if not yes:
msg = """
To enable DISPLAY be sure to connect to the server using 'ssh -X'
or else connect to the server's desktop using a local X server like X2Go.
"""
sys.stdout.write(msg)
sys.exit(1)
# Obtain the default/chosen language for the package.
lang = meta["languages"]
# Deal with malformed 'languages' field
lang_opts = {"python": "py", "R": "R"}
for k in list(lang_opts):
if lang in k:
lang = lang_opts[k]
break
# Obtain the specified script file.
script = cmd + "." + lang
logger = logging.getLogger(__name__)
logger.debug("Execute the script: " + os.path.join(path, script))
if cmd not in list(entry['commands']) or not os.path.exists(
os.path.join(path, script)):
raise utils.CommandNotFoundException(cmd, model)
# Determine the interpreter to use
#
# .R => Rscript; .py => python, etc.
interpreter = utils.interpreter(script)
# Change working dir if needed
if args.workding_dir is not None:
script = os.path.join(path, script)
path = args.workding_dir
# _MLHUB_CMD_CWD: a environment variable indicates current working
# directory where command `ml xxx` is invoked.
# _MLHUB_MODEL_NAME: env variable indicates the name of the model.
#
# The above two env vars can be obtained by helper function, such
# as utils.get_cmd_cwd(). And model package developer should be
# use the helper function instead of the env vars directly.
command = "export _MLHUB_CMD_CWD='{}'; export _MLHUB_MODEL_NAME='{}'; {} {} {}".format(
os.getcwd(), model, interpreter, script, param)
# Run script inside conda environment if specified
if conda_env_name is not None:
command = 'bash -c "source activate {}; {}"'.format(
conda_env_name, command)
logger.debug("(cd " + path + "; " + command + ")")
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
missing_r_dep = False
if proc.returncode != 0:
errors = errors.decode("utf-8")
# Check if it is Python dependency unsatisfied
dep_required = re.compile(
r"ModuleNotFoundError: No module named '(.*)'").search(errors)
# Check if R dependency unsatisified
if dep_required is None:
dep_required = re.compile(
r"there is no package called ‘(.*)’").search(errors)
if dep_required is not None:
missing_r_dep = True
# Check if required data resource not found
data_required = re.compile(
r"mlhub.utils.DataResourceNotFoundException").search(errors)
if dep_required is not None: # Dependency unsatisfied
dep_required = dep_required.group(1)
logger.error("Dependency unsatisfied: {}\n{}".format(
dep_required, errors))
raise utils.LackDependencyException(dep_required, missing_r_dep,
model)
elif data_required is not None: # Data not found
raise utils.DataResourceNotFoundException()
else: # Other unknown errors
print("An error was encountered:\n")
print(errors)
else:
# Suggest next step
if not args.quiet:
utils.print_next_step(cmd, description=entry, model=model)
def dispatch(args):
"""Dispatch other commands to the appropriate model provided script."""
cmd = args.cmd
model = args.model
path = utils.get_package_dir(model)
param = " ".join(args.param)
# Get working dir if any.
if args.workding_dir is not None:
utils.update_working_dir(model, args.workding_dir)
if args.workding_dir == '':
args.workding_dir = None
else:
args.working_dir = utils.get_working_dir(model)
# Get conda environment name if any.
conda_env_name = utils.get_conda_env_name(model)
# Check that the model is installed and has commands.
utils.check_model_installed(model)
entry = utils.load_description(model)
if 'commands' not in entry or len(entry['commands']) == 0:
raise utils.CommandNotFoundException(cmd, model)
# Correct misspelled command if possible.
matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands']))
if matched_cmd is not None:
cmd = matched_cmd
# Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml.
meta = entry['meta']
if 'display' in meta and cmd in meta['display'] and os.environ.get(
'DISPLAY', '') == '':
msg = "Graphic display is required but not available for command '{}'. Continue"
yes = utils.yes_or_no(msg, cmd, yes=False)
if not yes:
msg = """
To enable DISPLAY be sure to connect to the server using 'ssh -X'
or else connect to the server's desktop using a local X server like X2Go.
"""
sys.stdout.write(msg)
sys.exit(1)
# Obtain the default/chosen language for the package.
lang = meta["languages"]
# Deal with malformed 'languages' field
lang_opts = {"python": "py", "R": "R"}
for k in list(lang_opts):
if lang in k:
lang = lang_opts[k]
break
# Obtain the specified script file.
script = cmd + "." + lang
logger = logging.getLogger(__name__)
logger.debug("Execute the script: " + os.path.join(path, script))
if cmd not in list(entry['commands']) or not os.path.exists(
os.path.join(path, script)):
raise utils.CommandNotFoundException(cmd, model)
# Determine the interpreter to use
#
# .R => Rscript; .py => python, etc.
interpreter = utils.interpreter(script)
# Change working dir if needed
if args.workding_dir is not None:
script = os.path.join(path, script)
path = args.workding_dir
# Handle python environment
python_pkg_bin = None
python_pkg_path = None
if script.endswith('py'):
python_pkg_base = os.path.sep.join(
[utils.get_package_dir(model), '.python'])
python_pkg_path = python_pkg_base + site.USER_SITE
python_pkg_bin = python_pkg_base + site.USER_BASE + '/bin'
# TODO: Make sure to document:
# $ sudo apt-get install -y python3-pip
# $ /usr/bin/pip3 install mlhub
# Since in DSVM, the default pip is conda's pip, so if we stick to
# use system's command, then the installation of MLHub itself should
# be completed via system's pip, otherwise, MLHub will not work.
if sys.executable != SYS_PYTHON_CMD:
python_pkg_path = python_pkg_base + site.getsitepackages()[0]
python_pkg_bin = python_pkg_base + site.PREFIXES[0] + '/bin'
if utils.get_sys_python_pkg_usage(model):
utils.print_on_stderr(MSG_INCOMPATIBLE_PYTHON_ENV, model)
# _MLHUB_CMD_CWD: a environment variable indicates current working
# directory where command `ml xxx` is invoked.
# _MLHUB_MODEL_NAME: env variable indicates the name of the model.
#
# The above two env vars can be obtained by helper function, such
# as utils.get_cmd_cwd(). And model package developer should be
# use the helper function instead of the env vars directly.
env_var = "export _MLHUB_CMD_CWD='{}'; ".format(os.getcwd())
env_var += "export _MLHUB_MODEL_NAME='{}'; ".format(model)
env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable)
env_var += "export PYTHONPATH='{}'; ".format(
python_pkg_path) if python_pkg_path else ""
env_var += "export PATH=\"{}:$PATH\"; ".format(
python_pkg_bin) if python_pkg_bin else ""
command = "{}{} {} {}".format(env_var, interpreter, script, param)
# Run script inside conda environment if specified
if conda_env_name is not None:
command = '{} -c "source activate {}; {}"'.format(
BASH_CMD, conda_env_name, command)
logger.debug("(cd " + path + "; " + command + ")")
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
missing_r_dep = False
if proc.returncode != 0:
errors = errors.decode("utf-8")
# Check if it is Python dependency unsatisfied
dep_required = re.compile(
r"ModuleNotFoundError: No module named '(.*)'").search(errors)
# Check if R dependency unsatisified
if dep_required is None:
dep_required = re.compile(
r"there is no package called ‘(.*)’").search(errors)
if dep_required is not None:
missing_r_dep = True
# Check if required data resource not found
data_required = re.compile(
r"mlhub.utils.DataResourceNotFoundException").search(errors)
if dep_required is not None: # Dependency unsatisfied
dep_required = dep_required.group(1)
logger.error("Dependency unsatisfied: {}\n{}".format(
dep_required, errors))
raise utils.LackDependencyException(dep_required, missing_r_dep,
model)
elif data_required is not None: # Data not found
raise utils.DataResourceNotFoundException()
else: # Other unknown errors
print("An error was encountered:\n")
print(errors)
else:
# Suggest next step
if not args.quiet:
utils.print_next_step(cmd, description=entry, model=model)