def do_action_set(self, args):
if not args.contents:
return get_defaults(global_only=args.is_global)
for kv in args.contents:
key, value = kv.split('=')
assert key is not None and value is not None, "must specify a key=value pair"
update_default(key, value, is_global=args.is_global)
def __init__(self):
self.factory = dict()
defaults = get_defaults()
# cluster
self.add_argument('--cluster-alias', '-a', default=defaults.get('cluster-alias', None), help='cluster alias to select')
self.add_argument('--virtual-cluster', '--vc', default=defaults.get('virtual-cluster', None), help='virtual cluster to use')
self.add_argument('cluster_alias', help='cluster alias to select')
self.add_argument('--pai-uri', help="uri of openpai cluster, in format of http://x.x.x.x")
self.add_argument('--user', help='username')
self.add_argument('--password', help="password")
self.add_argument('--editor', default="code", help="path to your editor used to open files")
# storage
self.add_argument('--storage-alias', '-s', help='storage alias')
self.add_argument('--web-hdfs-uri', help="uri of web hdfs, in format of http://x.x.x.x:port")
# job spec
self.add_argument('--job-name', '-j', help='job name', default=defaults.get('job-name', None))
self.add_argument('--workspace', '-w', default=defaults.get('workspace', None), help='remote path for workspace (store code, output, ...)')
self.add_argument('--v2', action="store_true", default=False, help="use job protocol version 2")
self.add_argument('--sources', '-s', action='append', help='sources files')
# requirements
self.add_argument('--image', '-i', default=defaults.get('image', None), help='docker image')
# common
self.add_argument('--is-global', '-g', action="store_true", help="set globally (not limited to current working folder)", default=False)
self.add_argument('--details', help='if asserted, show details of the job (or cluster)', action='store_true', default=False)
self.add_argument('--default', help='set current as default', action='store_true', default=False)
self.add_argument('--update', '-u', action='append', help='replace current key-value pairs with new key=value (key1:key2:...=value for nested objects)')
self.add_argument('--preview', action='store_true', help='preview result before doing action')
self.add_argument('--no-browser', action='store_true', help='does not open the job link in web browser')
self.add_argument('--interactive', action='store_true', help='enter the interactive mode after job starts')
self.add_argument('--token', default="abcd", help='authentication token')
self.add_argument('--enable-sdk', action="store_true", default=False, help="enable sdk installation")
self.add_argument("--python", default="python", help="command or path of python, default is {python}, may be {python3}")
self.add_argument("--pip-installs", action="append", help="packages to be added by {pip install}")
# task role
self.add_argument('--task-role-name', '-t', default='main', help='task role name')
self.add_argument('--task-number', '-n', type=int, default=1, help='number of tasks per role')
self.add_argument('--cpu', type=int, default=defaults.get('cpu', 4), help='cpu number per instance')
self.add_argument('--gpu', type=int, default=defaults.get('gpu', 0), help='gpu number per instance')
self.add_argument('--memoryMB', type=int, default=defaults.get('memMB', 8192), help='memory #MB per instance')
self.add_argument('--cmd-sep', default="\s*&&\s*", help="command separator, default is (&&)")
self.add_argument('commands', nargs=argparse.REMAINDER, help='shell commands to execute')
# runtime
self.add_argument('config', nargs='?', help='job config file')
self.add_argument('notebook', nargs='?', help='Jupyter notebook file')
self.add_argument('--working-dir', default='', help="working directory")
# storage
self.add_argument('--storage-alias', help="alias of storage attached to cluster")
self.add_argument('--recursive', action='store_true', default=False, help="recursive target operation")
self.add_argument('--overwrite', action='store_true', default=False, help="enable overwrite if exists")
self.add_argument('local_path', help="local path")
self.add_argument('remote_path', help="remote path")
def __init__(self):
from openpaisdk import __cluster_config_file__ as openpai_ext_config
from openpaisdk.io_utils import from_file as openpai_ext_from_file
from openpaisdk.cluster import ClusterList as openpai_ext_ClusterList
from openpaisdk.io_utils import get_defaults, update_default
if get_defaults().get('container-sdk-branch') != 'notebook-extension':
update_default('container-sdk-branch', 'notebook-extension')
self.cll = openpai_ext_ClusterList(
openpai_ext_from_file(openpai_ext_config, default=[]))
def do_action_set(self, args):
defaults = get_defaults()
if not args.contents:
return defaults
for kv in args.contents:
key, value = kv.split('=')
assert key is not None and value is not None, "must specify a key=value pair"
defaults[key] = value
to_file(defaults, __local_default_file__)
return defaults
def do_action_unset(self, args):
result = []
defaults = get_defaults()
for key in args.variables:
if key not in defaults:
result.append(
"cannot unset default variable %s because it doesn't exist"
% key)
continue
value = defaults.pop(key, None)
result.append("default variable {} (previously {}) deleted".format(
key, value))
to_file(defaults, __local_default_file__)
return result
def deployment_for_sdk(self,
cluster_alias_lst: str,
workspace: str = None,
sources: list = [],
python: str = "python",
pip_installs: list = [],
name: str = "sdk_install",
taskRoles: list = ["main"]):
assert isinstance(sources, list) and isinstance(
pip_installs,
list), "sources and pip_installs must be list %s, %s" % (
sources, pip_installs)
# embed clusters and other info to secrets
clusters = [
c for c in ClusterList().load().clusters
if c["cluster_alias"] in cluster_alias_lst
]
version = get_defaults().get("sdk-branch", __sdk_branch__)
self.protocol["secrets"]["clusters"] = json.dumps(clusters)
self.protocol["secrets"]["cluster_alias"] = cluster_alias_lst[0]
if sources:
assert workspace, "must specify a workspace to transfer sources"
self.protocol["extras"]["__sources__"] = sources
if workspace:
self.protocol["secrets"].setdefault(
"work_directory", '{}/jobs/{}'.format(workspace, self.name))
# installing sdk and other packages
cmds = []
pip_installs += [get_install_uri(version)]
cmds += [
"{} -m pip install {}".format(python, p) for p in pip_installs
]
# restore clusters
c_dir = '~/{}'.format(__cache__)
c_file = '%s/%s' % (c_dir, os.path.basename(__cluster_config_file__))
cmds.extend([
"mkdir %s" % c_dir,
"echo \"write config to {}\"".format(c_file),
"echo <% $secrets.clusters %> > {}".format(c_file),
"opai cluster select <% $secrets.cluster_alias %>",
])
# download files
cmds += [
"opai storage download <% $secrets.work_directory %>/source/{} {}".
format(f, f) for f in sources
]
self.new_deployment(name, pre_commands=cmds, taskRoles=taskRoles)
def sdk_job_template(self,
cluster_alias_lst: str = [],
workspace: str = None,
sources: list = None,
pip_installs: list = None):
"generate the job template for a sdk-submitted job"
# secrets
clusters = [
get_cluster(alias, get_client=False) for alias in cluster_alias_lst
]
self.set_secret("clusters", json.dumps(clusters))
self.set_param("cluster_alias",
cluster_alias_lst[0] if cluster_alias_lst else None)
self.set_param(
"work_directory",
'{}/jobs/{}'.format(workspace, self.name) if workspace else None)
# parameters
self.set_param("python_path", "python")
# signature
self.add_tag(__internal_tags__["sdk"])
# sdk.plugins
sdk_install_uri = "-U {}".format(
get_install_uri(get_defaults().get("container-sdk-branch",
__container_sdk_branch__)))
c_dir = '~/{}'.format(__cache__)
c_file = '%s/%s' % (c_dir, os.path.basename(__cluster_config_file__))
plugins = []
if sources:
plugins.append({
"plugin": "local.uploadFiles",
"parameters": {
"files": sources,
},
})
plugins.extend([
{
"plugin":
"container.preCommands", # commands to install essential pip packages
"parameters": {
"commands": [
"<% $parameters.python_path %> -m pip install {}".
format(p)
for p in [sdk_install_uri] + na(pip_installs, [])
]
}
},
{
"plugin": "container.preCommands", # copy cluster information
"parameters": {
"commands": [
"mkdir %s" % c_dir,
"echo \"write config to {}\"".format(c_file),
"echo <% $secrets.clusters %> > {}".format(c_file),
"opai cluster select <% $parameters.cluster_alias %>",
]
}
}
])
if sources:
a_file = os.path.basename(self.temp_archive)
plugins.append({
"plugin": "container.preCommands",
"parameters": {
"commands": [
"opai storage download <% $parameters.work_directory %>/source/{} {}"
.format(a_file, a_file), "tar xvfz {}".format(a_file)
]
}
})
self.set_extra("sdk.plugins", plugins)
return self
