def do_action_set(self, args): if not args.contents: return get_defaults(global_only=args.is_global) for kv in args.contents: key, value = kv.split('=') assert key is not None and value is not None, "must specify a key=value pair" update_default(key, value, is_global=args.is_global)
def __init__(self): self.factory = dict() defaults = get_defaults() # cluster self.add_argument('--cluster-alias', '-a', default=defaults.get('cluster-alias', None), help='cluster alias to select') self.add_argument('--virtual-cluster', '--vc', default=defaults.get('virtual-cluster', None), help='virtual cluster to use') self.add_argument('cluster_alias', help='cluster alias to select') self.add_argument('--pai-uri', help="uri of openpai cluster, in format of http://x.x.x.x") self.add_argument('--user', help='username') self.add_argument('--password', help="password") self.add_argument('--editor', default="code", help="path to your editor used to open files") # storage self.add_argument('--storage-alias', '-s', help='storage alias') self.add_argument('--web-hdfs-uri', help="uri of web hdfs, in format of http://x.x.x.x:port") # job spec self.add_argument('--job-name', '-j', help='job name', default=defaults.get('job-name', None)) self.add_argument('--workspace', '-w', default=defaults.get('workspace', None), help='remote path for workspace (store code, output, ...)') self.add_argument('--v2', action="store_true", default=False, help="use job protocol version 2") self.add_argument('--sources', '-s', action='append', help='sources files') # requirements self.add_argument('--image', '-i', default=defaults.get('image', None), help='docker image') # common self.add_argument('--is-global', '-g', action="store_true", help="set globally (not limited to current working folder)", default=False) self.add_argument('--details', help='if asserted, show details of the job (or cluster)', action='store_true', default=False) self.add_argument('--default', help='set current as default', action='store_true', default=False) self.add_argument('--update', '-u', action='append', help='replace current key-value pairs with new key=value (key1:key2:...=value for nested objects)') self.add_argument('--preview', action='store_true', help='preview result before doing action') self.add_argument('--no-browser', action='store_true', help='does not open the job link in web browser') self.add_argument('--interactive', action='store_true', help='enter the interactive mode after job starts') self.add_argument('--token', default="abcd", help='authentication token') self.add_argument('--enable-sdk', action="store_true", default=False, help="enable sdk installation") self.add_argument("--python", default="python", help="command or path of python, default is {python}, may be {python3}") self.add_argument("--pip-installs", action="append", help="packages to be added by {pip install}") # task role self.add_argument('--task-role-name', '-t', default='main', help='task role name') self.add_argument('--task-number', '-n', type=int, default=1, help='number of tasks per role') self.add_argument('--cpu', type=int, default=defaults.get('cpu', 4), help='cpu number per instance') self.add_argument('--gpu', type=int, default=defaults.get('gpu', 0), help='gpu number per instance') self.add_argument('--memoryMB', type=int, default=defaults.get('memMB', 8192), help='memory #MB per instance') self.add_argument('--cmd-sep', default="\s*&&\s*", help="command separator, default is (&&)") self.add_argument('commands', nargs=argparse.REMAINDER, help='shell commands to execute') # runtime self.add_argument('config', nargs='?', help='job config file') self.add_argument('notebook', nargs='?', help='Jupyter notebook file') self.add_argument('--working-dir', default='', help="working directory") # storage self.add_argument('--storage-alias', help="alias of storage attached to cluster") self.add_argument('--recursive', action='store_true', default=False, help="recursive target operation") self.add_argument('--overwrite', action='store_true', default=False, help="enable overwrite if exists") self.add_argument('local_path', help="local path") self.add_argument('remote_path', help="remote path")
def __init__(self): from openpaisdk import __cluster_config_file__ as openpai_ext_config from openpaisdk.io_utils import from_file as openpai_ext_from_file from openpaisdk.cluster import ClusterList as openpai_ext_ClusterList from openpaisdk.io_utils import get_defaults, update_default if get_defaults().get('container-sdk-branch') != 'notebook-extension': update_default('container-sdk-branch', 'notebook-extension') self.cll = openpai_ext_ClusterList( openpai_ext_from_file(openpai_ext_config, default=[]))
def do_action_set(self, args): defaults = get_defaults() if not args.contents: return defaults for kv in args.contents: key, value = kv.split('=') assert key is not None and value is not None, "must specify a key=value pair" defaults[key] = value to_file(defaults, __local_default_file__) return defaults
def do_action_unset(self, args): result = [] defaults = get_defaults() for key in args.variables: if key not in defaults: result.append( "cannot unset default variable %s because it doesn't exist" % key) continue value = defaults.pop(key, None) result.append("default variable {} (previously {}) deleted".format( key, value)) to_file(defaults, __local_default_file__) return result
def deployment_for_sdk(self, cluster_alias_lst: str, workspace: str = None, sources: list = [], python: str = "python", pip_installs: list = [], name: str = "sdk_install", taskRoles: list = ["main"]): assert isinstance(sources, list) and isinstance( pip_installs, list), "sources and pip_installs must be list %s, %s" % ( sources, pip_installs) # embed clusters and other info to secrets clusters = [ c for c in ClusterList().load().clusters if c["cluster_alias"] in cluster_alias_lst ] version = get_defaults().get("sdk-branch", __sdk_branch__) self.protocol["secrets"]["clusters"] = json.dumps(clusters) self.protocol["secrets"]["cluster_alias"] = cluster_alias_lst[0] if sources: assert workspace, "must specify a workspace to transfer sources" self.protocol["extras"]["__sources__"] = sources if workspace: self.protocol["secrets"].setdefault( "work_directory", '{}/jobs/{}'.format(workspace, self.name)) # installing sdk and other packages cmds = [] pip_installs += [get_install_uri(version)] cmds += [ "{} -m pip install {}".format(python, p) for p in pip_installs ] # restore clusters c_dir = '~/{}'.format(__cache__) c_file = '%s/%s' % (c_dir, os.path.basename(__cluster_config_file__)) cmds.extend([ "mkdir %s" % c_dir, "echo \"write config to {}\"".format(c_file), "echo <% $secrets.clusters %> > {}".format(c_file), "opai cluster select <% $secrets.cluster_alias %>", ]) # download files cmds += [ "opai storage download <% $secrets.work_directory %>/source/{} {}". format(f, f) for f in sources ] self.new_deployment(name, pre_commands=cmds, taskRoles=taskRoles)
def sdk_job_template(self, cluster_alias_lst: str = [], workspace: str = None, sources: list = None, pip_installs: list = None): "generate the job template for a sdk-submitted job" # secrets clusters = [ get_cluster(alias, get_client=False) for alias in cluster_alias_lst ] self.set_secret("clusters", json.dumps(clusters)) self.set_param("cluster_alias", cluster_alias_lst[0] if cluster_alias_lst else None) self.set_param( "work_directory", '{}/jobs/{}'.format(workspace, self.name) if workspace else None) # parameters self.set_param("python_path", "python") # signature self.add_tag(__internal_tags__["sdk"]) # sdk.plugins sdk_install_uri = "-U {}".format( get_install_uri(get_defaults().get("container-sdk-branch", __container_sdk_branch__))) c_dir = '~/{}'.format(__cache__) c_file = '%s/%s' % (c_dir, os.path.basename(__cluster_config_file__)) plugins = [] if sources: plugins.append({ "plugin": "local.uploadFiles", "parameters": { "files": sources, }, }) plugins.extend([ { "plugin": "container.preCommands", # commands to install essential pip packages "parameters": { "commands": [ "<% $parameters.python_path %> -m pip install {}". format(p) for p in [sdk_install_uri] + na(pip_installs, []) ] } }, { "plugin": "container.preCommands", # copy cluster information "parameters": { "commands": [ "mkdir %s" % c_dir, "echo \"write config to {}\"".format(c_file), "echo <% $secrets.clusters %> > {}".format(c_file), "opai cluster select <% $parameters.cluster_alias %>", ] } } ]) if sources: a_file = os.path.basename(self.temp_archive) plugins.append({ "plugin": "container.preCommands", "parameters": { "commands": [ "opai storage download <% $parameters.work_directory %>/source/{} {}" .format(a_file, a_file), "tar xvfz {}".format(a_file) ] } }) self.set_extra("sdk.plugins", plugins) return self