def get(*args):
dic = LayeredSettings.as_dict()
if not args:
return dic
elif len(args) == 1:
return dic[args[0]]
else:
return [dic[a] for a in args]
def __init__(self):
self.factory = dict()
# deal with predefined defaults
for name, params in LayeredSettings.definitions.items():
args = ['--' + name]
abbr = params.get('abbreviation', None)
if abbr: # args = ['--{name}', '-{abbr}' or '--{abbr}']
args += [('-' if len(abbr) == 1 else '--') + abbr]
kwargs = {
k: v
for k, v in params.items()
if k not in ["name", "abbreviation"]
}
kwargs["default"] = LayeredSettings.get(name)
self.add_argument(*args, **kwargs)
# cluster
self.add_argument('cluster_alias', help='cluster alias to select')
self.add_argument(
'--pai-uri',
help="uri of openpai cluster, in format of http://x.x.x.x")
self.add_argument('--user', help='username')
self.add_argument('--password', help="password")
self.add_argument('--authen-token',
'--token',
dest='token',
help="authentication token")
self.add_argument('--editor',
default="code",
help="path to your editor used to open files")
# job spec
self.add_argument('--job-name', '-j', help='job name')
self.add_argument(
'--is-global',
'-g',
action="store_true",
help="set globally (not limited to current working folder)",
default=False)
self.add_argument(
'--update',
'-u',
action='append',
help=
'replace current key-value pairs with new key=value (key1:key2:...=value for nested objects)'
)
self.add_argument('--preview',
action='store_true',
help='preview result before doing action')
self.add_argument('--no-browser',
action='store_true',
help='does not open the job link in web browser')
self.add_argument('--interactive',
action='store_true',
help='enter the interactive mode after job starts')
self.add_argument('--notebook-token',
'--token',
dest='token',
default="abcd",
help='jupyter notebook authentication token')
self.add_argument(
"--python",
default="python",
help=
"command or path of python, default is {python}, may be {python3}")
self.add_argument('--cmd-sep',
default="\s*&&\s*",
help="command separator, default is (&&)")
self.add_argument('commands',
nargs=argparse.REMAINDER,
help='shell commands to execute')
# runtime
self.add_argument('config', nargs='?', help='job config file')
self.add_argument('notebook', nargs='?', help='Jupyter notebook file')
# storage
self.add_argument('--recursive',
action='store_true',
default=False,
help="recursive target operation")
self.add_argument('--overwrite',
action='store_true',
default=False,
help="enable overwrite if exists")
self.add_argument('local_path', help="local path")
self.add_argument('remote_path', help="remote path")
def sdk_job_template(self,
cluster_alias_lst: str = [],
workspace: str = None,
sources: list = None,
pip_installs: list = None):
"generate the job template for a sdk-submitted job"
# secrets
clusters = [
get_cluster(alias, get_client=False) for alias in cluster_alias_lst
]
workspace = na(workspace, LayeredSettings.get("workspace"))
workspace = na(workspace,
f"{__flags__.storage_root}/{clusters[0]['user']}")
self.set_secret("clusters", json.dumps(clusters))
self.set_param("cluster_alias",
cluster_alias_lst[0] if cluster_alias_lst else None)
self.set_param(
"work_directory",
'{}/jobs/{}'.format(workspace, self.name) if workspace else None)
# parameters
self.set_param("python_path", "python")
# signature
self.add_tag(__internal_tags__["sdk"])
# sdk.plugins
sdk_install_uri = "-U {}".format(get_install_uri())
c_dir = '~/{}'.format(__flags__.cache)
c_file = '%s/%s' % (c_dir, __flags__.cluster_cfg_file)
plugins = []
if sources:
plugins.append({
"plugin": "local.uploadFiles",
"parameters": {
"files": list(set([os.path.relpath(s) for s in sources])),
},
})
plugins.extend([
{
"plugin":
"container.preCommands", # commands to install essential pip packages
"parameters": {
"commands": [
"<% $parameters.python_path %> -m pip install {}".
format(p)
for p in [sdk_install_uri] + na(pip_installs, [])
]
}
},
{
"plugin": "container.preCommands", # copy cluster information
"parameters": {
"commands": [
"mkdir %s" % c_dir,
"echo \"write config to {}\"".format(c_file),
"echo <% $secrets.clusters %> > {}".format(c_file),
"opai cluster select <% $parameters.cluster_alias %>",
]
}
}
])
if sources:
a_file = os.path.basename(self.temp_archive)
plugins.append({
"plugin": "container.preCommands",
"parameters": {
"commands": [
"opai storage download <% $parameters.work_directory %>/source/{} {}"
.format(a_file, a_file), "tar xvfz {}".format(a_file)
]
}
})
self.set_extra("sdk.plugins", plugins)
return self
def set(key, value):
LayeredSettings.update("user_advaced", key, value)
def print_supported_items():
ret = LayeredSettings.print_supported_items()
if __flags__.disable_to_screen:
print(ret)
def reset():
LayeredSettings.reset()