def select_cluster(self,
cluster_alias: str = None,
virtual_cluster: str = None):
self._client = get_cluster(cluster_alias)
if virtual_cluster:
if self.protocolVersion == "1":
self.protocol["virtualCluster"] = virtual_cluster
else:
self.set_param("virtualCluster",
virtual_cluster,
field="defaults")
return self
def load(self,
fname: str = None,
job_name: str = None,
cluster_alias: str = None):
if cluster_alias: # load job config from cluster by REST api
job_name = na(job_name, self.name)
self.protocol = get_cluster(cluster_alias).rest_api_job_info(
job_name, 'config')
else: # load from local file
if not fname:
fname = Job(job_name).protocol_file
if os.path.isfile(fname):
self.protocol = from_file(fname, default="==FATAL==")
self.protocol.setdefault(
'protocolVersion',
'1') # v1 protocol (json) has no protocolVersion
return self
def client(self):
if self._client is None:
alias = self.param("cluster_alias")
if alias:
self._client = get_cluster(alias)
return self._client
def sdk_job_template(self,
cluster_alias_lst: str = [],
workspace: str = None,
sources: list = None,
pip_installs: list = None):
"generate the job template for a sdk-submitted job"
# secrets
clusters = [
get_cluster(alias, get_client=False) for alias in cluster_alias_lst
]
workspace = na(workspace, LayeredSettings.get("workspace"))
workspace = na(workspace,
f"{__flags__.storage_root}/{clusters[0]['user']}")
self.set_secret("clusters", json.dumps(clusters))
self.set_param("cluster_alias",
cluster_alias_lst[0] if cluster_alias_lst else None)
self.set_param(
"work_directory",
'{}/jobs/{}'.format(workspace, self.name) if workspace else None)
# parameters
self.set_param("python_path", "python")
# signature
self.add_tag(__internal_tags__["sdk"])
# sdk.plugins
sdk_install_uri = "-U {}".format(get_install_uri())
c_dir = '~/{}'.format(__flags__.cache)
c_file = '%s/%s' % (c_dir, __flags__.cluster_cfg_file)
plugins = []
if sources:
plugins.append({
"plugin": "local.uploadFiles",
"parameters": {
"files": list(set([os.path.relpath(s) for s in sources])),
},
})
plugins.extend([
{
"plugin":
"container.preCommands", # commands to install essential pip packages
"parameters": {
"commands": [
"<% $parameters.python_path %> -m pip install {}".
format(p)
for p in [sdk_install_uri] + na(pip_installs, [])
]
}
},
{
"plugin": "container.preCommands", # copy cluster information
"parameters": {
"commands": [
"mkdir %s" % c_dir,
"echo \"write config to {}\"".format(c_file),
"echo <% $secrets.clusters %> > {}".format(c_file),
"opai cluster select <% $parameters.cluster_alias %>",
]
}
}
])
if sources:
a_file = os.path.basename(self.temp_archive)
plugins.append({
"plugin": "container.preCommands",
"parameters": {
"commands": [
"opai storage download <% $parameters.work_directory %>/source/{} {}"
.format(a_file, a_file), "tar xvfz {}".format(a_file)
]
}
})
self.set_extra("sdk.plugins", plugins)
return self
