def select_cluster(self, cluster_alias: str = None, virtual_cluster: str = None): self._client = get_cluster(cluster_alias) if virtual_cluster: if self.protocolVersion == "1": self.protocol["virtualCluster"] = virtual_cluster else: self.set_param("virtualCluster", virtual_cluster, field="defaults") return self
def load(self, fname: str = None, job_name: str = None, cluster_alias: str = None): if cluster_alias: # load job config from cluster by REST api job_name = na(job_name, self.name) self.protocol = get_cluster(cluster_alias).rest_api_job_info( job_name, 'config') else: # load from local file if not fname: fname = Job(job_name).protocol_file if os.path.isfile(fname): self.protocol = from_file(fname, default="==FATAL==") self.protocol.setdefault( 'protocolVersion', '1') # v1 protocol (json) has no protocolVersion return self
def client(self): if self._client is None: alias = self.param("cluster_alias") if alias: self._client = get_cluster(alias) return self._client
def sdk_job_template(self, cluster_alias_lst: str = [], workspace: str = None, sources: list = None, pip_installs: list = None): "generate the job template for a sdk-submitted job" # secrets clusters = [ get_cluster(alias, get_client=False) for alias in cluster_alias_lst ] workspace = na(workspace, LayeredSettings.get("workspace")) workspace = na(workspace, f"{__flags__.storage_root}/{clusters[0]['user']}") self.set_secret("clusters", json.dumps(clusters)) self.set_param("cluster_alias", cluster_alias_lst[0] if cluster_alias_lst else None) self.set_param( "work_directory", '{}/jobs/{}'.format(workspace, self.name) if workspace else None) # parameters self.set_param("python_path", "python") # signature self.add_tag(__internal_tags__["sdk"]) # sdk.plugins sdk_install_uri = "-U {}".format(get_install_uri()) c_dir = '~/{}'.format(__flags__.cache) c_file = '%s/%s' % (c_dir, __flags__.cluster_cfg_file) plugins = [] if sources: plugins.append({ "plugin": "local.uploadFiles", "parameters": { "files": list(set([os.path.relpath(s) for s in sources])), }, }) plugins.extend([ { "plugin": "container.preCommands", # commands to install essential pip packages "parameters": { "commands": [ "<% $parameters.python_path %> -m pip install {}". format(p) for p in [sdk_install_uri] + na(pip_installs, []) ] } }, { "plugin": "container.preCommands", # copy cluster information "parameters": { "commands": [ "mkdir %s" % c_dir, "echo \"write config to {}\"".format(c_file), "echo <% $secrets.clusters %> > {}".format(c_file), "opai cluster select <% $parameters.cluster_alias %>", ] } } ]) if sources: a_file = os.path.basename(self.temp_archive) plugins.append({ "plugin": "container.preCommands", "parameters": { "commands": [ "opai storage download <% $parameters.work_directory %>/source/{} {}" .format(a_file, a_file), "tar xvfz {}".format(a_file) ] } }) self.set_extra("sdk.plugins", plugins) return self