def submit_it(self, args):
if args.preview:
return self.__job__.validate().get_config()
result = self.__job__.submit(args.cluster_alias, args.virtual_cluster)
if "job_link" in result and not getattr(args, 'no_browser', False):
browser_open(result["job_link"])
return result
def submit(self, cluster_alias: str):
__logger__.info("submit job %s to cluster %s", self.name,
cluster_alias)
self.validate()
client = ClusterList().load().get_client(cluster_alias)
# upload
for f in self.protocol.get("extras", {}).get("__sources__", []):
if not f:
continue
local_path, remote_path = f, '{}/source/{}'.format(
self.protocol["secrets"]["work_directory"], f)
client.get_storage().upload(local_path=local_path,
remote_path=remote_path,
overwrite=True)
client.get_token().rest_api_submit(self.get_config())
job_link = client.get_job_link(self.name)
browser_open(job_link)
return job_link
def from_notebook(self,
nb_file: str,
image: str,
cluster: dict,
resources: dict = None,
submit: bool = True,
interactive_mode: bool = True,
token: str = "abcd",
**kwargs):
if not nb_file:
interactive_mode, nb_file = True, ""
html_file = os.path.splitext(
nb_file)[0] + ".html" if not interactive_mode else ""
if interactive_mode:
resources.setdefault("ports", {})["jupyter"] = 1
cmds = [
"jupyter notebook --no-browser --ip 0.0.0.0 --port $PAI_CONTAINER_HOST_jupyter_PORT_LIST --NotebookApp.token={} --allow-root --NotebookApp.file_to_run={}"
.format(token, nb_file),
]
else:
cmds = [
'jupyter nbconvert --ExecutePreprocessor.timeout=-1 --ExecutePreprocessor.allow_errors=True --to html --execute %s'
% nb_file,
'opai storage upload {} <% $secrets.work_directory %>/output/{}'
.format(html_file, html_file),
]
job_info = self.one_liner(commands=cmds,
image=image,
resources=resources,
cluster=cluster,
submit=submit,
**kwargs)
if not submit:
return job_info
# post processing after Submitting
client = ClusterList().load().get_client(cluster["cluster_alias"])
print("waiting job to be started")
if interactive_mode:
# wait and get ip
state = client.wait([self.name],
exit_states=["SUCCEEDED", "FAILED",
"RUNNING"])[0]
assert state == "RUNNING", "why not running {}".format(state)
while True:
try:
status = client.jobs(
self.name)["taskRoles"]["main"]["taskStatuses"][0]
browser_open(
"http://%s:%s/notebooks/%s" %
(status["containerIp"],
status["containerPorts"]["jupyter"], nb_file))
break
except:
time.sleep(10)
else:
state = client.wait([self.name])[0]
if state != "SUCCEEDED":
__logger__.warn("job %s failed", self.name)
return
local_path = Job.job_cache_file(self.name,
os.path.join('output', html_file))
remote_path = '{}/output/{}'.format(
self.protocol["secrets"]["work_directory"], html_file)
client.get_storage().download(remote_path=remote_path,
local_path=local_path)
browser_open(local_path)
def connect_notebook(self):
result = self.__job__.wait()
if result.get("notebook", None) is not None:
browser_open(result["notebook"])
return result
from openpaisdk.io_utils import browser_open
try:
import nbmerge
except:
run_command([sys.executable, '-m pip install nbmerge'])
test_notebooks = [
'0-install-sdk-specify-openpai-cluster.ipynb',
'1-submit-and-query-via-command-line.ipynb',
# '2-submit-job-from-local-notebook.ipynb',
]
merged_file = "integrated_tests.ipynb"
html_file = os.path.splitext(merged_file)[0] + '.html'
shutil.rmtree(merged_file, ignore_errors=True)
shutil.rmtree(html_file, ignore_errors=True)
# clear output for committing
for f in test_notebooks:
os.system(
"jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace %s"
% f)
os.system('nbmerge %s -o %s' % (' '.join(test_notebooks), merged_file))
os.system(
'jupyter nbconvert --ExecutePreprocessor.timeout=-1 --ExecutePreprocessor.allow_errors=True --to html --execute %s'
% merged_file)
browser_open(html_file)