def get_url_filename_from_server(url):
try:
blah = urlopen(url).info()['Content-Disposition']
_, params = cgi.parse_header(blah)
return params["filename"]
except Exception as e:
__logger__.warn('Failed to get filename from server: %s', e)
return None
def check_arguments_notebook(self, args):
self.check_essentials(args)
assert args.notebook or args.interactive, "must specify a notebook name unless in interactive mode"
if not args.job_name:
assert args.notebook or args.interactive, "must specify a notebook if no job name defined"
args.job_name = os.path.splitext(os.path.basename(args.notebook))[
0] + "_" + randstr().hex if args.notebook else "jupyter_server_{}".format(randstr().hex)
if args.interactive and not args.token:
__logger__.warn("no authentication token is set")
def f(*args, default="==FATAL==", **kwargs):
try:
return func(*args, **kwargs)
except Exception as identifier:
if default == "==FATAL==":
__logger__.error('Error: %s', identifier, exc_info=True)
__logger__.warn(
'error occurs when reading %s (%s), return default (%s)', args,
identifier, default)
return default
def delete(lst: list, key: str, target, getter=dict.get) -> list:
indexes = OrganizedList.filter(lst, key, target, getter)["indexes"]
if not indexes:
__logger__.warn(
"element with %s = %s cannot be deleted due to non-existence",
key, target)
return False
for index in sorted(indexes, reverse=True):
del lst[index]
return True
def check_arguments_notebook(self, args):
append_options_to_list(args, ["sources", "pip_installs"])
assert args.notebook or args.interactive, "must specify a notebook name unless in interactive mode"
if not args.job_name:
assert args.notebook or args.interactive, "must specify a notebook if no job name defined"
args.job_name = os.path.splitext(os.path.basename(args.notebook))[0] + "_" + randstr().hex if args.notebook else "jupyter_server_{}".format(randstr().hex)
if args.interactive and not args.token:
__logger__.warn("no authentication token is set")
args.pip_installs = args.pip_installs + ["jupyter"]
args.sources = args.sources + [args.notebook]
def check_arguments_sub(self, args):
append_options_to_list(args, ["sources", "pip_installs"])
if args.sources or args.pip_installs:
if not args.enable_sdk:
__logger__.warn("upload local file requires --enable-sdk, assert automatically")
args.enable_sdk = True
if args.sources:
assert args.workspace, "must specify --workspace if --sources used"
for s in args.sources:
assert os.path.isfile(s), "file %s not found" % s
def select(self, alias: str=None):
"""return the cluster configuration (dict) with its alias equal to specified one
if only one cluster in the list and alias is not set, the only cluster would be returned"""
if not alias and len(self.clusters) == 1:
alias = self.clusters[0]["cluster_alias"]
__logger__.warn("cluster-alias is not set, the only defined cluster %s will be used", alias)
assert alias, "must specify a cluster_alias"
dic = ol.as_dict(self.clusters, "cluster_alias")
if alias not in dic:
os.system("cat %s" % __cluster_config_file__)
__logger__.error("cannot find %s from %s (%s)", alias, list(dic.keys()), __cluster_config_file__)
return dic[alias]
def update_default(key: str,
value: str = None,
is_global: bool = False,
to_delete: bool = False):
filename = __global_default_file__ if is_global else __local_default_file__
dic = get_global_defaults() if is_global else get_per_folder_defaults()
if to_delete:
if key not in dic:
__logger__.warn("key %s not found in %s, ignored", key, filename)
return
del dic[key]
else:
__logger__.info("key %s updated to %s in %s", key, value, filename)
dic[key] = value
to_file(dic, filename)
def make_string(pth: str, target: str, iters=(list, dict)):
flag = True
assert isinstance(
target, iters), "not supported type %s (%s)" % (pth, type(target))
for i in (target.keys() if isinstance(target, dict) else range(
len(target))):
pth_next = "%s[%s]" % (pth, i)
if isinstance(target[i], iters):
flag = flag and Cluster.make_string(pth_next, target[i])
elif not (target[i] is None or isinstance(target[i], str)):
__logger__.warn(
'only string is allowed in the cluster configuration, %s is %s, replace it by str()',
pth_next, type(target[i]))
target[i] = str(target[i])
flag = False
return flag
def plugin_uploadFiles(self, plugin: dict):
import tarfile
to_screen("archiving and uploading ...")
work_directory = self.param("work_directory")
assert work_directory, "must specify a storage to upload"
with safe_open(self.temp_archive, "w:gz", func=tarfile.open) as fn:
for src in plugin["parameters"]["files"]:
src = os.path.relpath(src)
if os.path.dirname(src) != "":
__logger__.warn(
"files not in current folder may cause wrong location in the container, please check it {}"
.format(src))
fn.add(src)
to_screen("{} archived and wait to be uploaded".format(src))
self.client.get_storage().upload(local_path=self.temp_archive,
remote_path="{}/source/{}".format(
work_directory,
os.path.basename(
self.temp_archive)),
overwrite=True)
def append_options_to_list(args: argparse.Namespace, keys: list):
for k in keys:
if not hasattr(args, k):
__logger__.warn("option %s not found, please check it", k)
if getattr(args, k, None) is None:
setattr(args, k, [])
def browser_open(url: str):
__logger__.info("open in browser: %s", url)
try:
open_new_tab(url)
except Exception as e:
__logger__.warn("failed to open %s due to %s", url, e)
def from_notebook(self,
nb_file: str,
image: str,
cluster: dict,
resources: dict = None,
submit: bool = True,
interactive_mode: bool = True,
token: str = "abcd",
**kwargs):
if not nb_file:
interactive_mode, nb_file = True, ""
html_file = os.path.splitext(
nb_file)[0] + ".html" if not interactive_mode else ""
if interactive_mode:
resources.setdefault("ports", {})["jupyter"] = 1
cmds = [
"jupyter notebook --no-browser --ip 0.0.0.0 --port $PAI_CONTAINER_HOST_jupyter_PORT_LIST --NotebookApp.token={} --allow-root --NotebookApp.file_to_run={}"
.format(token, nb_file),
]
else:
cmds = [
'jupyter nbconvert --ExecutePreprocessor.timeout=-1 --ExecutePreprocessor.allow_errors=True --to html --execute %s'
% nb_file,
'opai storage upload {} <% $secrets.work_directory %>/output/{}'
.format(html_file, html_file),
]
job_info = self.one_liner(commands=cmds,
image=image,
resources=resources,
cluster=cluster,
submit=submit,
**kwargs)
if not submit:
return job_info
# post processing after Submitting
client = ClusterList().load().get_client(cluster["cluster_alias"])
print("waiting job to be started")
if interactive_mode:
# wait and get ip
state = client.wait([self.name],
exit_states=["SUCCEEDED", "FAILED",
"RUNNING"])[0]
assert state == "RUNNING", "why not running {}".format(state)
while True:
try:
status = client.jobs(
self.name)["taskRoles"]["main"]["taskStatuses"][0]
browser_open(
"http://%s:%s/notebooks/%s" %
(status["containerIp"],
status["containerPorts"]["jupyter"], nb_file))
break
except:
time.sleep(10)
else:
state = client.wait([self.name])[0]
if state != "SUCCEEDED":
__logger__.warn("job %s failed", self.name)
return
local_path = Job.job_cache_file(self.name,
os.path.join('output', html_file))
remote_path = '{}/output/{}'.format(
self.protocol["secrets"]["work_directory"], html_file)
client.get_storage().download(remote_path=remote_path,
local_path=local_path)
browser_open(local_path)
def check_arguments_attach_hdfs(self, args):
assert args.cluster_alias and args.storage_alias, "must specify valid cluster-alias and storage-alias"
if not args.web_hdfs_uri.startswith("http://") or not args.web_hdfs_uri.startswith("https://"):
__logger__.warn("web-hdfs-uri not starts with http:// or https://")
def check_arguments_add(self, args):
if not args.pai_uri.startswith("http://") or not args.pai_uri.startswith("https://"):
__logger__.warn("pai-uri not starts with http:// or https://")
assert args.user and args.cluster_alias, "must specify an cluster-alias and user name"
