def exec_start_cmd(args):
app_descr = json.load(args.jsonfile)
app_validate(app_descr)
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(),
utils.zoe_pass())
ret = exec_api.execution_start(args.name, app_descr)
print(
"Application scheduled successfully with ID {}, use the exec-get command to check its status"
.format(ret))
def new_execution(request, exec_name, app_name, dict):
print("zoe api: new execution")
print("zoe api: new execution {} - {}: arguments {}".format(exec_name, app_name, dict))
exec_api = ZoeExecutionsAPI(ZOE_URL, ZOE_USER, ZOE_PWD)
if app_name == 'ipython':
try:
notebook_memory_limit = dict['notebook_mem_limit'] * (1024 ** 3) # GB
spark_master_memory_limit = dict['master_mem_limit'] * (1024 ** 2) # MB
spark_worker_memory_limit = dict['worker_memory'] * (1024 ** 3) # GB
spark_worker_cores = dict['worker_cores']
spark_worker_count = dict['worker_count']
app_descr = zapps.create_notebook_app(notebook_memory_limit=notebook_memory_limit,
spark_master_memory_limit=spark_master_memory_limit,
spark_worker_memory_limit=spark_worker_memory_limit,
spark_worker_cores=spark_worker_cores,
spark_worker_count=spark_worker_count
)
except:
app_descr = zapps.create_notebook_app()
exec_api.execution_start(exec_name, app_descr)
elif app_name == 'mpi':
try:
spark_worker_cores = dict['worker_cores']
spark_worker_count = dict['worker_count']
assert spark_worker_cores * spark_worker_count <= 8
in_wm = int(dict['worker_memory'])
assert in_wm > 0
wm = in_wm * (1024 ** 3)
app_descr = zapps.create_idiada_app(worker_memory=wm)
except:
app_descr = zapps.create_idiada_app()
exec_api.execution_start('mpidynademo', app_descr)
else:
print("App not supported.")
return
return "Done"
class ZoeWorkFlow:
def __init__(self, workspace_base_path, identity, name):
self.identity = identity
self.name = name
self.workspace = ZoeWorkspace(workspace_base_path, identity, name)
self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
self.cont_api = ZoeServiceAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
info_api = ZoeInfoAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
zoe_info = info_api.info()
self.hostname_prefix = zoe_info['name_prefix']
self.exec_counter = 0
def generate_hostname(self, process_name: str) -> str:
return self.hostname_prefix + '-' + process_name + '-' + self.identity[
'username']
def start_workflow(self):
self.workspace.create()
def end_workflow(self):
self.workspace.destroy()
def execution_start(self, app):
self.exec_counter += 1
return self.exec_api.execution_start(
self.name + '{}'.format(self.exec_counter), app)
def wait_termination(self, exec_id):
execution = self.exec_api.execution_get(exec_id)
while execution['status'] == 'submitted' or execution[
'status'] == 'running':
time.sleep(1)
execution = self.exec_api.execution_get(exec_id)
def __enter__(self):
self.start_workflow()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.end_workflow()
class ZoeWorkFlow:
def __init__(self, workspace_base_path, identity, name):
self.identity = identity
self.name = name
self.workspace = ZoeWorkspace(workspace_base_path, identity, name)
self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
self.cont_api = ZoeServiceAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
info_api = ZoeInfoAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
zoe_info = info_api.info()
self.hostname_prefix = zoe_info['name_prefix']
self.exec_counter = 0
def generate_hostname(self, process_name: str) -> str:
return self.hostname_prefix + '-' + process_name + '-' + self.identity['username']
def start_workflow(self):
self.workspace.create()
def end_workflow(self):
self.workspace.destroy()
def execution_start(self, app):
self.exec_counter += 1
return self.exec_api.execution_start(self.name + '{}'.format(self.exec_counter), app)
def wait_termination(self, exec_id):
execution = self.exec_api.execution_get(exec_id)
while execution['status'] == 'submitted' or execution['status'] == 'running':
time.sleep(1)
execution = self.exec_api.execution_get(exec_id)
def __enter__(self):
self.start_workflow()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.end_workflow()
def home_guest():
auth = request.authorization
if not auth:
return missing_auth()
guest_identifier = auth.username
guest_password = auth.password
match = guest_id_pattern.match(guest_identifier)
if match is None:
return missing_auth()
query_api = ZoeQueryAPI(get_conf().zoe_url, guest_identifier, guest_password)
template_vars = {
'refresh': randint(2, 8),
'user_gateway': 'Please wait...',
'execution_status': 'Please wait...',
'execution_urls': [],
'guest_identifier': guest_identifier
}
try:
user = query_api.query('user', name=guest_identifier)
except ZoeAPIException:
return missing_auth()
if len(user) == 0:
return missing_auth()
else:
user = user[0]
template_vars['user_gateway'] = user['gateway_urls'][0]
template_vars['gateway_ip'] = user['gateway_urls'][0].split('/')[2].split(':')[0]
app_api = ZoeApplicationAPI(get_conf().zoe_url, guest_identifier, guest_password)
exec_api = ZoeExecutionsAPI(get_conf().zoe_url, guest_identifier, guest_password)
app = query_api.query('application', name='spark-jupyter-lab')
if len(app) == 0:
app_descr = spark_jupyter_notebook_lab_app()
app_api.create(app_descr)
return render_template('home_guest.html', **template_vars)
else:
app = app[0]
execution = query_api.query('execution', name='guest-lab-{}'.format(guest_identifier))
if len(execution) == 0:
exec_api.execution_start('guest-lab-{}'.format(guest_identifier), app['name'])
template_vars['execution_status'] = 'submitted'
return render_template('home_guest.html', **template_vars)
else:
execution = execution[0]
if execution['status'] == 'terminated':
app_api.delete(app['id'])
return render_template('home_guest.html', **template_vars)
elif execution['status'] != 'running':
template_vars['execution_status'] = execution['status']
return render_template('home_guest.html', **template_vars)
else:
template_vars['refresh'] = -1
cont_api = ZoeContainerAPI(get_conf().zoe_url, guest_identifier, guest_password)
template_vars['execution_status'] = execution['status']
for c_id in execution['containers']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
for p in c['ports']:
template_vars['execution_urls'].append(('{}'.format(p['name']), '{}://{}:{}{}'.format(p['protocol'], ip, p['port_number'], p['path'])))
return render_template('home_guest.html', **template_vars)
def exec_start_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
ret = exec_api.execution_start(args.name, args.app_name)
print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(ret))
def exec_start_cmd(args):
app_descr = json.load(args.jsonfile)
app_validate(app_descr)
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
ret = exec_api.execution_start(args.name, app_descr)
print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(ret))
