def exec_start_cmd(args): app_descr = json.load(args.jsonfile) app_validate(app_descr) exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) ret = exec_api.execution_start(args.name, app_descr) print( "Application scheduled successfully with ID {}, use the exec-get command to check its status" .format(ret))
def new_execution(request, exec_name, app_name, dict): print("zoe api: new execution") print("zoe api: new execution {} - {}: arguments {}".format(exec_name, app_name, dict)) exec_api = ZoeExecutionsAPI(ZOE_URL, ZOE_USER, ZOE_PWD) if app_name == 'ipython': try: notebook_memory_limit = dict['notebook_mem_limit'] * (1024 ** 3) # GB spark_master_memory_limit = dict['master_mem_limit'] * (1024 ** 2) # MB spark_worker_memory_limit = dict['worker_memory'] * (1024 ** 3) # GB spark_worker_cores = dict['worker_cores'] spark_worker_count = dict['worker_count'] app_descr = zapps.create_notebook_app(notebook_memory_limit=notebook_memory_limit, spark_master_memory_limit=spark_master_memory_limit, spark_worker_memory_limit=spark_worker_memory_limit, spark_worker_cores=spark_worker_cores, spark_worker_count=spark_worker_count ) except: app_descr = zapps.create_notebook_app() exec_api.execution_start(exec_name, app_descr) elif app_name == 'mpi': try: spark_worker_cores = dict['worker_cores'] spark_worker_count = dict['worker_count'] assert spark_worker_cores * spark_worker_count <= 8 in_wm = int(dict['worker_memory']) assert in_wm > 0 wm = in_wm * (1024 ** 3) app_descr = zapps.create_idiada_app(worker_memory=wm) except: app_descr = zapps.create_idiada_app() exec_api.execution_start('mpidynademo', app_descr) else: print("App not supported.") return return "Done"
class ZoeWorkFlow: def __init__(self, workspace_base_path, identity, name): self.identity = identity self.name = name self.workspace = ZoeWorkspace(workspace_base_path, identity, name) self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password']) self.cont_api = ZoeServiceAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password']) info_api = ZoeInfoAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password']) zoe_info = info_api.info() self.hostname_prefix = zoe_info['name_prefix'] self.exec_counter = 0 def generate_hostname(self, process_name: str) -> str: return self.hostname_prefix + '-' + process_name + '-' + self.identity[ 'username'] def start_workflow(self): self.workspace.create() def end_workflow(self): self.workspace.destroy() def execution_start(self, app): self.exec_counter += 1 return self.exec_api.execution_start( self.name + '{}'.format(self.exec_counter), app) def wait_termination(self, exec_id): execution = self.exec_api.execution_get(exec_id) while execution['status'] == 'submitted' or execution[ 'status'] == 'running': time.sleep(1) execution = self.exec_api.execution_get(exec_id) def __enter__(self): self.start_workflow() return self def __exit__(self, exc_type, exc_val, exc_tb): self.end_workflow()
class ZoeWorkFlow: def __init__(self, workspace_base_path, identity, name): self.identity = identity self.name = name self.workspace = ZoeWorkspace(workspace_base_path, identity, name) self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password']) self.cont_api = ZoeServiceAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password']) info_api = ZoeInfoAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password']) zoe_info = info_api.info() self.hostname_prefix = zoe_info['name_prefix'] self.exec_counter = 0 def generate_hostname(self, process_name: str) -> str: return self.hostname_prefix + '-' + process_name + '-' + self.identity['username'] def start_workflow(self): self.workspace.create() def end_workflow(self): self.workspace.destroy() def execution_start(self, app): self.exec_counter += 1 return self.exec_api.execution_start(self.name + '{}'.format(self.exec_counter), app) def wait_termination(self, exec_id): execution = self.exec_api.execution_get(exec_id) while execution['status'] == 'submitted' or execution['status'] == 'running': time.sleep(1) execution = self.exec_api.execution_get(exec_id) def __enter__(self): self.start_workflow() return self def __exit__(self, exc_type, exc_val, exc_tb): self.end_workflow()
def home_guest(): auth = request.authorization if not auth: return missing_auth() guest_identifier = auth.username guest_password = auth.password match = guest_id_pattern.match(guest_identifier) if match is None: return missing_auth() query_api = ZoeQueryAPI(get_conf().zoe_url, guest_identifier, guest_password) template_vars = { 'refresh': randint(2, 8), 'user_gateway': 'Please wait...', 'execution_status': 'Please wait...', 'execution_urls': [], 'guest_identifier': guest_identifier } try: user = query_api.query('user', name=guest_identifier) except ZoeAPIException: return missing_auth() if len(user) == 0: return missing_auth() else: user = user[0] template_vars['user_gateway'] = user['gateway_urls'][0] template_vars['gateway_ip'] = user['gateway_urls'][0].split('/')[2].split(':')[0] app_api = ZoeApplicationAPI(get_conf().zoe_url, guest_identifier, guest_password) exec_api = ZoeExecutionsAPI(get_conf().zoe_url, guest_identifier, guest_password) app = query_api.query('application', name='spark-jupyter-lab') if len(app) == 0: app_descr = spark_jupyter_notebook_lab_app() app_api.create(app_descr) return render_template('home_guest.html', **template_vars) else: app = app[0] execution = query_api.query('execution', name='guest-lab-{}'.format(guest_identifier)) if len(execution) == 0: exec_api.execution_start('guest-lab-{}'.format(guest_identifier), app['name']) template_vars['execution_status'] = 'submitted' return render_template('home_guest.html', **template_vars) else: execution = execution[0] if execution['status'] == 'terminated': app_api.delete(app['id']) return render_template('home_guest.html', **template_vars) elif execution['status'] != 'running': template_vars['execution_status'] = execution['status'] return render_template('home_guest.html', **template_vars) else: template_vars['refresh'] = -1 cont_api = ZoeContainerAPI(get_conf().zoe_url, guest_identifier, guest_password) template_vars['execution_status'] = execution['status'] for c_id in execution['containers']: c = cont_api.get(c_id) ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one? for p in c['ports']: template_vars['execution_urls'].append(('{}'.format(p['name']), '{}://{}:{}{}'.format(p['protocol'], ip, p['port_number'], p['path']))) return render_template('home_guest.html', **template_vars)
def exec_start_cmd(args): exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) ret = exec_api.execution_start(args.name, args.app_name) print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(ret))
def exec_start_cmd(args): app_descr = json.load(args.jsonfile) app_validate(app_descr) exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) ret = exec_api.execution_start(args.name, app_descr) print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(ret))