def exec_get_cmd(args):
"""Gather information about an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error_message']))
print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
service = cont_api.get(c_id)
print('Service {} (ID: {})'.format(service['name'], service['id']))
print(' - zoe status: {}'.format(service['status']))
print(' - docker status: {}'.format(service['docker_status']))
if service['docker_status'] == 'started':
ip = service['ip_address']
for port in service['description']['ports']:
print(' - {}: {}://{}:{}{}'.format(port['name'], port['protocol'], ip, port['port_number'], port['path']))
def check_guests(swarm):
query_api = ZoeQueryAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
exec_api = ZoeExecutionsAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
cont_api = ZoeContainerAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
guests = query_api.query('user', role='guest')
execs = exec_api.list()
for guest in guests:
my_execs = [e for e in execs if e['owner'] == guest['id']]
for my_exec in my_execs:
if len(my_exec['containers']) == 0:
continue
my_exec_since_started = datetime.datetime.now() - dateutil.parser.parse(my_exec['time_started'])
my_exec_since_started = my_exec_since_started.total_seconds()
terminate = False
for c in my_exec['containers']:
c = cont_api.get(c)
for port in c['ports']:
if port['name'] == 'Spark application web interface':
if check_spark_job(swarm, c['docker_id'], my_exec_since_started):
log.info('Execution {} for user {} has been idle for too long, terminating...'.format(my_exec['name'], guest['name']))
terminate = True
break
if terminate:
break
if terminate:
exec_api.terminate(my_exec['id'])
def exec_get_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.execution_get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error']))
print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
print('Service {} (ID: {})'.format(c['name'], c['id']))
for p in c['ports']:
print(' - {}: {}://{}:{}{}'.format(p['name'], p['protocol'], ip, p['port_number'], p['path']))
def main():
"""Main."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
print(
'id,time_submit,time_start,time_end,cpu_usage,mem_usage,net_rx_usage,net_tx_usage,blkio_usage'
)
for e_id in execs:
e = exec_api.get(e_id)
if e['name'] != 'boinc-loader' or e['status'] != 'terminated':
continue
trace_line = {
'id': e['id'],
'time_submit': e['time_submit'],
'time_start': e['time_start'],
'time_end': e['time_end'],
'cpu_usage': get_influx_cpu_data(e_id),
'mem_usage': get_influx_mem_data(e_id),
'net_rx_usage': get_influx_net_rx_data(e_id),
'net_tx_usage': get_influx_net_tx_data(e_id),
'blkio_usage': get_influx_blkio_data(e_id)
}
print(
'{id},{time_submit},{time_start},{time_end},{cpu_usage},{mem_usage},{net_rx_usage},{net_tx_usage},{blkio_usage}'
.format(**trace_line))
def submit_zapp(zapp):
"""Submits one ZApp for execution."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
ret = exec_api.start('boinc-loader', zapp)
print(
"Application scheduled successfully with ID {}, use the exec-get command to check its status"
.format(ret))
def exec_list_cmd(_):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(),
utils.zoe_pass())
data = exec_api.list()
for e in data:
print('Execution {} (User: {}, ID: {}): {}'.format(
e['name'], e['user_id'], e['id'], e['status']))
def get_execution_details(exec_id):
print("zoe api: get_execution_details")
try:
print("zoe api: get_execution_details: found in vault with id = {}".format(exec_id))
return vault[exec_id]
except KeyError:
print("zoe api: get_execution_details: no execution found with id = {}".format(exec_id))
vault[exec_id] = {}
exec_api = ZoeExecutionsAPI(ZOE_URL, ZOE_USER, ZOE_PWD)
cont_api = ZoeServiceAPI(ZOE_URL, ZOE_USER, ZOE_PWD)
exec_details = exec_api.execution_get(exec_id)
owner, gateway = get_user_info(exec_details)
service_details = []
for c_id in exec_details['services']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
cont_id = c['id']
cont_name = c['name']
tmp = {'name': cont_name, 'details': {}}
for p in c['ports']:
url = "{}://{}:{}{}".format(p['protocol'], ip, p['port_number'], p['path'])
tmp['details'] = {'name': p['name'], 'url': url}
service_details.append(tmp)
exec_details.update({'service_details': service_details, 'owner': owner, 'gateway': _translate_gw(gateway)})
vault[exec_id].update(exec_details)
return vault[exec_id]
def exec_list_cmd(_):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
api_user = ZoeUserAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
data = exec_api.list()
for e in data:
user = api_user.get(e['owner'])
print('Execution {} (User: {}, ID: {}): {}'.format(e['name'], user['name'], e['id'], e['status']))
def exec_start_cmd(args):
"""Submit an execution."""
app_descr = json.load(args.jsonfile)
app_validate(app_descr)
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
ret = exec_api.start(args.name, app_descr)
print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(ret))
def exec_get_cmd(auth, args):
"""Gather information about an execution."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
cont_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
execution = exec_api.get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'],
execution['id']))
print('Application name: {}'.format(execution['description']['name']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error_message']))
print()
print('Time submit: {}'.format(
datetime.fromtimestamp(execution['time_submit'],
timezone.utc).astimezone()))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(
datetime.fromtimestamp(execution['time_start'],
timezone.utc).astimezone()))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(
datetime.fromtimestamp(execution['time_end'],
timezone.utc).astimezone()))
print()
endpoints = exec_api.endpoints(execution['id'])
if endpoints is not None and len(endpoints) > 0:
print('Exposed endpoints:')
for endpoint in endpoints:
print(' - {}: {}'.format(endpoint[0], endpoint[1]))
else:
print('This ZApp does not expose any endpoint')
print()
tabular_data = []
for c_id in execution['services']:
service = cont_api.get(c_id)
service_data = [
service['id'], service['name'],
'true' if service['essential'] else 'false', service['status'],
service['backend_status'], service['backend_host'],
service['error_message']
if service['error_message'] is not None else ''
]
tabular_data.append(service_data)
headers = [
'ID', 'Name', 'Essential', 'Zoe status', 'Backend status', 'Host',
'Error message'
]
print(tabulate(tabular_data, headers))
def app_get_cmd(args):
"""Extract an application description from an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print("no such execution")
else:
json.dump(execution['description'], sys.stdout, sort_keys=True, indent=4)
def app_get_cmd(args):
"""Extract an application description from an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print("no such execution")
else:
json.dump(execution['description'], sys.stdout, sort_keys=True, indent=4)
def exec_kill_user_cmd(auth, args):
"""Terminates all executions for the given user."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
filters = {'status': 'running', 'user_id': args.user_id}
data = exec_api.list(**filters)
print('Terminating {} executions belonging to user {}'.format(
len(data), args.user_id))
for execution in data:
exec_api.terminate(execution)
print('Execution {} terminated'.format(execution))
def delete_finished():
"""Delete finished executions from Zoe."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
for e_id in execs:
e = exec_api.get(e_id)
if e is None:
continue
if e['name'] == 'boinc-loader' and e['status'] == 'terminated':
print('Execution {} has finished, deleting...'.format(e_id))
exec_api.delete(e['id'])
def app_get_cmd(auth, args):
"""Extract an application description from an execution."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
execution = exec_api.get(args.id)
if execution is None:
print("no such execution")
else:
json.dump(execution['description'],
sys.stdout,
sort_keys=True,
indent=4)
def count_jobs():
"""Count how many zapps have already been submitted."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
count = 0
for e_id in execs:
e = exec_api.get(e_id)
if e['name'] != 'boinc-loader':
continue
if e['status'] != 'terminated':
count += 1
return count
def exec_list_cmd(request):
print("zoe api: exec_list_cmd")
token = zoe_api(request)
headers = dict()
headers["X-Auth-Token"] = str(token)
headers['Content-Type'] = "text/plain"
exec_api = ZoeExecutionsAPI(ZOE_URL, ZOE_USER, ZOE_PWD)
data = exec_api.list()
url = URL_BASIC + "/sdscontroller/executions"
r = requests.get(url, headers=headers)
return r, data
def count_jobs(all=False):
"""Count how many zapps have already been submitted."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
count = 0
for e_id in execs:
e = exec_api.get(e_id)
if e is None:
continue
elif not all and e['name'] != 'boinc-loader':
continue
if e['status'] != 'terminated':
count += 1
return count
class ZoeWorkFlow:
def __init__(self, workspace_base_path, identity, name):
self.identity = identity
self.name = name
self.workspace = ZoeWorkspace(workspace_base_path, identity, name)
self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
self.cont_api = ZoeServiceAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
info_api = ZoeInfoAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
zoe_info = info_api.info()
self.hostname_prefix = zoe_info['name_prefix']
self.exec_counter = 0
def generate_hostname(self, process_name: str) -> str:
return self.hostname_prefix + '-' + process_name + '-' + self.identity[
'username']
def start_workflow(self):
self.workspace.create()
def end_workflow(self):
self.workspace.destroy()
def execution_start(self, app):
self.exec_counter += 1
return self.exec_api.execution_start(
self.name + '{}'.format(self.exec_counter), app)
def wait_termination(self, exec_id):
execution = self.exec_api.execution_get(exec_id)
while execution['status'] == 'submitted' or execution[
'status'] == 'running':
time.sleep(1)
execution = self.exec_api.execution_get(exec_id)
def __enter__(self):
self.start_workflow()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.end_workflow()
class Test:
def __init__(self, user, pwd, url, name, zapp):
self.user = user
self.pwd = pwd
self.url = url
self.name = name
try:
with open(zapp, 'r') as infile:
self.zapp = json.load(infile)
except:
exit("Unable to load zapp file.")
self.exec_api = ZoeExecutionsAPI(self.url, self.user, self.pwd)
self.service_api = ZoeServiceAPI(self.url, self.user, self.pwd)
def start_exec(self):
exec_id = self.exec_api.start(self.name, self.zapp)
return exec_id
def get_services_id(self, exec_id):
services = self.exec_api.get(exec_id)
return services['services']
def get_submit_service(self, exec_id):
while len(self.get_services_id(exec_id)) < 4:
print('waiting')
time.sleep(0.5)
for service_id in self.get_services_id(exec_id):
srv = self.service_api.get(service_id)
if re.search('submit', srv['name']):
return srv['id']
def is_running(self, exec_id):
return self.exec_api.get(exec_id)['status'] == 'running'
def run_test(self):
ts = time.time()
exec_id = self.start_exec()
outfilename = './logs/{}'.format(exec_id)
submit_id = self.get_submit_service(exec_id)
while not self.is_running(exec_id):
time.sleep(0.5)
te = time.time()
with open(outfilename, 'w') as out:
out.write("PerfMeasure: Scheduling time: {}".format(te - ts))
for line in self.service_api.get_logs(submit_id):
out.write("{}\n".format(line))
self.exec_api.terminate(exec_id)
print('Terminated {}'.format(exec_id))
def exec_list_cmd(auth, args):
"""List executions"""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
filter_names = [
'status', 'name', 'limit', 'earlier_than_submit', 'earlier_than_start',
'earlier_than_end', 'later_than_submit', 'later_than_start',
'later_than_end'
]
filters = {'user_id': auth['user']}
for key, value in vars(args).items():
if key in filter_names:
filters[key] = value
data = exec_api.list(**filters)
if len(data) == 0:
return
tabular_data = [[e['id'], e['name'], e['user_id'], e['status']]
for e in sorted(data.values(), key=lambda x: x['id'])]
headers = ['ID', 'Name', 'User ID', 'Status']
print(tabulate(tabular_data, headers))
def __init__(self, workspace_base_path, identity, name):
self.identity = identity
self.name = name
self.workspace = ZoeWorkspace(workspace_base_path, identity, name)
self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
self.cont_api = ZoeServiceAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
info_api = ZoeInfoAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
zoe_info = info_api.info()
self.hostname_prefix = zoe_info['name_prefix']
self.exec_counter = 0
def exec_start_cmd(auth, args):
"""Submit an execution."""
app_descr = json.load(args.jsonfile)
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
exec_id = exec_api.start(args.name, app_descr)
if not args.synchronous:
print(
"Application scheduled successfully with ID {}, use the exec-get command to check its status"
.format(exec_id))
else:
print(
"Application scheduled successfully with ID {}, waiting for status change"
.format(exec_id))
old_status = 'submitted'
while True:
execution = exec_api.get(exec_id)
current_status = execution['status']
if old_status != current_status:
print('Execution is now {}'.format(current_status))
old_status = current_status
if current_status == 'running':
break
time.sleep(1)
monitor_service_id = None
service_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
for service_id in execution['services']:
service = service_api.get(service_id)
if service['description']['monitor']:
monitor_service_id = service['id']
break
print('\n>------ start of log streaming -------<\n')
why_stop = _log_stream_stdout(monitor_service_id, False, auth)
print('\n>------ end of log streaming -------<\n')
if why_stop == 'stream_end':
print('Execution finished')
exit(0)
elif why_stop == 'interrupt':
print('Do not worry, your execution ({}) is still running.'.format(
exec_id))
exit(1)
def main():
"""Main."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
print('id,time_submit,time_start,time_end,cpu_usage,mem_usage,net_rx_usage,net_tx_usage,blkio_usage')
for e_id in execs:
e = exec_api.get(e_id)
if e['name'] != 'boinc-loader' or e['status'] != 'terminated':
continue
trace_line = {
'id': e['id'],
'time_submit': e['time_submit'],
'time_start': e['time_start'],
'time_end': e['time_end'],
'cpu_usage': get_influx_cpu_data(e_id),
'mem_usage': get_influx_mem_data(e_id),
'net_rx_usage': get_influx_net_rx_data(e_id),
'net_tx_usage': get_influx_net_tx_data(e_id),
'blkio_usage': get_influx_blkio_data(e_id)
}
print('{id},{time_submit},{time_start},{time_end},{cpu_usage},{mem_usage},{net_rx_usage},{net_tx_usage},{blkio_usage}'.format(**trace_line))
def __init__(self, user, pwd, url, name, zapp):
self.user = user
self.pwd = pwd
self.url = url
self.name = name
try:
with open(zapp, 'r') as infile:
self.zapp = json.load(infile)
except:
exit("Unable to load zapp file.")
self.exec_api = ZoeExecutionsAPI(self.url, self.user, self.pwd)
self.service_api = ZoeServiceAPI(self.url, self.user, self.pwd)
def exec_get_cmd(args):
app_api = ZoeApplicationAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
cont_api = ZoeContainerAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.execution_get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
print('Status: {}'.format(execution['status']))
print('Time started: {}'.format(execution['time_started']))
print('Time scheduled: {}'.format(execution['time_scheduled']))
print('Time finished: {}'.format(execution['time_finished']))
app = app_api.get(execution['application_id'])
print('Application name: {}'.format(app['name']))
for c_id in execution['containers']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
print('Container {} (ID: {})'.format(c['name'], c['id']))
for p in c['ports']:
print(' - {}: {}://{}:{}{}'.format(p['name'], p['protocol'], ip, p['port_number'], p['path']))
def check_guests(swarm):
query_api = ZoeQueryAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
exec_api = ZoeExecutionsAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
cont_api = ZoeServiceAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
guests = query_api.query('user', role='guest')
execs = exec_api.list()
for guest in guests:
my_execs = [e for e in execs if e['owner'] == guest['name']]
for my_exec in my_execs:
if len(my_exec['services']) == 0:
continue
my_exec_since_started = datetime.datetime.now(
) - dateutil.parser.parse(my_exec['time_started'])
my_exec_since_started = my_exec_since_started.total_seconds()
terminate = False
for c in my_exec['services']:
c = cont_api.get(c)
for port in c['ports']:
if port['name'] == 'Spark application web interface':
idle_time = check_spark_job(swarm, c['docker_id'],
my_exec_since_started)
if check_if_kill(idle_time):
log.info(
'Execution {} for user {} has been idle for too long, terminating...'
.format(my_exec['name'], guest['name']))
terminate = True
break
else:
log.debug(
'Execution {} for user {} has been idle for {} seconds'
.format(my_exec['name'], guest['name'],
idle_time))
if terminate:
break
if terminate:
exec_api.terminate(my_exec['id'])
def exec_get_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(),
utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(),
utils.zoe_pass())
execution = exec_api.execution_get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'],
execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error']))
print('Time submit: {}'.format(
datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(
datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(
datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[
0] # FIXME how to decide which network is the right one?
print('Service {} (ID: {})'.format(c['name'], c['id']))
for p in c['ports']:
print(' - {}: {}://{}:{}{}'.format(p['name'], p['protocol'],
ip, p['port_number'],
p['path']))
class ZoeWorkFlow:
def __init__(self, workspace_base_path, identity, name):
self.identity = identity
self.name = name
self.workspace = ZoeWorkspace(workspace_base_path, identity, name)
self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
self.cont_api = ZoeServiceAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
info_api = ZoeInfoAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
zoe_info = info_api.info()
self.hostname_prefix = zoe_info['name_prefix']
self.exec_counter = 0
def generate_hostname(self, process_name: str) -> str:
return self.hostname_prefix + '-' + process_name + '-' + self.identity['username']
def start_workflow(self):
self.workspace.create()
def end_workflow(self):
self.workspace.destroy()
def execution_start(self, app):
self.exec_counter += 1
return self.exec_api.execution_start(self.name + '{}'.format(self.exec_counter), app)
def wait_termination(self, exec_id):
execution = self.exec_api.execution_get(exec_id)
while execution['status'] == 'submitted' or execution['status'] == 'running':
time.sleep(1)
execution = self.exec_api.execution_get(exec_id)
def __enter__(self):
self.start_workflow()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.end_workflow()
def __init__(self, workspace_base_path, identity, name):
self.identity = identity
self.name = name
self.workspace = ZoeWorkspace(workspace_base_path, identity, name)
self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
self.cont_api = ZoeServiceAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
info_api = ZoeInfoAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
zoe_info = info_api.info()
self.hostname_prefix = zoe_info['name_prefix']
self.exec_counter = 0
def new_execution(request, exec_name, app_name, dict):
print("zoe api: new execution")
print("zoe api: new execution {} - {}: arguments {}".format(exec_name, app_name, dict))
exec_api = ZoeExecutionsAPI(ZOE_URL, ZOE_USER, ZOE_PWD)
if app_name == 'ipython':
try:
notebook_memory_limit = dict['notebook_mem_limit'] * (1024 ** 3) # GB
spark_master_memory_limit = dict['master_mem_limit'] * (1024 ** 2) # MB
spark_worker_memory_limit = dict['worker_memory'] * (1024 ** 3) # GB
spark_worker_cores = dict['worker_cores']
spark_worker_count = dict['worker_count']
app_descr = zapps.create_notebook_app(notebook_memory_limit=notebook_memory_limit,
spark_master_memory_limit=spark_master_memory_limit,
spark_worker_memory_limit=spark_worker_memory_limit,
spark_worker_cores=spark_worker_cores,
spark_worker_count=spark_worker_count
)
except:
app_descr = zapps.create_notebook_app()
exec_api.execution_start(exec_name, app_descr)
elif app_name == 'mpi':
try:
spark_worker_cores = dict['worker_cores']
spark_worker_count = dict['worker_count']
assert spark_worker_cores * spark_worker_count <= 8
in_wm = int(dict['worker_memory'])
assert in_wm > 0
wm = in_wm * (1024 ** 3)
app_descr = zapps.create_idiada_app(worker_memory=wm)
except:
app_descr = zapps.create_idiada_app()
exec_api.execution_start('mpidynademo', app_descr)
else:
print("App not supported.")
return
return "Done"
def exec_get_cmd(args):
"""Gather information about an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error_message']))
print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
service = cont_api.get(c_id)
print('Service {} (ID: {})'.format(service['name'], service['id']))
print(' - zoe status: {}'.format(service['status']))
print(' - docker status: {}'.format(service['docker_status']))
if service['error_message'] is not None:
print(' - error: {}'.format(service['error_message']))
if service['docker_status'] == 'started':
ip = service['ip_address']
for port in service['description']['ports']:
print(' - {}: {}://{}:{}{}'.format(port['name'], port['protocol'], ip, port['port_number'], port['path']))
def exec_start_cmd(args):
"""Submit an execution."""
app_descr = json.load(args.jsonfile)
app_validate(app_descr)
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
exec_id = exec_api.start(args.name, app_descr)
if not args.synchronous:
print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(exec_id))
else:
print("Application scheduled successfully with ID {}, waiting for status change".format(exec_id))
old_status = 'submitted'
while True:
execution = exec_api.get(exec_id)
current_status = execution['status']
if old_status != current_status:
print('Execution is now {}'.format(current_status))
old_status = current_status
if current_status == 'running':
break
time.sleep(1)
monitor_service_id = None
service_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
for service_id in execution['services']:
service = service_api.get(service_id)
if service['description']['monitor']:
monitor_service_id = service['id']
break
print('\n>------ start of log streaming -------<\n')
why_stop = _log_stream_stdout(monitor_service_id, False)
print('\n>------ end of log streaming -------<\n')
if why_stop == 'stream_end':
print('Execution finished')
exit(0)
elif why_stop == 'interrupt':
print('Do not worry, your execution ({}) is still running.'.format(exec_id))
exit(1)
def delete_finished():
"""Delete finished executions from Zoe."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
for e_id in execs:
e = exec_api.get(e_id)
if e['name'] == 'boinc-loader' and e['status'] == 'terminated':
print('Execution {} has finished, deleting...'.format(e_id))
exec_api.delete(e['id'])
def submit_zapp(zapp):
"""Submits one ZApp for execution."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
ret = exec_api.start('boinc-loader', zapp)
return ret
def terminate_exec(request, exec_id):
print("zoe api: terminate_exec")
print("TO TERMINATE: ", exec_id)
exec_api = ZoeExecutionsAPI(ZOE_URL, ZOE_USER, ZOE_PWD)
return exec_api.terminate(exec_id)
def exec_rm_cmd(auth, args):
"""Delete an execution and kill it if necessary."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
exec_api.delete(args.id)
def exec_kill_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
exec_api.terminate(args.id)
def exec_kill_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(),
utils.zoe_pass())
exec_api.terminate(args.id)
def exec_start_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
ret = exec_api.execution_start(args.name, args.app_name)
print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(ret))
def submit_zapp(zapp):
"""Submits one ZApp for execution."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
ret = exec_api.start('boinc-loader', zapp)
print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(ret))
def exec_rm_cmd(args):
"""Delete an execution and kill it if necessary."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
exec_api.delete(args.id)
def home_guest():
auth = request.authorization
if not auth:
return missing_auth()
guest_identifier = auth.username
guest_password = auth.password
match = guest_id_pattern.match(guest_identifier)
if match is None:
return missing_auth()
query_api = ZoeQueryAPI(get_conf().zoe_url, guest_identifier, guest_password)
template_vars = {
'refresh': randint(2, 8),
'user_gateway': 'Please wait...',
'execution_status': 'Please wait...',
'execution_urls': [],
'guest_identifier': guest_identifier
}
try:
user = query_api.query('user', name=guest_identifier)
except ZoeAPIException:
return missing_auth()
if len(user) == 0:
return missing_auth()
else:
user = user[0]
template_vars['user_gateway'] = user['gateway_urls'][0]
template_vars['gateway_ip'] = user['gateway_urls'][0].split('/')[2].split(':')[0]
app_api = ZoeApplicationAPI(get_conf().zoe_url, guest_identifier, guest_password)
exec_api = ZoeExecutionsAPI(get_conf().zoe_url, guest_identifier, guest_password)
app = query_api.query('application', name='spark-jupyter-lab')
if len(app) == 0:
app_descr = spark_jupyter_notebook_lab_app()
app_api.create(app_descr)
return render_template('home_guest.html', **template_vars)
else:
app = app[0]
execution = query_api.query('execution', name='guest-lab-{}'.format(guest_identifier))
if len(execution) == 0:
exec_api.execution_start('guest-lab-{}'.format(guest_identifier), app['name'])
template_vars['execution_status'] = 'submitted'
return render_template('home_guest.html', **template_vars)
else:
execution = execution[0]
if execution['status'] == 'terminated':
app_api.delete(app['id'])
return render_template('home_guest.html', **template_vars)
elif execution['status'] != 'running':
template_vars['execution_status'] = execution['status']
return render_template('home_guest.html', **template_vars)
else:
template_vars['refresh'] = -1
cont_api = ZoeContainerAPI(get_conf().zoe_url, guest_identifier, guest_password)
template_vars['execution_status'] = execution['status']
for c_id in execution['containers']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
for p in c['ports']:
template_vars['execution_urls'].append(('{}'.format(p['name']), '{}://{}:{}{}'.format(p['protocol'], ip, p['port_number'], p['path'])))
return render_template('home_guest.html', **template_vars)
def exec_start_cmd(args):
app_descr = json.load(args.jsonfile)
app_validate(app_descr)
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
ret = exec_api.execution_start(args.name, app_descr)
print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(ret))
def exec_kill_cmd(auth, args):
"""Kill an execution."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
exec_api.terminate(args.id)
def submit_zapp(zapp):
"""Submits one ZApp for execution."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
ret = exec_api.start('boinc-loader', zapp)
return ret
def exec_rm_cmd(args):
"""Delete an execution and kill it if necessary."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
exec_api.delete(args.id)
def exec_list_cmd(_):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
data = exec_api.list()
for e in data:
print('Execution {} (User: {}, ID: {}): {}'.format(e['name'], e['user_id'], e['id'], e['status']))
def exec_list_cmd(args_):
"""List executions"""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
data = exec_api.list()
for e in sorted(data.values(), key=lambda x: x['id']):
print('Execution {} (User: {}, ID: {}): {}'.format(e['name'], e['user_id'], e['id'], e['status']))
