def exec_get_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.execution_get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error']))
print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
print('Service {} (ID: {})'.format(c['name'], c['id']))
for p in c['ports']:
print(' - {}: {}://{}:{}{}'.format(p['name'], p['protocol'], ip, p['port_number'], p['path']))
def exec_get_cmd(args):
"""Gather information about an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error_message']))
print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
service = cont_api.get(c_id)
print('Service {} (ID: {})'.format(service['name'], service['id']))
print(' - zoe status: {}'.format(service['status']))
print(' - docker status: {}'.format(service['docker_status']))
if service['docker_status'] == 'started':
ip = service['ip_address']
for port in service['description']['ports']:
print(' - {}: {}://{}:{}{}'.format(port['name'], port['protocol'], ip, port['port_number'], port['path']))
def check_guests(swarm):
query_api = ZoeQueryAPI(get_conf().master_url, 'zoeadmin', get_conf().zoeadmin_password)
exec_api = ZoeExecutionsAPI(get_conf().master_url, 'zoeadmin', get_conf().zoeadmin_password)
cont_api = ZoeServiceAPI(get_conf().master_url, 'zoeadmin', get_conf().zoeadmin_password)
guests = query_api.query('user', role='guest')
execs = exec_api.list()
for guest in guests:
my_execs = [e for e in execs if e['owner'] == guest['name']]
for my_exec in my_execs:
if len(my_exec['services']) == 0:
continue
my_exec_since_started = datetime.datetime.now() - dateutil.parser.parse(my_exec['time_started'])
my_exec_since_started = my_exec_since_started.total_seconds()
terminate = False
for c in my_exec['services']:
c = cont_api.get(c)
for port in c['ports']:
if port['name'] == 'Spark application web interface':
idle_time = check_spark_job(swarm, c['docker_id'], my_exec_since_started)
if check_if_kill(idle_time):
log.info('Execution {} for user {} has been idle for too long, terminating...'.format(my_exec['name'], guest['name']))
terminate = True
break
else:
log.debug('Execution {} for user {} has been idle for {} seconds'.format(my_exec['name'], guest['name'], idle_time))
if terminate:
break
if terminate:
exec_api.terminate(my_exec['id'])
def get_execution_details(exec_id):
print("zoe api: get_execution_details")
try:
print("zoe api: get_execution_details: found in vault with id = {}".format(exec_id))
return vault[exec_id]
except KeyError:
print("zoe api: get_execution_details: no execution found with id = {}".format(exec_id))
vault[exec_id] = {}
exec_api = ZoeExecutionsAPI(ZOE_URL, ZOE_USER, ZOE_PWD)
cont_api = ZoeServiceAPI(ZOE_URL, ZOE_USER, ZOE_PWD)
exec_details = exec_api.execution_get(exec_id)
owner, gateway = get_user_info(exec_details)
service_details = []
for c_id in exec_details['services']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
cont_id = c['id']
cont_name = c['name']
tmp = {'name': cont_name, 'details': {}}
for p in c['ports']:
url = "{}://{}:{}{}".format(p['protocol'], ip, p['port_number'], p['path'])
tmp['details'] = {'name': p['name'], 'url': url}
service_details.append(tmp)
exec_details.update({'service_details': service_details, 'owner': owner, 'gateway': _translate_gw(gateway)})
vault[exec_id].update(exec_details)
return vault[exec_id]
def exec_get_cmd(auth, args):
"""Gather information about an execution."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
cont_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
execution = exec_api.get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'],
execution['id']))
print('Application name: {}'.format(execution['description']['name']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error_message']))
print()
print('Time submit: {}'.format(
datetime.fromtimestamp(execution['time_submit'],
timezone.utc).astimezone()))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(
datetime.fromtimestamp(execution['time_start'],
timezone.utc).astimezone()))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(
datetime.fromtimestamp(execution['time_end'],
timezone.utc).astimezone()))
print()
endpoints = exec_api.endpoints(execution['id'])
if endpoints is not None and len(endpoints) > 0:
print('Exposed endpoints:')
for endpoint in endpoints:
print(' - {}: {}'.format(endpoint[0], endpoint[1]))
else:
print('This ZApp does not expose any endpoint')
print()
tabular_data = []
for c_id in execution['services']:
service = cont_api.get(c_id)
service_data = [
service['id'], service['name'],
'true' if service['essential'] else 'false', service['status'],
service['backend_status'], service['backend_host'],
service['error_message']
if service['error_message'] is not None else ''
]
tabular_data.append(service_data)
headers = [
'ID', 'Name', 'Essential', 'Zoe status', 'Backend status', 'Host',
'Error message'
]
print(tabulate(tabular_data, headers))
def container_died(service_id):
log.debug('A container died')
# tell the master via the rest api
cont_api = ZoeServiceAPI(get_conf().master_url, 'zoeadmin', get_conf().zoeadmin_password)
try:
cont_api.died(service_id)
except ZoeAPIException as e:
if e.message != "No such service":
log.exception('Error reporting a dead service')
def container_died(service_id):
log.debug('A container died')
# tell the master via the rest api
cont_api = ZoeServiceAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
try:
cont_api.died(service_id)
except ZoeAPIException as e:
if e.message != "No such service":
log.exception('Error reporting a dead service')
def logs_cmd(args):
"""Retrieves and streams the logs of a service."""
service_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
try:
for line in service_api.get_logs(args.service_id):
if args.timestamps:
print(line[0], line[1])
else:
print(line[1])
except KeyboardInterrupt:
print('CTRL-C detected, exiting...')
def _log_stream_stdout(service_id, timestamps):
service_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
try:
for line in service_api.get_logs(service_id):
if timestamps:
print(line[0], line[1])
else:
print(line[1])
except KeyboardInterrupt:
print('CTRL-C detected, exiting...')
return 'interrupt'
return 'stream_end'
def _log_stream_stdout(service_id, timestamps, auth):
service_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
try:
for line in service_api.get_logs(service_id):
if timestamps:
print(line[0], line[1])
else:
print(line[1])
except KeyboardInterrupt:
print('CTRL-C detected, exiting...')
return 'interrupt'
return 'stream_end'
class Test:
def __init__(self, user, pwd, url, name, zapp):
self.user = user
self.pwd = pwd
self.url = url
self.name = name
try:
with open(zapp, 'r') as infile:
self.zapp = json.load(infile)
except:
exit("Unable to load zapp file.")
self.exec_api = ZoeExecutionsAPI(self.url, self.user, self.pwd)
self.service_api = ZoeServiceAPI(self.url, self.user, self.pwd)
def start_exec(self):
exec_id = self.exec_api.start(self.name, self.zapp)
return exec_id
def get_services_id(self, exec_id):
services = self.exec_api.get(exec_id)
return services['services']
def get_submit_service(self, exec_id):
while len(self.get_services_id(exec_id)) < 4:
print('waiting')
time.sleep(0.5)
for service_id in self.get_services_id(exec_id):
srv = self.service_api.get(service_id)
if re.search('submit', srv['name']):
return srv['id']
def is_running(self, exec_id):
return self.exec_api.get(exec_id)['status'] == 'running'
def run_test(self):
ts = time.time()
exec_id = self.start_exec()
outfilename = './logs/{}'.format(exec_id)
submit_id = self.get_submit_service(exec_id)
while not self.is_running(exec_id):
time.sleep(0.5)
te = time.time()
with open(outfilename, 'w') as out:
out.write("PerfMeasure: Scheduling time: {}".format(te - ts))
for line in self.service_api.get_logs(submit_id):
out.write("{}\n".format(line))
self.exec_api.terminate(exec_id)
print('Terminated {}'.format(exec_id))
def exec_start_cmd(auth, args):
"""Submit an execution."""
app_descr = json.load(args.jsonfile)
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
exec_id = exec_api.start(args.name, app_descr)
if not args.synchronous:
print(
"Application scheduled successfully with ID {}, use the exec-get command to check its status"
.format(exec_id))
else:
print(
"Application scheduled successfully with ID {}, waiting for status change"
.format(exec_id))
old_status = 'submitted'
while True:
execution = exec_api.get(exec_id)
current_status = execution['status']
if old_status != current_status:
print('Execution is now {}'.format(current_status))
old_status = current_status
if current_status == 'running':
break
time.sleep(1)
monitor_service_id = None
service_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
for service_id in execution['services']:
service = service_api.get(service_id)
if service['description']['monitor']:
monitor_service_id = service['id']
break
print('\n>------ start of log streaming -------<\n')
why_stop = _log_stream_stdout(monitor_service_id, False, auth)
print('\n>------ end of log streaming -------<\n')
if why_stop == 'stream_end':
print('Execution finished')
exit(0)
elif why_stop == 'interrupt':
print('Do not worry, your execution ({}) is still running.'.format(
exec_id))
exit(1)
def __init__(self, user, pwd, url, name, zapp):
self.user = user
self.pwd = pwd
self.url = url
self.name = name
try:
with open(zapp, 'r') as infile:
self.zapp = json.load(infile)
except:
exit("Unable to load zapp file.")
self.exec_api = ZoeExecutionsAPI(self.url, self.user, self.pwd)
self.service_api = ZoeServiceAPI(self.url, self.user, self.pwd)
def check_guests(swarm):
query_api = ZoeQueryAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
exec_api = ZoeExecutionsAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
cont_api = ZoeServiceAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
guests = query_api.query('user', role='guest')
execs = exec_api.list()
for guest in guests:
my_execs = [e for e in execs if e['owner'] == guest['name']]
for my_exec in my_execs:
if len(my_exec['services']) == 0:
continue
my_exec_since_started = datetime.datetime.now(
) - dateutil.parser.parse(my_exec['time_started'])
my_exec_since_started = my_exec_since_started.total_seconds()
terminate = False
for c in my_exec['services']:
c = cont_api.get(c)
for port in c['ports']:
if port['name'] == 'Spark application web interface':
idle_time = check_spark_job(swarm, c['docker_id'],
my_exec_since_started)
if check_if_kill(idle_time):
log.info(
'Execution {} for user {} has been idle for too long, terminating...'
.format(my_exec['name'], guest['name']))
terminate = True
break
else:
log.debug(
'Execution {} for user {} has been idle for {} seconds'
.format(my_exec['name'], guest['name'],
idle_time))
if terminate:
break
if terminate:
exec_api.terminate(my_exec['id'])
def exec_get_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(),
utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(),
utils.zoe_pass())
execution = exec_api.execution_get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'],
execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error']))
print('Time submit: {}'.format(
datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(
datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(
datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[
0] # FIXME how to decide which network is the right one?
print('Service {} (ID: {})'.format(c['name'], c['id']))
for p in c['ports']:
print(' - {}: {}://{}:{}{}'.format(p['name'], p['protocol'],
ip, p['port_number'],
p['path']))
def home_user():
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
if role == 'user' or role == 'admin':
executions = config.api_endpoint.execution_list(uid, role)
template_vars = {
'executions': executions,
'is_admin': role == 'admin',
}
return render_template('home_user.html', **template_vars)
else:
template_vars = {
'refresh': randint(2, 8),
'execution_status': 'Please wait...',
'execution_urls': [],
}
app_descr = json.load(open('contrib/zoeapps/eurecom_aml_lab.json', 'r'))
execution = config.api_endpoint.execution_list(uid, role, name='aml-lab')
if len(execution) == 0 or execution[0]['status'] == 'terminated' or execution[0]['status'] == 'finished':
config.api_endpoint.execution_start(uid, role, 'aml-lab', app_descr)
template_vars['execution_status'] = 'submitted'
return render_template('home_guest.html', **template_vars)
else:
execution = execution[0]
if execution['status'] != 'running':
template_vars['execution_status'] = execution['status']
return render_template('home_guest.html', **template_vars)
else:
template_vars['refresh'] = -1
cont_api = ZoeServiceAPI(get_conf().master_url, guest_identifier, guest_password)
template_vars['execution_status'] = execution['status']
for c_id in execution['services']:
c = cont_api.get(c_id)
ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
for p in c['ports']:
template_vars['execution_urls'].append(('{}'.format(p['name']), '{}://{}:{}{}'.format(p['protocol'], ip, p['port_number'], p['path'])))
return render_template('home_guest.html', **template_vars)
def exec_get_cmd(args):
"""Gather information about an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error_message']))
print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
service = cont_api.get(c_id)
print('Service {} (ID: {})'.format(service['name'], service['id']))
print(' - zoe status: {}'.format(service['status']))
print(' - docker status: {}'.format(service['docker_status']))
if service['error_message'] is not None:
print(' - error: {}'.format(service['error_message']))
if service['docker_status'] == 'started':
ip = service['ip_address']
for port in service['description']['ports']:
print(' - {}: {}://{}:{}{}'.format(port['name'], port['protocol'], ip, port['port_number'], port['path']))
def exec_start_cmd(args):
"""Submit an execution."""
app_descr = json.load(args.jsonfile)
app_validate(app_descr)
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
exec_id = exec_api.start(args.name, app_descr)
if not args.synchronous:
print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(exec_id))
else:
print("Application scheduled successfully with ID {}, waiting for status change".format(exec_id))
old_status = 'submitted'
while True:
execution = exec_api.get(exec_id)
current_status = execution['status']
if old_status != current_status:
print('Execution is now {}'.format(current_status))
old_status = current_status
if current_status == 'running':
break
time.sleep(1)
monitor_service_id = None
service_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
for service_id in execution['services']:
service = service_api.get(service_id)
if service['description']['monitor']:
monitor_service_id = service['id']
break
print('\n>------ start of log streaming -------<\n')
why_stop = _log_stream_stdout(monitor_service_id, False)
print('\n>------ end of log streaming -------<\n')
if why_stop == 'stream_end':
print('Execution finished')
exit(0)
elif why_stop == 'interrupt':
print('Do not worry, your execution ({}) is still running.'.format(exec_id))
exit(1)
def __init__(self, workspace_base_path, identity, name):
self.identity = identity
self.name = name
self.workspace = ZoeWorkspace(workspace_base_path, identity, name)
self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
self.cont_api = ZoeServiceAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
info_api = ZoeInfoAPI(self.identity['zoe_url'],
self.identity['username'],
self.identity['password'])
zoe_info = info_api.info()
self.hostname_prefix = zoe_info['name_prefix']
self.exec_counter = 0