class Test:
def __init__(self, user, pwd, url, name, zapp):
self.user = user
self.pwd = pwd
self.url = url
self.name = name
try:
with open(zapp, 'r') as infile:
self.zapp = json.load(infile)
except:
exit("Unable to load zapp file.")
self.exec_api = ZoeExecutionsAPI(self.url, self.user, self.pwd)
self.service_api = ZoeServiceAPI(self.url, self.user, self.pwd)
def start_exec(self):
exec_id = self.exec_api.start(self.name, self.zapp)
return exec_id
def get_services_id(self, exec_id):
services = self.exec_api.get(exec_id)
return services['services']
def get_submit_service(self, exec_id):
while len(self.get_services_id(exec_id)) < 4:
print('waiting')
time.sleep(0.5)
for service_id in self.get_services_id(exec_id):
srv = self.service_api.get(service_id)
if re.search('submit', srv['name']):
return srv['id']
def is_running(self, exec_id):
return self.exec_api.get(exec_id)['status'] == 'running'
def run_test(self):
ts = time.time()
exec_id = self.start_exec()
outfilename = './logs/{}'.format(exec_id)
submit_id = self.get_submit_service(exec_id)
while not self.is_running(exec_id):
time.sleep(0.5)
te = time.time()
with open(outfilename, 'w') as out:
out.write("PerfMeasure: Scheduling time: {}".format(te - ts))
for line in self.service_api.get_logs(submit_id):
out.write("{}\n".format(line))
self.exec_api.terminate(exec_id)
print('Terminated {}'.format(exec_id))
def main():
"""Main."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
print(
'id,time_submit,time_start,time_end,cpu_usage,mem_usage,net_rx_usage,net_tx_usage,blkio_usage'
)
for e_id in execs:
e = exec_api.get(e_id)
if e['name'] != 'boinc-loader' or e['status'] != 'terminated':
continue
trace_line = {
'id': e['id'],
'time_submit': e['time_submit'],
'time_start': e['time_start'],
'time_end': e['time_end'],
'cpu_usage': get_influx_cpu_data(e_id),
'mem_usage': get_influx_mem_data(e_id),
'net_rx_usage': get_influx_net_rx_data(e_id),
'net_tx_usage': get_influx_net_tx_data(e_id),
'blkio_usage': get_influx_blkio_data(e_id)
}
print(
'{id},{time_submit},{time_start},{time_end},{cpu_usage},{mem_usage},{net_rx_usage},{net_tx_usage},{blkio_usage}'
.format(**trace_line))
def exec_get_cmd(args):
"""Gather information about an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error_message']))
print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
service = cont_api.get(c_id)
print('Service {} (ID: {})'.format(service['name'], service['id']))
print(' - zoe status: {}'.format(service['status']))
print(' - docker status: {}'.format(service['docker_status']))
if service['docker_status'] == 'started':
ip = service['ip_address']
for port in service['description']['ports']:
print(' - {}: {}://{}:{}{}'.format(port['name'], port['protocol'], ip, port['port_number'], port['path']))
def exec_get_cmd(auth, args):
"""Gather information about an execution."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
cont_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
execution = exec_api.get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'],
execution['id']))
print('Application name: {}'.format(execution['description']['name']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error_message']))
print()
print('Time submit: {}'.format(
datetime.fromtimestamp(execution['time_submit'],
timezone.utc).astimezone()))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(
datetime.fromtimestamp(execution['time_start'],
timezone.utc).astimezone()))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(
datetime.fromtimestamp(execution['time_end'],
timezone.utc).astimezone()))
print()
endpoints = exec_api.endpoints(execution['id'])
if endpoints is not None and len(endpoints) > 0:
print('Exposed endpoints:')
for endpoint in endpoints:
print(' - {}: {}'.format(endpoint[0], endpoint[1]))
else:
print('This ZApp does not expose any endpoint')
print()
tabular_data = []
for c_id in execution['services']:
service = cont_api.get(c_id)
service_data = [
service['id'], service['name'],
'true' if service['essential'] else 'false', service['status'],
service['backend_status'], service['backend_host'],
service['error_message']
if service['error_message'] is not None else ''
]
tabular_data.append(service_data)
headers = [
'ID', 'Name', 'Essential', 'Zoe status', 'Backend status', 'Host',
'Error message'
]
print(tabulate(tabular_data, headers))
def app_get_cmd(args):
"""Extract an application description from an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print("no such execution")
else:
json.dump(execution['description'], sys.stdout, sort_keys=True, indent=4)
def app_get_cmd(args):
"""Extract an application description from an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print("no such execution")
else:
json.dump(execution['description'], sys.stdout, sort_keys=True, indent=4)
def delete_finished():
"""Delete finished executions from Zoe."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
for e_id in execs:
e = exec_api.get(e_id)
if e['name'] == 'boinc-loader' and e['status'] == 'terminated':
print('Execution {} has finished, deleting...'.format(e_id))
exec_api.delete(e['id'])
def delete_finished():
"""Delete finished executions from Zoe."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
for e_id in execs:
e = exec_api.get(e_id)
if e is None:
continue
if e['name'] == 'boinc-loader' and e['status'] == 'terminated':
print('Execution {} has finished, deleting...'.format(e_id))
exec_api.delete(e['id'])
def app_get_cmd(auth, args):
"""Extract an application description from an execution."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
execution = exec_api.get(args.id)
if execution is None:
print("no such execution")
else:
json.dump(execution['description'],
sys.stdout,
sort_keys=True,
indent=4)
def count_jobs():
"""Count how many zapps have already been submitted."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
count = 0
for e_id in execs:
e = exec_api.get(e_id)
if e['name'] != 'boinc-loader':
continue
if e['status'] != 'terminated':
count += 1
return count
def count_jobs(all=False):
"""Count how many zapps have already been submitted."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
count = 0
for e_id in execs:
e = exec_api.get(e_id)
if e is None:
continue
elif not all and e['name'] != 'boinc-loader':
continue
if e['status'] != 'terminated':
count += 1
return count
def exec_start_cmd(auth, args):
"""Submit an execution."""
app_descr = json.load(args.jsonfile)
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
exec_id = exec_api.start(args.name, app_descr)
if not args.synchronous:
print(
"Application scheduled successfully with ID {}, use the exec-get command to check its status"
.format(exec_id))
else:
print(
"Application scheduled successfully with ID {}, waiting for status change"
.format(exec_id))
old_status = 'submitted'
while True:
execution = exec_api.get(exec_id)
current_status = execution['status']
if old_status != current_status:
print('Execution is now {}'.format(current_status))
old_status = current_status
if current_status == 'running':
break
time.sleep(1)
monitor_service_id = None
service_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
for service_id in execution['services']:
service = service_api.get(service_id)
if service['description']['monitor']:
monitor_service_id = service['id']
break
print('\n>------ start of log streaming -------<\n')
why_stop = _log_stream_stdout(monitor_service_id, False, auth)
print('\n>------ end of log streaming -------<\n')
if why_stop == 'stream_end':
print('Execution finished')
exit(0)
elif why_stop == 'interrupt':
print('Do not worry, your execution ({}) is still running.'.format(
exec_id))
exit(1)
def main():
"""Main."""
exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass())
execs = exec_api.list()
print('id,time_submit,time_start,time_end,cpu_usage,mem_usage,net_rx_usage,net_tx_usage,blkio_usage')
for e_id in execs:
e = exec_api.get(e_id)
if e['name'] != 'boinc-loader' or e['status'] != 'terminated':
continue
trace_line = {
'id': e['id'],
'time_submit': e['time_submit'],
'time_start': e['time_start'],
'time_end': e['time_end'],
'cpu_usage': get_influx_cpu_data(e_id),
'mem_usage': get_influx_mem_data(e_id),
'net_rx_usage': get_influx_net_rx_data(e_id),
'net_tx_usage': get_influx_net_tx_data(e_id),
'blkio_usage': get_influx_blkio_data(e_id)
}
print('{id},{time_submit},{time_start},{time_end},{cpu_usage},{mem_usage},{net_rx_usage},{net_tx_usage},{blkio_usage}'.format(**trace_line))
def exec_start_cmd(args):
"""Submit an execution."""
app_descr = json.load(args.jsonfile)
app_validate(app_descr)
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
exec_id = exec_api.start(args.name, app_descr)
if not args.synchronous:
print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(exec_id))
else:
print("Application scheduled successfully with ID {}, waiting for status change".format(exec_id))
old_status = 'submitted'
while True:
execution = exec_api.get(exec_id)
current_status = execution['status']
if old_status != current_status:
print('Execution is now {}'.format(current_status))
old_status = current_status
if current_status == 'running':
break
time.sleep(1)
monitor_service_id = None
service_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
for service_id in execution['services']:
service = service_api.get(service_id)
if service['description']['monitor']:
monitor_service_id = service['id']
break
print('\n>------ start of log streaming -------<\n')
why_stop = _log_stream_stdout(monitor_service_id, False)
print('\n>------ end of log streaming -------<\n')
if why_stop == 'stream_end':
print('Execution finished')
exit(0)
elif why_stop == 'interrupt':
print('Do not worry, your execution ({}) is still running.'.format(exec_id))
exit(1)
def exec_get_cmd(args):
"""Gather information about an execution."""
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
execution = exec_api.get(args.id)
if execution is None:
print('Execution not found')
else:
print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
print('Status: {}'.format(execution['status']))
if execution['status'] == 'error':
print('Last error: {}'.format(execution['error_message']))
print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit'])))
if execution['time_start'] is None:
print('Time start: {}'.format('not yet'))
else:
print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start'])))
if execution['time_end'] is None:
print('Time end: {}'.format('not yet'))
else:
print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end'])))
app = execution['description']
print('Application name: {}'.format(app['name']))
for c_id in execution['services']:
service = cont_api.get(c_id)
print('Service {} (ID: {})'.format(service['name'], service['id']))
print(' - zoe status: {}'.format(service['status']))
print(' - docker status: {}'.format(service['docker_status']))
if service['error_message'] is not None:
print(' - error: {}'.format(service['error_message']))
if service['docker_status'] == 'started':
ip = service['ip_address']
for port in service['description']['ports']:
print(' - {}: {}://{}:{}{}'.format(port['name'], port['protocol'], ip, port['port_number'], port['path']))
