class Test: def __init__(self, user, pwd, url, name, zapp): self.user = user self.pwd = pwd self.url = url self.name = name try: with open(zapp, 'r') as infile: self.zapp = json.load(infile) except: exit("Unable to load zapp file.") self.exec_api = ZoeExecutionsAPI(self.url, self.user, self.pwd) self.service_api = ZoeServiceAPI(self.url, self.user, self.pwd) def start_exec(self): exec_id = self.exec_api.start(self.name, self.zapp) return exec_id def get_services_id(self, exec_id): services = self.exec_api.get(exec_id) return services['services'] def get_submit_service(self, exec_id): while len(self.get_services_id(exec_id)) < 4: print('waiting') time.sleep(0.5) for service_id in self.get_services_id(exec_id): srv = self.service_api.get(service_id) if re.search('submit', srv['name']): return srv['id'] def is_running(self, exec_id): return self.exec_api.get(exec_id)['status'] == 'running' def run_test(self): ts = time.time() exec_id = self.start_exec() outfilename = './logs/{}'.format(exec_id) submit_id = self.get_submit_service(exec_id) while not self.is_running(exec_id): time.sleep(0.5) te = time.time() with open(outfilename, 'w') as out: out.write("PerfMeasure: Scheduling time: {}".format(te - ts)) for line in self.service_api.get_logs(submit_id): out.write("{}\n".format(line)) self.exec_api.terminate(exec_id) print('Terminated {}'.format(exec_id))
def main(): """Main.""" exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass()) execs = exec_api.list() print( 'id,time_submit,time_start,time_end,cpu_usage,mem_usage,net_rx_usage,net_tx_usage,blkio_usage' ) for e_id in execs: e = exec_api.get(e_id) if e['name'] != 'boinc-loader' or e['status'] != 'terminated': continue trace_line = { 'id': e['id'], 'time_submit': e['time_submit'], 'time_start': e['time_start'], 'time_end': e['time_end'], 'cpu_usage': get_influx_cpu_data(e_id), 'mem_usage': get_influx_mem_data(e_id), 'net_rx_usage': get_influx_net_rx_data(e_id), 'net_tx_usage': get_influx_net_tx_data(e_id), 'blkio_usage': get_influx_blkio_data(e_id) } print( '{id},{time_submit},{time_start},{time_end},{cpu_usage},{mem_usage},{net_rx_usage},{net_tx_usage},{blkio_usage}' .format(**trace_line))
def exec_get_cmd(args): """Gather information about an execution.""" exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) execution = exec_api.get(args.id) if execution is None: print('Execution not found') else: print('Execution {} (ID: {})'.format(execution['name'], execution['id'])) print('Status: {}'.format(execution['status'])) if execution['status'] == 'error': print('Last error: {}'.format(execution['error_message'])) print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit']))) if execution['time_start'] is None: print('Time start: {}'.format('not yet')) else: print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start']))) if execution['time_end'] is None: print('Time end: {}'.format('not yet')) else: print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end']))) app = execution['description'] print('Application name: {}'.format(app['name'])) for c_id in execution['services']: service = cont_api.get(c_id) print('Service {} (ID: {})'.format(service['name'], service['id'])) print(' - zoe status: {}'.format(service['status'])) print(' - docker status: {}'.format(service['docker_status'])) if service['docker_status'] == 'started': ip = service['ip_address'] for port in service['description']['ports']: print(' - {}: {}://{}:{}{}'.format(port['name'], port['protocol'], ip, port['port_number'], port['path']))
def exec_get_cmd(auth, args): """Gather information about an execution.""" exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass']) cont_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass']) execution = exec_api.get(args.id) if execution is None: print('Execution not found') else: print('Execution {} (ID: {})'.format(execution['name'], execution['id'])) print('Application name: {}'.format(execution['description']['name'])) print('Status: {}'.format(execution['status'])) if execution['status'] == 'error': print('Last error: {}'.format(execution['error_message'])) print() print('Time submit: {}'.format( datetime.fromtimestamp(execution['time_submit'], timezone.utc).astimezone())) if execution['time_start'] is None: print('Time start: {}'.format('not yet')) else: print('Time start: {}'.format( datetime.fromtimestamp(execution['time_start'], timezone.utc).astimezone())) if execution['time_end'] is None: print('Time end: {}'.format('not yet')) else: print('Time end: {}'.format( datetime.fromtimestamp(execution['time_end'], timezone.utc).astimezone())) print() endpoints = exec_api.endpoints(execution['id']) if endpoints is not None and len(endpoints) > 0: print('Exposed endpoints:') for endpoint in endpoints: print(' - {}: {}'.format(endpoint[0], endpoint[1])) else: print('This ZApp does not expose any endpoint') print() tabular_data = [] for c_id in execution['services']: service = cont_api.get(c_id) service_data = [ service['id'], service['name'], 'true' if service['essential'] else 'false', service['status'], service['backend_status'], service['backend_host'], service['error_message'] if service['error_message'] is not None else '' ] tabular_data.append(service_data) headers = [ 'ID', 'Name', 'Essential', 'Zoe status', 'Backend status', 'Host', 'Error message' ] print(tabulate(tabular_data, headers))
def app_get_cmd(args): """Extract an application description from an execution.""" exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) execution = exec_api.get(args.id) if execution is None: print("no such execution") else: json.dump(execution['description'], sys.stdout, sort_keys=True, indent=4)
def app_get_cmd(args): """Extract an application description from an execution.""" exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) execution = exec_api.get(args.id) if execution is None: print("no such execution") else: json.dump(execution['description'], sys.stdout, sort_keys=True, indent=4)
def delete_finished(): """Delete finished executions from Zoe.""" exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass()) execs = exec_api.list() for e_id in execs: e = exec_api.get(e_id) if e['name'] == 'boinc-loader' and e['status'] == 'terminated': print('Execution {} has finished, deleting...'.format(e_id)) exec_api.delete(e['id'])
def delete_finished(): """Delete finished executions from Zoe.""" exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass()) execs = exec_api.list() for e_id in execs: e = exec_api.get(e_id) if e is None: continue if e['name'] == 'boinc-loader' and e['status'] == 'terminated': print('Execution {} has finished, deleting...'.format(e_id)) exec_api.delete(e['id'])
def app_get_cmd(auth, args): """Extract an application description from an execution.""" exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass']) execution = exec_api.get(args.id) if execution is None: print("no such execution") else: json.dump(execution['description'], sys.stdout, sort_keys=True, indent=4)
def count_jobs(): """Count how many zapps have already been submitted.""" exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass()) execs = exec_api.list() count = 0 for e_id in execs: e = exec_api.get(e_id) if e['name'] != 'boinc-loader': continue if e['status'] != 'terminated': count += 1 return count
def count_jobs(all=False): """Count how many zapps have already been submitted.""" exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass()) execs = exec_api.list() count = 0 for e_id in execs: e = exec_api.get(e_id) if e is None: continue elif not all and e['name'] != 'boinc-loader': continue if e['status'] != 'terminated': count += 1 return count
def exec_start_cmd(auth, args): """Submit an execution.""" app_descr = json.load(args.jsonfile) exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass']) exec_id = exec_api.start(args.name, app_descr) if not args.synchronous: print( "Application scheduled successfully with ID {}, use the exec-get command to check its status" .format(exec_id)) else: print( "Application scheduled successfully with ID {}, waiting for status change" .format(exec_id)) old_status = 'submitted' while True: execution = exec_api.get(exec_id) current_status = execution['status'] if old_status != current_status: print('Execution is now {}'.format(current_status)) old_status = current_status if current_status == 'running': break time.sleep(1) monitor_service_id = None service_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass']) for service_id in execution['services']: service = service_api.get(service_id) if service['description']['monitor']: monitor_service_id = service['id'] break print('\n>------ start of log streaming -------<\n') why_stop = _log_stream_stdout(monitor_service_id, False, auth) print('\n>------ end of log streaming -------<\n') if why_stop == 'stream_end': print('Execution finished') exit(0) elif why_stop == 'interrupt': print('Do not worry, your execution ({}) is still running.'.format( exec_id)) exit(1)
def main(): """Main.""" exec_api = ZoeExecutionsAPI(zoe_url(), zoe_user(), zoe_pass()) execs = exec_api.list() print('id,time_submit,time_start,time_end,cpu_usage,mem_usage,net_rx_usage,net_tx_usage,blkio_usage') for e_id in execs: e = exec_api.get(e_id) if e['name'] != 'boinc-loader' or e['status'] != 'terminated': continue trace_line = { 'id': e['id'], 'time_submit': e['time_submit'], 'time_start': e['time_start'], 'time_end': e['time_end'], 'cpu_usage': get_influx_cpu_data(e_id), 'mem_usage': get_influx_mem_data(e_id), 'net_rx_usage': get_influx_net_rx_data(e_id), 'net_tx_usage': get_influx_net_tx_data(e_id), 'blkio_usage': get_influx_blkio_data(e_id) } print('{id},{time_submit},{time_start},{time_end},{cpu_usage},{mem_usage},{net_rx_usage},{net_tx_usage},{blkio_usage}'.format(**trace_line))
def exec_start_cmd(args): """Submit an execution.""" app_descr = json.load(args.jsonfile) app_validate(app_descr) exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) exec_id = exec_api.start(args.name, app_descr) if not args.synchronous: print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(exec_id)) else: print("Application scheduled successfully with ID {}, waiting for status change".format(exec_id)) old_status = 'submitted' while True: execution = exec_api.get(exec_id) current_status = execution['status'] if old_status != current_status: print('Execution is now {}'.format(current_status)) old_status = current_status if current_status == 'running': break time.sleep(1) monitor_service_id = None service_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) for service_id in execution['services']: service = service_api.get(service_id) if service['description']['monitor']: monitor_service_id = service['id'] break print('\n>------ start of log streaming -------<\n') why_stop = _log_stream_stdout(monitor_service_id, False) print('\n>------ end of log streaming -------<\n') if why_stop == 'stream_end': print('Execution finished') exit(0) elif why_stop == 'interrupt': print('Do not worry, your execution ({}) is still running.'.format(exec_id)) exit(1)
def exec_get_cmd(args): """Gather information about an execution.""" exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) cont_api = ZoeServiceAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass()) execution = exec_api.get(args.id) if execution is None: print('Execution not found') else: print('Execution {} (ID: {})'.format(execution['name'], execution['id'])) print('Status: {}'.format(execution['status'])) if execution['status'] == 'error': print('Last error: {}'.format(execution['error_message'])) print('Time submit: {}'.format(datetime.datetime.fromtimestamp(execution['time_submit']))) if execution['time_start'] is None: print('Time start: {}'.format('not yet')) else: print('Time start: {}'.format(datetime.datetime.fromtimestamp(execution['time_start']))) if execution['time_end'] is None: print('Time end: {}'.format('not yet')) else: print('Time end: {}'.format(datetime.datetime.fromtimestamp(execution['time_end']))) app = execution['description'] print('Application name: {}'.format(app['name'])) for c_id in execution['services']: service = cont_api.get(c_id) print('Service {} (ID: {})'.format(service['name'], service['id'])) print(' - zoe status: {}'.format(service['status'])) print(' - docker status: {}'.format(service['docker_status'])) if service['error_message'] is not None: print(' - error: {}'.format(service['error_message'])) if service['docker_status'] == 'started': ip = service['ip_address'] for port in service['description']['ports']: print(' - {}: {}://{}:{}{}'.format(port['name'], port['protocol'], ip, port['port_number'], port['path']))