def check_guests(swarm):
query_api = ZoeQueryAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
exec_api = ZoeExecutionsAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
cont_api = ZoeContainerAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
guests = query_api.query('user', role='guest')
execs = exec_api.list()
for guest in guests:
my_execs = [e for e in execs if e['owner'] == guest['id']]
for my_exec in my_execs:
if len(my_exec['containers']) == 0:
continue
my_exec_since_started = datetime.datetime.now() - dateutil.parser.parse(my_exec['time_started'])
my_exec_since_started = my_exec_since_started.total_seconds()
terminate = False
for c in my_exec['containers']:
c = cont_api.get(c)
for port in c['ports']:
if port['name'] == 'Spark application web interface':
if check_spark_job(swarm, c['docker_id'], my_exec_since_started):
log.info('Execution {} for user {} has been idle for too long, terminating...'.format(my_exec['name'], guest['name']))
terminate = True
break
if terminate:
break
if terminate:
exec_api.terminate(my_exec['id'])
def exec_kill_user_cmd(auth, args):
"""Terminates all executions for the given user."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
filters = {'status': 'running', 'user_id': args.user_id}
data = exec_api.list(**filters)
print('Terminating {} executions belonging to user {}'.format(
len(data), args.user_id))
for execution in data:
exec_api.terminate(execution)
print('Execution {} terminated'.format(execution))
class Test:
def __init__(self, user, pwd, url, name, zapp):
self.user = user
self.pwd = pwd
self.url = url
self.name = name
try:
with open(zapp, 'r') as infile:
self.zapp = json.load(infile)
except:
exit("Unable to load zapp file.")
self.exec_api = ZoeExecutionsAPI(self.url, self.user, self.pwd)
self.service_api = ZoeServiceAPI(self.url, self.user, self.pwd)
def start_exec(self):
exec_id = self.exec_api.start(self.name, self.zapp)
return exec_id
def get_services_id(self, exec_id):
services = self.exec_api.get(exec_id)
return services['services']
def get_submit_service(self, exec_id):
while len(self.get_services_id(exec_id)) < 4:
print('waiting')
time.sleep(0.5)
for service_id in self.get_services_id(exec_id):
srv = self.service_api.get(service_id)
if re.search('submit', srv['name']):
return srv['id']
def is_running(self, exec_id):
return self.exec_api.get(exec_id)['status'] == 'running'
def run_test(self):
ts = time.time()
exec_id = self.start_exec()
outfilename = './logs/{}'.format(exec_id)
submit_id = self.get_submit_service(exec_id)
while not self.is_running(exec_id):
time.sleep(0.5)
te = time.time()
with open(outfilename, 'w') as out:
out.write("PerfMeasure: Scheduling time: {}".format(te - ts))
for line in self.service_api.get_logs(submit_id):
out.write("{}\n".format(line))
self.exec_api.terminate(exec_id)
print('Terminated {}'.format(exec_id))
def check_guests(swarm):
query_api = ZoeQueryAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
exec_api = ZoeExecutionsAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
cont_api = ZoeServiceAPI(get_conf().master_url, 'zoeadmin',
get_conf().zoeadmin_password)
guests = query_api.query('user', role='guest')
execs = exec_api.list()
for guest in guests:
my_execs = [e for e in execs if e['owner'] == guest['name']]
for my_exec in my_execs:
if len(my_exec['services']) == 0:
continue
my_exec_since_started = datetime.datetime.now(
) - dateutil.parser.parse(my_exec['time_started'])
my_exec_since_started = my_exec_since_started.total_seconds()
terminate = False
for c in my_exec['services']:
c = cont_api.get(c)
for port in c['ports']:
if port['name'] == 'Spark application web interface':
idle_time = check_spark_job(swarm, c['docker_id'],
my_exec_since_started)
if check_if_kill(idle_time):
log.info(
'Execution {} for user {} has been idle for too long, terminating...'
.format(my_exec['name'], guest['name']))
terminate = True
break
else:
log.debug(
'Execution {} for user {} has been idle for {} seconds'
.format(my_exec['name'], guest['name'],
idle_time))
if terminate:
break
if terminate:
exec_api.terminate(my_exec['id'])
def terminate_exec(request, exec_id):
print("zoe api: terminate_exec")
print("TO TERMINATE: ", exec_id)
exec_api = ZoeExecutionsAPI(ZOE_URL, ZOE_USER, ZOE_PWD)
return exec_api.terminate(exec_id)
def exec_kill_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
exec_api.terminate(args.id)
def exec_kill_cmd(args):
exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(),
utils.zoe_pass())
exec_api.terminate(args.id)
def exec_kill_cmd(auth, args):
"""Kill an execution."""
exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
exec_api.terminate(args.id)
