def start_domain_from_id(self, id, ssl=True):
# INFO TO DEVELOPER, QUE DE UN ERROR SI EL ID NO EXISTE
id_domain = id
pool_id = get_pool_from_domain(id_domain)
cpu_host_model = self.manager.pools[pool_id].conf.get(
'cpu_host_model', 'host-model')
# TODO: Read the cpu_host_model value from hypervisor_pool database
try:
xml = recreate_xml_to_start(id_domain, ssl, cpu_host_model)
except Exception as e:
log.error('recreate_xml_to_start in domain {}'.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
xml = False
if xml is False:
update_domain_status(
'Failed',
id_domain,
detail="DomainXML can not parse and modify xml to start")
return False
else:
hyp = self.start_domain_from_xml(xml, id_domain, pool_id=pool_id)
return hyp
def _start_domains(self):
domains_id_list = EvalController.get_domains_id_randomized(self.user_id, self.id_pool, self.defined_domains,
self.templates)
total_domains = len(domains_id_list)
threshold = floor(total_domains / 2)
keep_starting = True
started_domains = 0
while (len(domains_id_list) and keep_starting):
n = max(int(len(domains_id_list) / 4), 1)
# upper_limit_random = len(domains_id_list) if len(domains_id_list) < threshold else threshold
# n = randint(1, upper_limit_random)
logs.eval.debug("Starting {} random domains".format(n))
started_domains += n
sleep_time = 20
while (n):
id = domains_id_list.pop()
update_domain_status('Starting', id)
n -= 1
logs.eval.debug("Waiting for next start {} seconds".format(sleep_time))
while (sleep_time > 0 and keep_starting):
sleep(5)
keep_starting = self._evaluate()
sleep_time -= 5
keep_starting = self._evaluate()
sleep(20)
return {"started_domains": started_domains}
def stop_domain_from_id(self, id):
# INFO TO DEVELOPER. puede pasar que alguna actualización en algún otro hilo del status haga que
# durante un corto período de tiempo devuelva None,para evitarlo durante un segundo vamos a ir pidiendo cada
# 100 ms que nos de el hypervisor
time_wait = 0.0
while time_wait <= 20.0:
hyp_id = get_domain_hyp_started(id)
if hyp_id is not None:
if len(hyp_id) > 0:
break
else:
time_wait = time_wait + 0.1
sleep(0.1)
log.debug(
'waiting {} seconds to find hypervisor started for domain {}'
.format(time_wait, id))
log.debug('stop domain id {} in {}'.format(id, hyp_id))
# hyp_id = get_domain_hyp_started(id)
# log.debug('stop domain id {} in {}'.format(id,hyp_id))
if hyp_id is None:
hyp_id = ''
if len(hyp_id) <= 0:
log.debug(
'hypervisor where domain {} is started not finded'.format(id))
update_domain_status(
status='Unknown',
id_domain=id_domain,
hyp_id=None,
detail='hypervisor where domain {} is started not finded'.
format(id))
else:
self.stop_domain(id, hyp_id)
def _calcule_ux_background(self, domain_id, results, thread_id, step):
# logs.eval.info("_calcule_ux_background: domain:{}, thread_id: {}".format(domain_id, thread_id))
try:
# Start
update_domain_status('Starting', domain_id)
self._wait_starting(domain_id)
# Get hyp_id where is running
hyp_id, hyp, template_id = self._get_hyp_domain_running(domain_id)
logs.eval.info(
"_calcule_ux_background: domain:{}, thread_id: {}, hyp_id: {}".format(domain_id, thread_id, hyp_id))
# Wait until stop
et_initial = self.initial_ux[template_id][hyp_id]["execution_time"]
et_inc_max = self.ux_scorers["execution_time"]["max"]
timeout = et_initial * (1 + et_inc_max / 100)
logs.eval.info("TIMEOUT: {}, ET: {}".format(timeout, et_initial))
stats, et, stop_by_timeout = self._wait_stop(domain_id, hyp, timeout=timeout)
# et = self._calcule_random_et(et_initial, et_inc_max, step)
# Calcule domain ux
ux = self._calcule_ux_domain(stats, et, hyp.cpu_power)
self.sender.send(template_id + '.execution_time', ux["execution_time"])
self.sender.send(template_id + '.performance', ux["performance"])
# logs.eval.debug("UX: domain_id: {}, hyp_id:{} , pformat(ux): {}".format(domain_id, hyp_id, pformat(ux)))
# Get increment data: actual/initial
initial_ux = self.initial_ux[template_id][hyp_id]
increment = self._get_inc(ux, initial_ux)
self.sender.send(hyp_id + '.inc_hyp_cpu_usage', increment["cpu_hyp_usage"]["mean"])
self.sender.send(hyp_id + '.inc_hyp_cpu_iowait', increment["cpu_hyp_iowait"]["mean"])
# Calcule some data
data = self._calcule_data_from_ux(domain_id, hyp_id, template_id, ux, increment)
results[thread_id] = (data, stop_by_timeout)
except:
logs.eval.error(format_exc())
def analize_backing_chains_outputs(array_out_err=[], path_to_write_json=None, path_to_read_json=None):
if path_to_write_json is not None:
f = open(path_to_write_json, 'w')
json.dump(array_out_err, f)
f.close()
if path_to_read_json is not None:
f = open(path_to_read_json, 'r')
array_out_err = json.load(f)
log.debug(len(array_out_err))
f.close()
domains_ok = 0
domains_err = 0
for d in array_out_err:
id = d['title']
if len(d['err']) > 0:
domains_err += 1
log.info(d['err'])
update_domain_status('Failed', id, detail=d['err'])
else:
log.debug(id)
domains_ok += 1
if type(d['out']) is not str:
out = out.decode('utf-8')
else:
out = d['out']
l = json.loads(out)
from pprint import pprint
pprint(l)
update_disk_backing_chain(id, 0, l[0]['filename'], l)
return ({'ok': domains_ok, 'err': domains_err})
def stop_domain_from_id(self, id):
# INFO TO DEVELOPER. puede pasar que alguna actualización en algún otro hilo del status haga que
# durante un corto período de tiempo devuelva None,para evitarlo durante un segundo vamos a ir pidiendo cada
# 100 ms que nos de el hypervisor
time_wait = 0.0
while time_wait <= 20.0:
hyp_id = get_domain_hyp_started(id)
if hyp_id is not None:
if len(hyp_id) > 0:
break
else:
time_wait = time_wait + 0.1
sleep(0.1)
log.debug('waiting {} seconds to find hypervisor started for domain {}'.format(time_wait, id))
log.debug('stop domain id {} in {}'.format(id, hyp_id))
# hyp_id = get_domain_hyp_started(id)
# log.debug('stop domain id {} in {}'.format(id,hyp_id))
if hyp_id is None:
hyp_id = ''
if len(hyp_id) <= 0:
log.debug('hypervisor where domain {} is started not finded'.format(id))
update_domain_status(status='Unknown',
id_domain=id_domain,
hyp_id=None,
detail='hypervisor where domain {} is started not finded'.format(id))
else:
self.stop_domain(id, hyp_id)
def _wait_stop(self, domain_id, hyp, timeout=150):
stats = []
i = 0
start_time = time.time()
execution_time = time.time() - start_time
while (get_domain_status(domain_id) == "Started"
and (self.real_stop or i < self.time_to_stop)
and execution_time < timeout):
tmp = []
s = self._process_stats(hyp, domain_id)
# logs.eval.debug("UX Stats: {}".format(pformat(s)))
for name in self.names:
value = s.get(name)
tmp.append(value)
if value:
self.sender.send(hyp.id + '.' + name, value)
# logs.eval.debug(tmp)
stats.append(tmp)
# logs.eval.debug("Domain {} is started and i : {}".format(domain_id, i))
sleep(1)
i += 1
execution_time = time.time() - start_time
if get_domain_status(domain_id) == "Started":
update_domain_status('Stopping', domain_id, hyp.id) # Force stop
j = 0
while ((get_domain_status(domain_id) == "Stopping") and j < 10):
sleep(1)
j += 1
stop_by_timeout = execution_time > timeout
logs.eval.debug(
"Execution_time: {}, stop_by_time_out: {}, i_value: {}, timeout: {}"
.format(execution_time, stop_by_timeout, i, timeout))
return stats, execution_time, stop_by_timeout
def _wait_stop(self, domain_id, hyp, timeout=150):
stats = []
i = 0
start_time = time.time()
execution_time = time.time() - start_time
while (get_domain_status(domain_id) == "Started"
and (self.real_stop or i < self.time_to_stop)
and execution_time < timeout):
tmp = []
s = self._process_stats(hyp, domain_id)
# logs.eval.debug("UX Stats: {}".format(pformat(s)))
for name in self.names:
value = s.get(name)
tmp.append(value)
if value:
self.sender.send(hyp.id + '.' + name, value)
# logs.eval.debug(tmp)
stats.append(tmp)
# logs.eval.debug("Domain {} is started and i : {}".format(domain_id, i))
sleep(1)
i += 1
execution_time = time.time() - start_time
if get_domain_status(domain_id) == "Started":
update_domain_status('Stopping', domain_id, hyp.id) # Force stop
j = 0
while ((get_domain_status(domain_id) == "Stopping") and j < 10):
sleep(1)
j += 1
stop_by_timeout = execution_time > timeout
logs.eval.debug(
"Execution_time: {}, stop_by_time_out: {}, i_value: {}, timeout: {}".format(execution_time, stop_by_timeout,
i, timeout))
return stats, execution_time, stop_by_timeout
def deleting_disks_from_domain(self, id_domain, force=False):
# ALBERTO FALTA ACABAR
dict_domain = get_domain(id_domain)
if dict_domain['kind'] != 'desktop' and force is False:
log.info('{} is a template, disks will be deleted')
if len(dict_domain['hardware']['disks']) > 0:
index_disk = 0
for d in dict_domain['hardware']['disks']:
disk_path = d['file']
pool_id = dict_domain['hypervisors_pools'][0]
if pool_id not in self.manager.pools.keys():
log.error(
'hypervisor pool {} nor running in manager, can\'t delete disks in domain {}'.format(pool_id,
id_domain))
return False
next_hyp = self.manager.pools[pool_id].get_next()
log.debug('hypervisor where delete disk {}: {}'.format(disk_path, next_hyp))
cmds = create_cmds_delete_disk(disk_path)
action = dict()
action['id_domain'] = id_domain
action['type'] = 'delete_disk'
action['disk_path'] = disk_path
action['domain'] = id_domain
action['ssh_commands'] = cmds
action['index_disk'] = index_disk
try:
update_domain_status(status='DeletingDomainDisk',
id_domain=id_domain,
hyp_id=False,
detail='Deleting disk {} in domain {}, queued in hypervisor thread {}'.format(
disk_path,
id_domain,
next_hyp
))
self.manager.q.workers[next_hyp].put(action)
except Exception as e:
update_domain_status(status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail='Creating template operation failed when insert action in queue for disk operations')
log.error(
'Creating disk operation failed when insert action in queue for disk operations in host {}. Exception: {}'.format(
next_hyp, e))
return False
index_disk += 1
else:
log.debug('no disk to delete in domain {}'.format(id_domain))
return True
def updating_from_create_dict(self, id_domain):
try:
populate_dict_hardware_from_create_dict(id_domain)
except Exception as e:
log.error(
'error when populate dict hardware from create dict in domain {}'
.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
update_domain_status(
'Failed',
id_domain,
detail=
'Updating aborted, failed when populate hardware dictionary')
return False
try:
xml_raw = update_xml_from_dict_domain(id_domain)
except Exception as e:
log.error(
'error when populate dict hardware from create dict in domain {}'
.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
update_domain_status(
'Failed',
id_domain,
detail=
'Updating aborted, failed when updating xml from hardware dictionary'
)
return False
update_domain_status(
'Updating',
id_domain,
detail=
'xml and hardware dict updated, waiting to test if domain start paused in hypervisor'
)
pool_id = get_pool_from_domain(id_domain)
if pool_id is False:
update_domain_status(
'Failed',
id_domain,
detail='Updating aborted, domain has not pool')
return False
kind = get_domain_kind(id_domain)
if kind == 'desktop':
self.start_paused_domain_from_xml(xml=xml_raw,
id_domain=id_domain,
pool_id=pool_id)
else:
update_domain_status(
'Stopped',
id_domain,
detail='Updating finalished, ready to derivate desktops')
return True
def clear_domains(self):
"""
Just for testing purposes.
:return:
"""
domains = get_domains(self.user["id"])
for d in domains:
update_domain_status('Stopped', d['id'])
return "Okey"
def destroy_domains(self):
"""
Remove all eval domains.
Must be removed on each defined pool.
:return:
"""
ids = get_domains_id(self.user["id"], self.id_pool)
for a in ids:
update_domain_status('Deleting', a['id'])
def clear_domains(self):
"""
Just for testing purposes.
:return:
"""
domains = get_domains(self.user["id"])
for d in domains:
update_domain_status('Stopped', d['id'])
return "Okey"
def destroy_domains(self):
"""
Remove all eval domains.
Must be removed on each defined pool.
:return:
"""
ids = get_domains_id(self.user["id"], self.id_pool)
for a in ids:
update_domain_status('Deleting', a['id'])
def disk_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
if action['type'] in ['create_disk']:
launch_action_disk(action,
self.hostname,
self.user,
self.port)
elif action['type'] in ['create_disk_from_scratch']:
launch_action_disk(action,
self.hostname,
self.user,
self.port,
from_scratch=True)
elif action['type'] in ['delete_disk']:
launch_delete_disk_action(action,
self.hostname,
self.user,
self.port)
elif action['type'] in ['create_template_disk_from_domain']:
launch_action_create_template_disk(action,
self.hostname,
self.user,
self.port)
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def start_domain_from_xml(self, xml, id_domain, pool_id='default'):
failed = False
if pool_id in self.manager.pools.keys():
force_hyp = get_domain_force_hyp(id_domain)
if force_hyp is not False:
hyps_in_pool = get_hypers_in_pool(pool_id, only_online=False)
if force_hyp in hyps_in_pool:
next_hyp = force_hyp
else:
log.error(
'force hypervisor failed for doomain {}: {} not in hypervisors pool {}'
.format(id_domain, force_hyp, pool_id))
next_hyp = self.manager.pools[pool_id].get_next(
domain_id=id_domain)
else:
next_hyp = self.manager.pools[pool_id].get_next(
domain_id=id_domain)
if next_hyp is not False:
# update_domain_status(status='Starting',
# id_domain=id_domain,
# hyp_id=next_hyp,
# detail='desktop starting paused in pool {} on hypervisor {}'.format(pool_id,
# next_hyp))
self.manager.q.workers[next_hyp].put({
'type': 'start_domain',
'xml': xml,
'id_domain': id_domain
})
else:
log.error(
'get next hypervisor in pool {} failed'.format(pool_id))
failed = True
else:
log.error('pool_id {} does not exists??'.format(pool_id))
failed = True
if failed is True:
update_domain_status(
status='Failed',
id_domain=id_domain,
hyp_id=next_hyp,
detail='desktop not started: no hypervisors online in pool {}'.
format(pool_id))
log.error(
'desktop not started: no hypervisors online in pool {}'.format(
pool_id))
return False
else:
return next_hyp
def stop_domain(self, id_domain, hyp_id, delete_after_stopped=False):
update_domain_status(status='Stopping',
id_domain=id_domain,
hyp_id=hyp_id,
detail='desktop stopping in hyp {}'.format(hyp_id))
from pprint import pprint
action = {'type': 'stop_domain',
'id_domain': id_domain,
'delete_after_stopped': delete_after_stopped}
self.manager.q.workers[hyp_id].put(action)
return True
def stop_domains(cls, user_id, stop_sleep_time):
"""
Stop domains if they are started.
:return:
"""
domains = get_domains(user_id, status="Started")
while(len(domains) > 0):
eval_log.info("Stoping {} domains".format(len(domains)))
for d in domains:
update_domain_status('Stopping', d['id'], hyp_id=d['hyp_started'])
sleep(stop_sleep_time)
domains = get_domains(user_id, status="Started")
return {"total_stopped_domains": len(domains),
"data": None}
def start_paused_domain_from_xml(self, xml, id_domain, pool_id):
#def start_paused_domain_from_xml(self, xml, id_domain, pool_id, start_after_created=False):
failed = False
if pool_id in self.manager.pools.keys():
next_hyp = self.manager.pools[pool_id].get_next(
domain_id=id_domain)
log.debug('//////////////////////')
if next_hyp is not False:
log.debug('next_hyp={}'.format(next_hyp))
dict_action = {
'type': 'start_paused_domain',
'xml': xml,
'id_domain': id_domain
}
# if start_after_created is True:
# dict_action['start_after_created'] = True
#else:
self.manager.q.workers[next_hyp].put(dict_action)
update_domain_status(
status='CreatingDomain',
id_domain=id_domain,
hyp_id=False,
detail=
'Waiting to try starting paused in hypervisor {} in pool {} ({} operations in queue)'
.format(next_hyp, pool_id,
self.manager.q.workers[next_hyp].qsize()))
else:
log.error(
'get next hypervisor in pool {} failed'.format(pool_id))
failed = True
else:
log.error('pool_id {} does not exists??'.format(pool_id))
failed = True
if failed is True:
update_domain_status(
status='FailedCreatingDomain',
id_domain=id_domain,
hyp_id=next_hyp,
detail='desktop not started: no hypervisors online in pool {}'.
format(pool_id))
log.error(
'desktop not started: no hypervisors online in pool {}'.format(
pool_id))
return False
else:
return next_hyp
def stop_domains(cls, user_id, stop_sleep_time):
"""
Stop domains if they are started.
:return:
"""
domains = get_domains(user_id, status="Started")
while (len(domains) > 0):
eval_log.info("Stoping {} domains".format(len(domains)))
for d in domains:
update_domain_status('Stopping',
d['id'],
hyp_id=d['hyp_started'])
sleep(stop_sleep_time)
domains = get_domains(user_id, status="Started")
return {"total_stopped_domains": len(domains), "data": None}
def start_domain_from_xml(self, xml, id_domain, pool_id='default'):
failed = False
if pool_id in self.manager.pools.keys():
force_hyp = get_domain_force_hyp(id_domain)
if force_hyp is not False:
hyps_in_pool = get_hypers_in_pool(pool_id, only_online=False)
if force_hyp in hyps_in_pool:
next_hyp = force_hyp
else:
log.error('force hypervisor failed for doomain {}: {} not in hypervisors pool {}'.format(id_domain,
force_hyp,
pool_id))
next_hyp = self.manager.pools[pool_id].get_next(domain_id=id_domain)
else:
next_hyp = self.manager.pools[pool_id].get_next(domain_id=id_domain)
if next_hyp is not False:
# update_domain_status(status='Starting',
# id_domain=id_domain,
# hyp_id=next_hyp,
# detail='desktop starting paused in pool {} on hypervisor {}'.format(pool_id,
# next_hyp))
if LOG_LEVEL == 'DEBUG':
print(f'%%%% DOMAIN: {id_domain} -- XML TO START IN HYPERVISOR: {next_hyp} %%%%')
print(xml)
update_table_field('domains',id_domain,'xml_to_start',xml)
print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
self.manager.q.workers[next_hyp].put({'type': 'start_domain', 'xml': xml, 'id_domain': id_domain})
else:
log.error('get next hypervisor in pool {} failed'.format(pool_id))
failed = True
else:
log.error('pool_id {} does not exists??'.format(pool_id))
failed = True
if failed is True:
update_domain_status(status='Failed',
id_domain=id_domain,
hyp_id=next_hyp,
detail='desktop not started: no hypervisors online in pool {}'.format(pool_id))
log.error('desktop not started: no hypervisors online in pool {}'.format(pool_id))
return False
else:
return next_hyp
def _get_stats_background(self, domain_id, results, i, hyp):
# hyp.launch_eval_statistics()
try:
stats = []
et_mean = []
for j in range(self.initial_ux_iterations):
sleep(2)
update_domain_status('Starting', domain_id)
self._wait_starting(domain_id)
tmp_stats, et, stop_by_timeout = self._wait_stop(domain_id, hyp)
stats.extend(tmp_stats)
et_mean.append(et)
sleep(5)
et = round(mean(et_mean), 2)
results[i] = (stats, et)
except:
logs.eval.error(format_exc())
def start_paused_domain_from_xml(self, xml, id_domain, pool_id):
#def start_paused_domain_from_xml(self, xml, id_domain, pool_id, start_after_created=False):
failed = False
if pool_id in self.manager.pools.keys():
next_hyp = self.manager.pools[pool_id].get_next(domain_id=id_domain)
log.debug('//////////////////////')
if next_hyp is not False:
log.debug('next_hyp={}'.format(next_hyp))
dict_action = {'type': 'start_paused_domain', 'xml': xml, 'id_domain': id_domain}
# if start_after_created is True:
# dict_action['start_after_created'] = True
#else:
if LOG_LEVEL == 'DEBUG':
print(f'%%%% DOMAIN CREATING:{id_domain} -- XML TO START PAUSED IN HYPERVISOR {next_hyp} %%%%')
print(xml)
update_table_field('domains', id_domain, 'xml_to_start', xml)
print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
self.manager.q.workers[next_hyp].put(
dict_action)
update_domain_status(status='CreatingDomain',
id_domain=id_domain,
hyp_id=False,
detail='Waiting to try starting paused in hypervisor {} in pool {} ({} operations in queue)'.format(
next_hyp,
pool_id,
self.manager.q.workers[next_hyp].qsize()))
else:
log.error('get next hypervisor in pool {} failed'.format(pool_id))
failed = True
else:
log.error('pool_id {} does not exists??'.format(pool_id))
failed = True
if failed is True:
update_domain_status(status='FailedCreatingDomain',
id_domain=id_domain,
hyp_id=next_hyp,
detail='desktop not started: no hypervisors online in pool {}'.format(pool_id))
log.error('desktop not started: no hypervisors online in pool {}'.format(pool_id))
return False
else:
return next_hyp
def _initial_ux_sequencial(self, template_id, domain_id):
self.initial_ux[template_id] = {}
for hyp in self.hyps:
logs.eval.info("Calculing ux for template: {} in hypervisor {}".format(template_id, hyp.id))
update_domain_force_hyp(domain_id, hyp.id)
update_domain_status('Starting', domain_id)
stats = []
et_mean = []
for i in range(self.initial_ux_iterations):
sleep(2)
self._wait_starting(domain_id)
tmp_stats, et = self._wait_stop(domain_id, hyp)
sleep(2)
stats.extend(tmp_stats)
et_mean.append(et)
et = round(mean(et_mean), 2)
self.initial_ux[template_id][hyp.id] = self._calcule_ux_domain(stats, et, hyp.cpu_power)
update_domain_force_hyp(domain_id, '') # Clean force_hyp
def _get_stats_background(self, domain_id, results, i, hyp):
# hyp.launch_eval_statistics()
try:
stats = []
et_mean = []
for j in range(self.initial_ux_iterations):
sleep(2)
update_domain_status('Starting', domain_id)
self._wait_starting(domain_id)
tmp_stats, et, stop_by_timeout = self._wait_stop(
domain_id, hyp)
stats.extend(tmp_stats)
et_mean.append(et)
sleep(5)
et = round(mean(et_mean), 2)
results[i] = (stats, et)
except:
logs.eval.error(format_exc())
def _calcule_ux_background(self, domain_id, results, thread_id, step):
# logs.eval.info("_calcule_ux_background: domain:{}, thread_id: {}".format(domain_id, thread_id))
try:
# Start
update_domain_status('Starting', domain_id)
self._wait_starting(domain_id)
# Get hyp_id where is running
hyp_id, hyp, template_id = self._get_hyp_domain_running(domain_id)
logs.eval.info(
"_calcule_ux_background: domain:{}, thread_id: {}, hyp_id: {}".
format(domain_id, thread_id, hyp_id))
# Wait until stop
et_initial = self.initial_ux[template_id][hyp_id]["execution_time"]
et_inc_max = self.ux_scorers["execution_time"]["max"]
timeout = et_initial * (1 + et_inc_max / 100)
logs.eval.info("TIMEOUT: {}, ET: {}".format(timeout, et_initial))
stats, et, stop_by_timeout = self._wait_stop(domain_id,
hyp,
timeout=timeout)
# et = self._calcule_random_et(et_initial, et_inc_max, step)
# Calcule domain ux
ux = self._calcule_ux_domain(stats, et, hyp.cpu_power)
self.sender.send(template_id + '.execution_time',
ux["execution_time"])
self.sender.send(template_id + '.performance', ux["performance"])
# logs.eval.debug("UX: domain_id: {}, hyp_id:{} , pformat(ux): {}".format(domain_id, hyp_id, pformat(ux)))
# Get increment data: actual/initial
initial_ux = self.initial_ux[template_id][hyp_id]
increment = self._get_inc(ux, initial_ux)
self.sender.send(hyp_id + '.inc_hyp_cpu_usage',
increment["cpu_hyp_usage"]["mean"])
self.sender.send(hyp_id + '.inc_hyp_cpu_iowait',
increment["cpu_hyp_iowait"]["mean"])
# Calcule some data
data = self._calcule_data_from_ux(domain_id, hyp_id, template_id,
ux, increment)
results[thread_id] = (data, stop_by_timeout)
except:
logs.eval.error(format_exc())
def clean_intermediate_status():
#status_to_delete = ['DownloadAborting']
status_to_delete = []
status_to_failed = ['Updating','Deleting']
all_domains = get_all_domains_with_id_and_status()
[delete_domain(d['id']) for d in all_domains if d['status'] in status_to_delete]
[update_domain_status('Failed', d['id'],
detail='change status from {} when isard engine restart'.format(d['status'])) for d in
all_domains if d['status'] in status_to_failed]
def _initial_ux_sequencial(self, template_id, domain_id):
self.initial_ux[template_id] = {}
for hyp in self.hyps:
logs.eval.info(
"Calculing ux for template: {} in hypervisor {}".format(
template_id, hyp.id))
update_domain_force_hyp(domain_id, hyp.id)
update_domain_status('Starting', domain_id)
stats = []
et_mean = []
for i in range(self.initial_ux_iterations):
sleep(2)
self._wait_starting(domain_id)
tmp_stats, et = self._wait_stop(domain_id, hyp)
sleep(2)
stats.extend(tmp_stats)
et_mean.append(et)
et = round(mean(et_mean), 2)
self.initial_ux[template_id][hyp.id] = self._calcule_ux_domain(
stats, et, hyp.cpu_power)
update_domain_force_hyp(domain_id, '') # Clean force_hyp
def create_template_in_db(self, id_domain):
domain_dict = get_domain(id_domain)
template_dict = domain_dict['create_dict']['template_dict']
template_dict['status'] = 'CreatingNewTemplateInDB'
template_id = template_dict['id']
if insert_domain(template_dict)['inserted'] == 1:
hw_dict = domain_dict['hardware'].copy()
for i in range(len(hw_dict['disks'])):
hw_dict['disks'][i]['file'] = template_dict['create_dict'][
'hardware']['disks'][i]['file']
update_table_field('domains',
template_id,
'hardware',
hw_dict,
merge_dict=False)
xml_parsed = update_xml_from_dict_domain(id_domain=template_id,
xml=domain_dict['xml'])
remove_disk_template_created_list_in_domain(id_domain)
remove_dict_new_template_from_domain(id_domain)
# update_table_field('domains', template_id, 'xml', xml_parsed, merge_dict=False)
update_domain_status(
status='Stopped',
id_domain=template_id,
hyp_id=False,
detail=
'Template created, ready to create domains from this template')
update_domain_status(
status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail=
'Template created from this domain, now domain is ready to start again'
)
else:
log.error(
'template {} can not be inserted in rethink, domain_id duplicated??'
.format(template_id))
return False
def start_domain_from_id(self, id, ssl=True):
# INFO TO DEVELOPER, QUE DE UN ERROR SI EL ID NO EXISTE
id_domain = id
pool_id = get_pool_from_domain(id_domain)
cpu_host_model = self.manager.pools[pool_id].conf.get('cpu_host_model',DEFAULT_HOST_MODE)
# TODO: Read the cpu_host_model value from hypervisor_pool database
try:
xml = recreate_xml_to_start(id_domain, ssl, cpu_host_model)
except Exception as e:
log.error('recreate_xml_to_start in domain {}'.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
xml = False
if xml is False:
update_domain_status('Failed', id_domain,
detail="DomainXML can not parse and modify xml to start")
return False
else:
hyp = self.start_domain_from_xml(xml, id_domain, pool_id=pool_id)
return hyp
def clean_intermediate_status():
status_to_delete = ['DownloadAborting']
status_to_failed = ['Updating']
all_domains = get_all_domains_with_id_and_status()
[
delete_domain(d['id']) for d in all_domains
if d['status'] in status_to_delete
]
[
update_domain_status(
'Failed',
d['id'],
detail='change status from {} when isard engine restart'.format(
d['status'])) for d in all_domains
if d['status'] in status_to_failed
]
def create_template_in_db(self, id_domain):
domain_dict = get_domain(id_domain)
template_dict = domain_dict['create_dict']['template_dict']
template_dict['status'] = 'CreatingNewTemplateInDB'
template_id = template_dict['id']
if insert_domain(template_dict)['inserted'] == 1:
hw_dict = domain_dict['hardware'].copy()
for i in range(len(hw_dict['disks'])):
hw_dict['disks'][i]['file'] = template_dict['create_dict']['hardware']['disks'][i]['file']
update_table_field('domains', template_id, 'hardware', hw_dict, merge_dict=False)
xml_parsed = update_xml_from_dict_domain(id_domain=template_id, xml=domain_dict['xml'])
if xml_parsed is False:
update_domain_status(status='Failed',
id_domain=template_id,
hyp_id=False,
detail='XML Parser Error, xml is not valid')
return False
remove_disk_template_created_list_in_domain(id_domain)
remove_dict_new_template_from_domain(id_domain)
if 'parents' in domain_dict.keys():
domain_parents_chain_update = domain_dict['parents'].copy()
else:
domain_parents_chain_update = []
domain_parents_chain_update.append(template_id)
update_table_field('domains', id_domain, 'parents', domain_parents_chain_update)
update_origin_and_parents_to_new_template(id_domain,template_id)
# update_table_field('domains', template_id, 'xml', xml_parsed, merge_dict=False)
update_domain_status(status='Stopped',
id_domain=template_id,
hyp_id=False,
detail='Template created, ready to create domains from this template')
update_domain_status(status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail='Template created from this domain, now domain is ready to start again')
else:
log.error('template {} can not be inserted in rethink, domain_id duplicated??'.format(template_id))
return False
import engine.services.db.domains
from engine.services import db
[db.update_domain_status('Starting', a['id']) for a in engine.services.db.domains.get_domains_from_user('test1')]
def creating_disk_from_virtbuilder(self, id_new):
dict_domain = get_domain(id_new)
pool_var = dict_domain['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
dict_to_create = dict_domain['create_dict']
relative_path = dict_to_create['hardware']['disks'][0]['file']
path_new_disk, path_selected = get_path_to_disk(relative_path,
pool=pool_id)
# UPDATE PATH IN DOMAIN
dict_to_create['hardware']['disks'][0]['file'] = path_new_disk
dict_to_create['hardware']['disks'][0]['path_selected'] = path_selected
size_str = dict_to_create['hardware']['disks'][0]['size']
memory_in_mb = int(dict_to_create['hardware']['memory'] / 1024)
options_virt_builder = dict_to_create['builder']['options']
options_virt_install = dict_to_create['install']['options']
id_domains_virt_builder = dict_to_create['builder']['id']
id_os_virt_install = dict_to_create['install']['id']
# UPDATE HARDWARE DICT
hardware_update = {}
hardware_update['disks'] = dict_to_create['hardware']['disks']
update_domain_dict_hardware(id_new, hardware_update)
hyp_to_disk_create = get_host_long_operations_from_path(
path_selected, pool=pool_id, type_path='groups')
cmds = create_cmd_disk_from_virtbuilder(
path_new_qcow=path_new_disk,
os_version=id_domains_virt_builder,
id_os_virt_install=id_os_virt_install,
name_domain_in_xml=id_new,
size_str=size_str,
memory_in_mb=memory_in_mb,
options_cmd=options_virt_builder)
# cmds = [{'cmd':'ls -lah > /tmp/prova.txt','title':'es un ls'}]
action = {}
action['type'] = 'create_disk_virt_builder'
action['disk_path'] = path_new_disk
action['index_disk'] = 0
action['domain'] = id_new
action['ssh_commands'] = cmds
try:
update_domain_status(
status='RunningVirtBuilder',
id_domain=id_new,
hyp_id=False,
detail=
'Creating virt-builder image operation is launched in hypervisor {} ({} operations in queue)'
.format(
hyp_to_disk_create, self.manager.
q_long_operations[hyp_to_disk_create].qsize()))
self.manager.q_long_operations[hyp_to_disk_create].put(action)
except Exception as e:
update_domain_status(
status='FailedCreatingDomain',
id_domain=id_new,
hyp_id=False,
detail=
'Creating disk operation failed when insert action in queue for disk operations'
)
log.error(
'Creating disk operation failed when insert action in queue for disk operations. Exception: {}'
.format(e))
import engine.services.db.domains
from engine.services import db
[db.update_domain_status('Deleting', a['id']) for a in engine.services.db.domains.get_domains_from_user('test1')]
def creating_and_test_xml_start(self, id_domain, creating_from_create_dict=False,
xml_from_virt_install=False,
xml_string=None,ssl=True):
if creating_from_create_dict is True:
try:
populate_dict_hardware_from_create_dict(id_domain)
except Exception as e:
log.error('error when populate dict hardware from create dict in domain {}'.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
domain = get_domain(id_domain)
#create_dict_hw = domain['create_dict']['hardware']
# for media in ['isos','floppies']
# if 'isos' in create_dict_hw.keys():
# for index_disk in range(len(create_dict_hw['isos'])):
# update_hw['hardware']['isos'][index_disk]['file'] = new_file
if type(xml_string) is str:
xml_from = xml_string
elif 'create_from_virt_install_xml' in domain['create_dict']:
xml_from = get_dict_from_item_in_table('virt_install',domain['create_dict']['create_from_virt_install_xml'])['xml']
elif xml_from_virt_install is False:
id_template = domain['create_dict']['origin']
template = get_domain(id_template)
xml_from = template['xml']
parents_chain = template.get('parents',[]) + domain.get('parents',[])
#when creating template from domain, the domain would be inserted as a parent while template is creating
# parent_chain never can't have id_domain as parent
if id_domain in parents_chain:
for i in range(parents_chain.count('a')):
parents_chain.remove(id_domain)
update_table_field('domains', id_domain, 'parents', parents_chain)
elif xml_from_virt_install is True:
xml_from = domain['xml_virt_install']
else:
return False
update_table_field('domains', id_domain, 'xml', xml_from)
xml_raw = update_xml_from_dict_domain(id_domain)
if xml_raw is False:
update_domain_status(status='FailedCreatingDomain',
id_domain=id_domain,
detail='XML Parser Error, xml is not valid')
return False
update_domain_status('CreatingDomain', id_domain,
detail='xml and hardware dict updated, waiting to test if domain start paused in hypervisor')
pool_id = get_pool_from_domain(id_domain)
if 'start_after_created' in domain.keys():
if domain['start_after_created'] is True:
update_domain_status('StartingDomainDisposable', id_domain,
detail='xml and hardware dict updated, starting domain disposable')
self.start_domain_from_id(id_domain)
else:
#change viewer password, remove selinux options and recreate network interfaces
try:
cpu_host_model = self.manager.pools[pool_id].conf.get('cpu_host_model', DEFAULT_HOST_MODE)
xml = recreate_xml_to_start(id_domain,ssl,cpu_host_model)
except Exception as e:
log.error('recreate_xml_to_start in domain {}'.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
xml = False
if xml is False:
update_domain_status('Failed', id_domain,
detail="DomainXML can't parse and modify xml to start")
else:
self.start_paused_domain_from_xml(xml=xml,
id_domain=id_domain,
pool_id=pool_id)
def updating_from_create_dict(self, id_domain,ssl=True):
try:
populate_dict_hardware_from_create_dict(id_domain)
except Exception as e:
log.error('error when populate dict hardware from create dict in domain {}'.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
update_domain_status('Failed', id_domain,
detail='Updating aborted, failed when populate hardware dictionary')
return False
try:
xml_raw = update_xml_from_dict_domain(id_domain)
if xml_raw is False:
update_domain_status(status='Failed',
id_domain=id_domain,
detail='XML Parser Error, xml is not valid')
return False
except Exception as e:
log.error('error when populate dict hardware from create dict in domain {}'.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
update_domain_status('Failed', id_domain,
detail='Updating aborted, failed when updating xml from hardware dictionary')
return False
update_domain_status('Updating', id_domain,
detail='xml and hardware dict updated, waiting to test if domain start paused in hypervisor')
pool_id = get_pool_from_domain(id_domain)
if pool_id is False:
update_domain_status('Failed', id_domain, detail='Updating aborted, domain has not pool')
return False
kind = get_domain_kind(id_domain)
if kind == 'desktop':
cpu_host_model = self.manager.pools[pool_id].conf.get('cpu_host_model', DEFAULT_HOST_MODE)
xml_to_test = recreate_xml_to_start(id_domain,ssl,cpu_host_model)
self.start_paused_domain_from_xml(xml=xml_to_test, id_domain=id_domain, pool_id=pool_id)
else:
update_domain_status('Stopped', id_domain,
detail='Updating finalished, ready to derivate desktops')
return True
def creating_disk_from_scratch(self,id_new):
dict_domain = get_domain(id_new)
pool_var = dict_domain['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
dict_to_create = dict_domain['create_dict']
if 'disks' in dict_to_create['hardware'].keys():
if len(dict_to_create['hardware']['disks']) > 0:
# for index_disk in range(len(dict_to_create['hardware']['disks'])):
# relative_path = dict_to_create['hardware']['disks'][index_disk]['file']
# path_new_file, path_selected = get_path_to_disk(relative_path, pool=pool_id)
# # UPDATE PATH IN DOMAIN
# dict_to_create['hardware']['disks'][index_disk]['file'] = new_file
# dict_to_create['hardware']['disks'][index_disk]['path_selected'] = path_selected
relative_path = dict_to_create['hardware']['disks'][0]['file']
path_new_disk, path_selected = get_path_to_disk(relative_path, pool=pool_id)
# UPDATE PATH IN DOMAIN
d_update_domain = {'hardware':{'disks':[{}]}}
if len(dict_to_create['hardware']['disks']) > 0:
## supplementary disks
for i,dict_other_disk in enumerate(dict_to_create['hardware']['disks'][1:]):
path_other_disk, path_other_disk_selected = get_path_to_disk(dict_other_disk['file'],
pool=pool_id,
type_path=dict_other_disk['type_path'])
d_update_domain['hardware']['disks'].append({})
d_update_domain['hardware']['disks'][i+1]['file'] = path_other_disk
d_update_domain['hardware']['disks'][i+1]['path_selected'] = path_other_disk_selected
d_update_domain['hardware']['disks'][i + 1]['bus'] = dict_other_disk.get('bus','virtio')
if dict_other_disk.get('readonly',True) is True:
d_update_domain['hardware']['disks'][i + 1]['readonly'] = True
else:
pass
# TODO
# update_media_write_access_by_domain(id_media,id_domain)
d_update_domain['hardware']['disks'][0]['file'] = path_new_disk
d_update_domain['hardware']['disks'][0]['path_selected'] = path_selected
d_update_domain['hardware']['disks'][0]['size'] = dict_to_create['hardware']['disks'][0]['size']
if 'bus' in dict_to_create['hardware']['disks'][0].keys():
if dict_to_create['hardware']['disks'][0]['bus'] in BUS_TYPES:
d_update_domain['hardware']['disks'][0]['bus'] = dict_to_create['hardware']['disks'][0]['bus']
update_domain_dict_hardware(id_new, d_update_domain)
update_domain_dict_create_dict(id_new, d_update_domain)
size_str = dict_to_create['hardware']['disks'][0]['size']
hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=pool_id, type_path='groups')
cmds = create_cmd_disk_from_scratch(path_new_disk=path_new_disk,
size_str=size_str)
action = {}
action['type'] = 'create_disk_from_scratch'
action['disk_path'] = path_new_disk
action['index_disk'] = 0
action['domain'] = id_new
action['ssh_commands'] = cmds
try:
update_domain_status(status='CreatingDiskFromScratch',
id_domain=id_new,
hyp_id=False,
detail='Creating disk commands are launched in hypervisor {} ({} operations in queue)'.format(
hyp_to_disk_create,
self.manager.q_disk_operations[hyp_to_disk_create].qsize()))
self.manager.q_disk_operations[hyp_to_disk_create].put(action)
except Exception as e:
update_domain_status(status='FailedCreatingDomain',
id_domain=id_new,
hyp_id=False,
detail='Creating disk operation failed when insert action in queue for disk operations')
log.error(
'Creating disk operation failed when insert action in queue for disk operations. Exception: {}'.format(
e))
else:
update_domain_status(status='CreatingDomain',
id_domain=id_new,
hyp_id=False,
detail='Creating domain withouth disks')
import engine.services.db.domains
from engine.services import db
[
db.update_domain_status('Stopping', a['id'])
for a in engine.services.db.domains.get_domains_from_user('test1')
]
def creating_disk_from_scratch(self, id_new):
dict_domain = get_domain(id_new)
pool_var = dict_domain['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
dict_to_create = dict_domain['create_dict']
if 'disks' in dict_to_create['hardware'].keys():
if len(dict_to_create['hardware']['disks']) > 0:
# for index_disk in range(len(dict_to_create['hardware']['disks'])):
# relative_path = dict_to_create['hardware']['disks'][index_disk]['file']
# path_new_file, path_selected = get_path_to_disk(relative_path, pool=pool_id)
# # UPDATE PATH IN DOMAIN
# dict_to_create['hardware']['disks'][index_disk]['file'] = new_file
# dict_to_create['hardware']['disks'][index_disk]['path_selected'] = path_selected
relative_path = dict_to_create['hardware']['disks'][0]['file']
path_new_disk, path_selected = get_path_to_disk(relative_path,
pool=pool_id)
# UPDATE PATH IN DOMAIN
d_update_domain = {'hardware': {'disks': [{}]}}
d_update_domain['hardware']['disks'][0]['file'] = path_new_disk
d_update_domain['hardware']['disks'][0][
'path_selected'] = path_selected
update_domain_dict_hardware(id_new, d_update_domain)
update_domain_dict_create_dict(id_new, d_update_domain)
size_str = dict_to_create['hardware']['disks'][0]['size']
hyp_to_disk_create = get_host_disk_operations_from_path(
path_selected, pool=pool_id, type_path='groups')
cmds = create_cmd_disk_from_scratch(
path_new_disk=path_new_disk, size_str=size_str)
action = {}
action['type'] = 'create_disk_from_scratch'
action['disk_path'] = path_new_disk
action['index_disk'] = 0
action['domain'] = id_new
action['ssh_commands'] = cmds
try:
update_domain_status(
status='CreatingDiskFromScratch',
id_domain=id_new,
hyp_id=False,
detail=
'Creating disk commands are launched in hypervisor {} ({} operations in queue)'
.format(
hyp_to_disk_create, self.manager.
q_disk_operations[hyp_to_disk_create].qsize()))
self.manager.q_disk_operations[hyp_to_disk_create].put(
action)
except Exception as e:
update_domain_status(
status='FailedCreatingDomain',
id_domain=id_new,
hyp_id=False,
detail=
'Creating disk operation failed when insert action in queue for disk operations'
)
log.error(
'Creating disk operation failed when insert action in queue for disk operations. Exception: {}'
.format(e))
def creating_disk_from_virtbuilder(self,
id_new):
dict_domain = get_domain(id_new)
pool_var = dict_domain['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
dict_to_create = dict_domain['create_dict']
relative_path = dict_to_create['hardware']['disks'][0]['file']
path_new_disk, path_selected = get_path_to_disk(relative_path, pool=pool_id)
# UPDATE PATH IN DOMAIN
dict_to_create['hardware']['disks'][0]['file'] = path_new_disk
dict_to_create['hardware']['disks'][0]['path_selected'] = path_selected
size_str = dict_to_create['hardware']['disks'][0]['size']
memory_in_mb = int(dict_to_create['hardware']['memory'] / 1024)
options_virt_builder = dict_to_create['builder']['options']
options_virt_install = dict_to_create['install']['options']
id_domains_virt_builder = dict_to_create['builder']['id']
id_os_virt_install = dict_to_create['install']['id']
# UPDATE HARDWARE DICT
hardware_update = {}
hardware_update['disks'] = dict_to_create['hardware']['disks']
update_domain_dict_hardware(id_new, hardware_update)
hyp_to_disk_create = get_host_long_operations_from_path(path_selected, pool=pool_id, type_path='groups')
cmds = create_cmd_disk_from_virtbuilder(path_new_qcow=path_new_disk,
os_version=id_domains_virt_builder,
id_os_virt_install=id_os_virt_install,
name_domain_in_xml=id_new,
size_str=size_str,
memory_in_mb=memory_in_mb,
options_cmd=options_virt_builder)
# cmds = [{'cmd':'ls -lah > /tmp/prova.txt','title':'es un ls'}]
action = {}
action['type'] = 'create_disk_virt_builder'
action['disk_path'] = path_new_disk
action['index_disk'] = 0
action['domain'] = id_new
action['ssh_commands'] = cmds
try:
update_domain_status(status='RunningVirtBuilder',
id_domain=id_new,
hyp_id=False,
detail='Creating virt-builder image operation is launched in hypervisor {} ({} operations in queue)'.format(
hyp_to_disk_create,
self.manager.q_long_operations[hyp_to_disk_create].qsize()))
self.manager.q_long_operations[hyp_to_disk_create].put(action)
except Exception as e:
update_domain_status(status='FailedCreatingDomain',
id_domain=id_new,
hyp_id=False,
detail='Creating disk operation failed when insert action in queue for disk operations')
log.error(
'Creating disk operation failed when insert action in queue for disk operations. Exception: {}'.format(
e))
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port
)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format(
action['disk_path'], action['domain'], self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status('Failed', id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'],
action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install)
update_domain_status('CreatingDomainFromBuilder', id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def creating_and_test_xml_start(self,
id_domain,
creating_from_create_dict=False,
xml_from_virt_install=False,
xml_string=None):
if creating_from_create_dict is True:
try:
populate_dict_hardware_from_create_dict(id_domain)
except Exception as e:
log.error(
'error when populate dict hardware from create dict in domain {}'
.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
domain = get_domain(id_domain)
#create_dict_hw = domain['create_dict']['hardware']
# for media in ['isos','floppies']
# if 'isos' in create_dict_hw.keys():
# for index_disk in range(len(create_dict_hw['isos'])):
# update_hw['hardware']['isos'][index_disk]['file'] = new_file
if type(xml_string) is str:
xml_from = xml_string
elif 'create_from_virt_install_xml' in domain['create_dict']:
xml_from = get_dict_from_item_in_table(
'virt_install',
domain['create_dict']['create_from_virt_install_xml'])['xml']
elif xml_from_virt_install is False:
id_template = domain['create_dict']['origin']
template = get_domain(id_template)
xml_from = template['xml']
elif xml_from_virt_install is True:
xml_from = domain['xml_virt_install']
else:
return False
update_table_field('domains', id_domain, 'xml', xml_from)
xml_raw = update_xml_from_dict_domain(id_domain)
update_domain_status(
'CreatingDomain',
id_domain,
detail=
'xml and hardware dict updated, waiting to test if domain start paused in hypervisor'
)
pool_id = get_pool_from_domain(id_domain)
if 'start_after_created' in domain.keys():
if domain['start_after_created'] is True:
update_domain_status(
'StartingDomainDisposable',
id_domain,
detail=
'xml and hardware dict updated, starting domain disposable'
)
self.start_domain_from_id(id_domain)
else:
#change viewer password, remove selinux options and recreate network interfaces
xml = recreate_xml_to_start(id_domain)
self.start_paused_domain_from_xml(xml=xml,
id_domain=id_domain,
pool_id=pool_id)
def create_template_disks_from_domain(self, id_domain):
dict_domain = get_domain(id_domain)
create_dict = dict_domain['create_dict']
pool_var = create_dict['template_dict']['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
try:
dict_new_template = create_dict['template_dict']
except KeyError as e:
update_domain_status(status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail='Action Creating Template from domain failed. No template_json in domain dictionary')
log.error(
'No template_dict in keys of domain dictionary, when creating template form domain {}. Exception: {}'.format(
id_domain, str(e)))
return False
disk_index_in_bus = 0
if 'disks' in dict_domain['hardware']:
list_disk_template_path_relative = [d['file'] for d in create_dict['hardware']['disks']]
create_disk_template_created_list_in_domain(id_domain)
for i in range(len(list_disk_template_path_relative)):
# for disk in dict_domain['hardware']['disks']:
path_domain_disk = dict_domain['hardware']['disks'][i]['file']
try:
# path_template_disk_relative = dict_new_template['create_dict']['hardware']['disks'][i]['file']
path_template_disk_relative = list_disk_template_path_relative[i]
except KeyError as e:
update_domain_status(status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail='Action Creating Template from domain failed. No disks in template_json in domain dictionary')
log.error(
'No disks in template_json in keys of domain dictionary, when creating template form domain {}. Exception: {}'.format(
id_domain, str(e)))
return False
if dict_new_template['kind'] == 'base':
type_path_selected = 'bases'
else:
type_path_selected = 'templates'
new_file, path_selected = get_path_to_disk(path_template_disk_relative, pool=pool_id,
type_path=type_path_selected)
path_absolute_template_disk = new_file = new_file.replace('//', '/')
dict_new_template['create_dict']['hardware']['disks'][i]['file'] = new_file
dict_new_template['create_dict']['hardware']['disks'][i]['path_selected'] = path_selected
update_table_field('domains', id_domain, 'create_dict', create_dict)
action = {}
action['id_domain'] = id_domain
action['type'] = 'create_template_disk_from_domain'
action['path_template_disk'] = path_absolute_template_disk
action['path_domain_disk'] = path_domain_disk
action['disk_index'] = disk_index_in_bus
hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=pool_id,
type_path=type_path_selected)
# INFO TO DEVELOPER: falta terminar de ver que hacemos con el pool para crear
# discos, debería haber un disk operations por pool
try:
update_domain_status(status='CreatingTemplateDisk',
id_domain=id_domain,
hyp_id=False,
detail='Creating template disk operation is launched in hostname {} ({} operations in queue)'.format(
hyp_to_disk_create,
self.manager.q_disk_operations[hyp_to_disk_create].qsize()))
self.manager.q_disk_operations[hyp_to_disk_create].put(action)
except Exception as e:
update_domain_status(status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail='Creating template operation failed when insert action in queue for disk operations')
log.error(
'Creating disk operation failed when insert action in queue for disk operations in host {}. Exception: {}'.format(
hyp_to_disk_create, e))
return False
disk_index_in_bus = disk_index_in_bus + 1
return True
def creating_disks_from_template(self, id_new):
dict_domain = get_domain(id_new)
if 'create_dict' in dict_domain.keys():
dict_to_create = dict_domain['create_dict']
pool_var = dict_domain['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
# INFO TO DEVELOPER DEBERÍA SER UN FOR PARA CADA DISCO
# y si el disco no tiene backing_chain, crear un disco vacío
# del tamño que marcase
# d['hardware']['disks'][0]['size']
# el backing_file debería estar asociado a cada disco:
# d['hardware']['disks'][0]['backing_file']
for index_disk in range(len(dict_to_create['hardware']['disks'])):
relative_path = dict_to_create['hardware']['disks'][index_disk][
'file']
new_file, path_selected = get_path_to_disk(relative_path,
pool=pool_id)
# UPDATE PATH IN DOMAIN
dict_to_create['hardware']['disks'][index_disk]['file'] = new_file
dict_to_create['hardware']['disks'][index_disk][
'path_selected'] = path_selected
update_table_field('domains', id_new, 'create_dict', dict_to_create)
#TODO: REVISAR SI RELAMENTE ES NECESARIO o esta acción responde a versiones antiguas de nuestras funciones de creación
hardware_update = {}
hardware_update['disks'] = dict_to_create['hardware']['disks']
update_domain_dict_hardware(id_new, hardware_update)
##################
for index_disk in range(len(dict_to_create['hardware']['disks'])):
backing_file = dict_to_create['hardware']['disks'][index_disk][
'parent']
new_file = dict_to_create['hardware']['disks'][index_disk]['file']
path_selected = dict_to_create['hardware']['disks'][index_disk][
'path_selected']
hyp_to_disk_create = get_host_disk_operations_from_path(
path_selected, pool=pool_id, type_path='groups')
cmds = create_cmds_disk_from_base(path_base=backing_file,
path_new=new_file)
log.debug(
'commands to disk create to launch in disk_operations: \n{}'.
format('\n'.join(cmds)))
action = {}
action['type'] = 'create_disk'
action['disk_path'] = new_file
action['index_disk'] = index_disk
action['domain'] = id_new
action['ssh_commands'] = cmds
try:
update_domain_status(
status='CreatingDisk',
id_domain=id_new,
hyp_id=False,
detail=
'Creating disk operation is launched in hypervisor {} ({} operations in queue)'
.format(
hyp_to_disk_create, self.manager.
q_disk_operations[hyp_to_disk_create].qsize()))
self.manager.q_disk_operations[hyp_to_disk_create].put(action)
except Exception as e:
update_domain_status(
status='FailedCreatingDomain',
id_domain=id_new,
hyp_id=False,
detail=
'Creating disk operation failed when insert action in queue for disk operations'
)
log.error(
'Creating disk operation failed when insert action in queue for disk operations. Exception: {}'
.format(e))
def polling(self):
while self.stop is not True:
interval = 0.0
while interval < self.polling_interval:
sleep(0.1)
interval += 0.1
if self.stop is True:
break
if self.manager.check_actions_domains_enabled() is False:
continue
l = get_domains_with_transitional_status()
list_domains_without_hyp = [d for d in l if 'hyp_started' not in d.keys()]
list_domains = [d for d in l if 'hyp_started' in d.keys()]
for d in list_domains_without_hyp:
logs.broom.error('DOMAIN {} WITH STATUS {} without HYPERVISOR'.format(d['id'], d['status']))
update_domain_status('Unknown', d['id'], detail='starting or stoping status witouth hypervisor')
hyps_to_try = set([d['hyp_started'] for d in list_domains if d is str])
hyps_domain_started = {}
for hyp_id in hyps_to_try:
try:
hostname, port, user = get_hyp_hostname_from_id(hyp_id)
if hostname is False:
logs.broom.error('hyp {} with id has not hostname or is nos in database'.format(hyp_id))
else:
h = hyp(hostname, user=user, port=port)
if h.connected:
hyps_domain_started[hyp_id] = {}
hyps_domain_started[hyp_id]['hyp'] = h
list_domains_from_hyp = h.get_domains()
if list_domains_from_hyp is None:
list_domains_from_hyp = []
hyps_domain_started[hyp_id]['active_domains'] = list_domains_from_hyp
else:
logs.broom.error('HYPERVISOR {} libvirt connection failed')
hyps_domain_started[hyp_id] = False
except Exception as e:
logs.broom.error('Exception when try to hypervisor {}: {}'.format(hyp_id, e))
logs.broom.error('Traceback: {}'.format(traceback.format_exc()))
for d in list_domains_without_hyp:
domain_id = d['id']
status = d['status']
if status == 'Stopping':
logs.broom.debug('DOMAIN: {} STATUS STOPPING WITHOUTH HYPERVISOR, UNKNOWN REASON'.format(domain_id))
update_domain_status('Stopped', domain_id,
detail='Stopped by broom thread because has not hypervisor')
for d in list_domains:
domain_id = d['id']
status = d['status']
hyp_started = d['hyp_started']
if type(hyp_started) is bool:
continue
if len(hyp_started) == 0:
continue
# TODO bug sometimes hyp_started not in hyps_domain_started keys... why?
if hyp_started in hyps_domain_started.keys() and len(hyp_started) > 0:
if hyps_domain_started[hyp_started] is not False:
if status == 'Starting':
logs.broom.debug(
'DOMAIN: {} STATUS STARTING TO RUN IN HYPERVISOR: {}'.format(domain_id,
hyp_started))
# try:
# if domain_id in hyps_domain_started[hyp_started]['active_domains']:
# print(domain_id)
# except Exception as e:
# logs.broom.error(e)
if domain_id in hyps_domain_started[hyp_started]['active_domains']:
logs.broom.debug('DOMAIN: {} ACTIVE IN HYPERVISOR: {}'.format(domain_id, hyp_started))
state_libvirt = hyps_domain_started[hyp_started]['hyp'].domains[domain_id].state()
state_str, cause = state_and_cause_to_str(state_libvirt[0], state_libvirt[1])
status = dict_domain_libvirt_state_to_isard_state(state_str)
logs.broom.debug(
'DOMAIN: {} ACTIVE IN HYPERVISOR: {} WITH STATUS: {}'.format(domain_id,
hyp_started,
status))
update_domain_hyp_started(domain_id, hyp_started)
else:
logs.broom.debug('DOMAIN: {} NOT ACTIVE YET IN HYPERVISOR: {} '.format(domain_id, hyp_started))
elif status == 'Stopping':
logs.broom.debug('DOMAIN: {} STATUS STOPPING IN HYPERVISOR: {}'.format(domain_id, hyp_started))
if domain_id not in hyps_domain_started[hyp_started]['active_domains']:
update_domain_status('Stopped', domain_id, detail='Stopped by broom thread')
else:
logs.broom.debug('DOMAIN: {} NOT ACTIVE YET IN HYPERVISOR: {} '.format(domain_id, hyp_started))
else:
if len(hyps_domain_started) > 0:
logs.broom.error('hyp_started: {} NOT IN hyps_domain_started keys:'.format(hyp_started))
def creating_disks_from_template(self,
id_new):
dict_domain = get_domain(id_new)
if 'create_dict' in dict_domain.keys():
dict_to_create = dict_domain['create_dict']
pool_var = dict_domain['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
# INFO TO DEVELOPER DEBERÍA SER UN FOR PARA CADA DISCO
# y si el disco no tiene backing_chain, crear un disco vacío
# del tamño que marcase
# d['hardware']['disks'][0]['size']
# el backing_file debería estar asociado a cada disco:
# d['hardware']['disks'][0]['backing_file']
for index_disk in range(len(dict_to_create['hardware']['disks'])):
relative_path = dict_to_create['hardware']['disks'][index_disk]['file']
new_file, path_selected = get_path_to_disk(relative_path, pool=pool_id)
# UPDATE PATH IN DOMAIN
dict_to_create['hardware']['disks'][index_disk]['file'] = new_file
dict_to_create['hardware']['disks'][index_disk]['path_selected'] = path_selected
update_table_field('domains',id_new,'create_dict',dict_to_create)
#TODO: REVISAR SI RELAMENTE ES NECESARIO o esta acción responde a versiones antiguas de nuestras funciones de creación
hardware_update = {}
hardware_update['disks'] = dict_to_create['hardware']['disks']
update_domain_dict_hardware(id_new, hardware_update)
##################
for index_disk in range(len(dict_to_create['hardware']['disks'])):
backing_file = dict_to_create['hardware']['disks'][index_disk]['parent']
new_file = dict_to_create['hardware']['disks'][index_disk]['file']
path_selected = dict_to_create['hardware']['disks'][index_disk]['path_selected']
hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=pool_id, type_path='groups')
cmds = create_cmds_disk_from_base(path_base=backing_file, path_new=new_file)
log.debug('commands to disk create to launch in disk_operations: \n{}'.format('\n'.join(cmds)))
action = {}
action['type'] = 'create_disk'
action['disk_path'] = new_file
action['index_disk'] = index_disk
action['domain'] = id_new
action['ssh_commands'] = cmds
try:
update_domain_status(status='CreatingDisk',
id_domain=id_new,
hyp_id=False,
detail='Creating disk operation is launched in hypervisor {} ({} operations in queue)'.format(
hyp_to_disk_create,
self.manager.q_disk_operations[hyp_to_disk_create].qsize()))
self.manager.q_disk_operations[hyp_to_disk_create].put(action)
except Exception as e:
update_domain_status(status='FailedCreatingDomain',
id_domain=id_new,
hyp_id=False,
detail='Creating disk operation failed when insert action in queue for disk operations')
log.error(
'Creating disk operation failed when insert action in queue for disk operations. Exception: {}'.format(
e))
def run(self):
self.tid = get_tid()
logs.workers.info('starting thread: {} (TID {})'.format(self.name, self.tid))
host, port, user = get_hyp_hostname_from_id(self.hyp_id)
port = int(port)
self.hostname = host
self.h = hyp(self.hostname, user=user, port=port)
# self.h.get_kvm_mod()
# self.h.get_hyp_info()
update_db_hyp_info(self.hyp_id, self.h.info)
hyp_id = self.hyp_id
while self.stop is not True:
try:
# do={type:'start_domain','xml':'xml','id_domain'='prova'}
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
logs.workers.debug('received action in working thread {}'.format(action['type']))
if action['type'] == 'start_paused_domain':
logs.workers.debug('xml to start paused some lines...: {}'.format(action['xml'][30:100]))
try:
self.h.conn.createXML(action['xml'], flags=VIR_DOMAIN_START_PAUSED)
# 32 is the constant for domains paused
# reference: https://libvirt.org/html/libvirt-libvirt-domain.html#VIR_CONNECT_LIST_DOMAINS_PAUSED
FLAG_LIST_DOMAINS_PAUSED = 32
list_all_domains = self.h.conn.listAllDomains(FLAG_LIST_DOMAINS_PAUSED)
list_names_domains = [d.name() for d in list_all_domains]
dict_domains = dict(zip(list_names_domains,list_all_domains))
if action['id_domain'] in list_names_domains:
# domain started in pause mode
domain = dict_domains[action['id_domain']]
domain_active = True
try:
domain.isActive()
domain.destroy()
try:
domain.isActive()
except Exception as e:
logs.workers.debug('verified domain {} is destroyed'.format(action['id_domain']))
domain_active = False
except libvirtError as e:
from pprint import pformat
error_msg = pformat(e.get_error_message())
update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id,
detail='domain {} failed when try to destroy from paused domain in hypervisor {}. creating domain operation is aborted')
logs.workers.error(
'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format(
action['id_domain'], self.hyp_id, error_msg))
continue
if domain_active is False:
# domain is destroyed, all ok
update_domain_status('CreatingDomain', action['id_domain'], hyp_id='',
detail='Domain created and test OK: Started, paused and now stopped in hyp {}'.format(self.hyp_id))
logs.workers.debug(
'domain {} creating operation finalished. Started paused and destroyed in hypervisor {}. Now status is Stopped. READY TO USE'.format(
action['id_domain'], self.hyp_id))
else:
update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id,
detail='Domain is created, started in pause mode but not destroyed,creating domain operation is aborted')
logs.workers.error(
'domain {} started paused but not destroyed in hypervisor {}, must be destroyed'.format(
action['id_domain'], self.hyp_id))
else:
update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id,
detail='XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted by unknown cause'.format(
action['id_domain'], self.hyp_id))
logs.workers.error(
'XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted, not exception, rare case, unknown cause'.format(
action['id_domain'], self.hyp_id))
except libvirtError as e:
from pprint import pformat
error_msg = pformat(e.get_error_message())
update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id,
detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted')
logs.workers.error(
'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format(
action['id_domain'], self.hyp_id, error_msg))
except Exception as e:
update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id,
detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted')
logs.workers.error(
'Exception starting paused xml for domain {} in hypervisor {}. NOT LIBVIRT EXCEPTION, RARE CASE. Exception message: {}'.format(
action['id_domain'], self.hyp_id, str(e)))
## START DOMAIN
elif action['type'] == 'start_domain':
logs.workers.debug('xml to start some lines...: {}'.format(action['xml'][30:100]))
try:
self.h.conn.createXML(action['xml'])
# wait to event started to save state in database
#update_domain_status('Started', action['id_domain'], hyp_id=self.hyp_id, detail='Domain has started in worker thread')
logs.workers.debug('STARTED domain {}: createdXML action in hypervisor {} has been sent'.format(
action['id_domain'], host))
except libvirtError as e:
update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id,
detail=("Hypervisor can not create domain with libvirt exception: " + str(e)))
logs.workers.debug('exception in starting domain {}: '.format(e))
except Exception as e:
update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=("Exception when starting domain: " + str(e)))
logs.workers.debug('exception in starting domain {}: '.format(e))
## STOP DOMAIN
elif action['type'] == 'stop_domain':
logs.workers.debug('action stop domain: {}'.format(action['id_domain'][30:100]))
try:
self.h.conn.lookupByName(action['id_domain']).destroy()
logs.workers.debug('STOPPED domain {}'.format(action['id_domain']))
check_if_delete = action.get('delete_after_stopped',False)
if check_if_delete is True:
update_domain_status('Stopped', action['id_domain'], hyp_id='')
update_domain_status('Deleting', action['id_domain'], hyp_id='')
else:
update_domain_status('Stopped', action['id_domain'], hyp_id='')
except Exception as e:
update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=str(e))
logs.workers.debug('exception in stopping domain {}: '.format(e))
elif action['type'] in ['create_disk', 'delete_disk']:
launch_action_disk(action,
self.hostname,
user,
port)
elif action['type'] in ['add_media_hot']:
pass
elif action['type'] in ['killall_curl']:
launch_killall_curl(self.hostname,
user,
port)
elif action['type'] in ['delete_media']:
final_status = action.get('final_status','Deleted')
launch_delete_media (action,
self.hostname,
user,
port,
final_status=final_status)
# ## DESTROY THREAD
# elif action['type'] == 'destroy_thread':
# list_works_in_queue = list(self.queue_actions.queue)
# if self.queue_master is not None:
# self.queue_master.put(['destroy_working_thread',self.hyp_id,list_works_in_queue])
# #INFO TO DEVELOPER, si entra aquí es porque no quedaba nada en cola, si no ya lo habrán matado antes
#
# logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id))
#
# raise 'destoyed'
elif action['type'] == 'create_disk':
pass
elif action['type'] == 'hyp_info':
self.h.get_hyp_info()
logs.workers.debug('hypervisor motherboard: {}'.format(self.h.info['motherboard_manufacturer']))
## DESTROY THREAD
elif action['type'] == 'stop_thread':
self.stop = True
else:
logs.workers.error('type action {} not supported in queue actions'.format(action['type']))
# time.sleep(0.1)
## TRY DOMAIN
except queue.Empty:
try:
self.h.conn.getLibVersion()
pass
# logs.workers.debug('hypervisor {} is alive'.format(host))
except:
logs.workers.info('trying to reconnect hypervisor {}, alive test in working thread failed'.format(host))
alive = False
for i in range(RETRIES_HYP_IS_ALIVE):
try:
time.sleep(TIMEOUT_BETWEEN_RETRIES_HYP_IS_ALIVE)
self.h.conn.getLibVersion()
alive = True
logs.workers.info('hypervisor {} is alive'.format(host))
break
except:
logs.workers.info('hypervisor {} is NOT alive'.format(host))
if alive is False:
try:
self.h.connect_to_hyp()
self.h.conn.getLibVersion()
update_hyp_status(self.hyp_id, 'Online')
except:
logs.workers.debug('hypervisor {} failed'.format(host))
logs.workers.error('fail reconnecting to hypervisor {} in working thread'.format(host))
reason = self.h.fail_connected_reason
update_hyp_status(self.hyp_id, 'Error', reason)
update_domains_started_in_hyp_to_unknown(self.hyp_id)
list_works_in_queue = list(self.queue_actions.queue)
if self.queue_master is not None:
self.queue_master.put(['error_working_thread', self.hyp_id, list_works_in_queue])
logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id))
self.active = False
break
def myDomainEventCallbackRethink(conn, dom, event, detail, opaque):
now = time.time()
dom_id = dom.name()
hyp_id = get_id_hyp_from_uri(conn.getURI())
dict_event = {'domain': dom_id,
'hyp_id': hyp_id,
'event' : domEventToString(event),
'detail': domDetailToString(event, detail),
'when' : now}
logs.status.debug("EVENT: {domain} - {event} ({detail}) - {hyp}".format(domain=dom_id,
event=dict_event['event'],
detail=dict_event['detail'],
hyp=hyp_id
))
domain_status = get_domain_status(dom_id)
if domain_status is not None:
if hyp_id is None or hyp_id == '':
logs.status.debug('event in Hypervisor not in database with uri. hyp_id:{}, uri:{}'.dom_id, conn.getURI())
r_status = opaque
if dict_event['event'] in ('Started', 'Resumed'):
if domain_status == 'StartingDomainDisposable' and dict_event['event'] == 'Resumed':
logs.status.debug('Event Resumed Received but waiting for Started')
elif domain_status == 'CreatingDomain' and dict_event['event'] == 'Started':
logs.status.debug('Event Started Received but waiting for Paused')
elif domain_status == 'Stopped' and dict_event['event'] == 'Resumed':
logs.status.debug('Event Resumed Received but waiting for Paused to update status in database')
elif domain_status == 'Started' and dict_event['event'] == 'Resumed':
logs.status.debug('Event Resumed Received but is state is started in database')
else:
try:
xml_started = dom.XMLDesc()
vm = DomainXML(xml_started)
spice, spice_tls, vnc, vnc_websocket = vm.get_graphics_port()
update_domain_viewer_started_values(dom_id, hyp_id=hyp_id,
spice=spice, spice_tls=spice_tls,
vnc = vnc,vnc_websocket=vnc_websocket)
logs.status.info(f'DOMAIN STARTED - {dom_id} in {hyp_id} (spice: {spice} / spicetls:{spice_tls} / vnc: {vnc} / vnc_websocket: {vnc_websocket})')
update_domain_status(id_domain=dom_id,
status=domEventToString(event),
hyp_id=hyp_id,
detail="Event received: " + domDetailToString(event, detail)
)
except Exception as e:
logs.status.debug(
'Domain {} has been destroyed while event started is processing, typical if try domain with starting paused and destroyed'.format(
dom_id))
logs.status.debug('Exception: ' + str(e))
if dict_event['event'] in ('Suspended'):
if domain_status == 'CreatingDomain' and dict_event['event'] == 'Suspended':
logs.status.debug('Event Paused Received but waiting for Stoped to update status')
else:
update_domain_status(id_domain=dom_id,
status='Paused',
hyp_id=hyp_id,
detail="Event received: " + domDetailToString(event, detail)
)
if dict_event['event'] in ('Stopped', 'Shutdown'):
remove_domain_viewer_values(dom_id)
if get_domain_status(dict_event['domain']) != 'Stopped':
logs.status.debug('event {} ({}) in hypervisor {} changes status to Stopped in domain {}'.format(
dict_event['event'],
dict_event['detail'],
hyp_id,
dict_event['domain']
))
update_domain_status(status='Stopped', id_domain=dict_event['domain'], hyp_id=False,
detail='Ready to Start')
r_status.insert_event_in_db(dict_event)
if dict_event['event'] in (
"Defined",
"Undefined",
# "Started",
# "Suspended",
# "Resumed",
# "Stopped",
# "Shutdown",
"PMSuspended",
"Crashed"
):
logs.status.error('event strange, why?? event: {}, domain: {}, hyp_id: {}, detail: {}'.format(
dict_event['event'],
dict_event['domain'],
hyp_id,
dict_event['detail']
))
else:
logs.status.info('domain {} launch event in hyervisor {}, but id_domain is not in database'.format(dom_id, hyp_id))
logs.status.info('event: {}; detail: {}'.format(domEventToString(event), domDetailToString(event, detail)))