def set_domains_coherence(dict_hyps_ready):
for hyp_id, hostname in dict_hyps_ready.items():
hyp_obj = hyp_from_hyp_id(hyp_id)
try:
hyp_obj.get_domains()
except:
log.error('hypervisor {} can not get domains'.format(hyp_id))
update_hyp_status(hyp_id, 'Error')
break
# update domain_status
update_all_domains_status(reset_status='Stopped', from_status=['Starting'])
update_all_domains_status(reset_status='Started', from_status=['Stopping'])
domains_started_in_rethink = get_domains_started_in_hyp(hyp_id)
domains_are_started = []
for domain_name, domain_obj in hyp_obj.domains.items():
domain_state_libvirt = domain_obj.state()
state, reason = state_and_cause_to_str(domain_state_libvirt[0], domain_state_libvirt[1])
status_isard = dict_domain_libvirt_state_to_isard_state[state]
update_domain_status(status=status_isard, id_domain=domain_name, hyp_id=hyp_id, detail=reason)
domains_are_started.append(domain_name)
if len(domains_started_in_rethink) > 0:
domains_are_shutdown = list(set(domains_started_in_rethink).difference(set(domains_are_started)))
for domain_stopped in domains_are_shutdown:
update_domain_status(status='Stopped', id_domain=domain_stopped, hyp_id='')
# TODO INFO TO DEVELOPER: faltaría revisar que ningún dominio está duplicado en y started en dos hypervisores
# a nivel de libvirt, porque a nivel de rethink es imposible, y si pasa poner un erroraco gigante
# a parte de dejarlo en unknown
update_hyp_status(hyp_id, 'ReadyToStart')
def test_hyps_and_start_threads(self):
l_hyps_to_test = get_hyps_with_status(
list_status=['Error', 'Offline'], empty=True)
dict_hyps_to_test = {
d['id']: {
'hostname': d['hostname'],
'port': d['port'] if 'port' in d.keys() else 22,
'user': d['user'] if 'user' in d.keys() else 'root'
}
for d in l_hyps_to_test
}
launch_try_hyps(dict_hyps_to_test)
dict_hyps_ready = self.manager.dict_hyps_ready = get_hyps_ready_to_start(
)
if len(dict_hyps_ready) > 0:
logs.main.debug('hyps_ready_to_start: ' +
pprint.pformat(dict_hyps_ready))
#launch thread events
if self.manager.t_events is None:
logs.main.info('launching hypervisor events thread')
self.manager.t_events = launch_thread_hyps_event(
dict_hyps_ready)
else:
#if new hypervisor has added then add hypervisor to receive events
logs.main.info('hypervisors added to thread events')
logs.main.info(pprint.pformat(dict_hyps_ready))
self.manager.t_events.hyps.update(dict_hyps_ready)
for hyp_id, hostname in self.manager.t_events.hyps.items():
self.manager.t_events.add_hyp_to_receive_events(hyp_id)
set_unknown_domains_not_in_hyps(dict_hyps_ready.keys())
set_domains_coherence(dict_hyps_ready)
pools = set()
for hyp_id, hostname in dict_hyps_ready.items():
update_hyp_status(hyp_id, 'StartingThreads')
# start worker thread
self.manager.t_workers[hyp_id], self.manager.q.workers[
hyp_id] = launch_thread_worker(hyp_id)
if self.manager.with_status_threads is True:
self.manager.t_status[hyp_id] = launch_thread_status(
hyp_id, self.manager.STATUS_POLLING_INTERVAL)
# self.manager.launch_threads(hyp_id)
# INFO TO DEVELOPER FALTA VERIFICAR QUE REALMENTE ESTÁN ARRANCADOS LOS THREADS??
# comprobar alguna variable a true en alguno de los threads
update_hyp_status(hyp_id, 'Online')
pools.update(get_pools_from_hyp(hyp_id))
for id_pool in pools:
if id_pool not in self.manager.pools.keys():
self.manager.pools[id_pool] = PoolHypervisors(
id_pool, self.manager, len(dict_hyps_ready))
def set_domains_coherence(dict_hyps_ready):
for hyp_id, hostname in dict_hyps_ready.items():
hyp_obj = hyp_from_hyp_id(hyp_id)
try:
hyp_obj.get_domains()
except:
log.error('hypervisor {} can not get domains'.format(hyp_id))
update_hyp_status(hyp_id, 'Error')
break
# update domain_status
update_all_domains_status(reset_status='Stopped',
from_status=['Starting'])
update_all_domains_status(reset_status='Started',
from_status=['Stopping'])
domains_started_in_rethink = get_domains_started_in_hyp(hyp_id)
domains_are_started = []
for domain_name, domain_obj in hyp_obj.domains.items():
domain_state_libvirt = domain_obj.state()
state, reason = state_and_cause_to_str(domain_state_libvirt[0],
domain_state_libvirt[1])
status_isard = dict_domain_libvirt_state_to_isard_state[state]
update_domain_status(status=status_isard,
id_domain=domain_name,
hyp_id=hyp_id,
detail=reason)
domains_are_started.append(domain_name)
if len(domains_started_in_rethink) > 0:
domains_are_shutdown = list(
set(domains_started_in_rethink).difference(
set(domains_are_started)))
for domain_stopped in domains_are_shutdown:
update_domain_status(status='Stopped',
id_domain=domain_stopped,
hyp_id='')
# TODO INFO TO DEVELOPER: faltaría revisar que ningún dominio está duplicado en y started en dos hypervisores
# a nivel de libvirt, porque a nivel de rethink es imposible, y si pasa poner un erroraco gigante
# a parte de dejarlo en unknown
update_hyp_status(hyp_id, 'ReadyToStart')
def try_hyp_connection(hyp_id, hostname, port, user):
update_hyp_status(hyp_id, 'TryConnection')
log.debug('Starting trying to connect to hypervisor {} '.format(hostname))
# INFO TO DEVELOPER, VOLVER A ACTIVAR CUANDO NO FALLE LA AUTENTICACIÓN CON ALGORITMOS MODERNOS DE SSH
# hyp_obj = hyp(hostname,user=user,port=port,try_ssh_autologin=True)
hyp_obj = hyp(hostname, user=user, port=port, try_ssh_autologin=True)
log.debug('####@@@@$$$$$$$$$$$$$$$$')
log.debug('hostname: {} , reason: {}'.format(
hostname, hyp_obj.fail_connected_reason))
try:
reason = hyp_obj.fail_connected_reason
except Exception as e:
log.error('try hyp {}, error: {}'.format(hyp_id, e))
reason = 'no reason available'
update_hypervisor_failed_connection(hyp_id, reason)
if hyp_obj.connected is True:
ok = True
log.debug('hypervisor {} ready'.format(hyp_id))
update_hyp_status(hyp_id, 'ReadyToStart')
hyp_obj.disconnect()
else:
ok = False
log.error('hypervisor {} failed when trying to connect'.format(hyp_id))
log.error('fail_connected_reason: {}'.format(reason))
update_hyp_status(hyp_id, 'Error', detail=reason)
return hyp_obj, ok
def try_hyp_connection(hyp_id, hostname, port, user):
update_hyp_status(hyp_id, 'TryConnection')
log.debug('Starting trying to connect to hypervisor {} '.format(hostname))
# INFO TO DEVELOPER, VOLVER A ACTIVAR CUANDO NO FALLE LA AUTENTICACIÓN CON ALGORITMOS MODERNOS DE SSH
# hyp_obj = hyp(hostname,user=user,port=port,try_ssh_autologin=True)
hyp_obj = hyp(hostname, user=user, port=port, try_ssh_autologin=True)
log.debug('####@@@@$$$$$$$$$$$$$$$$')
log.debug('hostname: {} , reason: {}'.format(
hostname, hyp_obj.fail_connected_reason))
try:
reason = hyp_obj.fail_connected_reason
except Exception as e:
log.error('try hyp {}, error: {}'.format(hyp_id, e))
reason = 'no reason available'
update_hypervisor_failed_connection(hyp_id, reason)
if hyp_obj.connected is True:
log.debug('hypervisor {} libvirt connection ready'.format(hyp_id))
hyp_obj.get_kvm_mod()
hyp_obj.get_hyp_info()
update_db_hyp_info(hyp_id, hyp_obj.info)
if hyp_obj.info['kvm_module'] == 'intel' or hyp_obj.info[
'kvm_module'] == 'amd':
ok = True
update_hyp_status(hyp_id, 'ReadyToStart')
else:
ok = False
log.error(
'hypervisor {} has not virtualization support (VT-x for Intel processors and AMD-V for AMD processors). '
.format(hyp_id))
update_hyp_status(
hyp_id,
'Error',
detail=
"KVM requires that the virtual machine host's processor has virtualization "
+
"support (named VT-x for Intel processors and AMD-V for AMD processors). "
+
"Check CPU capabilities and enable virtualization support in your BIOS."
)
hyp_obj.disconnect()
else:
ok = False
log.error('hypervisor {} failed when trying to connect'.format(hyp_id))
log.error('fail_connected_reason: {}'.format(reason))
update_hyp_status(hyp_id, 'Error', detail=reason)
return hyp_obj, ok
def try_hyp_connection(hyp_id, hostname, port, user):
update_hyp_status(hyp_id, 'TryConnection')
log.debug('Starting trying to connect to hypervisor {} '.format(hostname))
# INFO TO DEVELOPER, VOLVER A ACTIVAR CUANDO NO FALLE LA AUTENTICACIÓN CON ALGORITMOS MODERNOS DE SSH
# hyp_obj = hyp(hostname,user=user,port=port,try_ssh_autologin=True)
hyp_obj = hyp(hostname, user=user, port=port, try_ssh_autologin=True)
log.debug('####@@@@$$$$$$$$$$$$$$$$')
log.debug('hostname: {} , reason: {}'.format(hostname, hyp_obj.fail_connected_reason))
try:
reason = hyp_obj.fail_connected_reason
except Exception as e:
log.error('try hyp {}, error: {}'.format(hyp_id, e))
reason = 'no reason available'
update_hypervisor_failed_connection(hyp_id, reason)
if hyp_obj.connected is True:
log.debug('hypervisor {} libvirt connection ready'.format(hyp_id))
hyp_obj.get_kvm_mod()
hyp_obj.get_hyp_info()
update_db_hyp_info(hyp_id,hyp_obj.info)
if hyp_obj.info['kvm_module'] == 'intel' or hyp_obj.info['kvm_module'] == 'amd':
ok = True
update_hyp_status(hyp_id, 'ReadyToStart')
else:
ok = False
log.error('hypervisor {} has not virtualization support (VT-x for Intel processors and AMD-V for AMD processors). '.format(hyp_id))
update_hyp_status(hyp_id, 'Error', detail="KVM requires that the virtual machine host's processor has virtualization " +
"support (named VT-x for Intel processors and AMD-V for AMD processors). " +
"Check CPU capabilities and enable virtualization support in your BIOS.")
hyp_obj.disconnect()
else:
ok = False
log.error('hypervisor {} failed when trying to connect'.format(hyp_id))
log.error('fail_connected_reason: {}'.format(reason))
update_hyp_status(hyp_id, 'Error', detail=reason)
return hyp_obj, ok
def test_hyps_and_start_threads(self):
"""If status of hypervisor is Error or Offline and are enabled,
this function try to connect and launch threads.
If hypervisor pass connection test, status change to ReadyToStart,
then change to StartingThreads previous to launch threads, when
threads are running state is Online. Status sequence is:
(Offline,Error) => ReadyToStart => StartingThreads => (Online,Error)"""
# DISK_OPERATIONS: launch threads if test disk operations passed and is not launched
self.launch_threads_disk_and_long_operations()
l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
dict_hyps_to_test = {d['id']: {'hostname': d['hostname'],
'port': d['port'] if 'port' in d.keys() else 22,
'user': d['user'] if 'user' in d.keys() else 'root'} for d in
l_hyps_to_test}
#TRY hypervisor connexion and UPDATE hypervisors status
# update status: ReadyToStart if all ok
launch_try_hyps(dict_hyps_to_test)
#hyp_hostnames of hyps ready to start
dict_hyps_ready = self.manager.dict_hyps_ready = get_hyps_ready_to_start()
if len(dict_hyps_ready) > 0:
logs.main.debug('hyps_ready_to_start: ' + pprint.pformat(dict_hyps_ready))
#launch thread events if is None
if self.manager.t_events is None:
logs.main.info('launching hypervisor events thread')
self.manager.t_events = launch_thread_hyps_event()
# else:
# #if new hypervisor has added then add hypervisor to receive events
# logs.main.info('hypervisors added to thread events')
# logs.main.info(pprint.pformat(dict_hyps_ready))
# self.manager.t_events.hyps.update(dict_hyps_ready)
# for hyp_id, hostname in self.manager.t_events.hyps.items():
# self.manager.t_events.add_hyp_to_receive_events(hyp_id)
set_unknown_domains_not_in_hyps(dict_hyps_ready.keys())
set_domains_coherence(dict_hyps_ready)
pools = set()
for hyp_id, hostname in dict_hyps_ready.items():
update_hyp_status(hyp_id, 'StartingThreads')
# launch worker thread
self.manager.t_workers[hyp_id], self.manager.q.workers[hyp_id] = launch_thread_worker(hyp_id)
# LAUNCH status thread
if self.manager.with_status_threads is True:
self.manager.t_status[hyp_id] = launch_thread_status(hyp_id,
self.manager.STATUS_POLLING_INTERVAL)
# ADD hyp to receive_events
self.manager.t_events.q_event_register.put({'type': 'add_hyp_to_receive_events', 'hyp_id': hyp_id})
# self.manager.launch_threads(hyp_id)
# INFO TO DEVELOPER FALTA VERIFICAR QUE REALMENTE ESTÁN ARRANCADOS LOS THREADS??
# comprobar alguna variable a true en alguno de los threads
update_hyp_status(hyp_id, 'Online')
pools.update(get_pools_from_hyp(hyp_id))
#if hypervisor not in pools defined in manager add it
for id_pool in pools:
if id_pool not in self.manager.pools.keys():
self.manager.pools[id_pool] = PoolHypervisors(id_pool, self.manager, len(dict_hyps_ready))
def test_hyps_and_start_threads(self):
"""If status of hypervisor is Error or Offline and are enabled,
this function try to connect and launch threads.
If hypervisor pass connection test, status change to ReadyToStart,
then change to StartingThreads previous to launch threads, when
threads are running state is Online. Status sequence is:
(Offline,Error) => ReadyToStart => StartingThreads => (Online,Error)"""
# DISK_OPERATIONS: launch threads if test disk operations passed and is not launched
self.launch_threads_disk_and_long_operations()
l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
dict_hyps_to_test = {d['id']: {'hostname': d['hostname'],
'port': d['port'] if 'port' in d.keys() else 22,
'user': d['user'] if 'user' in d.keys() else 'root'} for d in
l_hyps_to_test}
#TRY hypervisor connexion and UPDATE hypervisors status
# update status: ReadyToStart if all ok
launch_try_hyps(dict_hyps_to_test)
#hyp_hostnames of hyps ready to start
dict_hyps_ready = self.manager.dict_hyps_ready = get_hyps_ready_to_start()
if len(dict_hyps_ready) > 0:
logs.main.debug('hyps_ready_to_start: ' + pprint.pformat(dict_hyps_ready))
#launch thread events if is None
if self.manager.t_events is None:
logs.main.info('launching hypervisor events thread')
self.manager.t_events = launch_thread_hyps_event()
# else:
# #if new hypervisor has added then add hypervisor to receive events
# logs.main.info('hypervisors added to thread events')
# logs.main.info(pprint.pformat(dict_hyps_ready))
# self.manager.t_events.hyps.update(dict_hyps_ready)
# for hyp_id, hostname in self.manager.t_events.hyps.items():
# self.manager.t_events.add_hyp_to_receive_events(hyp_id)
set_unknown_domains_not_in_hyps(dict_hyps_ready.keys())
set_domains_coherence(dict_hyps_ready)
pools = set()
for hyp_id, hostname in dict_hyps_ready.items():
update_hyp_status(hyp_id, 'StartingThreads')
# launch worker thread
self.manager.t_workers[hyp_id], self.manager.q.workers[hyp_id] = launch_thread_worker(hyp_id)
# LAUNCH status thread
if self.manager.with_status_threads is True:
self.manager.t_status[hyp_id] = launch_thread_status(hyp_id,
self.manager.STATUS_POLLING_INTERVAL)
# ADD hyp to receive_events
self.manager.t_events.q_event_register.put({'type': 'add_hyp_to_receive_events', 'hyp_id': hyp_id})
# self.manager.launch_threads(hyp_id)
# INFO TO DEVELOPER FALTA VERIFICAR QUE REALMENTE ESTÁN ARRANCADOS LOS THREADS??
# comprobar alguna variable a true en alguno de los threads
update_hyp_status(hyp_id, 'Online')
pools.update(get_pools_from_hyp(hyp_id))
#if hypervisor not in pools defined in manager add it
for id_pool in pools:
if id_pool not in self.manager.pools.keys():
self.manager.pools[id_pool] = PoolHypervisors(id_pool, self.manager, len(dict_hyps_ready))