def send_stats(self):
#hypervisors
send_stats_to_rethink = True
if self.id_hyp_rethink is None:
#self.id_hyp_rethink = get_id_hyp_from_uri('qemu+ssh://root@isard-hypervisor:22/system')
self.id_hyp_rethink = get_id_hyp_from_uri(
hostname_to_uri(self.hostname, user=self.user, port=self.port))
if send_stats_to_rethink:
update_actual_stats_hyp(self.id_hyp_rethink, self.stats_hyp_now)
for id_domain, s in self.stats_domains_now.items():
means = {
'near':
self.stats_domains[id_domain].get('means_near', False),
'medium':
self.stats_domains[id_domain].get('means_medium', False),
'long':
self.stats_domains[id_domain].get('means_long', False),
'total':
self.stats_domains[id_domain].get('means_total', False),
'boot':
self.stats_domains[id_domain].get('means_boot', False)
}
update_actual_stats_domain(id_domain, s, means)
def add_hyp_to_receive_events(self, hyp_id):
d_hyp_parameters = get_hyp_hostname_user_port_from_id(hyp_id)
hostname = d_hyp_parameters['hostname']
user = d_hyp_parameters.get('user', 'root')
port = d_hyp_parameters.get('port', 22)
uri = hostname_to_uri(hostname, user=user, port=port)
conn_ok = False
try:
self.hyps_conn[hyp_id] = libvirt.openReadOnly(uri)
logs.status.debug(
'####################connection to {} ready in events thread'.
format(hyp_id))
update_uri_hyp(hyp_id, uri)
conn_ok = True
except Exception as e:
logs.status.error(
'libvirt connection read only in events thread in hypervisor: {}'
.format(hyp_id))
logs.status.error(e)
if conn_ok is True:
self.events_ids[hyp_id] = self.register_events(
self.hyps_conn[hyp_id])
self.hyps[hyp_id] = hostname
def update_domains_started_and_stopped(self,domains_with_stats):
if self.id_hyp_rethink is None:
try:
self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port))
except Exception as e:
log.error('error when hypervisor have not rethink id. {}'.format(e))
return False
l_all_domains = get_domains_with_status_in_list(list_status=['Started', 'Stopped', 'Failed'])
for d in l_all_domains:
if d['id'] in domains_with_stats:
if d['status'] == 'Started':
#if status started check if has the same hypervisor
if d['hyp_started'] != self.id_hyp_rethink:
log.error(f"Domain {d['id']} started in hypervisor ({self.id_hyp_rethink}) but database says that is started in {d['hyp_started']} !! ")
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Started in other hypervisor!! {self.id_hyp_rethink}. Updated by status thread',
hyp_id=self.id_hyp_rethink)
else:
#if status is Stopped or Failed update, the domain is started
log.info('Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread')
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread',
hyp_id=self.id_hyp_rethink)
elif d['hyp_started'] == self.id_hyp_rethink:
#Domain is started in this hypervisor in database, but is stopped
if d['status'] == 'Started':
update_domain_status(status='Stopped',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is stopped in {self.id_hyp_rethink} but in database was Started, updated by status thread',
)
def update_domains_started_and_stopped(self,domains_with_stats):
if self.id_hyp_rethink is None:
try:
self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port))
except Exception as e:
log.error('error when hypervisor have not rethink id. {}'.format(e))
return False
l_all_domains = get_domains_with_status_in_list(list_status=['Started', 'Stopped', 'Failed'])
for d in l_all_domains:
if d['id'] in domains_with_stats:
if d['status'] == 'Started':
#if status started check if has the same hypervisor
if d['hyp_started'] != self.id_hyp_rethink:
log.error(f"Domain {d['id']} started in hypervisor ({self.id_hyp_rethink}) but database says that is started in {d['hyp_started']} !! ")
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Started in other hypervisor!! {self.id_hyp_rethink}. Updated by status thread',
hyp_id=self.id_hyp_rethink)
else:
#if status is Stopped or Failed update, the domain is started
log.info('Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread')
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread',
hyp_id=self.id_hyp_rethink)
elif d['hyp_started'] == self.id_hyp_rethink:
#Domain is started in this hypervisor in database, but is stopped
if d['status'] == 'Started':
update_domain_status(status='Stopped',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is stopped in {self.id_hyp_rethink} but in database was Started, updated by status thread',
)
def add_hyp_to_receive_events(self, hyp_id):
d_hyp_parameters = get_hyp_hostname_user_port_from_id(hyp_id)
hostname = d_hyp_parameters['hostname']
user = d_hyp_parameters.get('user', 'root')
port = d_hyp_parameters.get('port', 22)
uri = hostname_to_uri(hostname, user=user, port=port)
conn_ok = False
try:
self.hyps_conn[hyp_id] = libvirt.openReadOnly(uri)
logs.status.debug(
'####################connection to {} ready in events thread'.
format(hyp_id))
update_uri_hyp(hyp_id, uri)
conn_ok = True
except Exception as e:
logs.status.error(
'libvirt connection read only in events thread in hypervisor: {}'
.format(hyp_id))
logs.status.error(e)
if conn_ok is True:
for i in range(NUM_TRY_REGISTER_EVENTS):
#try 5
try:
self.events_ids[hyp_id] = self.register_events(
self.hyps_conn[hyp_id])
self.hyps[hyp_id] = hostname
break
except libvirt.libvirtError as e:
logs.status.error(
f'Error when register_events, wait {SLEEP_BETWEEN_TRY_REGISTER_EVENTS}, try {i+1} of {NUM_TRY_REGISTER_EVENTS}'
)
logs.status.error(e)
time.sleep(SLEEP_BETWEEN_TRY_REGISTER_EVENTS)
def add_hyp_to_receive_events(self, hyp_id):
d_hyp_parameters = get_hyp_hostname_user_port_from_id(hyp_id)
hostname = d_hyp_parameters['hostname']
user = d_hyp_parameters.get('user', 'root')
port = d_hyp_parameters.get('port', 22)
uri = hostname_to_uri(hostname, user=user, port=port)
conn_ok = False
try:
self.hyps_conn[hyp_id] = libvirt.openReadOnly(uri)
logs.status.debug('####################connection to {} ready in events thread'.format(hyp_id))
update_uri_hyp(hyp_id, uri)
conn_ok = True
except Exception as e:
logs.status.error('libvirt connection read only in events thread in hypervisor: {}'.format(hyp_id))
logs.status.error(e)
if conn_ok is True:
for i in range(NUM_TRY_REGISTER_EVENTS):
#try 5
try:
self.events_ids[hyp_id] = self.register_events(self.hyps_conn[hyp_id])
self.hyps[hyp_id] = hostname
break
except libvirt.libvirtError as e:
logs.status.error(f'Error when register_events, wait {SLEEP_BETWEEN_TRY_REGISTER_EVENTS}, try {i+1} of {NUM_TRY_REGISTER_EVENTS}')
logs.status.error(e)
time.sleep(SLEEP_BETWEEN_TRY_REGISTER_EVENTS)
def send_stats(self):
#hypervisors
send_stats_to_rethink = True
if self.id_hyp_rethink is None:
#self.id_hyp_rethink = get_id_hyp_from_uri('qemu+ssh://root@isard-hypervisor:22/system')
self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port))
if send_stats_to_rethink:
update_actual_stats_hyp(self.id_hyp_rethink,
self.stats_hyp_now)
for id_domain, s in self.stats_domains_now.items():
means = {'near': self.stats_domains[id_domain].get('means_near',False),
'medium': self.stats_domains[id_domain].get('means_medium',False),
'long': self.stats_domains[id_domain].get('means_long',False),
'total': self.stats_domains[id_domain].get('means_total',False),
'boot': self.stats_domains[id_domain].get('means_boot',False)}
update_actual_stats_domain(id_domain, s, means)
def connect_to_hyp(self):
try:
self.ip = socket.gethostbyname(self.hostname)
if self.try_ssh_autologin == True:
self.try_ssh()
if self.ssh_autologin_fail is False:
try:
self.uri = hostname_to_uri(self.hostname, user=self.user, port=self.port)
timeout_libvirt = float(CONFIG_DICT['TIMEOUTS']['libvirt_hypervisor_timeout_connection'])
self.conn = timelimit(timeout_libvirt, test_hypervisor_conn, self.uri)
# timeout = float(CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection'])
if (self.conn != False):
self.connected = True
# prueba de alberto para que indique cuando ha caído y para que mantenga alive la conexión
# OJO INFO TO DEVELOPER
# self.startEvent()
# este setKeepAlive no tengo claro que haga algo, pero bueno...
# y al ponerlo da error lo dejo comentado, pero en futuro hay que quitar
# esta línea si no sabemos bien que hace...
# self.conn.setKeepAlive(5, 3)
log.debug("connected to hypervisor: %s" % self.hostname)
self.set_status(HYP_STATUS_CONNECTED)
self.fail_connected_reason = ''
# para que le de tiempo a los eventos a quedarse registrados hay que esperar un poquillo, ya
# que se arranca otro thread
# self.get_hyp_info()
return True
else:
log.error('libvirt can\'t connect to hypervisor {}'.format(self.hostname))
log.info("""connection to hypervisor fail, try policykit or permissions,
or try in the hypervisor if libvirtd service is started
(in Fedora/Centos: systemctl status libvirtd )
or if the port 22 is open""")
self.set_status(HYP_STATUS_ERROR_WHEN_CONNECT)
self.fail_connected_reason = 'Hypervisor policykit or permissions or libvirtd has not started'
return False
# except TimeLimitExpired:
# log.error("""Time Limit Expired connecting to hypervisor""")
# self.set_status(HYP_STATUS_ERROR_WHEN_CONNECT_TIMELIMIT)
# self.fail_connected_reason = 'Time Limit Expired connecting to hypervisor'
# return False
except Exception as e:
log.error('connection to hypervisor {} fail with unexpected error: {}'.format(self.hostname, e))
log.error('libvirt uri: {}'.format(self.uri))
self.set_status(HYP_STATUS_ERROR_WHEN_CONNECT)
self.fail_connected_reason = 'connection to hypervisor {} fail with unexpected error'.format(
self.hostname)
return False
except socket.error as e:
log.error(e)
log.error('not resolves ip from hostname: {}'.format(self.hostname))
self.fail_connected_reason = 'not resolves ip from hostname: {}'.format(self.hostname)
return False
except Exception as e:
log.error(e)
def connect_to_hyp(self):
try:
self.ip = socket.gethostbyname(self.hostname)
if self.try_ssh_autologin == True:
self.try_ssh()
if self.ssh_autologin_fail is False:
try:
self.uri = hostname_to_uri(self.hostname,
user=self.user,
port=self.port)
timeout_libvirt = float(
CONFIG_DICT['TIMEOUTS']
['libvirt_hypervisor_timeout_connection'])
self.conn = timelimit(timeout_libvirt,
test_hypervisor_conn, self.uri)
# timeout = float(CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection'])
if (self.conn != False):
self.connected = True
# prueba de alberto para que indique cuando ha caído y para que mantenga alive la conexión
# OJO INFO TO DEVELOPER
# self.startEvent()
# este setKeepAlive no tengo claro que haga algo, pero bueno...
# y al ponerlo da error lo dejo comentado, pero en futuro hay que quitar
# esta línea si no sabemos bien que hace...
# self.conn.setKeepAlive(5, 3)
log.debug("connected to hypervisor: %s" %
self.hostname)
self.set_status(HYP_STATUS_CONNECTED)
self.fail_connected_reason = ''
# para que le de tiempo a los eventos a quedarse registrados hay que esperar un poquillo, ya
# que se arranca otro thread
# self.get_hyp_info()
return True
else:
log.error(
'libvirt can\'t connect to hypervisor {}'.format(
self.hostname))
log.info(
"""connection to hypervisor fail, try policykit or permissions,
or try in the hypervisor if libvirtd service is started
(in Fedora/Centos: systemctl status libvirtd )
or if the port 22 is open""")
self.set_status(HYP_STATUS_ERROR_WHEN_CONNECT)
self.fail_connected_reason = 'Hypervisor policykit or permissions or libvirtd has not started'
return False
# except TimeLimitExpired:
# log.error("""Time Limit Expired connecting to hypervisor""")
# self.set_status(HYP_STATUS_ERROR_WHEN_CONNECT_TIMELIMIT)
# self.fail_connected_reason = 'Time Limit Expired connecting to hypervisor'
# return False
except Exception as e:
log.error(
'connection to hypervisor {} fail with unexpected error: {}'
.format(self.hostname, e))
log.error('libvirt uri: {}'.format(self.uri))
self.set_status(HYP_STATUS_ERROR_WHEN_CONNECT)
self.fail_connected_reason = 'connection to hypervisor {} fail with unexpected error'.format(
self.hostname)
return False
except socket.error as e:
log.error(e)
log.error('not resolves ip from hostname: {}'.format(
self.hostname))
self.fail_connected_reason = 'not resolves ip from hostname: {}'.format(
self.hostname)
return False
except Exception as e:
log.error(e)