def update_domains_started_and_stopped(self,domains_with_stats):
if self.id_hyp_rethink is None:
try:
self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port))
except Exception as e:
log.error('error when hypervisor have not rethink id. {}'.format(e))
return False
l_all_domains = get_domains_with_status_in_list(list_status=['Started', 'Stopped', 'Failed'])
for d in l_all_domains:
if d['id'] in domains_with_stats:
if d['status'] == 'Started':
#if status started check if has the same hypervisor
if d['hyp_started'] != self.id_hyp_rethink:
log.error(f"Domain {d['id']} started in hypervisor ({self.id_hyp_rethink}) but database says that is started in {d['hyp_started']} !! ")
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Started in other hypervisor!! {self.id_hyp_rethink}. Updated by status thread',
hyp_id=self.id_hyp_rethink)
else:
#if status is Stopped or Failed update, the domain is started
log.info('Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread')
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread',
hyp_id=self.id_hyp_rethink)
elif d['hyp_started'] == self.id_hyp_rethink:
#Domain is started in this hypervisor in database, but is stopped
if d['status'] == 'Started':
update_domain_status(status='Stopped',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is stopped in {self.id_hyp_rethink} but in database was Started, updated by status thread',
)
def send_stats(self):
#hypervisors
send_stats_to_rethink = True
if self.id_hyp_rethink is None:
#self.id_hyp_rethink = get_id_hyp_from_uri('qemu+ssh://root@isard-hypervisor:22/system')
self.id_hyp_rethink = get_id_hyp_from_uri(
hostname_to_uri(self.hostname, user=self.user, port=self.port))
if send_stats_to_rethink:
update_actual_stats_hyp(self.id_hyp_rethink, self.stats_hyp_now)
for id_domain, s in self.stats_domains_now.items():
means = {
'near':
self.stats_domains[id_domain].get('means_near', False),
'medium':
self.stats_domains[id_domain].get('means_medium', False),
'long':
self.stats_domains[id_domain].get('means_long', False),
'total':
self.stats_domains[id_domain].get('means_total', False),
'boot':
self.stats_domains[id_domain].get('means_boot', False)
}
update_actual_stats_domain(id_domain, s, means)
def update_domains_started_and_stopped(self,domains_with_stats):
if self.id_hyp_rethink is None:
try:
self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port))
except Exception as e:
log.error('error when hypervisor have not rethink id. {}'.format(e))
return False
l_all_domains = get_domains_with_status_in_list(list_status=['Started', 'Stopped', 'Failed'])
for d in l_all_domains:
if d['id'] in domains_with_stats:
if d['status'] == 'Started':
#if status started check if has the same hypervisor
if d['hyp_started'] != self.id_hyp_rethink:
log.error(f"Domain {d['id']} started in hypervisor ({self.id_hyp_rethink}) but database says that is started in {d['hyp_started']} !! ")
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Started in other hypervisor!! {self.id_hyp_rethink}. Updated by status thread',
hyp_id=self.id_hyp_rethink)
else:
#if status is Stopped or Failed update, the domain is started
log.info('Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread')
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread',
hyp_id=self.id_hyp_rethink)
elif d['hyp_started'] == self.id_hyp_rethink:
#Domain is started in this hypervisor in database, but is stopped
if d['status'] == 'Started':
update_domain_status(status='Stopped',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is stopped in {self.id_hyp_rethink} but in database was Started, updated by status thread',
)
def send_stats(self):
#hypervisors
send_stats_to_rethink = True
if self.id_hyp_rethink is None:
#self.id_hyp_rethink = get_id_hyp_from_uri('qemu+ssh://root@isard-hypervisor:22/system')
self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port))
if send_stats_to_rethink:
update_actual_stats_hyp(self.id_hyp_rethink,
self.stats_hyp_now)
for id_domain, s in self.stats_domains_now.items():
means = {'near': self.stats_domains[id_domain].get('means_near',False),
'medium': self.stats_domains[id_domain].get('means_medium',False),
'long': self.stats_domains[id_domain].get('means_long',False),
'total': self.stats_domains[id_domain].get('means_total',False),
'boot': self.stats_domains[id_domain].get('means_boot',False)}
update_actual_stats_domain(id_domain, s, means)
def myDomainEventCallbackRethink(conn, dom, event, detail, opaque):
now = time.time()
dom_id = dom.name()
hyp_id = get_id_hyp_from_uri(conn.getURI())
dict_event = {
'domain': dom_id,
'hyp_id': hyp_id,
'event': domEventToString(event),
'detail': domDetailToString(event, detail),
'when': now
}
logs.status.debug("EVENT: {domain} - {event} ({detail}) - {hyp}".format(
domain=dom_id,
event=dict_event['event'],
detail=dict_event['detail'],
hyp=hyp_id))
domain_status = get_domain_status(dom_id)
if domain_status is not None:
if hyp_id is None or hyp_id == '':
logs.status.debug(
'event in Hypervisor not in database with uri. hyp_id:{}, uri:{}'
.dom_id, conn.getURI())
r_status = opaque
if dict_event['event'] in ('Started', 'Resumed'):
if domain_status == 'StartingDomainDisposable' and dict_event[
'event'] == 'Resumed':
logs.status.debug(
'Event Resumed Received but waiting for Started')
elif domain_status == 'CreatingDomain' and dict_event[
'event'] == 'Started':
logs.status.debug(
'Event Started Received but waiting for Paused')
elif domain_status == 'Stopped' and dict_event[
'event'] == 'Resumed':
logs.status.debug(
'Event Resumed Received but waiting for Paused to update status in database'
)
elif domain_status == 'Started' and dict_event[
'event'] == 'Resumed':
logs.status.debug(
'Event Resumed Received but is state is started in database'
)
else:
try:
xml_started = dom.XMLDesc()
vm = DomainXML(xml_started)
spice, spice_tls, vnc, vnc_websocket = vm.get_graphics_port(
)
update_domain_viewer_started_values(
dom_id,
hyp_id=hyp_id,
spice=spice,
spice_tls=spice_tls,
vnc=vnc,
vnc_websocket=vnc_websocket)
logs.status.info(
f'DOMAIN STARTED - {dom_id} in {hyp_id} (spice: {spice} / spicetls:{spice_tls} / vnc: {vnc} / vnc_websocket: {vnc_websocket})'
)
update_domain_status(id_domain=dom_id,
status=domEventToString(event),
hyp_id=hyp_id,
detail="Event received: " +
domDetailToString(event, detail))
except Exception as e:
logs.status.debug(
'Domain {} has been destroyed while event started is processing, typical if try domain with starting paused and destroyed'
.format(dom_id))
logs.status.debug('Exception: ' + str(e))
if dict_event['event'] in ('Suspended'):
if domain_status == 'CreatingDomain' and dict_event[
'event'] == 'Suspended':
logs.status.debug(
'Event Paused Received but waiting for Stoped to update status'
)
else:
update_domain_status(id_domain=dom_id,
status='Paused',
hyp_id=hyp_id,
detail="Event received: " +
domDetailToString(event, detail))
if dict_event['event'] in ('Stopped', 'Shutdown'):
remove_domain_viewer_values(dom_id)
if get_domain_status(dict_event['domain']) != 'Stopped':
logs.status.debug(
'event {} ({}) in hypervisor {} changes status to Stopped in domain {}'
.format(dict_event['event'], dict_event['detail'], hyp_id,
dict_event['domain']))
update_domain_status(status='Stopped',
id_domain=dict_event['domain'],
hyp_id=False,
detail='Ready to Start')
r_status.insert_event_in_db(dict_event)
if dict_event['event'] in (
"Defined",
"Undefined",
# "Started",
# "Suspended",
# "Resumed",
# "Stopped",
# "Shutdown",
"PMSuspended",
"Crashed"):
logs.status.error(
'event strange, why?? event: {}, domain: {}, hyp_id: {}, detail: {}'
.format(dict_event['event'], dict_event['domain'], hyp_id,
dict_event['detail']))
else:
logs.status.info(
'domain {} launch event in hyervisor {}, but id_domain is not in database'
.format(dom_id, hyp_id))
logs.status.info('event: {}; detail: {}'.format(
domEventToString(event), domDetailToString(event, detail)))
def myDomainEventCallbackRethink(conn, dom, event, detail, opaque):
now = time.time()
dom_id = dom.name()
hyp_id = get_id_hyp_from_uri(conn.getURI())
dict_event = {'domain': dom_id,
'hyp_id': hyp_id,
'event' : domEventToString(event),
'detail': domDetailToString(event, detail),
'when' : now}
logs.status.debug("EVENT: {domain} - {event} ({detail}) - {hyp}".format(domain=dom_id,
event=dict_event['event'],
detail=dict_event['detail'],
hyp=hyp_id
))
domain_status = get_domain_status(dom_id)
if domain_status is not None:
if hyp_id is None or hyp_id == '':
logs.status.debug('event in Hypervisor not in database with uri. hyp_id:{}, uri:{}'.dom_id, conn.getURI())
r_status = opaque
if dict_event['event'] in ('Started', 'Resumed'):
if domain_status == 'StartingDomainDisposable' and dict_event['event'] == 'Resumed':
logs.status.debug('Event Resumed Received but waiting for Started')
elif domain_status == 'CreatingDomain' and dict_event['event'] == 'Started':
logs.status.debug('Event Started Received but waiting for Paused')
elif domain_status == 'Stopped' and dict_event['event'] == 'Resumed':
logs.status.debug('Event Resumed Received but waiting for Paused to update status in database')
elif domain_status == 'Started' and dict_event['event'] == 'Resumed':
logs.status.debug('Event Resumed Received but is state is started in database')
else:
try:
xml_started = dom.XMLDesc()
vm = DomainXML(xml_started)
spice, spice_tls, vnc, vnc_websocket = vm.get_graphics_port()
update_domain_viewer_started_values(dom_id, hyp_id=hyp_id,
spice=spice, spice_tls=spice_tls,
vnc = vnc,vnc_websocket=vnc_websocket)
logs.status.info(f'DOMAIN STARTED - {dom_id} in {hyp_id} (spice: {spice} / spicetls:{spice_tls} / vnc: {vnc} / vnc_websocket: {vnc_websocket})')
update_domain_status(id_domain=dom_id,
status=domEventToString(event),
hyp_id=hyp_id,
detail="Event received: " + domDetailToString(event, detail)
)
except Exception as e:
logs.status.debug(
'Domain {} has been destroyed while event started is processing, typical if try domain with starting paused and destroyed'.format(
dom_id))
logs.status.debug('Exception: ' + str(e))
if dict_event['event'] in ('Suspended'):
if domain_status == 'CreatingDomain' and dict_event['event'] == 'Suspended':
logs.status.debug('Event Paused Received but waiting for Stoped to update status')
else:
update_domain_status(id_domain=dom_id,
status='Paused',
hyp_id=hyp_id,
detail="Event received: " + domDetailToString(event, detail)
)
if dict_event['event'] in ('Stopped', 'Shutdown'):
remove_domain_viewer_values(dom_id)
if get_domain_status(dict_event['domain']) != 'Stopped':
logs.status.debug('event {} ({}) in hypervisor {} changes status to Stopped in domain {}'.format(
dict_event['event'],
dict_event['detail'],
hyp_id,
dict_event['domain']
))
update_domain_status(status='Stopped', id_domain=dict_event['domain'], hyp_id=False,
detail='Ready to Start')
r_status.insert_event_in_db(dict_event)
if dict_event['event'] in (
"Defined",
"Undefined",
# "Started",
# "Suspended",
# "Resumed",
# "Stopped",
# "Shutdown",
"PMSuspended",
"Crashed"
):
logs.status.error('event strange, why?? event: {}, domain: {}, hyp_id: {}, detail: {}'.format(
dict_event['event'],
dict_event['domain'],
hyp_id,
dict_event['detail']
))
else:
logs.status.info('domain {} launch event in hyervisor {}, but id_domain is not in database'.format(dom_id, hyp_id))
logs.status.info('event: {}; detail: {}'.format(domEventToString(event), domDetailToString(event, detail)))
