def update_domains_started_and_stopped(self,domains_with_stats): if self.id_hyp_rethink is None: try: self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port)) except Exception as e: log.error('error when hypervisor have not rethink id. {}'.format(e)) return False l_all_domains = get_domains_with_status_in_list(list_status=['Started', 'Stopped', 'Failed']) for d in l_all_domains: if d['id'] in domains_with_stats: if d['status'] == 'Started': #if status started check if has the same hypervisor if d['hyp_started'] != self.id_hyp_rethink: log.error(f"Domain {d['id']} started in hypervisor ({self.id_hyp_rethink}) but database says that is started in {d['hyp_started']} !! ") update_domain_status(status='Started', id_domain='_admin_downloaded_tetros', detail=f'Started in other hypervisor!! {self.id_hyp_rethink}. Updated by status thread', hyp_id=self.id_hyp_rethink) else: #if status is Stopped or Failed update, the domain is started log.info('Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread') update_domain_status(status='Started', id_domain='_admin_downloaded_tetros', detail=f'Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread', hyp_id=self.id_hyp_rethink) elif d['hyp_started'] == self.id_hyp_rethink: #Domain is started in this hypervisor in database, but is stopped if d['status'] == 'Started': update_domain_status(status='Stopped', id_domain='_admin_downloaded_tetros', detail=f'Domain is stopped in {self.id_hyp_rethink} but in database was Started, updated by status thread', )
def send_stats(self): #hypervisors send_stats_to_rethink = True if self.id_hyp_rethink is None: #self.id_hyp_rethink = get_id_hyp_from_uri('qemu+ssh://root@isard-hypervisor:22/system') self.id_hyp_rethink = get_id_hyp_from_uri( hostname_to_uri(self.hostname, user=self.user, port=self.port)) if send_stats_to_rethink: update_actual_stats_hyp(self.id_hyp_rethink, self.stats_hyp_now) for id_domain, s in self.stats_domains_now.items(): means = { 'near': self.stats_domains[id_domain].get('means_near', False), 'medium': self.stats_domains[id_domain].get('means_medium', False), 'long': self.stats_domains[id_domain].get('means_long', False), 'total': self.stats_domains[id_domain].get('means_total', False), 'boot': self.stats_domains[id_domain].get('means_boot', False) } update_actual_stats_domain(id_domain, s, means)
def update_domains_started_and_stopped(self,domains_with_stats): if self.id_hyp_rethink is None: try: self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port)) except Exception as e: log.error('error when hypervisor have not rethink id. {}'.format(e)) return False l_all_domains = get_domains_with_status_in_list(list_status=['Started', 'Stopped', 'Failed']) for d in l_all_domains: if d['id'] in domains_with_stats: if d['status'] == 'Started': #if status started check if has the same hypervisor if d['hyp_started'] != self.id_hyp_rethink: log.error(f"Domain {d['id']} started in hypervisor ({self.id_hyp_rethink}) but database says that is started in {d['hyp_started']} !! ") update_domain_status(status='Started', id_domain='_admin_downloaded_tetros', detail=f'Started in other hypervisor!! {self.id_hyp_rethink}. Updated by status thread', hyp_id=self.id_hyp_rethink) else: #if status is Stopped or Failed update, the domain is started log.info('Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread') update_domain_status(status='Started', id_domain='_admin_downloaded_tetros', detail=f'Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread', hyp_id=self.id_hyp_rethink) elif d['hyp_started'] == self.id_hyp_rethink: #Domain is started in this hypervisor in database, but is stopped if d['status'] == 'Started': update_domain_status(status='Stopped', id_domain='_admin_downloaded_tetros', detail=f'Domain is stopped in {self.id_hyp_rethink} but in database was Started, updated by status thread', )
def send_stats(self): #hypervisors send_stats_to_rethink = True if self.id_hyp_rethink is None: #self.id_hyp_rethink = get_id_hyp_from_uri('qemu+ssh://root@isard-hypervisor:22/system') self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port)) if send_stats_to_rethink: update_actual_stats_hyp(self.id_hyp_rethink, self.stats_hyp_now) for id_domain, s in self.stats_domains_now.items(): means = {'near': self.stats_domains[id_domain].get('means_near',False), 'medium': self.stats_domains[id_domain].get('means_medium',False), 'long': self.stats_domains[id_domain].get('means_long',False), 'total': self.stats_domains[id_domain].get('means_total',False), 'boot': self.stats_domains[id_domain].get('means_boot',False)} update_actual_stats_domain(id_domain, s, means)
def myDomainEventCallbackRethink(conn, dom, event, detail, opaque): now = time.time() dom_id = dom.name() hyp_id = get_id_hyp_from_uri(conn.getURI()) dict_event = { 'domain': dom_id, 'hyp_id': hyp_id, 'event': domEventToString(event), 'detail': domDetailToString(event, detail), 'when': now } logs.status.debug("EVENT: {domain} - {event} ({detail}) - {hyp}".format( domain=dom_id, event=dict_event['event'], detail=dict_event['detail'], hyp=hyp_id)) domain_status = get_domain_status(dom_id) if domain_status is not None: if hyp_id is None or hyp_id == '': logs.status.debug( 'event in Hypervisor not in database with uri. hyp_id:{}, uri:{}' .dom_id, conn.getURI()) r_status = opaque if dict_event['event'] in ('Started', 'Resumed'): if domain_status == 'StartingDomainDisposable' and dict_event[ 'event'] == 'Resumed': logs.status.debug( 'Event Resumed Received but waiting for Started') elif domain_status == 'CreatingDomain' and dict_event[ 'event'] == 'Started': logs.status.debug( 'Event Started Received but waiting for Paused') elif domain_status == 'Stopped' and dict_event[ 'event'] == 'Resumed': logs.status.debug( 'Event Resumed Received but waiting for Paused to update status in database' ) elif domain_status == 'Started' and dict_event[ 'event'] == 'Resumed': logs.status.debug( 'Event Resumed Received but is state is started in database' ) else: try: xml_started = dom.XMLDesc() vm = DomainXML(xml_started) spice, spice_tls, vnc, vnc_websocket = vm.get_graphics_port( ) update_domain_viewer_started_values( dom_id, hyp_id=hyp_id, spice=spice, spice_tls=spice_tls, vnc=vnc, vnc_websocket=vnc_websocket) logs.status.info( f'DOMAIN STARTED - {dom_id} in {hyp_id} (spice: {spice} / spicetls:{spice_tls} / vnc: {vnc} / vnc_websocket: {vnc_websocket})' ) update_domain_status(id_domain=dom_id, status=domEventToString(event), hyp_id=hyp_id, detail="Event received: " + domDetailToString(event, detail)) except Exception as e: logs.status.debug( 'Domain {} has been destroyed while event started is processing, typical if try domain with starting paused and destroyed' .format(dom_id)) logs.status.debug('Exception: ' + str(e)) if dict_event['event'] in ('Suspended'): if domain_status == 'CreatingDomain' and dict_event[ 'event'] == 'Suspended': logs.status.debug( 'Event Paused Received but waiting for Stoped to update status' ) else: update_domain_status(id_domain=dom_id, status='Paused', hyp_id=hyp_id, detail="Event received: " + domDetailToString(event, detail)) if dict_event['event'] in ('Stopped', 'Shutdown'): remove_domain_viewer_values(dom_id) if get_domain_status(dict_event['domain']) != 'Stopped': logs.status.debug( 'event {} ({}) in hypervisor {} changes status to Stopped in domain {}' .format(dict_event['event'], dict_event['detail'], hyp_id, dict_event['domain'])) update_domain_status(status='Stopped', id_domain=dict_event['domain'], hyp_id=False, detail='Ready to Start') r_status.insert_event_in_db(dict_event) if dict_event['event'] in ( "Defined", "Undefined", # "Started", # "Suspended", # "Resumed", # "Stopped", # "Shutdown", "PMSuspended", "Crashed"): logs.status.error( 'event strange, why?? event: {}, domain: {}, hyp_id: {}, detail: {}' .format(dict_event['event'], dict_event['domain'], hyp_id, dict_event['detail'])) else: logs.status.info( 'domain {} launch event in hyervisor {}, but id_domain is not in database' .format(dom_id, hyp_id)) logs.status.info('event: {}; detail: {}'.format( domEventToString(event), domDetailToString(event, detail)))
def myDomainEventCallbackRethink(conn, dom, event, detail, opaque): now = time.time() dom_id = dom.name() hyp_id = get_id_hyp_from_uri(conn.getURI()) dict_event = {'domain': dom_id, 'hyp_id': hyp_id, 'event' : domEventToString(event), 'detail': domDetailToString(event, detail), 'when' : now} logs.status.debug("EVENT: {domain} - {event} ({detail}) - {hyp}".format(domain=dom_id, event=dict_event['event'], detail=dict_event['detail'], hyp=hyp_id )) domain_status = get_domain_status(dom_id) if domain_status is not None: if hyp_id is None or hyp_id == '': logs.status.debug('event in Hypervisor not in database with uri. hyp_id:{}, uri:{}'.dom_id, conn.getURI()) r_status = opaque if dict_event['event'] in ('Started', 'Resumed'): if domain_status == 'StartingDomainDisposable' and dict_event['event'] == 'Resumed': logs.status.debug('Event Resumed Received but waiting for Started') elif domain_status == 'CreatingDomain' and dict_event['event'] == 'Started': logs.status.debug('Event Started Received but waiting for Paused') elif domain_status == 'Stopped' and dict_event['event'] == 'Resumed': logs.status.debug('Event Resumed Received but waiting for Paused to update status in database') elif domain_status == 'Started' and dict_event['event'] == 'Resumed': logs.status.debug('Event Resumed Received but is state is started in database') else: try: xml_started = dom.XMLDesc() vm = DomainXML(xml_started) spice, spice_tls, vnc, vnc_websocket = vm.get_graphics_port() update_domain_viewer_started_values(dom_id, hyp_id=hyp_id, spice=spice, spice_tls=spice_tls, vnc = vnc,vnc_websocket=vnc_websocket) logs.status.info(f'DOMAIN STARTED - {dom_id} in {hyp_id} (spice: {spice} / spicetls:{spice_tls} / vnc: {vnc} / vnc_websocket: {vnc_websocket})') update_domain_status(id_domain=dom_id, status=domEventToString(event), hyp_id=hyp_id, detail="Event received: " + domDetailToString(event, detail) ) except Exception as e: logs.status.debug( 'Domain {} has been destroyed while event started is processing, typical if try domain with starting paused and destroyed'.format( dom_id)) logs.status.debug('Exception: ' + str(e)) if dict_event['event'] in ('Suspended'): if domain_status == 'CreatingDomain' and dict_event['event'] == 'Suspended': logs.status.debug('Event Paused Received but waiting for Stoped to update status') else: update_domain_status(id_domain=dom_id, status='Paused', hyp_id=hyp_id, detail="Event received: " + domDetailToString(event, detail) ) if dict_event['event'] in ('Stopped', 'Shutdown'): remove_domain_viewer_values(dom_id) if get_domain_status(dict_event['domain']) != 'Stopped': logs.status.debug('event {} ({}) in hypervisor {} changes status to Stopped in domain {}'.format( dict_event['event'], dict_event['detail'], hyp_id, dict_event['domain'] )) update_domain_status(status='Stopped', id_domain=dict_event['domain'], hyp_id=False, detail='Ready to Start') r_status.insert_event_in_db(dict_event) if dict_event['event'] in ( "Defined", "Undefined", # "Started", # "Suspended", # "Resumed", # "Stopped", # "Shutdown", "PMSuspended", "Crashed" ): logs.status.error('event strange, why?? event: {}, domain: {}, hyp_id: {}, detail: {}'.format( dict_event['event'], dict_event['domain'], hyp_id, dict_event['detail'] )) else: logs.status.info('domain {} launch event in hyervisor {}, but id_domain is not in database'.format(dom_id, hyp_id)) logs.status.info('event: {}; detail: {}'.format(domEventToString(event), domDetailToString(event, detail)))