def run(self): self.tid = get_tid() logs.main.info('starting thread: {} (TID {})'.format( self.name, self.tid)) self.r_conn = new_rethink_connection() # rtable=r.table('disk_operations') # for c in r.table('hypervisors').changes(include_initial=True, include_states=True).run(r_conn): for c in r.table('hypervisors').pluck('capabilities', 'enabled', 'hostname', 'hypervisors_pools', 'port', 'user', 'viewer_hostname', 'viewer_nat_hostname').merge({'table': 'hypervisors'}).changes().\ union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes())\ .run(self.r_conn): #stop thread if self.stop is True: break if c['new_val'] is not None: if c['new_val']['table'] == 'engine': if c['new_val']['status_all_threads'] == 'Stopping': break # hypervisor deleted if c['new_val'] is None: if c['old_val'].get('table', False) == 'hypervisors': logs.main.info('hypervisor deleted in rethink') logs.main.info(pprint.pformat(c)) #TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if # domains are running and hypevisor communication have lost engine_restart() # hypervisor created elif c['old_val'] is None: if c['new_val'].get('table', False) == 'hypervisors': logs.main.info('hypervisor created in rethink') logs.main.info(pprint.pformat(c)) engine_restart() else: if c['new_val'].get('table', False) == 'hypervisors': #TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if # domains are running and hypevisor communication have lost logs.main.info('hypervisor fields modified in rethink') logs.main.info(pprint.pformat(c)) engine_restart() #self.manager.q.background.put({'type': 'add_hyp'}) self.r_conn.close()
def run(self): self.tid = get_tid() logs.main.info('starting thread: {} (TID {})'.format(self.name, self.tid)) self.r_conn = new_rethink_connection() # rtable=r.table('disk_operations') # for c in r.table('hypervisors').changes(include_initial=True, include_states=True).run(r_conn): for c in r.table('hypervisors').pluck('capabilities', 'enabled', 'hostname', 'hypervisors_pools', 'port', 'user').merge({'table': 'hypervisors'}).changes().\ union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes())\ .run(self.r_conn): #stop thread if self.stop is True: break if c['new_val'] is not None: if c['new_val']['table'] == 'engine': if c['new_val']['status_all_threads'] == 'Stopping': break # hypervisor deleted if c['new_val'] is None: if c['old_val'].get('table',False) == 'hypervisors': logs.main.info('hypervisor deleted in rethink') logs.main.info(pprint.pformat(c)) #TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if # domains are running and hypevisor communication have lost engine_restart() # hypervisor created elif c['old_val'] is None: if c['new_val'].get('table',False) == 'hypervisors': logs.main.info('hypervisor created in rethink') logs.main.info(pprint.pformat(c)) engine_restart() else: if c['new_val'].get('table', False) == 'hypervisors': #TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if # domains are running and hypevisor communication have lost logs.main.info('hypervisor fields modified in rethink') logs.main.info(pprint.pformat(c)) engine_restart() #self.manager.q.background.put({'type': 'add_hyp'}) self.r_conn.close()
def run(self): self.tid = get_tid() logs.workers.info('starting thread: {} (TID {})'.format(self.name, self.tid)) host, port, user = get_hyp_hostname_from_id(self.hyp_id) port = int(port) self.hostname = host self.h = hyp(self.hostname, user=user, port=port) self.h.get_hyp_info() if self.h.info['virtualization_bios_enabled'] is True: update_db_hyp_info(self.hyp_id, self.h.info) hyp_id = self.hyp_id while self.stop is not True: try: # do={type:'start_domain','xml':'xml','id_domain'='prova'} action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) logs.workers.debug('received action in working thread {}'.format(action['type'])) if action['type'] == 'start_paused_domain': logs.workers.debug('xml to start some lines...: {}'.format(action['xml'][30:100])) try: self.h.conn.createXML(action['xml'], flags=VIR_DOMAIN_START_PAUSED) # 32 is the constant for domains paused # reference: https://libvirt.org/html/libvirt-libvirt-domain.html#VIR_CONNECT_LIST_DOMAINS_PAUSED FLAG_LIST_DOMAINS_PAUSED = 32 list_all_domains = self.h.conn.listAllDomains(FLAG_LIST_DOMAINS_PAUSED) list_names_domains = [d.name() for d in list_all_domains] dict_domains = dict(zip(list_names_domains,list_all_domains)) if action['id_domain'] in list_names_domains: # domain started in pause mode domain = dict_domains[action['id_domain']] domain_active = True try: domain.isActive() domain.destroy() try: domain.isActive() except Exception as e: logs.workers.debug('verified domain {} is destroyed'.format(action['id_domain'])) domain_active = False except libvirtError as e: from pprint import pformat error_msg = pformat(e.get_error_message()) update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id, detail='domain {} failed when try to destroy from paused domain in hypervisor {}. creating domain operation is aborted') logs.workers.error( 'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format( action['id_domain'], self.hyp_id, error_msg)) continue if domain_active is False: # domain is destroyed, all ok update_domain_status('CreatingDomain', action['id_domain'], hyp_id='', detail='Domain created and test OK: Started, paused and now stopped in hyp {}'.format(self.hyp_id)) logs.workers.debug( 'domain {} creating operation finalished. Started paused and destroyed in hypervisor {}. Now status is Stopped. READY TO USE'.format( action['id_domain'], self.hyp_id)) else: update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id, detail='Domain is created, started in pause mode but not destroyed,creating domain operation is aborted') logs.workers.error( 'domain {} started paused but not destroyed in hypervisor {}, must be destroyed'.format( action['id_domain'], self.hyp_id)) else: update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id, detail='XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted by unknown cause'.format( action['id_domain'], self.hyp_id)) logs.workers.error( 'XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted, not exception, rare case, unknown cause'.format( action['id_domain'], self.hyp_id)) except libvirtError as e: from pprint import pformat error_msg = pformat(e.get_error_message()) update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id, detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted') logs.workers.error( 'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format( action['id_domain'], self.hyp_id, error_msg)) except Exception as e: update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id, detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted') logs.workers.error( 'Exception starting paused xml for domain {} in hypervisor {}. NOT LIBVIRT EXCEPTION, RARE CASE. Exception message: {}'.format( action['id_domain'], self.hyp_id, str(e))) ## START DOMAIN elif action['type'] == 'start_domain': logs.workers.debug('xml to start some lines...: {}'.format(action['xml'][30:100])) try: self.h.conn.createXML(action['xml']) # wait to event started to save state in database #update_domain_status('Started', action['id_domain'], hyp_id=self.hyp_id, detail='Domain has started in worker thread') logs.workers.debug('STARTED domain {}: createdXML action in hypervisor {} has been sent'.format( action['id_domain'], host)) except libvirtError as e: update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=("Hypervisor can not create domain with libvirt exception: " + str(e))) logs.workers.debug('exception in starting domain {}: '.format(e)) except Exception as e: update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=("Exception when starting domain: " + str(e))) logs.workers.debug('exception in starting domain {}: '.format(e)) ## STOP DOMAIN elif action['type'] == 'stop_domain': logs.workers.debug('action stop domain: {}'.format(action['id_domain'][30:100])) try: self.h.conn.lookupByName(action['id_domain']).destroy() logs.workers.debug('STOPPED domain {}'.format(action['id_domain'])) check_if_delete = action.get('delete_after_stopped',False) if check_if_delete is True: update_domain_status('Stopped', action['id_domain'], hyp_id='') update_domain_status('Deleting', action['id_domain'], hyp_id='') else: update_domain_status('Stopped', action['id_domain'], hyp_id='') except Exception as e: update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=str(e)) logs.workers.debug('exception in stopping domain {}: '.format(e)) elif action['type'] in ['create_disk', 'delete_disk']: launch_action_disk(action, self.hostname, user, port) elif action['type'] in ['add_media_hot']: pass elif action['type'] in ['killall_curl']: launch_killall_curl(self.hostname, user, port) elif action['type'] in ['delete_media']: final_status = action.get('final_status','Deleted') launch_delete_media (action, self.hostname, user, port, final_status=final_status) # ## DESTROY THREAD # elif action['type'] == 'destroy_thread': # list_works_in_queue = list(self.queue_actions.queue) # if self.queue_master is not None: # self.queue_master.put(['destroy_working_thread',self.hyp_id,list_works_in_queue]) # #INFO TO DEVELOPER, si entra aquí es porque no quedaba nada en cola, si no ya lo habrán matado antes # # logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id)) # # raise 'destoyed' elif action['type'] == 'create_disk': pass elif action['type'] == 'hyp_info': self.h.get_hyp_info() logs.workers.debug('hypervisor motherboard: {}'.format(self.h.info['motherboard_manufacturer'])) ## DESTROY THREAD elif action['type'] == 'stop_thread': self.stop = True else: logs.workers.error('type action {} not supported in queue actions'.format(action['type'])) # time.sleep(0.1) ## TRY DOMAIN except queue.Empty: try: self.h.conn.getLibVersion() pass # logs.workers.debug('hypervisor {} is alive'.format(host)) except: logs.workers.info('trying to reconnect hypervisor {}, alive test in working thread failed'.format(host)) alive = False for i in range(RETRIES_HYP_IS_ALIVE): try: time.sleep(TIMEOUT_BETWEEN_RETRIES_HYP_IS_ALIVE) self.h.conn.getLibVersion() alive = True logs.workers.info('hypervisor {} is alive'.format(host)) break except: logs.workers.info('hypervisor {} is NOT alive'.format(host)) if alive is False: try: self.h.connect_to_hyp() self.h.conn.getLibVersion() update_hyp_status(self.hyp_id, 'Online') except: logs.workers.debug('hypervisor {} failed'.format(host)) logs.workers.error('fail reconnecting to hypervisor {} in working thread'.format(host)) reason = self.h.fail_connected_reason update_hyp_status(self.hyp_id, 'Error', reason) update_domains_started_in_hyp_to_unknown(self.hyp_id) list_works_in_queue = list(self.queue_actions.queue) if self.queue_master is not None: self.queue_master.put(['error_working_thread', self.hyp_id, list_works_in_queue]) logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id)) self.active = False break else: update_hyp_status(self.hyp_id, 'Error','bios vmx or svm virtualization capabilities not activated') update_table_field('hypervisors',self.hyp_id,'enabled',False) logs.workers.error('hypervisor {} disabled: bios vmx or svm virtualization capabilities not activated') #restart when engine is started (not starting) timeout = 10 i = 0.0 while i < timeout: if get_engine()['status_all_threads'] == 'Started': engine_restart() break else: i + 0.2 sleep(0.2)