def run(self): self.tid = get_tid() logs.main.info('starting thread: {} (TID {})'.format(self.name, self.tid)) #get hostname grafana config self.get_hostname_grafana() hyps_online = [] elapsed = self.send_static_values_interval while self.stop is False: sleep(self.send_to_grafana_interval) elapsed += self.send_to_grafana_interval if self.restart_send_config is True: self.restart_send_config = False self.get_hostname_grafana() if self.active is True: for i,id_hyp in enumerate(self.t_status.keys()): try: if self.t_status[id_hyp].status_obj.hyp_obj.connected is True: if id_hyp not in hyps_online: hyps_online.append(id_hyp) check_hyp = True except: logs.main.error(f'hypervisor {id_hyp} problem checking if is connected') check_hyp = False if len(hyps_online) > 0 and check_hyp is True: #send static values of hypervisors if elapsed >= self.send_static_values_interval: d_hyps_info = dict() for i, id_hyp in enumerate(hyps_online): d_hyps_info[f'hyp-info-{i}'] = self.t_status[id_hyp].status_obj.hyp_obj.info # ~ self.send(d_hyps_info) elapsed = 0 #send stats dict_to_send = dict() j=0 for i, id_hyp in enumerate(hyps_online): if id_hyp in self.t_status.keys(): #stats_hyp = self.t_status[id_hyp].status_obj.hyp_obj.stats_hyp stats_hyp_now = self.t_status[id_hyp].status_obj.hyp_obj.stats_hyp_now #stats_domains = self.t_status[id_hyp].status_obj.hyp_obj.stats_domains if len(stats_hyp_now) > 0: dict_to_send[f'hypers.'+id_hyp] = {'stats':stats_hyp_now,'info':d_hyps_info['hyp-info-'+str(i)],'domains':{}} stats_domains_now = self.t_status[id_hyp].status_obj.hyp_obj.stats_domains_now # ~ for id_domain,d_stats in stats_domains_now.items(): # ~ if len(stats_hyp_now) > 0: # ~ for id_domain,d_stats in stats_domains_now.items(): # ~ dict_to_send[f'domain-stats-{j}'] = {'domain-id':{id_domain:1},'last': d_stats,} dict_to_send[f'hypers.'+id_hyp]['domains']=stats_domains_now #{x:0 for x in stats_domains_now} # ~ print(stats_domains_now) # ~ j+=1 if len(dict_to_send) > 0: self.send(dict_to_send)
def disk_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands if action['type'] in ['create_disk']: launch_action_disk(action, self.hostname, self.user, self.port) elif action['type'] in ['create_disk_from_scratch']: launch_action_disk(action, self.hostname, self.user, self.port, from_scratch=True) elif action['type'] in ['delete_disk']: launch_delete_disk_action(action, self.hostname, self.user, self.port) elif action['type'] in ['create_template_disk_from_domain']: launch_action_create_template_disk(action, self.hostname, self.user, self.port) elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format(action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format( disk_path, id_domain)) log.error('\n'.join( ['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))])) update_domain_status('Failed', id_domain, detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def run(self): self.tid = get_tid() log.info('starting thread: {} (TID {})'.format(self.name, self.tid)) while self.stop is not True: self.hyp_obj.get_load() interval = 0.0 while interval < self.polling_interval: time.sleep(0.1) interval += 0.1 if self.stop is True: break
def run(self): self.tid = get_tid() log.info('starting thread: {} (TID {})'.format(self.name, self.tid)) while self.stop is not True: self.hyp_obj.get_load() interval = 0.0 while interval < self.polling_interval: time.sleep(0.1) interval += 0.1 if self.stop is True: break
def run(self): self.tid = get_tid() log.info('starting thread: {} (TID {})'.format(self.name, self.tid)) self.hyp_obj, self.ok = try_hyp_connection(self.hyp_id, self.hostname, self.port, self.user) log.debug('Exiting from thread {} try_hyp {}'.format( self.name, self.hostname)) return self.ok
def run(self): self.tid = get_tid() logs.main.info('starting thread: {} (TID {})'.format( self.name, self.tid)) self.r_conn = new_rethink_connection() # rtable=r.table('disk_operations') # for c in r.table('hypervisors').changes(include_initial=True, include_states=True).run(r_conn): for c in r.table('hypervisors').pluck('capabilities', 'enabled', 'hostname', 'hypervisors_pools', 'port', 'user', 'viewer_hostname', 'viewer_nat_hostname').merge({'table': 'hypervisors'}).changes().\ union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes())\ .run(self.r_conn): #stop thread if self.stop is True: break if c['new_val'] is not None: if c['new_val']['table'] == 'engine': if c['new_val']['status_all_threads'] == 'Stopping': break # hypervisor deleted if c['new_val'] is None: if c['old_val'].get('table', False) == 'hypervisors': logs.main.info('hypervisor deleted in rethink') logs.main.info(pprint.pformat(c)) #TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if # domains are running and hypevisor communication have lost engine_restart() # hypervisor created elif c['old_val'] is None: if c['new_val'].get('table', False) == 'hypervisors': logs.main.info('hypervisor created in rethink') logs.main.info(pprint.pformat(c)) engine_restart() else: if c['new_val'].get('table', False) == 'hypervisors': #TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if # domains are running and hypevisor communication have lost logs.main.info('hypervisor fields modified in rethink') logs.main.info(pprint.pformat(c)) engine_restart() #self.manager.q.background.put({'type': 'add_hyp'}) self.r_conn.close()
def run(self): self.tid = get_tid() logs.main.info('starting thread: {} (TID {})'.format(self.name, self.tid)) self.r_conn = new_rethink_connection() # rtable=r.table('disk_operations') # for c in r.table('hypervisors').changes(include_initial=True, include_states=True).run(r_conn): for c in r.table('hypervisors').pluck('capabilities', 'enabled', 'hostname', 'hypervisors_pools', 'port', 'user').merge({'table': 'hypervisors'}).changes().\ union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes())\ .run(self.r_conn): #stop thread if self.stop is True: break if c['new_val'] is not None: if c['new_val']['table'] == 'engine': if c['new_val']['status_all_threads'] == 'Stopping': break # hypervisor deleted if c['new_val'] is None: if c['old_val'].get('table',False) == 'hypervisors': logs.main.info('hypervisor deleted in rethink') logs.main.info(pprint.pformat(c)) #TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if # domains are running and hypevisor communication have lost engine_restart() # hypervisor created elif c['old_val'] is None: if c['new_val'].get('table',False) == 'hypervisors': logs.main.info('hypervisor created in rethink') logs.main.info(pprint.pformat(c)) engine_restart() else: if c['new_val'].get('table', False) == 'hypervisors': #TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if # domains are running and hypevisor communication have lost logs.main.info('hypervisor fields modified in rethink') logs.main.info(pprint.pformat(c)) engine_restart() #self.manager.q.background.put({'type': 'add_hyp'}) self.r_conn.close()
def run(self): # Close connection on exit (to test cleanup paths) self.tid = get_tid() logs.status.info('starting thread: {} (TID {})'.format( self.name, self.tid)) old_exitfunc = getattr(sys, 'exitfunc', None) def exit(): logs.status.info('Closing hypervisors connexions') for hyp_id, hostname in self.hyps.items(): self.hyps_conn[hyp_id].close() if (old_exitfunc): old_exitfunc() sys.exitfunc = exit self.thread_event_loop = virEventLoopNativeStart(self.stop_event_loop) # self.r_status = RethinkHypEvent() while self.stop is not True: try: action = self.q_event_register.get( timeout=TIMEOUT_QUEUE_REGISTER_EVENTS) if action['type'] in ['add_hyp_to_receive_events']: hyp_id = action['hyp_id'] self.add_hyp_to_receive_events(hyp_id) elif action['type'] in ['del_hyp_to_receive_events']: hyp_id = action['hyp_id'] self.del_hyp_to_receive_events(hyp_id) elif action['type'] == 'stop_thread': self.stop = True else: logs.status.error('type action {} not supported'.format( action['type'])) except queue.Empty: pass except Exception as e: log.error( 'Exception in ThreadHypEvents main loop: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False self.stop_event_loop[0] = True while self.thread_event_loop.is_alive(): pass
def run(self): # Close connection on exit (to test cleanup paths) self.tid = get_tid() logs.status.info('starting thread: {} (TID {})'.format(self.name, self.tid)) old_exitfunc = getattr(sys, 'exitfunc', None) def exit(): logs.status.info('Closing hypervisors connexions') for hyp_id, hostname in self.hyps.items(): self.hyps_conn[hyp_id].close() if (old_exitfunc): old_exitfunc() sys.exitfunc = exit self.thread_event_loop = virEventLoopNativeStart(self.stop_event_loop) # self.r_status = RethinkHypEvent() while self.stop is not True: try: action = self.q_event_register.get(timeout=TIMEOUT_QUEUE_REGISTER_EVENTS) if action['type'] in ['add_hyp_to_receive_events']: hyp_id = action['hyp_id'] self.add_hyp_to_receive_events(hyp_id) elif action['type'] in ['del_hyp_to_receive_events']: hyp_id = action['hyp_id'] self.del_hyp_to_receive_events(hyp_id) elif action['type'] == 'stop_thread': self.stop = True else: logs.status.error('type action {} not supported'.format(action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception in ThreadHypEvents main loop: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False self.stop_event_loop[0] = True while self.thread_event_loop.is_alive(): pass
def __init__(self, launch_threads=True, with_status_threads=True, status_polling_interval=STATUS_POLLING_INTERVAL, test_hyp_fail_interval=TEST_HYP_FAIL_INTERVAL): logs.main.info('MAIN TID: {}'.format(get_tid())) self.time_between_polling = TIME_BETWEEN_POLLING self.polling_interval_background = POLLING_INTERVAL_BACKGROUND self.with_status_threads = with_status_threads self.q = self.QueuesThreads() self.t_workers = {} self.t_status = {} self.pools = {} self.t_disk_operations = {} self.q_disk_operations = {} self.t_long_operations = {} self.q_long_operations = {} self.t_changes_hyps = None self.t_events = None self.t_changes_domains = None self.t_broom = None self.t_background = None self.t_downloads_changes = None self.t_grafana = None self.quit = False self.threads_info_main = {} self.threads_info_hyps = {} self.hypers_disk_operations_tested = [] self.num_workers = 0 self.threads_main_started = False self.STATUS_POLLING_INTERVAL = status_polling_interval self.TEST_HYP_FAIL_INTERVAL = test_hyp_fail_interval update_all_hyps_status(reset_status='Offline') if launch_threads is True: self.launch_thread_background_polling()
def __init__(self, launch_threads=True, with_status_threads=True, status_polling_interval=STATUS_POLLING_INTERVAL, test_hyp_fail_interval=TEST_HYP_FAIL_INTERVAL): logs.main.info('MAIN TID: {}'.format(get_tid())) self.time_between_polling = TIME_BETWEEN_POLLING self.polling_interval_background = POLLING_INTERVAL_BACKGROUND self.with_status_threads = with_status_threads self.q = self.QueuesThreads() self.t_workers = {} self.t_status = {} self.pools = {} self.t_disk_operations = {} self.q_disk_operations = {} self.t_long_operations = {} self.q_long_operations = {} self.t_changes_hyps = None self.t_events = None self.t_changes_domains = None self.t_broom = None self.t_background = None self.t_downloads_changes = None self.t_grafana = None self.quit = False self.threads_info_main = {} self.threads_info_hyps = {} self.hypers_disk_operations_tested = [] self.num_workers = 0 self.threads_main_started = False self.STATUS_POLLING_INTERVAL = status_polling_interval self.TEST_HYP_FAIL_INTERVAL = test_hyp_fail_interval update_all_hyps_status(reset_status='Offline') if launch_threads is True: self.launch_thread_background_polling()
def run(self): # Close connection on exit (to test cleanup paths) self.tid = get_tid() logs.status.info('starting thread: {} (TID {})'.format( self.name, self.tid)) old_exitfunc = getattr(sys, 'exitfunc', None) def exit(): logs.status.info('Closing hypervisors connexions') for hyp_id, hostname in self.hyps.items(): self.hyps_conn[hyp_id].close() if (old_exitfunc): old_exitfunc() sys.exitfunc = exit # self.r_status = RethinkHypEvent() while True: if len(self.hyps) == 0: if self.stop: break time.sleep(0.1) else: self.thread_event_loop = virEventLoopNativeStart( self.stop_event_loop) for hyp_id, hostname in self.hyps.items(): self.add_hyp_to_receive_events(hyp_id) while self.stop is not True: time.sleep(0.1) if self.stop is True: for hyp_id in list(self.hyps): self.del_hyp_to_receive_events(hyp_id) self.stop_event_loop[0] = True while self.thread_event_loop.is_alive(): pass break
def run(self): self.tid = get_tid() logs.broom.info('starting thread: {} (TID {})'.format(self.name, self.tid)) self.polling()
def long_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug( 'Thread to launchdisks operations in host {} with TID: {}...'. format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands id_domain = action['domain'] if action['type'] in ['create_disk_virt_builder']: cmds_done = execute_commands( host=self.hostname, ssh_commands=action['ssh_commands'], dict_mode=True, user=self.user, port=self.port) if len([d for d in cmds_done if len(d['err']) > 0]) > 1: log.error('some error in virt builder operations') log.error( 'Virt Builder Failed creating disk file {} in domain {} in hypervisor {}' .format(action['disk_path'], action['domain'], self.hyp_id)) log.debug('print cmds_done:') log.debug(pprint.pprint(cmds_done)) log.debug('print ssh_commands:') log.debug(pprint.pprint(action['ssh_commands'])) update_domain_status( 'Failed', id_domain, detail='Virt Builder Failed creating disk file') else: log.info( 'Disk created from virt-builder. Domain: {} , disk: {}' .format(action['domain'], action['disk_path'])) xml_virt_install = cmds_done[-1]['out'] update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install) update_domain_status( 'CreatingDomainFromBuilder', id_domain, detail='disk created from virt-builder') elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format( action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}' .format(disk_path, id_domain)) log.error('\n'.join([ 'cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands'])) ])) update_domain_status( 'Failed', id_domain, detail= 'new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs' )
def run(self): self.tid = get_tid() logs.downloads.debug( 'RUN-DOWNLOAD-THREAD-------------------------------------') pool_id = 'default' first_loop = True if self.stop is False: if first_loop is True: # if domains or media have status Downloading when engine restart # we need to resetdownloading deleting file and first_loop = False # wait a hyp to downloads next_hyp = False while next_hyp is False: logs.downloads.info( 'waiting an hypervisor online to launch downloading actions' ) if pool_id in self.manager.pools.keys(): next_hyp = self.manager.pools[pool_id].get_next() sleep(1) for hyp_id in get_hypers_in_pool(): self.killall_curl(hyp_id) domains_status_downloading = get_domains_with_status( 'Downloading') medias_status_downloading = get_media_with_status( 'Downloading') for id_domain in domains_status_downloading: create_dict = get_domain(id_domain)['create_dict'] dict_changes = { 'id': id_domain, 'table': 'domains', 'create_dict': create_dict } update_domain_status('ResetDownloading', id_domain) self.abort_download(dict_changes, final_status='DownloadFailed') for id_media in medias_status_downloading: dict_media = get_media(id_media) dict_changes = { 'id': id_media, 'table': 'media', 'path': dict_media['path'], 'hypervisors_pools': dict_media['hypervisors_pools'] } update_status_table('media', 'ResetDownloading', id_media) self.abort_download(dict_changes, final_status='DownloadFailed') self.r_conn = new_rethink_connection() update_table_field('hypervisors_pools', pool_id, 'download_changes', 'Started') for c in r.table('media').get_all(r.args( ['Deleting', 'Deleted', 'Downloaded', 'DownloadFailed', 'DownloadStarting', 'Downloading', 'Download', 'DownloadAborting','ResetDownloading']), index='status'). \ pluck('id', 'path', 'url-isard', 'url-web', 'status' ).merge( {'table': 'media'}).changes(include_initial=True).union( r.table('domains').get_all( r.args(['Downloaded', 'DownloadFailed','DownloadStarting', 'Downloading', 'DownloadAborting','ResetDownloading']), index='status'). \ pluck('id', 'create_dict', 'url-isard', 'url-web', 'status').merge( {"table": "domains"}).changes(include_initial=True)).union( r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).run( self.r_conn): if self.stop: break if c.get('new_val', None) is not None: if c['new_val'].get('table', False) == 'engine': if c['new_val']['status_all_threads'] == 'Stopping': break else: continue logs.downloads.debug('DOWNLOAD CHANGES DETECTED:') logs.downloads.debug(pprint.pformat(c)) if c.get('old_val', None) is None: if c['new_val']['status'] == 'DownloadStarting': self.start_download(c['new_val']) elif c.get('new_val', None) is None: if c['old_val']['status'] in ['DownloadAborting']: self.remove_download_thread(c['old_val']) elif 'old_val' in c and 'new_val' in c: if c['old_val']['status'] == 'DownloadFailed' and c[ 'new_val']['status'] == 'DownloadStarting': self.start_download(c['new_val']) elif c['old_val']['status'] == 'Downloaded' and c[ 'new_val']['status'] == 'Deleting': if c['new_val']['table'] == 'media': self.delete_media(c['new_val']) elif c['old_val']['status'] == 'Deleting' and c['new_val'][ 'status'] == 'Deleted': if c['new_val']['table'] == 'media': remove_media(c['new_val']['id']) elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'DownloadFailed': pass elif c['old_val']['status'] == 'DownloadStarting' and c[ 'new_val']['status'] == 'Downloading': pass elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'Downloaded': pass elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'DownloadAborting': self.abort_download(c['new_val']) elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'ResetDownloading': self.abort_download(c['new_val'], final_status='DownloadFailed')
def run(self): self.tid = get_tid() logs.main.info('starting thread background: {} (TID {})'.format(self.name, self.tid)) q = self.manager.q.background first_loop = True pool_id = 'default' #can't launch downloads if download changes thread is not ready and hyps are not online update_table_field('hypervisors_pools', pool_id, 'download_changes', 'Stopped') # if domains have intermedite states (updating, download_aborting...) # to Failed or Delete clean_intermediate_status() l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True) # while len(l_hyps_to_test) == 0: # logs.main.error('no hypervisor enable, waiting for one hypervisor') # sleep(0.5) # l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True) while self.manager.quit is False: #################################################################### ### MAIN LOOP ###################################################### # ONLY FOR DEBUG logs.main.debug('##### THREADS ##################') threads_running = get_threads_running() #pprint.pprint(threads_running) #self.manager.update_info_threads_engine() # Threads that must be running always, with or withouth hypervisor: # - changes_hyp # - changes_domains # - downloads_changes # - broom # - events # - grafana # Threads that depends on hypervisors availavility: # - disk_operations # - long_operations # - for every hypervisor: # - worker # - status # LAUNCH MAIN THREADS if first_loop is True: update_table_field('engine', 'engine', 'status_all_threads', 'Starting') # launch changes_hyp thread self.manager.t_changes_hyps = self.manager.HypervisorChangesThread('changes_hyp', self.manager) self.manager.t_changes_hyps.daemon = True self.manager.t_changes_hyps.start() #launch changes_domains_thread self.manager.t_changes_domains = self.manager.DomainsChangesThread('changes_domains', self.manager) self.manager.t_changes_domains.daemon = True self.manager.t_changes_domains.start() #launch downloads changes thread logs.main.debug('Launching Download Changes Thread') self.manager.t_downloads_changes = launch_thread_download_changes(self.manager) #launch brom thread self.manager.t_broom = launch_thread_broom(self.manager) #launch events thread logs.main.debug('launching hypervisor events thread') self.manager.t_events = launch_thread_hyps_event() #launch grafana thread logs.main.debug('launching grafana thread') self.manager.t_grafana = launch_grafana_thread(self.manager.t_status) logs.main.info('THREADS LAUNCHED FROM BACKGROUND THREAD') update_table_field('engine', 'engine', 'status_all_threads', 'Starting') while True: #wait all sleep(0.1) self.manager.update_info_threads_engine() #if len(self.manager.threads_info_main['not_defined']) > 0 and len(self.manager.dict_hyps_ready) == 0: if len(self.manager.threads_info_main['not_defined']) > 0 or len(self.manager.threads_info_main['dead']) > 0: print('MAIN THREADS starting, wait a second extra') sleep(1) self.manager.update_info_threads_engine() pprint.pprint(self.manager.threads_info_main) #self.test_hyps_and_start_threads() if len(self.manager.threads_info_main['not_defined']) == 0 and len(self.manager.threads_info_main['dead']) == 0: update_table_field('engine', 'engine', 'status_all_threads', 'Started') self.manager.threads_main_started = True break # TEST HYPS AND START THREADS FOR HYPERVISORS self.test_hyps_and_start_threads() self.manager.num_workers = len(self.manager.t_workers) # Test hypervisor disk operations # Create Test disk in hypervisor disk operations if first_loop is True: first_loop = False # virtio_test_disk_relative_path = 'admin/admin/admin/virtio_testdisk.qcow2' # ui.creating_test_disk(test_disk_relative_route=virtio_test_disk_relative_path) self.manager.update_info_threads_engine() if len(self.manager.threads_info_hyps['not_defined']) > 0: logs.main.error('something was wrong when launching threads for hypervisors, threads not defined') logs.main.error(pprint.pformat(self.manager.threads_info_hyps)) if len(self.manager.threads_info_hyps['dead']) > 0: logs.main.error('something was wrong when launching threads for hypervisors, threads are dead') logs.main.error(pprint.pformat(self.manager.threads_info_hyps)) if len(self.manager.threads_info_hyps['dead']) == 0 and len(self.manager.threads_info_hyps['not_defined']) == 0: pass try: if len(self.manager.t_workers) == 0: timeout_queue = WAIT_HYP_ONLINE else: timeout_queue = TEST_HYP_FAIL_INTERVAL action = q.get(timeout=timeout_queue) if action['type'] == 'stop': self.manager.quit = True logs.main.info('engine end') except queue.Empty: pass except Exception as e: logs.main.error(e) return False
def run(self): self.tid = get_tid() log.info('starting thread: {} (TID {})'.format(self.name, self.tid)) self.disk_operations_thread()
def run(self): self.tid = get_tid() logs.main.info('starting thread background: {} (TID {})'.format(self.name, self.tid)) q = self.manager.q.background first_loop = True pool_id = 'default' #can't launch downloads if download changes thread is not ready and hyps are not online update_table_field('hypervisors_pools', pool_id, 'download_changes', 'Stopped') # if domains have intermedite states (updating, download_aborting...) # to Failed or Delete clean_intermediate_status() l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True) # while len(l_hyps_to_test) == 0: # logs.main.error('no hypervisor enable, waiting for one hypervisor') # sleep(0.5) # l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True) while self.manager.quit is False: #################################################################### ### MAIN LOOP ###################################################### # ONLY FOR DEBUG logs.main.debug('##### THREADS ##################') threads_running = get_threads_running() #pprint.pprint(threads_running) #self.manager.update_info_threads_engine() # Threads that must be running always, with or withouth hypervisor: # - changes_hyp # - changes_domains # - downloads_changes # - broom # - events # - grafana # Threads that depends on hypervisors availavility: # - disk_operations # - long_operations # - for every hypervisor: # - worker # - status # LAUNCH MAIN THREADS if first_loop is True: update_table_field('engine', 'engine', 'status_all_threads', 'Starting') # launch changes_hyp thread self.manager.t_changes_hyps = self.manager.HypervisorChangesThread('changes_hyp', self.manager) self.manager.t_changes_hyps.daemon = True self.manager.t_changes_hyps.start() #launch changes_domains_thread self.manager.t_changes_domains = self.manager.DomainsChangesThread('changes_domains', self.manager) self.manager.t_changes_domains.daemon = True self.manager.t_changes_domains.start() #launch downloads changes thread logs.main.debug('Launching Download Changes Thread') self.manager.t_downloads_changes = launch_thread_download_changes(self.manager) #launch brom thread self.manager.t_broom = launch_thread_broom(self.manager) #launch events thread logs.main.debug('launching hypervisor events thread') self.manager.t_events = launch_thread_hyps_event() #launch grafana thread logs.main.debug('launching grafana thread') self.manager.t_grafana = launch_grafana_thread(self.manager.t_status) logs.main.info('THREADS LAUNCHED FROM BACKGROUND THREAD') update_table_field('engine', 'engine', 'status_all_threads', 'Starting') while True: #wait all sleep(0.1) self.manager.update_info_threads_engine() #if len(self.manager.threads_info_main['not_defined']) > 0 and len(self.manager.dict_hyps_ready) == 0: if len(self.manager.threads_info_main['not_defined']) > 0 or len(self.manager.threads_info_main['dead']) > 0: print('MAIN THREADS starting, wait a second extra') sleep(1) self.manager.update_info_threads_engine() pprint.pprint(self.manager.threads_info_main) #self.test_hyps_and_start_threads() if len(self.manager.threads_info_main['not_defined']) == 0 and len(self.manager.threads_info_main['dead']) == 0: update_table_field('engine', 'engine', 'status_all_threads', 'Started') self.manager.threads_main_started = True break # TEST HYPS AND START THREADS FOR HYPERVISORS self.test_hyps_and_start_threads() self.manager.num_workers = len(self.manager.t_workers) # Test hypervisor disk operations # Create Test disk in hypervisor disk operations if first_loop is True: first_loop = False # virtio_test_disk_relative_path = 'admin/admin/admin/virtio_testdisk.qcow2' # ui.creating_test_disk(test_disk_relative_route=virtio_test_disk_relative_path) self.manager.update_info_threads_engine() if len(self.manager.threads_info_hyps['not_defined']) > 0: logs.main.error('something was wrong when launching threads for hypervisors, threads not defined') logs.main.error(pprint.pformat(self.manager.threads_info_hyps)) if len(self.manager.threads_info_hyps['dead']) > 0: logs.main.error('something was wrong when launching threads for hypervisors, threads are dead') logs.main.error(pprint.pformat(self.manager.threads_info_hyps)) if len(self.manager.threads_info_hyps['dead']) == 0 and len(self.manager.threads_info_hyps['not_defined']) == 0: pass try: if len(self.manager.t_workers) == 0: timeout_queue = WAIT_HYP_ONLINE else: timeout_queue = TEST_HYP_FAIL_INTERVAL action = q.get(timeout=timeout_queue) if action['type'] == 'stop': self.manager.quit = True logs.main.info('engine end') except queue.Empty: pass except Exception as e: logs.main.error(e) return False
def long_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands id_domain = action['domain'] if action['type'] in ['create_disk_virt_builder']: cmds_done = execute_commands(host=self.hostname, ssh_commands=action['ssh_commands'], dict_mode=True, user=self.user, port=self.port ) if len([d for d in cmds_done if len(d['err']) > 0]) > 1: log.error('some error in virt builder operations') log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format( action['disk_path'], action['domain'], self.hyp_id)) log.debug('print cmds_done:') log.debug(pprint.pprint(cmds_done)) log.debug('print ssh_commands:') log.debug(pprint.pprint(action['ssh_commands'])) update_domain_status('Failed', id_domain, detail='Virt Builder Failed creating disk file') else: log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'], action['disk_path'])) xml_virt_install = cmds_done[-1]['out'] update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install) update_domain_status('CreatingDomainFromBuilder', id_domain, detail='disk created from virt-builder') elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format(action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format( disk_path, id_domain)) log.error('\n'.join( ['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))])) update_domain_status('Failed', id_domain, detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def run(self): self.tid = get_tid() logs.changes.info('starting thread: {} (TID {})'.format( self.name, self.tid)) logs.changes.debug( '^^^^^^^^^^^^^^^^^^^ DOMAIN CHANGES THREAD ^^^^^^^^^^^^^^^^^') ui = UiActions(self.manager) self.r_conn = new_rethink_connection() cursor = r.table('domains').pluck('id', 'kind', 'status', 'detail').merge({'table': 'domains'}).changes().\ union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).\ run(self.r_conn) for c in cursor: if self.stop is True: break if c.get('new_val', None) is not None: if c['new_val']['table'] == 'engine': if c['new_val']['status_all_threads'] == 'Stopping': break else: continue logs.changes.debug('domain changes detected in main thread') detail_msg_if_no_hyps_online = 'No hypervisors Online in pool' if self.manager.check_actions_domains_enabled() is False: if c.get('new_val', None) is not None: if c.get('old_val', None) is not None: if c['new_val']['status'][-3:] == 'ing': update_domain_status( c['old_val']['status'], c['old_val']['id'], detail=detail_msg_if_no_hyps_online) #if no hypervisor availables no check status changes continue new_domain = False new_status = False old_status = False import pprint logs.changes.debug(pprint.pformat(c)) # action deleted if c.get('new_val', None) is None: pass # action created if c.get('old_val', None) is None: new_domain = True new_status = c['new_val']['status'] domain_id = c['new_val']['id'] logs.changes.debug('domain_id: {}'.format(new_domain)) # if engine is stopped/restarting or not hypervisors online if self.manager.check_actions_domains_enabled() is False: continue pass if c.get('new_val', None) is not None and c.get( 'old_val', None) is not None: old_status = c['old_val']['status'] new_status = c['new_val']['status'] new_detail = c['new_val']['detail'] domain_id = c['new_val']['id'] logs.changes.debug('domain_id: {}'.format(domain_id)) # if engine is stopped/restarting or not hypervisors online if old_status != new_status: # print('&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.format(domain_id,old_status,new_status, new_detail)) # if new_status[-3:] == 'ing': if 1 > 0: date_now = datetime.now() update_domain_history_from_id_domain( domain_id, new_status, new_detail, date_now) else: # print('&&&&&&&ESTADOS IGUALES OJO &&&&&&&\n&&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'. # format(domain_id,old_status,new_status,new_detail)) pass if (new_domain is True and new_status == "CreatingDiskFromScratch") or \ (old_status == 'FailedCreatingDomain' and new_status == "CreatingDiskFromScratch"): ui.creating_disk_from_scratch(domain_id) if (new_domain is True and new_status == "Creating") or \ (old_status == 'FailedCreatingDomain' and new_status == "Creating"): ui.creating_disks_from_template(domain_id) if (new_domain is True and new_status == "CreatingAndStarting"): update_domain_start_after_created(domain_id) ui.creating_disks_from_template(domain_id) # INFO TO DEVELOPER # recoger template de la que hay que derivar # verificar que realmente es una template # hay que recoger ram?? cpu?? o si no hay nada copiamos de la template?? if (new_domain is True and new_status == "CreatingFromBuilder") or \ (old_status == 'FailedCreatingDomain' and new_status == "CreatingFromBuilder"): ui.creating_disk_from_virtbuilder(domain_id) if (old_status in ['CreatingDisk','CreatingDiskFromScratch'] and new_status == "CreatingDomain") or \ (old_status == 'RunningVirtBuilder' and new_status == "CreatingDomainFromBuilder"): logs.changes.debug( 'llamo a creating_and_test_xml con domain id {}'. format(domain_id)) if new_status == "CreatingDomainFromBuilder": ui.creating_and_test_xml_start( domain_id, creating_from_create_dict=True, xml_from_virt_install=True) if new_status == "CreatingDomain": ui.creating_and_test_xml_start( domain_id, creating_from_create_dict=True) if old_status == 'Stopped' and new_status == "CreatingTemplate": ui.create_template_disks_from_domain(domain_id) if old_status == 'Stopped' and new_status == "Deleting": ui.deleting_disks_from_domain(domain_id) if (old_status == 'Stopped' and new_status == "Updating") or \ (old_status == 'Downloaded' and new_status == "Updating"): ui.updating_from_create_dict(domain_id) if old_status == 'DeletingDomainDisk' and new_status == "DiskDeleted": logs.changes.debug( 'disk deleted, mow remove domain form database') remove_domain(domain_id) if get_domain(domain_id) is None: logs.changes.info( 'domain {} deleted from database'.format( domain_id)) else: update_domain_status( 'Failed', id_domain, detail='domain {} can not be deleted from database' .format(domain_id)) if old_status == 'CreatingTemplateDisk' and new_status == "TemplateDiskCreated": # create_template_from_dict(dict_new_template) if get_if_all_disk_template_created(domain_id): ui.create_template_in_db(domain_id) else: # INFO TO DEVELOPER, este else no se si tiene mucho sentido, hay que hacer pruebas con la # creación de una template con dos discos y ver si pasa por aquí # waiting to create other disks update_domain_status( status='CreatingTemplateDisk', id_domain=domain_id, hyp_id=False, detail='Waiting to create more disks for template') if (old_status == 'Stopped' and new_status == "Starting") or \ (old_status == 'Failed' and new_status == "Starting"): ui.start_domain_from_id(id=domain_id, ssl=True) if (old_status == 'Started' and new_status == "Stopping" ) or \ (old_status == 'Suspended' and new_status == "Stopping" ): # INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started # ui.stop_domain_from_id(id=domain_id) hyp_started = get_domain_hyp_started(domain_id) if hyp_started: ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started) else: logs.main.error( 'domain without hyp_started can not be stopped: {}. ' .format(domain_id)) if (old_status == 'Started' and new_status == "StoppingAndDeleting" ) or \ (old_status == 'Suspended' and new_status == "StoppingAndDeleting" ): # INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started # ui.stop_domain_from_id(id=domain_id) hyp_started = get_domain_hyp_started(domain_id) print(hyp_started) ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started, delete_after_stopped=True) logs.main.info('finalished thread domain changes')
def run(self): self.tid = get_tid() logs.main.info('starting thread: {} (TID {})'.format( self.name, self.tid)) #get hostname grafana config self.get_hostname_grafana() hyps_online = [] elapsed = self.send_static_values_interval while self.stop is False: sleep(self.send_to_grafana_interval) elapsed += self.send_to_grafana_interval if self.restart_send_config is True: self.restart_send_config = False self.get_hostname_grafana() if self.active is True: for i, id_hyp in enumerate(self.t_status.keys()): try: if self.t_status[ id_hyp].status_obj.hyp_obj.connected is True: if id_hyp not in hyps_online: hyps_online.append(id_hyp) check_hyp = True except: logs.main.error( f'hypervisor {id_hyp} problem checking if is connected' ) check_hyp = False if len(hyps_online) > 0 and check_hyp is True: #send static values of hypervisors if elapsed >= self.send_static_values_interval: d_hyps_info = dict() for i, id_hyp in enumerate(hyps_online): d_hyps_info[f'hyp-info-{i}'] = self.t_status[ id_hyp].status_obj.hyp_obj.info # ~ self.send(d_hyps_info) elapsed = 0 #send stats dict_to_send = dict() j = 0 for i, id_hyp in enumerate(hyps_online): if id_hyp in self.t_status.keys(): #stats_hyp = self.t_status[id_hyp].status_obj.hyp_obj.stats_hyp stats_hyp_now = self.t_status[ id_hyp].status_obj.hyp_obj.stats_hyp_now #stats_domains = self.t_status[id_hyp].status_obj.hyp_obj.stats_domains if len(stats_hyp_now) > 0: dict_to_send[f'hypers.' + id_hyp] = { 'stats': stats_hyp_now, 'info': d_hyps_info['hyp-info-' + str(i)], 'domains': {} } stats_domains_now = self.t_status[ id_hyp].status_obj.hyp_obj.stats_domains_now # ~ for id_domain,d_stats in stats_domains_now.items(): # ~ if len(stats_hyp_now) > 0: # ~ for id_domain,d_stats in stats_domains_now.items(): # ~ dict_to_send[f'domain-stats-{j}'] = {'domain-id':{id_domain:1},'last': d_stats,} dict_to_send[f'hypers.' + id_hyp][ 'domains'] = stats_domains_now #{x:0 for x in stats_domains_now} # ~ print(stats_domains_now) # ~ j+=1 if len(dict_to_send) > 0: self.send(dict_to_send)
def run(self): self.tid = get_tid() logs.main.info('starting thread: {} (TID {})'.format( self.name, self.tid)) q = self.manager.q.background first_loop = True while self.manager.quit is False: # ONLY FOR DEBUG logs.main.debug('##### THREADS ##################') get_threads_running() self.manager.update_info_threads_engine() # DISK_OPERATIONS: if len(self.manager.t_disk_operations) == 0: self.launch_threads_disk_and_long_operations() # TEST HYPS AND START THREADS FROM RETHINK self.test_hyps_and_start_threads() # LAUNCH CHANGES THREADS if first_loop is True: update_table_field('engine', 'engine', 'status_all_threads', 'Starting') self.manager.t_changes_hyps = self.manager.HypervisorChangesThread( 'changes_hyp', self.manager) self.manager.t_changes_hyps.daemon = True self.manager.t_changes_hyps.start() self.manager.t_changes_domains = self.manager.DomainsChangesThread( 'changes_domains', self.manager) self.manager.t_changes_domains.daemon = True self.manager.t_changes_domains.start() logs.main.debug('Launching Download Changes Thread') self.manager.t_downloads_changes = launch_thread_download_changes( self.manager) self.manager.t_broom = launch_thread_broom(self.manager) first_loop = False logs.main.info('THREADS LAUNCHED FROM BACKGROUND THREAD') update_table_field('engine', 'engine', 'status_all_threads', 'Starting') while True: sleep(0.1) alive, dead, not_defined = self.manager.update_info_threads_engine( ) pprint.pprint({ 'alive': alive, 'dead': dead, 'not_defined': not_defined }) if len(not_defined) == 0 and len(dead) == 0: update_table_field('engine', 'engine', 'status_all_threads', 'Started') self.manager.num_workers = len( self.manager.t_workers) self.manager.threads_started = True break try: action = q.get(timeout=self.manager.TEST_HYP_FAIL_INTERVAL) if action['type'] == 'stop': self.manager.quit = True logs.main.info('engine end') except queue.Empty: pass except Exception as e: logs.main.error(e) return False
def run(self): self.tid = get_tid() logs.changes.info('starting thread: {} (TID {})'.format(self.name, self.tid)) logs.changes.debug('^^^^^^^^^^^^^^^^^^^ DOMAIN CHANGES THREAD ^^^^^^^^^^^^^^^^^') ui = UiActions(self.manager) self.r_conn = new_rethink_connection() cursor = r.table('domains').pluck('id', 'kind', 'status', 'detail').merge({'table': 'domains'}).changes().\ union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).\ run(self.r_conn) for c in cursor: if self.stop is True: break if c.get('new_val', None) is not None: if c['new_val']['table'] == 'engine': if c['new_val']['status_all_threads'] == 'Stopping': break else: continue logs.changes.debug('domain changes detected in main thread') detail_msg_if_no_hyps_online = 'No hypervisors Online in pool' if self.manager.check_actions_domains_enabled() is False: if c.get('new_val', None) is not None: if c.get('old_val', None) is not None: if c['new_val']['status'][-3:] == 'ing': update_domain_status(c['old_val']['status'], c['old_val']['id'], detail=detail_msg_if_no_hyps_online) #if no hypervisor availables no check status changes continue new_domain = False new_status = False old_status = False logs.changes.debug(pprint.pformat(c)) # action deleted if c.get('new_val', None) is None: pass # action created if c.get('old_val', None) is None: new_domain = True new_status = c['new_val']['status'] domain_id = c['new_val']['id'] logs.changes.debug('domain_id: {}'.format(new_domain)) # if engine is stopped/restarting or not hypervisors online if self.manager.check_actions_domains_enabled() is False: continue pass if c.get('new_val', None) is not None and c.get('old_val', None) is not None: old_status = c['old_val']['status'] new_status = c['new_val']['status'] new_detail = c['new_val']['detail'] domain_id = c['new_val']['id'] logs.changes.debug('domain_id: {}'.format(domain_id)) # if engine is stopped/restarting or not hypervisors online if old_status != new_status: # print('&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.format(domain_id,old_status,new_status, new_detail)) # if new_status[-3:] == 'ing': if 1 > 0: date_now = datetime.now() update_domain_history_from_id_domain(domain_id, new_status, new_detail, date_now) else: # print('&&&&&&&ESTADOS IGUALES OJO &&&&&&&\n&&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'. # format(domain_id,old_status,new_status,new_detail)) pass if (new_domain is True and new_status == "CreatingDiskFromScratch") or \ (old_status == 'FailedCreatingDomain' and new_status == "CreatingDiskFromScratch"): ui.creating_disk_from_scratch(domain_id) if (new_domain is True and new_status == "Creating") or \ (old_status == 'FailedCreatingDomain' and new_status == "Creating"): ui.creating_disks_from_template(domain_id) if (new_domain is True and new_status == "CreatingAndStarting"): update_domain_start_after_created(domain_id) ui.creating_disks_from_template(domain_id) # INFO TO DEVELOPER # recoger template de la que hay que derivar # verificar que realmente es una template # hay que recoger ram?? cpu?? o si no hay nada copiamos de la template?? if (new_domain is True and new_status == "CreatingFromBuilder") or \ (old_status == 'FailedCreatingDomain' and new_status == "CreatingFromBuilder"): ui.creating_disk_from_virtbuilder(domain_id) if (old_status in ['CreatingDisk','CreatingDiskFromScratch'] and new_status == "CreatingDomain") or \ (old_status == 'RunningVirtBuilder' and new_status == "CreatingDomainFromBuilder"): logs.changes.debug('llamo a creating_and_test_xml con domain id {}'.format(domain_id)) if new_status == "CreatingDomainFromBuilder": ui.creating_and_test_xml_start(domain_id, creating_from_create_dict=True, xml_from_virt_install=True,ssl=True) if new_status == "CreatingDomain": ui.creating_and_test_xml_start(domain_id, creating_from_create_dict=True,ssl=True) if old_status == 'Stopped' and new_status == "CreatingTemplate": ui.create_template_disks_from_domain(domain_id) if old_status != 'Started' and new_status == "Deleting": # or \ # old_status == 'Failed' and new_status == "Deleting" or \ # old_status == 'Downloaded' and new_status == "Deleting": ui.deleting_disks_from_domain(domain_id) if (old_status == 'Stopped' and new_status == "Updating") or \ (old_status == 'Downloaded' and new_status == "Updating"): ui.updating_from_create_dict(domain_id,ssl=True) if old_status == 'DeletingDomainDisk' and new_status == "DiskDeleted": logs.changes.debug('disk deleted, mow remove domain form database') remove_domain(domain_id) if get_domain(domain_id) is None: logs.changes.info('domain {} deleted from database'.format(domain_id)) else: update_domain_status('Failed', domain_id, detail='domain {} can not be deleted from database'.format(domain_id)) if old_status == 'CreatingTemplateDisk' and new_status == "TemplateDiskCreated": # create_template_from_dict(dict_new_template) if get_if_all_disk_template_created(domain_id): ui.create_template_in_db(domain_id) else: # INFO TO DEVELOPER, este else no se si tiene mucho sentido, hay que hacer pruebas con la # creación de una template con dos discos y ver si pasa por aquí # waiting to create other disks update_domain_status(status='CreatingTemplateDisk', id_domain=domain_id, hyp_id=False, detail='Waiting to create more disks for template') if (old_status == 'Stopped' and new_status == "Starting") or \ (old_status == 'Failed' and new_status == "Starting"): ui.start_domain_from_id(id=domain_id, ssl=True) if (old_status == 'Started' and new_status == "Stopping" ) or \ (old_status == 'Suspended' and new_status == "Stopping" ): # INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started # ui.stop_domain_from_id(id=domain_id) hyp_started = get_domain_hyp_started(domain_id) if hyp_started: ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started) else: logs.main.error('domain without hyp_started can not be stopped: {}. '.format(domain_id)) if (old_status == 'Started' and new_status == "StoppingAndDeleting" ) or \ (old_status == 'Suspended' and new_status == "StoppingAndDeleting" ): # INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started # ui.stop_domain_from_id(id=domain_id) hyp_started = get_domain_hyp_started(domain_id) print(hyp_started) ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started, delete_after_stopped=True) logs.main.info('finalished thread domain changes')
def run(self): self.tid = get_tid() logs.broom.info('starting thread: {} (TID {})'.format( self.name, self.tid)) self.polling()
def run(self): self.tid = get_tid() logs.workers.info('starting thread: {} (TID {})'.format(self.name, self.tid)) host, port, user = get_hyp_hostname_from_id(self.hyp_id) port = int(port) self.hostname = host self.h = hyp(self.hostname, user=user, port=port) # self.h.get_kvm_mod() # self.h.get_hyp_info() update_db_hyp_info(self.hyp_id, self.h.info) hyp_id = self.hyp_id while self.stop is not True: try: # do={type:'start_domain','xml':'xml','id_domain'='prova'} action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) logs.workers.debug('received action in working thread {}'.format(action['type'])) if action['type'] == 'start_paused_domain': logs.workers.debug('xml to start paused some lines...: {}'.format(action['xml'][30:100])) try: self.h.conn.createXML(action['xml'], flags=VIR_DOMAIN_START_PAUSED) # 32 is the constant for domains paused # reference: https://libvirt.org/html/libvirt-libvirt-domain.html#VIR_CONNECT_LIST_DOMAINS_PAUSED FLAG_LIST_DOMAINS_PAUSED = 32 list_all_domains = self.h.conn.listAllDomains(FLAG_LIST_DOMAINS_PAUSED) list_names_domains = [d.name() for d in list_all_domains] dict_domains = dict(zip(list_names_domains,list_all_domains)) if action['id_domain'] in list_names_domains: # domain started in pause mode domain = dict_domains[action['id_domain']] domain_active = True try: domain.isActive() domain.destroy() try: domain.isActive() except Exception as e: logs.workers.debug('verified domain {} is destroyed'.format(action['id_domain'])) domain_active = False except libvirtError as e: from pprint import pformat error_msg = pformat(e.get_error_message()) update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id, detail='domain {} failed when try to destroy from paused domain in hypervisor {}. creating domain operation is aborted') logs.workers.error( 'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format( action['id_domain'], self.hyp_id, error_msg)) continue if domain_active is False: # domain is destroyed, all ok update_domain_status('CreatingDomain', action['id_domain'], hyp_id='', detail='Domain created and test OK: Started, paused and now stopped in hyp {}'.format(self.hyp_id)) logs.workers.debug( 'domain {} creating operation finalished. Started paused and destroyed in hypervisor {}. Now status is Stopped. READY TO USE'.format( action['id_domain'], self.hyp_id)) else: update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id, detail='Domain is created, started in pause mode but not destroyed,creating domain operation is aborted') logs.workers.error( 'domain {} started paused but not destroyed in hypervisor {}, must be destroyed'.format( action['id_domain'], self.hyp_id)) else: update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id, detail='XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted by unknown cause'.format( action['id_domain'], self.hyp_id)) logs.workers.error( 'XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted, not exception, rare case, unknown cause'.format( action['id_domain'], self.hyp_id)) except libvirtError as e: from pprint import pformat error_msg = pformat(e.get_error_message()) update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id, detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted') logs.workers.error( 'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format( action['id_domain'], self.hyp_id, error_msg)) except Exception as e: update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id, detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted') logs.workers.error( 'Exception starting paused xml for domain {} in hypervisor {}. NOT LIBVIRT EXCEPTION, RARE CASE. Exception message: {}'.format( action['id_domain'], self.hyp_id, str(e))) ## START DOMAIN elif action['type'] == 'start_domain': logs.workers.debug('xml to start some lines...: {}'.format(action['xml'][30:100])) try: self.h.conn.createXML(action['xml']) # wait to event started to save state in database #update_domain_status('Started', action['id_domain'], hyp_id=self.hyp_id, detail='Domain has started in worker thread') logs.workers.debug('STARTED domain {}: createdXML action in hypervisor {} has been sent'.format( action['id_domain'], host)) except libvirtError as e: update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=("Hypervisor can not create domain with libvirt exception: " + str(e))) logs.workers.debug('exception in starting domain {}: '.format(e)) except Exception as e: update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=("Exception when starting domain: " + str(e))) logs.workers.debug('exception in starting domain {}: '.format(e)) ## STOP DOMAIN elif action['type'] == 'stop_domain': logs.workers.debug('action stop domain: {}'.format(action['id_domain'][30:100])) try: self.h.conn.lookupByName(action['id_domain']).destroy() logs.workers.debug('STOPPED domain {}'.format(action['id_domain'])) check_if_delete = action.get('delete_after_stopped',False) if check_if_delete is True: update_domain_status('Stopped', action['id_domain'], hyp_id='') update_domain_status('Deleting', action['id_domain'], hyp_id='') else: update_domain_status('Stopped', action['id_domain'], hyp_id='') except Exception as e: update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=str(e)) logs.workers.debug('exception in stopping domain {}: '.format(e)) elif action['type'] in ['create_disk', 'delete_disk']: launch_action_disk(action, self.hostname, user, port) elif action['type'] in ['add_media_hot']: pass elif action['type'] in ['killall_curl']: launch_killall_curl(self.hostname, user, port) elif action['type'] in ['delete_media']: final_status = action.get('final_status','Deleted') launch_delete_media (action, self.hostname, user, port, final_status=final_status) # ## DESTROY THREAD # elif action['type'] == 'destroy_thread': # list_works_in_queue = list(self.queue_actions.queue) # if self.queue_master is not None: # self.queue_master.put(['destroy_working_thread',self.hyp_id,list_works_in_queue]) # #INFO TO DEVELOPER, si entra aquí es porque no quedaba nada en cola, si no ya lo habrán matado antes # # logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id)) # # raise 'destoyed' elif action['type'] == 'create_disk': pass elif action['type'] == 'hyp_info': self.h.get_hyp_info() logs.workers.debug('hypervisor motherboard: {}'.format(self.h.info['motherboard_manufacturer'])) ## DESTROY THREAD elif action['type'] == 'stop_thread': self.stop = True else: logs.workers.error('type action {} not supported in queue actions'.format(action['type'])) # time.sleep(0.1) ## TRY DOMAIN except queue.Empty: try: self.h.conn.getLibVersion() pass # logs.workers.debug('hypervisor {} is alive'.format(host)) except: logs.workers.info('trying to reconnect hypervisor {}, alive test in working thread failed'.format(host)) alive = False for i in range(RETRIES_HYP_IS_ALIVE): try: time.sleep(TIMEOUT_BETWEEN_RETRIES_HYP_IS_ALIVE) self.h.conn.getLibVersion() alive = True logs.workers.info('hypervisor {} is alive'.format(host)) break except: logs.workers.info('hypervisor {} is NOT alive'.format(host)) if alive is False: try: self.h.connect_to_hyp() self.h.conn.getLibVersion() update_hyp_status(self.hyp_id, 'Online') except: logs.workers.debug('hypervisor {} failed'.format(host)) logs.workers.error('fail reconnecting to hypervisor {} in working thread'.format(host)) reason = self.h.fail_connected_reason update_hyp_status(self.hyp_id, 'Error', reason) update_domains_started_in_hyp_to_unknown(self.hyp_id) list_works_in_queue = list(self.queue_actions.queue) if self.queue_master is not None: self.queue_master.put(['error_working_thread', self.hyp_id, list_works_in_queue]) logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id)) self.active = False break
def run(self): self.tid = get_tid() logs.downloads.debug( 'RUN-DOWNLOAD-THREAD-------------------------------------') if self.stop is False: self.r_conn = new_rethink_connection() for c in r.table('media').get_all(r.args(['Deleting','Deleted','Downloaded','DownloadStarting', 'Downloading','Download','DownloadAborting']), index='status').\ pluck('id', 'path', 'url-isard', 'url-web', 'status' ).merge( {'table': 'media'}).changes(include_initial=True).union( r.table('domains').get_all(r.args(['Downloaded','DownloadStarting', 'Downloading','DownloadAborting']), index='status').\ pluck('id', 'create_dict', 'url-isard', 'url-web', 'status').merge( {"table": "domains"}).changes(include_initial=True)).union( r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).run(self.r_conn): if self.stop: break if c.get('new_val', None) is not None: if c['new_val'].get('table', False) == 'engine': if c['new_val']['status_all_threads'] == 'Stopping': break else: continue logs.downloads.debug('DOWNLOAD CHANGES DETECTED:') logs.downloads.debug(pprint.pformat(c)) if c.get('old_val', None) is None: if c['new_val']['status'] == 'DownloadStarting': self.start_download(c['new_val']) elif c.get('new_val', None) is None: if c['old_val']['status'] in ['DownloadAborting']: self.remove_download_thread(c['old_val']) elif 'old_val' in c and 'new_val' in c: if c['old_val']['status'] == 'FailedDownload' and c[ 'new_val']['status'] == 'DownloadStarting': self.start_download(c['new_val']) elif c['old_val']['status'] == 'Downloaded' and c[ 'new_val']['status'] == 'Deleting': if c['new_val']['table'] == 'media': self.delete_media(c['new_val']) elif c['old_val']['status'] == 'Deleting' and c['new_val'][ 'status'] == 'Deleted': if c['new_val']['table'] == 'media': remove_media(c['new_val']['id']) elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'FailedDownload': pass elif c['old_val']['status'] == 'DownloadStarting' and c[ 'new_val']['status'] == 'Downloading': pass elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'Downloaded': pass elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'DownloadAborting': self.abort_download(c['new_val'])