def update_info_threads_engine(self):
d = {}
alive = []
dead = []
not_defined = []
#events,broom
for name in [
'events', 'broom', 'downloads_changes', 'changes_hyps',
'changes_domains'
]:
try:
alive.append(name) if self.__getattribute__(
't_' + name).is_alive() else dead.append(name)
except:
#thread not defined
not_defined.append(name)
for name in [
'workers', 'status', 'disk_operations', 'long_operations'
]:
for hyp, t in self.__getattribute__('t_' + name).items():
try:
alive.append(name + '_' +
hyp) if t.is_alive() else dead.append(name +
'_' +
hyp)
except:
not_defined.append(name)
pass
d['alive'] = alive
d['dead'] = dead
d['not_defined'] = not_defined
update_table_field('engine', 'engine', 'threads', d)
return alive, dead, not_defined
def stop_threads(self):
# events and broom
#try:
self.t_events.stop = True
while True:
if self.t_events.is_alive() is False:
break
sleep(0.1)
#except TypeError:
# pass
self.t_broom.stop = True
# operations / status
for k, v in self.t_long_operations.items():
v.stop = True
for k, v in self.t_disk_operations.items():
v.stop = True
for k, v in self.t_workers.items():
v.stop = True
for k, v in self.t_status.items():
v.stop = True
self.q_disk_operations
# changes
update_table_field('engine', 'engine', 'status_all_threads',
'Stopping')
def stop_threads(self):
# events and broom
#try:
self.t_events.stop = True
while True:
if self.t_events.is_alive() is False:
break
sleep(0.1)
#except TypeError:
# pass
self.t_broom.stop = True
# operations / status
for k,v in self.t_long_operations.items():
v.stop = True
for k, v in self.t_disk_operations.items():
v.stop = True
for k, v in self.t_workers.items():
v.stop = True
for k, v in self.t_status.items():
v.stop = True
self.q_disk_operations
#self.t_downloads_changes.stop = True
# changes
update_table_field('engine', 'engine', 'status_all_threads', 'Stopping')
def engine_restart():
app.m.stop_threads()
while True:
app.m.update_info_threads_engine()
if len(app.m.threads_info_main['alive']) == 0 and len(app.m.threads_info_hyps['alive']) == 0:
action = {}
action['type'] = 'stop'
app.m.q.background.put(action)
break
sleep(0.5)
while True:
if app.m.t_background.is_alive():
sleep(0.2)
else:
update_table_field('engine', 'engine', 'status_all_threads', 'Stopped')
delattr(app, 'm')
app.m = ManagerHypervisors()
break
return jsonify({'engine_restart':True}), 200
def update_info_threads_engine(self):
d_mains = {}
alive=[]
dead=[]
not_defined=[]
#events,broom
for name in ['events','broom','downloads_changes','changes_hyps','changes_domains']:
try:
alive.append(name) if self.__getattribute__('t_'+name).is_alive() else dead.append(name)
except:
#thread not defined
not_defined.append(name)
d_mains['alive']=alive
d_mains['dead']=dead
d_mains['not_defined']=not_defined
self.threads_info_main = d_mains.copy()
d_hyps = {}
alive=[]
dead=[]
not_defined=[]
for name in ['workers','status','disk_operations','long_operations']:
for hyp,t in self.__getattribute__('t_'+name).items():
try:
alive.append(name + '_' + hyp) if t.is_alive() else dead.append(name + '_' + hyp)
except:
not_defined.append(name)
d_hyps['alive']=alive
d_hyps['dead']=dead
d_hyps['not_defined']=not_defined
self.threads_info_hyps = d_hyps.copy()
update_table_field('engine', 'engine', 'threads_info_main', d_mains)
update_table_field('engine', 'engine', 'threads_info_hyps', d_hyps)
return True
def run(self):
self.tid = get_tid()
logs.main.info('starting thread: {} (TID {})'.format(
self.name, self.tid))
q = self.manager.q.background
first_loop = True
while self.manager.quit is False:
# ONLY FOR DEBUG
logs.main.debug('##### THREADS ##################')
get_threads_running()
self.manager.update_info_threads_engine()
# DISK_OPERATIONS:
if len(self.manager.t_disk_operations) == 0:
self.launch_threads_disk_and_long_operations()
# TEST HYPS AND START THREADS FROM RETHINK
self.test_hyps_and_start_threads()
# LAUNCH CHANGES THREADS
if first_loop is True:
update_table_field('engine', 'engine',
'status_all_threads', 'Starting')
self.manager.t_changes_hyps = self.manager.HypervisorChangesThread(
'changes_hyp', self.manager)
self.manager.t_changes_hyps.daemon = True
self.manager.t_changes_hyps.start()
self.manager.t_changes_domains = self.manager.DomainsChangesThread(
'changes_domains', self.manager)
self.manager.t_changes_domains.daemon = True
self.manager.t_changes_domains.start()
logs.main.debug('Launching Download Changes Thread')
self.manager.t_downloads_changes = launch_thread_download_changes(
self.manager)
self.manager.t_broom = launch_thread_broom(self.manager)
first_loop = False
logs.main.info('THREADS LAUNCHED FROM BACKGROUND THREAD')
update_table_field('engine', 'engine',
'status_all_threads', 'Starting')
while True:
sleep(0.1)
alive, dead, not_defined = self.manager.update_info_threads_engine(
)
pprint.pprint({
'alive': alive,
'dead': dead,
'not_defined': not_defined
})
if len(not_defined) == 0 and len(dead) == 0:
update_table_field('engine', 'engine',
'status_all_threads', 'Started')
self.manager.num_workers = len(
self.manager.t_workers)
self.manager.threads_started = True
break
try:
action = q.get(timeout=self.manager.TEST_HYP_FAIL_INTERVAL)
if action['type'] == 'stop':
self.manager.quit = True
logs.main.info('engine end')
except queue.Empty:
pass
except Exception as e:
logs.main.error(e)
return False
def run(self):
self.tid = get_tid()
logs.main.info('starting thread background: {} (TID {})'.format(self.name, self.tid))
q = self.manager.q.background
first_loop = True
pool_id = 'default'
#can't launch downloads if download changes thread is not ready and hyps are not online
update_table_field('hypervisors_pools', pool_id, 'download_changes', 'Stopped')
# if domains have intermedite states (updating, download_aborting...)
# to Failed or Delete
clean_intermediate_status()
l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
# while len(l_hyps_to_test) == 0:
# logs.main.error('no hypervisor enable, waiting for one hypervisor')
# sleep(0.5)
# l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
while self.manager.quit is False:
####################################################################
### MAIN LOOP ######################################################
# ONLY FOR DEBUG
logs.main.debug('##### THREADS ##################')
threads_running = get_threads_running()
#pprint.pprint(threads_running)
#self.manager.update_info_threads_engine()
# Threads that must be running always, with or withouth hypervisor:
# - changes_hyp
# - changes_domains
# - downloads_changes
# - broom
# - events
# - grafana
# Threads that depends on hypervisors availavility:
# - disk_operations
# - long_operations
# - for every hypervisor:
# - worker
# - status
# LAUNCH MAIN THREADS
if first_loop is True:
update_table_field('engine', 'engine', 'status_all_threads', 'Starting')
# launch changes_hyp thread
self.manager.t_changes_hyps = self.manager.HypervisorChangesThread('changes_hyp', self.manager)
self.manager.t_changes_hyps.daemon = True
self.manager.t_changes_hyps.start()
#launch changes_domains_thread
self.manager.t_changes_domains = self.manager.DomainsChangesThread('changes_domains', self.manager)
self.manager.t_changes_domains.daemon = True
self.manager.t_changes_domains.start()
#launch downloads changes thread
logs.main.debug('Launching Download Changes Thread')
self.manager.t_downloads_changes = launch_thread_download_changes(self.manager)
#launch brom thread
self.manager.t_broom = launch_thread_broom(self.manager)
#launch events thread
logs.main.debug('launching hypervisor events thread')
self.manager.t_events = launch_thread_hyps_event()
#launch grafana thread
logs.main.debug('launching grafana thread')
self.manager.t_grafana = launch_grafana_thread(self.manager.t_status)
logs.main.info('THREADS LAUNCHED FROM BACKGROUND THREAD')
update_table_field('engine', 'engine', 'status_all_threads', 'Starting')
while True:
#wait all
sleep(0.1)
self.manager.update_info_threads_engine()
#if len(self.manager.threads_info_main['not_defined']) > 0 and len(self.manager.dict_hyps_ready) == 0:
if len(self.manager.threads_info_main['not_defined']) > 0 or len(self.manager.threads_info_main['dead']) > 0:
print('MAIN THREADS starting, wait a second extra')
sleep(1)
self.manager.update_info_threads_engine()
pprint.pprint(self.manager.threads_info_main)
#self.test_hyps_and_start_threads()
if len(self.manager.threads_info_main['not_defined']) == 0 and len(self.manager.threads_info_main['dead']) == 0:
update_table_field('engine', 'engine', 'status_all_threads', 'Started')
self.manager.threads_main_started = True
break
# TEST HYPS AND START THREADS FOR HYPERVISORS
self.test_hyps_and_start_threads()
self.manager.num_workers = len(self.manager.t_workers)
# Test hypervisor disk operations
# Create Test disk in hypervisor disk operations
if first_loop is True:
first_loop = False
# virtio_test_disk_relative_path = 'admin/admin/admin/virtio_testdisk.qcow2'
# ui.creating_test_disk(test_disk_relative_route=virtio_test_disk_relative_path)
self.manager.update_info_threads_engine()
if len(self.manager.threads_info_hyps['not_defined']) > 0:
logs.main.error('something was wrong when launching threads for hypervisors, threads not defined')
logs.main.error(pprint.pformat(self.manager.threads_info_hyps))
if len(self.manager.threads_info_hyps['dead']) > 0:
logs.main.error('something was wrong when launching threads for hypervisors, threads are dead')
logs.main.error(pprint.pformat(self.manager.threads_info_hyps))
if len(self.manager.threads_info_hyps['dead']) == 0 and len(self.manager.threads_info_hyps['not_defined']) == 0:
pass
try:
if len(self.manager.t_workers) == 0:
timeout_queue = WAIT_HYP_ONLINE
else:
timeout_queue = TEST_HYP_FAIL_INTERVAL
action = q.get(timeout=timeout_queue)
if action['type'] == 'stop':
self.manager.quit = True
logs.main.info('engine end')
except queue.Empty:
pass
except Exception as e:
logs.main.error(e)
return False
def run(self):
self.tid = get_tid()
logs.main.info('starting thread background: {} (TID {})'.format(self.name, self.tid))
q = self.manager.q.background
first_loop = True
pool_id = 'default'
#can't launch downloads if download changes thread is not ready and hyps are not online
update_table_field('hypervisors_pools', pool_id, 'download_changes', 'Stopped')
# if domains have intermedite states (updating, download_aborting...)
# to Failed or Delete
clean_intermediate_status()
l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
# while len(l_hyps_to_test) == 0:
# logs.main.error('no hypervisor enable, waiting for one hypervisor')
# sleep(0.5)
# l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
while self.manager.quit is False:
####################################################################
### MAIN LOOP ######################################################
# ONLY FOR DEBUG
logs.main.debug('##### THREADS ##################')
threads_running = get_threads_running()
#pprint.pprint(threads_running)
#self.manager.update_info_threads_engine()
# Threads that must be running always, with or withouth hypervisor:
# - changes_hyp
# - changes_domains
# - downloads_changes
# - broom
# - events
# - grafana
# Threads that depends on hypervisors availavility:
# - disk_operations
# - long_operations
# - for every hypervisor:
# - worker
# - status
# LAUNCH MAIN THREADS
if first_loop is True:
update_table_field('engine', 'engine', 'status_all_threads', 'Starting')
# launch changes_hyp thread
self.manager.t_changes_hyps = self.manager.HypervisorChangesThread('changes_hyp', self.manager)
self.manager.t_changes_hyps.daemon = True
self.manager.t_changes_hyps.start()
#launch changes_domains_thread
self.manager.t_changes_domains = self.manager.DomainsChangesThread('changes_domains', self.manager)
self.manager.t_changes_domains.daemon = True
self.manager.t_changes_domains.start()
#launch downloads changes thread
logs.main.debug('Launching Download Changes Thread')
self.manager.t_downloads_changes = launch_thread_download_changes(self.manager)
#launch brom thread
self.manager.t_broom = launch_thread_broom(self.manager)
#launch events thread
logs.main.debug('launching hypervisor events thread')
self.manager.t_events = launch_thread_hyps_event()
#launch grafana thread
logs.main.debug('launching grafana thread')
self.manager.t_grafana = launch_grafana_thread(self.manager.t_status)
logs.main.info('THREADS LAUNCHED FROM BACKGROUND THREAD')
update_table_field('engine', 'engine', 'status_all_threads', 'Starting')
while True:
#wait all
sleep(0.1)
self.manager.update_info_threads_engine()
#if len(self.manager.threads_info_main['not_defined']) > 0 and len(self.manager.dict_hyps_ready) == 0:
if len(self.manager.threads_info_main['not_defined']) > 0 or len(self.manager.threads_info_main['dead']) > 0:
print('MAIN THREADS starting, wait a second extra')
sleep(1)
self.manager.update_info_threads_engine()
pprint.pprint(self.manager.threads_info_main)
#self.test_hyps_and_start_threads()
if len(self.manager.threads_info_main['not_defined']) == 0 and len(self.manager.threads_info_main['dead']) == 0:
update_table_field('engine', 'engine', 'status_all_threads', 'Started')
self.manager.threads_main_started = True
break
# TEST HYPS AND START THREADS FOR HYPERVISORS
self.test_hyps_and_start_threads()
self.manager.num_workers = len(self.manager.t_workers)
# Test hypervisor disk operations
# Create Test disk in hypervisor disk operations
if first_loop is True:
first_loop = False
# virtio_test_disk_relative_path = 'admin/admin/admin/virtio_testdisk.qcow2'
# ui.creating_test_disk(test_disk_relative_route=virtio_test_disk_relative_path)
self.manager.update_info_threads_engine()
if len(self.manager.threads_info_hyps['not_defined']) > 0:
logs.main.error('something was wrong when launching threads for hypervisors, threads not defined')
logs.main.error(pprint.pformat(self.manager.threads_info_hyps))
if len(self.manager.threads_info_hyps['dead']) > 0:
logs.main.error('something was wrong when launching threads for hypervisors, threads are dead')
logs.main.error(pprint.pformat(self.manager.threads_info_hyps))
if len(self.manager.threads_info_hyps['dead']) == 0 and len(self.manager.threads_info_hyps['not_defined']) == 0:
pass
try:
if len(self.manager.t_workers) == 0:
timeout_queue = WAIT_HYP_ONLINE
else:
timeout_queue = TEST_HYP_FAIL_INTERVAL
action = q.get(timeout=timeout_queue)
if action['type'] == 'stop':
self.manager.quit = True
logs.main.info('engine end')
except queue.Empty:
pass
except Exception as e:
logs.main.error(e)
return False