def run(self):
self.tid = get_tid()
logs.main.info('starting thread: {} (TID {})'.format(self.name, self.tid))
#get hostname grafana config
self.get_hostname_grafana()
hyps_online = []
elapsed = self.send_static_values_interval
while self.stop is False:
sleep(self.send_to_grafana_interval)
elapsed += self.send_to_grafana_interval
if self.restart_send_config is True:
self.restart_send_config = False
self.get_hostname_grafana()
if self.active is True:
for i,id_hyp in enumerate(self.t_status.keys()):
try:
if self.t_status[id_hyp].status_obj.hyp_obj.connected is True:
if id_hyp not in hyps_online:
hyps_online.append(id_hyp)
check_hyp = True
except:
logs.main.error(f'hypervisor {id_hyp} problem checking if is connected')
check_hyp = False
if len(hyps_online) > 0 and check_hyp is True:
#send static values of hypervisors
if elapsed >= self.send_static_values_interval:
d_hyps_info = dict()
for i, id_hyp in enumerate(hyps_online):
d_hyps_info[f'hyp-info-{i}'] = self.t_status[id_hyp].status_obj.hyp_obj.info
# ~ self.send(d_hyps_info)
elapsed = 0
#send stats
dict_to_send = dict()
j=0
for i, id_hyp in enumerate(hyps_online):
if id_hyp in self.t_status.keys():
#stats_hyp = self.t_status[id_hyp].status_obj.hyp_obj.stats_hyp
stats_hyp_now = self.t_status[id_hyp].status_obj.hyp_obj.stats_hyp_now
#stats_domains = self.t_status[id_hyp].status_obj.hyp_obj.stats_domains
if len(stats_hyp_now) > 0:
dict_to_send[f'hypers.'+id_hyp] = {'stats':stats_hyp_now,'info':d_hyps_info['hyp-info-'+str(i)],'domains':{}}
stats_domains_now = self.t_status[id_hyp].status_obj.hyp_obj.stats_domains_now
# ~ for id_domain,d_stats in stats_domains_now.items():
# ~ if len(stats_hyp_now) > 0:
# ~ for id_domain,d_stats in stats_domains_now.items():
# ~ dict_to_send[f'domain-stats-{j}'] = {'domain-id':{id_domain:1},'last': d_stats,}
dict_to_send[f'hypers.'+id_hyp]['domains']=stats_domains_now #{x:0 for x in stats_domains_now}
# ~ print(stats_domains_now)
# ~ j+=1
if len(dict_to_send) > 0:
self.send(dict_to_send)
def disk_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
if action['type'] in ['create_disk']:
launch_action_disk(action,
self.hostname,
self.user,
self.port)
elif action['type'] in ['create_disk_from_scratch']:
launch_action_disk(action,
self.hostname,
self.user,
self.port,
from_scratch=True)
elif action['type'] in ['delete_disk']:
launch_delete_disk_action(action,
self.hostname,
self.user,
self.port)
elif action['type'] in ['create_template_disk_from_domain']:
launch_action_create_template_disk(action,
self.hostname,
self.user,
self.port)
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def run(self):
self.tid = get_tid()
log.info('starting thread: {} (TID {})'.format(self.name, self.tid))
while self.stop is not True:
self.hyp_obj.get_load()
interval = 0.0
while interval < self.polling_interval:
time.sleep(0.1)
interval += 0.1
if self.stop is True:
break
def run(self):
self.tid = get_tid()
log.info('starting thread: {} (TID {})'.format(self.name, self.tid))
while self.stop is not True:
self.hyp_obj.get_load()
interval = 0.0
while interval < self.polling_interval:
time.sleep(0.1)
interval += 0.1
if self.stop is True:
break
def run(self):
self.tid = get_tid()
log.info('starting thread: {} (TID {})'.format(self.name, self.tid))
self.hyp_obj, self.ok = try_hyp_connection(self.hyp_id, self.hostname,
self.port, self.user)
log.debug('Exiting from thread {} try_hyp {}'.format(
self.name, self.hostname))
return self.ok
def run(self):
self.tid = get_tid()
logs.main.info('starting thread: {} (TID {})'.format(
self.name, self.tid))
self.r_conn = new_rethink_connection()
# rtable=r.table('disk_operations')
# for c in r.table('hypervisors').changes(include_initial=True, include_states=True).run(r_conn):
for c in r.table('hypervisors').pluck('capabilities',
'enabled',
'hostname',
'hypervisors_pools',
'port',
'user',
'viewer_hostname',
'viewer_nat_hostname').merge({'table': 'hypervisors'}).changes().\
union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes())\
.run(self.r_conn):
#stop thread
if self.stop is True:
break
if c['new_val'] is not None:
if c['new_val']['table'] == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
# hypervisor deleted
if c['new_val'] is None:
if c['old_val'].get('table', False) == 'hypervisors':
logs.main.info('hypervisor deleted in rethink')
logs.main.info(pprint.pformat(c))
#TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if
# domains are running and hypevisor communication have lost
engine_restart()
# hypervisor created
elif c['old_val'] is None:
if c['new_val'].get('table', False) == 'hypervisors':
logs.main.info('hypervisor created in rethink')
logs.main.info(pprint.pformat(c))
engine_restart()
else:
if c['new_val'].get('table', False) == 'hypervisors':
#TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if
# domains are running and hypevisor communication have lost
logs.main.info('hypervisor fields modified in rethink')
logs.main.info(pprint.pformat(c))
engine_restart()
#self.manager.q.background.put({'type': 'add_hyp'})
self.r_conn.close()
def run(self):
self.tid = get_tid()
logs.main.info('starting thread: {} (TID {})'.format(self.name, self.tid))
self.r_conn = new_rethink_connection()
# rtable=r.table('disk_operations')
# for c in r.table('hypervisors').changes(include_initial=True, include_states=True).run(r_conn):
for c in r.table('hypervisors').pluck('capabilities',
'enabled',
'hostname',
'hypervisors_pools',
'port',
'user').merge({'table': 'hypervisors'}).changes().\
union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes())\
.run(self.r_conn):
#stop thread
if self.stop is True:
break
if c['new_val'] is not None:
if c['new_val']['table'] == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
# hypervisor deleted
if c['new_val'] is None:
if c['old_val'].get('table',False) == 'hypervisors':
logs.main.info('hypervisor deleted in rethink')
logs.main.info(pprint.pformat(c))
#TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if
# domains are running and hypevisor communication have lost
engine_restart()
# hypervisor created
elif c['old_val'] is None:
if c['new_val'].get('table',False) == 'hypervisors':
logs.main.info('hypervisor created in rethink')
logs.main.info(pprint.pformat(c))
engine_restart()
else:
if c['new_val'].get('table', False) == 'hypervisors':
#TODO: verify no domains in hypervisor running (front end and backend) and fence or unknown if
# domains are running and hypevisor communication have lost
logs.main.info('hypervisor fields modified in rethink')
logs.main.info(pprint.pformat(c))
engine_restart()
#self.manager.q.background.put({'type': 'add_hyp'})
self.r_conn.close()
def run(self):
# Close connection on exit (to test cleanup paths)
self.tid = get_tid()
logs.status.info('starting thread: {} (TID {})'.format(
self.name, self.tid))
old_exitfunc = getattr(sys, 'exitfunc', None)
def exit():
logs.status.info('Closing hypervisors connexions')
for hyp_id, hostname in self.hyps.items():
self.hyps_conn[hyp_id].close()
if (old_exitfunc): old_exitfunc()
sys.exitfunc = exit
self.thread_event_loop = virEventLoopNativeStart(self.stop_event_loop)
# self.r_status = RethinkHypEvent()
while self.stop is not True:
try:
action = self.q_event_register.get(
timeout=TIMEOUT_QUEUE_REGISTER_EVENTS)
if action['type'] in ['add_hyp_to_receive_events']:
hyp_id = action['hyp_id']
self.add_hyp_to_receive_events(hyp_id)
elif action['type'] in ['del_hyp_to_receive_events']:
hyp_id = action['hyp_id']
self.del_hyp_to_receive_events(hyp_id)
elif action['type'] == 'stop_thread':
self.stop = True
else:
logs.status.error('type action {} not supported'.format(
action['type']))
except queue.Empty:
pass
except Exception as e:
log.error(
'Exception in ThreadHypEvents main loop: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
self.stop_event_loop[0] = True
while self.thread_event_loop.is_alive():
pass
def run(self):
# Close connection on exit (to test cleanup paths)
self.tid = get_tid()
logs.status.info('starting thread: {} (TID {})'.format(self.name, self.tid))
old_exitfunc = getattr(sys, 'exitfunc', None)
def exit():
logs.status.info('Closing hypervisors connexions')
for hyp_id, hostname in self.hyps.items():
self.hyps_conn[hyp_id].close()
if (old_exitfunc): old_exitfunc()
sys.exitfunc = exit
self.thread_event_loop = virEventLoopNativeStart(self.stop_event_loop)
# self.r_status = RethinkHypEvent()
while self.stop is not True:
try:
action = self.q_event_register.get(timeout=TIMEOUT_QUEUE_REGISTER_EVENTS)
if action['type'] in ['add_hyp_to_receive_events']:
hyp_id = action['hyp_id']
self.add_hyp_to_receive_events(hyp_id)
elif action['type'] in ['del_hyp_to_receive_events']:
hyp_id = action['hyp_id']
self.del_hyp_to_receive_events(hyp_id)
elif action['type'] == 'stop_thread':
self.stop = True
else:
logs.status.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception in ThreadHypEvents main loop: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
self.stop_event_loop[0] = True
while self.thread_event_loop.is_alive():
pass
def __init__(self, launch_threads=True, with_status_threads=True,
status_polling_interval=STATUS_POLLING_INTERVAL,
test_hyp_fail_interval=TEST_HYP_FAIL_INTERVAL):
logs.main.info('MAIN TID: {}'.format(get_tid()))
self.time_between_polling = TIME_BETWEEN_POLLING
self.polling_interval_background = POLLING_INTERVAL_BACKGROUND
self.with_status_threads = with_status_threads
self.q = self.QueuesThreads()
self.t_workers = {}
self.t_status = {}
self.pools = {}
self.t_disk_operations = {}
self.q_disk_operations = {}
self.t_long_operations = {}
self.q_long_operations = {}
self.t_changes_hyps = None
self.t_events = None
self.t_changes_domains = None
self.t_broom = None
self.t_background = None
self.t_downloads_changes = None
self.t_grafana = None
self.quit = False
self.threads_info_main = {}
self.threads_info_hyps = {}
self.hypers_disk_operations_tested = []
self.num_workers = 0
self.threads_main_started = False
self.STATUS_POLLING_INTERVAL = status_polling_interval
self.TEST_HYP_FAIL_INTERVAL = test_hyp_fail_interval
update_all_hyps_status(reset_status='Offline')
if launch_threads is True:
self.launch_thread_background_polling()
def __init__(self, launch_threads=True, with_status_threads=True,
status_polling_interval=STATUS_POLLING_INTERVAL,
test_hyp_fail_interval=TEST_HYP_FAIL_INTERVAL):
logs.main.info('MAIN TID: {}'.format(get_tid()))
self.time_between_polling = TIME_BETWEEN_POLLING
self.polling_interval_background = POLLING_INTERVAL_BACKGROUND
self.with_status_threads = with_status_threads
self.q = self.QueuesThreads()
self.t_workers = {}
self.t_status = {}
self.pools = {}
self.t_disk_operations = {}
self.q_disk_operations = {}
self.t_long_operations = {}
self.q_long_operations = {}
self.t_changes_hyps = None
self.t_events = None
self.t_changes_domains = None
self.t_broom = None
self.t_background = None
self.t_downloads_changes = None
self.t_grafana = None
self.quit = False
self.threads_info_main = {}
self.threads_info_hyps = {}
self.hypers_disk_operations_tested = []
self.num_workers = 0
self.threads_main_started = False
self.STATUS_POLLING_INTERVAL = status_polling_interval
self.TEST_HYP_FAIL_INTERVAL = test_hyp_fail_interval
update_all_hyps_status(reset_status='Offline')
if launch_threads is True:
self.launch_thread_background_polling()
def run(self):
# Close connection on exit (to test cleanup paths)
self.tid = get_tid()
logs.status.info('starting thread: {} (TID {})'.format(
self.name, self.tid))
old_exitfunc = getattr(sys, 'exitfunc', None)
def exit():
logs.status.info('Closing hypervisors connexions')
for hyp_id, hostname in self.hyps.items():
self.hyps_conn[hyp_id].close()
if (old_exitfunc): old_exitfunc()
sys.exitfunc = exit
# self.r_status = RethinkHypEvent()
while True:
if len(self.hyps) == 0:
if self.stop:
break
time.sleep(0.1)
else:
self.thread_event_loop = virEventLoopNativeStart(
self.stop_event_loop)
for hyp_id, hostname in self.hyps.items():
self.add_hyp_to_receive_events(hyp_id)
while self.stop is not True:
time.sleep(0.1)
if self.stop is True:
for hyp_id in list(self.hyps):
self.del_hyp_to_receive_events(hyp_id)
self.stop_event_loop[0] = True
while self.thread_event_loop.is_alive():
pass
break
def run(self):
self.tid = get_tid()
logs.broom.info('starting thread: {} (TID {})'.format(self.name, self.tid))
self.polling()
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug(
'Thread to launchdisks operations in host {} with TID: {}...'.
format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(
host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error(
'Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'
.format(action['disk_path'], action['domain'],
self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status(
'Failed',
id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info(
'Disk created from virt-builder. Domain: {} , disk: {}'
.format(action['domain'], action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain,
'xml_virt_install',
xml_virt_install)
update_domain_status(
'CreatingDomainFromBuilder',
id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(
action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'
.format(disk_path, id_domain))
log.error('\n'.join([
'cmd: {}'.format(action['ssh_commands'][i])
for i in range(len(action['ssh_commands']))
]))
update_domain_status(
'Failed',
id_domain,
detail=
'new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs'
)
def run(self):
self.tid = get_tid()
logs.downloads.debug(
'RUN-DOWNLOAD-THREAD-------------------------------------')
pool_id = 'default'
first_loop = True
if self.stop is False:
if first_loop is True:
# if domains or media have status Downloading when engine restart
# we need to resetdownloading deleting file and
first_loop = False
# wait a hyp to downloads
next_hyp = False
while next_hyp is False:
logs.downloads.info(
'waiting an hypervisor online to launch downloading actions'
)
if pool_id in self.manager.pools.keys():
next_hyp = self.manager.pools[pool_id].get_next()
sleep(1)
for hyp_id in get_hypers_in_pool():
self.killall_curl(hyp_id)
domains_status_downloading = get_domains_with_status(
'Downloading')
medias_status_downloading = get_media_with_status(
'Downloading')
for id_domain in domains_status_downloading:
create_dict = get_domain(id_domain)['create_dict']
dict_changes = {
'id': id_domain,
'table': 'domains',
'create_dict': create_dict
}
update_domain_status('ResetDownloading', id_domain)
self.abort_download(dict_changes,
final_status='DownloadFailed')
for id_media in medias_status_downloading:
dict_media = get_media(id_media)
dict_changes = {
'id': id_media,
'table': 'media',
'path': dict_media['path'],
'hypervisors_pools': dict_media['hypervisors_pools']
}
update_status_table('media', 'ResetDownloading', id_media)
self.abort_download(dict_changes,
final_status='DownloadFailed')
self.r_conn = new_rethink_connection()
update_table_field('hypervisors_pools', pool_id,
'download_changes', 'Started')
for c in r.table('media').get_all(r.args(
['Deleting', 'Deleted', 'Downloaded', 'DownloadFailed', 'DownloadStarting', 'Downloading', 'Download',
'DownloadAborting','ResetDownloading']), index='status'). \
pluck('id',
'path',
'url-isard',
'url-web',
'status'
).merge(
{'table': 'media'}).changes(include_initial=True).union(
r.table('domains').get_all(
r.args(['Downloaded', 'DownloadFailed','DownloadStarting', 'Downloading', 'DownloadAborting','ResetDownloading']), index='status'). \
pluck('id',
'create_dict',
'url-isard',
'url-web',
'status').merge(
{"table": "domains"}).changes(include_initial=True)).union(
r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).run(
self.r_conn):
if self.stop:
break
if c.get('new_val', None) is not None:
if c['new_val'].get('table', False) == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
else:
continue
logs.downloads.debug('DOWNLOAD CHANGES DETECTED:')
logs.downloads.debug(pprint.pformat(c))
if c.get('old_val', None) is None:
if c['new_val']['status'] == 'DownloadStarting':
self.start_download(c['new_val'])
elif c.get('new_val', None) is None:
if c['old_val']['status'] in ['DownloadAborting']:
self.remove_download_thread(c['old_val'])
elif 'old_val' in c and 'new_val' in c:
if c['old_val']['status'] == 'DownloadFailed' and c[
'new_val']['status'] == 'DownloadStarting':
self.start_download(c['new_val'])
elif c['old_val']['status'] == 'Downloaded' and c[
'new_val']['status'] == 'Deleting':
if c['new_val']['table'] == 'media':
self.delete_media(c['new_val'])
elif c['old_val']['status'] == 'Deleting' and c['new_val'][
'status'] == 'Deleted':
if c['new_val']['table'] == 'media':
remove_media(c['new_val']['id'])
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'DownloadFailed':
pass
elif c['old_val']['status'] == 'DownloadStarting' and c[
'new_val']['status'] == 'Downloading':
pass
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'Downloaded':
pass
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'DownloadAborting':
self.abort_download(c['new_val'])
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'ResetDownloading':
self.abort_download(c['new_val'],
final_status='DownloadFailed')
def run(self):
self.tid = get_tid()
logs.main.info('starting thread background: {} (TID {})'.format(self.name, self.tid))
q = self.manager.q.background
first_loop = True
pool_id = 'default'
#can't launch downloads if download changes thread is not ready and hyps are not online
update_table_field('hypervisors_pools', pool_id, 'download_changes', 'Stopped')
# if domains have intermedite states (updating, download_aborting...)
# to Failed or Delete
clean_intermediate_status()
l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
# while len(l_hyps_to_test) == 0:
# logs.main.error('no hypervisor enable, waiting for one hypervisor')
# sleep(0.5)
# l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
while self.manager.quit is False:
####################################################################
### MAIN LOOP ######################################################
# ONLY FOR DEBUG
logs.main.debug('##### THREADS ##################')
threads_running = get_threads_running()
#pprint.pprint(threads_running)
#self.manager.update_info_threads_engine()
# Threads that must be running always, with or withouth hypervisor:
# - changes_hyp
# - changes_domains
# - downloads_changes
# - broom
# - events
# - grafana
# Threads that depends on hypervisors availavility:
# - disk_operations
# - long_operations
# - for every hypervisor:
# - worker
# - status
# LAUNCH MAIN THREADS
if first_loop is True:
update_table_field('engine', 'engine', 'status_all_threads', 'Starting')
# launch changes_hyp thread
self.manager.t_changes_hyps = self.manager.HypervisorChangesThread('changes_hyp', self.manager)
self.manager.t_changes_hyps.daemon = True
self.manager.t_changes_hyps.start()
#launch changes_domains_thread
self.manager.t_changes_domains = self.manager.DomainsChangesThread('changes_domains', self.manager)
self.manager.t_changes_domains.daemon = True
self.manager.t_changes_domains.start()
#launch downloads changes thread
logs.main.debug('Launching Download Changes Thread')
self.manager.t_downloads_changes = launch_thread_download_changes(self.manager)
#launch brom thread
self.manager.t_broom = launch_thread_broom(self.manager)
#launch events thread
logs.main.debug('launching hypervisor events thread')
self.manager.t_events = launch_thread_hyps_event()
#launch grafana thread
logs.main.debug('launching grafana thread')
self.manager.t_grafana = launch_grafana_thread(self.manager.t_status)
logs.main.info('THREADS LAUNCHED FROM BACKGROUND THREAD')
update_table_field('engine', 'engine', 'status_all_threads', 'Starting')
while True:
#wait all
sleep(0.1)
self.manager.update_info_threads_engine()
#if len(self.manager.threads_info_main['not_defined']) > 0 and len(self.manager.dict_hyps_ready) == 0:
if len(self.manager.threads_info_main['not_defined']) > 0 or len(self.manager.threads_info_main['dead']) > 0:
print('MAIN THREADS starting, wait a second extra')
sleep(1)
self.manager.update_info_threads_engine()
pprint.pprint(self.manager.threads_info_main)
#self.test_hyps_and_start_threads()
if len(self.manager.threads_info_main['not_defined']) == 0 and len(self.manager.threads_info_main['dead']) == 0:
update_table_field('engine', 'engine', 'status_all_threads', 'Started')
self.manager.threads_main_started = True
break
# TEST HYPS AND START THREADS FOR HYPERVISORS
self.test_hyps_and_start_threads()
self.manager.num_workers = len(self.manager.t_workers)
# Test hypervisor disk operations
# Create Test disk in hypervisor disk operations
if first_loop is True:
first_loop = False
# virtio_test_disk_relative_path = 'admin/admin/admin/virtio_testdisk.qcow2'
# ui.creating_test_disk(test_disk_relative_route=virtio_test_disk_relative_path)
self.manager.update_info_threads_engine()
if len(self.manager.threads_info_hyps['not_defined']) > 0:
logs.main.error('something was wrong when launching threads for hypervisors, threads not defined')
logs.main.error(pprint.pformat(self.manager.threads_info_hyps))
if len(self.manager.threads_info_hyps['dead']) > 0:
logs.main.error('something was wrong when launching threads for hypervisors, threads are dead')
logs.main.error(pprint.pformat(self.manager.threads_info_hyps))
if len(self.manager.threads_info_hyps['dead']) == 0 and len(self.manager.threads_info_hyps['not_defined']) == 0:
pass
try:
if len(self.manager.t_workers) == 0:
timeout_queue = WAIT_HYP_ONLINE
else:
timeout_queue = TEST_HYP_FAIL_INTERVAL
action = q.get(timeout=timeout_queue)
if action['type'] == 'stop':
self.manager.quit = True
logs.main.info('engine end')
except queue.Empty:
pass
except Exception as e:
logs.main.error(e)
return False
def run(self):
self.tid = get_tid()
log.info('starting thread: {} (TID {})'.format(self.name, self.tid))
self.disk_operations_thread()
def run(self):
self.tid = get_tid()
logs.main.info('starting thread background: {} (TID {})'.format(self.name, self.tid))
q = self.manager.q.background
first_loop = True
pool_id = 'default'
#can't launch downloads if download changes thread is not ready and hyps are not online
update_table_field('hypervisors_pools', pool_id, 'download_changes', 'Stopped')
# if domains have intermedite states (updating, download_aborting...)
# to Failed or Delete
clean_intermediate_status()
l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
# while len(l_hyps_to_test) == 0:
# logs.main.error('no hypervisor enable, waiting for one hypervisor')
# sleep(0.5)
# l_hyps_to_test = get_hyps_with_status(list_status=['Error', 'Offline'], empty=True)
while self.manager.quit is False:
####################################################################
### MAIN LOOP ######################################################
# ONLY FOR DEBUG
logs.main.debug('##### THREADS ##################')
threads_running = get_threads_running()
#pprint.pprint(threads_running)
#self.manager.update_info_threads_engine()
# Threads that must be running always, with or withouth hypervisor:
# - changes_hyp
# - changes_domains
# - downloads_changes
# - broom
# - events
# - grafana
# Threads that depends on hypervisors availavility:
# - disk_operations
# - long_operations
# - for every hypervisor:
# - worker
# - status
# LAUNCH MAIN THREADS
if first_loop is True:
update_table_field('engine', 'engine', 'status_all_threads', 'Starting')
# launch changes_hyp thread
self.manager.t_changes_hyps = self.manager.HypervisorChangesThread('changes_hyp', self.manager)
self.manager.t_changes_hyps.daemon = True
self.manager.t_changes_hyps.start()
#launch changes_domains_thread
self.manager.t_changes_domains = self.manager.DomainsChangesThread('changes_domains', self.manager)
self.manager.t_changes_domains.daemon = True
self.manager.t_changes_domains.start()
#launch downloads changes thread
logs.main.debug('Launching Download Changes Thread')
self.manager.t_downloads_changes = launch_thread_download_changes(self.manager)
#launch brom thread
self.manager.t_broom = launch_thread_broom(self.manager)
#launch events thread
logs.main.debug('launching hypervisor events thread')
self.manager.t_events = launch_thread_hyps_event()
#launch grafana thread
logs.main.debug('launching grafana thread')
self.manager.t_grafana = launch_grafana_thread(self.manager.t_status)
logs.main.info('THREADS LAUNCHED FROM BACKGROUND THREAD')
update_table_field('engine', 'engine', 'status_all_threads', 'Starting')
while True:
#wait all
sleep(0.1)
self.manager.update_info_threads_engine()
#if len(self.manager.threads_info_main['not_defined']) > 0 and len(self.manager.dict_hyps_ready) == 0:
if len(self.manager.threads_info_main['not_defined']) > 0 or len(self.manager.threads_info_main['dead']) > 0:
print('MAIN THREADS starting, wait a second extra')
sleep(1)
self.manager.update_info_threads_engine()
pprint.pprint(self.manager.threads_info_main)
#self.test_hyps_and_start_threads()
if len(self.manager.threads_info_main['not_defined']) == 0 and len(self.manager.threads_info_main['dead']) == 0:
update_table_field('engine', 'engine', 'status_all_threads', 'Started')
self.manager.threads_main_started = True
break
# TEST HYPS AND START THREADS FOR HYPERVISORS
self.test_hyps_and_start_threads()
self.manager.num_workers = len(self.manager.t_workers)
# Test hypervisor disk operations
# Create Test disk in hypervisor disk operations
if first_loop is True:
first_loop = False
# virtio_test_disk_relative_path = 'admin/admin/admin/virtio_testdisk.qcow2'
# ui.creating_test_disk(test_disk_relative_route=virtio_test_disk_relative_path)
self.manager.update_info_threads_engine()
if len(self.manager.threads_info_hyps['not_defined']) > 0:
logs.main.error('something was wrong when launching threads for hypervisors, threads not defined')
logs.main.error(pprint.pformat(self.manager.threads_info_hyps))
if len(self.manager.threads_info_hyps['dead']) > 0:
logs.main.error('something was wrong when launching threads for hypervisors, threads are dead')
logs.main.error(pprint.pformat(self.manager.threads_info_hyps))
if len(self.manager.threads_info_hyps['dead']) == 0 and len(self.manager.threads_info_hyps['not_defined']) == 0:
pass
try:
if len(self.manager.t_workers) == 0:
timeout_queue = WAIT_HYP_ONLINE
else:
timeout_queue = TEST_HYP_FAIL_INTERVAL
action = q.get(timeout=timeout_queue)
if action['type'] == 'stop':
self.manager.quit = True
logs.main.info('engine end')
except queue.Empty:
pass
except Exception as e:
logs.main.error(e)
return False
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port
)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format(
action['disk_path'], action['domain'], self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status('Failed', id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'],
action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install)
update_domain_status('CreatingDomainFromBuilder', id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def run(self):
self.tid = get_tid()
logs.changes.info('starting thread: {} (TID {})'.format(
self.name, self.tid))
logs.changes.debug(
'^^^^^^^^^^^^^^^^^^^ DOMAIN CHANGES THREAD ^^^^^^^^^^^^^^^^^')
ui = UiActions(self.manager)
self.r_conn = new_rethink_connection()
cursor = r.table('domains').pluck('id', 'kind', 'status', 'detail').merge({'table': 'domains'}).changes().\
union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).\
run(self.r_conn)
for c in cursor:
if self.stop is True:
break
if c.get('new_val', None) is not None:
if c['new_val']['table'] == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
else:
continue
logs.changes.debug('domain changes detected in main thread')
detail_msg_if_no_hyps_online = 'No hypervisors Online in pool'
if self.manager.check_actions_domains_enabled() is False:
if c.get('new_val', None) is not None:
if c.get('old_val', None) is not None:
if c['new_val']['status'][-3:] == 'ing':
update_domain_status(
c['old_val']['status'],
c['old_val']['id'],
detail=detail_msg_if_no_hyps_online)
#if no hypervisor availables no check status changes
continue
new_domain = False
new_status = False
old_status = False
import pprint
logs.changes.debug(pprint.pformat(c))
# action deleted
if c.get('new_val', None) is None:
pass
# action created
if c.get('old_val', None) is None:
new_domain = True
new_status = c['new_val']['status']
domain_id = c['new_val']['id']
logs.changes.debug('domain_id: {}'.format(new_domain))
# if engine is stopped/restarting or not hypervisors online
if self.manager.check_actions_domains_enabled() is False:
continue
pass
if c.get('new_val', None) is not None and c.get(
'old_val', None) is not None:
old_status = c['old_val']['status']
new_status = c['new_val']['status']
new_detail = c['new_val']['detail']
domain_id = c['new_val']['id']
logs.changes.debug('domain_id: {}'.format(domain_id))
# if engine is stopped/restarting or not hypervisors online
if old_status != new_status:
# print('&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.format(domain_id,old_status,new_status, new_detail))
# if new_status[-3:] == 'ing':
if 1 > 0:
date_now = datetime.now()
update_domain_history_from_id_domain(
domain_id, new_status, new_detail, date_now)
else:
# print('&&&&&&&ESTADOS IGUALES OJO &&&&&&&\n&&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.
# format(domain_id,old_status,new_status,new_detail))
pass
if (new_domain is True and new_status == "CreatingDiskFromScratch") or \
(old_status == 'FailedCreatingDomain' and new_status == "CreatingDiskFromScratch"):
ui.creating_disk_from_scratch(domain_id)
if (new_domain is True and new_status == "Creating") or \
(old_status == 'FailedCreatingDomain' and new_status == "Creating"):
ui.creating_disks_from_template(domain_id)
if (new_domain is True
and new_status == "CreatingAndStarting"):
update_domain_start_after_created(domain_id)
ui.creating_disks_from_template(domain_id)
# INFO TO DEVELOPER
# recoger template de la que hay que derivar
# verificar que realmente es una template
# hay que recoger ram?? cpu?? o si no hay nada copiamos de la template??
if (new_domain is True and new_status == "CreatingFromBuilder") or \
(old_status == 'FailedCreatingDomain' and new_status == "CreatingFromBuilder"):
ui.creating_disk_from_virtbuilder(domain_id)
if (old_status in ['CreatingDisk','CreatingDiskFromScratch'] and new_status == "CreatingDomain") or \
(old_status == 'RunningVirtBuilder' and new_status == "CreatingDomainFromBuilder"):
logs.changes.debug(
'llamo a creating_and_test_xml con domain id {}'.
format(domain_id))
if new_status == "CreatingDomainFromBuilder":
ui.creating_and_test_xml_start(
domain_id,
creating_from_create_dict=True,
xml_from_virt_install=True)
if new_status == "CreatingDomain":
ui.creating_and_test_xml_start(
domain_id, creating_from_create_dict=True)
if old_status == 'Stopped' and new_status == "CreatingTemplate":
ui.create_template_disks_from_domain(domain_id)
if old_status == 'Stopped' and new_status == "Deleting":
ui.deleting_disks_from_domain(domain_id)
if (old_status == 'Stopped' and new_status == "Updating") or \
(old_status == 'Downloaded' and new_status == "Updating"):
ui.updating_from_create_dict(domain_id)
if old_status == 'DeletingDomainDisk' and new_status == "DiskDeleted":
logs.changes.debug(
'disk deleted, mow remove domain form database')
remove_domain(domain_id)
if get_domain(domain_id) is None:
logs.changes.info(
'domain {} deleted from database'.format(
domain_id))
else:
update_domain_status(
'Failed',
id_domain,
detail='domain {} can not be deleted from database'
.format(domain_id))
if old_status == 'CreatingTemplateDisk' and new_status == "TemplateDiskCreated":
# create_template_from_dict(dict_new_template)
if get_if_all_disk_template_created(domain_id):
ui.create_template_in_db(domain_id)
else:
# INFO TO DEVELOPER, este else no se si tiene mucho sentido, hay que hacer pruebas con la
# creación de una template con dos discos y ver si pasa por aquí
# waiting to create other disks
update_domain_status(
status='CreatingTemplateDisk',
id_domain=domain_id,
hyp_id=False,
detail='Waiting to create more disks for template')
if (old_status == 'Stopped' and new_status == "Starting") or \
(old_status == 'Failed' and new_status == "Starting"):
ui.start_domain_from_id(id=domain_id, ssl=True)
if (old_status == 'Started' and new_status == "Stopping" ) or \
(old_status == 'Suspended' and new_status == "Stopping" ):
# INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started
# ui.stop_domain_from_id(id=domain_id)
hyp_started = get_domain_hyp_started(domain_id)
if hyp_started:
ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started)
else:
logs.main.error(
'domain without hyp_started can not be stopped: {}. '
.format(domain_id))
if (old_status == 'Started' and new_status == "StoppingAndDeleting" ) or \
(old_status == 'Suspended' and new_status == "StoppingAndDeleting" ):
# INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started
# ui.stop_domain_from_id(id=domain_id)
hyp_started = get_domain_hyp_started(domain_id)
print(hyp_started)
ui.stop_domain(id_domain=domain_id,
hyp_id=hyp_started,
delete_after_stopped=True)
logs.main.info('finalished thread domain changes')
def run(self):
self.tid = get_tid()
logs.main.info('starting thread: {} (TID {})'.format(
self.name, self.tid))
#get hostname grafana config
self.get_hostname_grafana()
hyps_online = []
elapsed = self.send_static_values_interval
while self.stop is False:
sleep(self.send_to_grafana_interval)
elapsed += self.send_to_grafana_interval
if self.restart_send_config is True:
self.restart_send_config = False
self.get_hostname_grafana()
if self.active is True:
for i, id_hyp in enumerate(self.t_status.keys()):
try:
if self.t_status[
id_hyp].status_obj.hyp_obj.connected is True:
if id_hyp not in hyps_online:
hyps_online.append(id_hyp)
check_hyp = True
except:
logs.main.error(
f'hypervisor {id_hyp} problem checking if is connected'
)
check_hyp = False
if len(hyps_online) > 0 and check_hyp is True:
#send static values of hypervisors
if elapsed >= self.send_static_values_interval:
d_hyps_info = dict()
for i, id_hyp in enumerate(hyps_online):
d_hyps_info[f'hyp-info-{i}'] = self.t_status[
id_hyp].status_obj.hyp_obj.info
# ~ self.send(d_hyps_info)
elapsed = 0
#send stats
dict_to_send = dict()
j = 0
for i, id_hyp in enumerate(hyps_online):
if id_hyp in self.t_status.keys():
#stats_hyp = self.t_status[id_hyp].status_obj.hyp_obj.stats_hyp
stats_hyp_now = self.t_status[
id_hyp].status_obj.hyp_obj.stats_hyp_now
#stats_domains = self.t_status[id_hyp].status_obj.hyp_obj.stats_domains
if len(stats_hyp_now) > 0:
dict_to_send[f'hypers.' + id_hyp] = {
'stats': stats_hyp_now,
'info': d_hyps_info['hyp-info-' + str(i)],
'domains': {}
}
stats_domains_now = self.t_status[
id_hyp].status_obj.hyp_obj.stats_domains_now
# ~ for id_domain,d_stats in stats_domains_now.items():
# ~ if len(stats_hyp_now) > 0:
# ~ for id_domain,d_stats in stats_domains_now.items():
# ~ dict_to_send[f'domain-stats-{j}'] = {'domain-id':{id_domain:1},'last': d_stats,}
dict_to_send[f'hypers.' + id_hyp][
'domains'] = stats_domains_now #{x:0 for x in stats_domains_now}
# ~ print(stats_domains_now)
# ~ j+=1
if len(dict_to_send) > 0:
self.send(dict_to_send)
def run(self):
self.tid = get_tid()
logs.main.info('starting thread: {} (TID {})'.format(
self.name, self.tid))
q = self.manager.q.background
first_loop = True
while self.manager.quit is False:
# ONLY FOR DEBUG
logs.main.debug('##### THREADS ##################')
get_threads_running()
self.manager.update_info_threads_engine()
# DISK_OPERATIONS:
if len(self.manager.t_disk_operations) == 0:
self.launch_threads_disk_and_long_operations()
# TEST HYPS AND START THREADS FROM RETHINK
self.test_hyps_and_start_threads()
# LAUNCH CHANGES THREADS
if first_loop is True:
update_table_field('engine', 'engine',
'status_all_threads', 'Starting')
self.manager.t_changes_hyps = self.manager.HypervisorChangesThread(
'changes_hyp', self.manager)
self.manager.t_changes_hyps.daemon = True
self.manager.t_changes_hyps.start()
self.manager.t_changes_domains = self.manager.DomainsChangesThread(
'changes_domains', self.manager)
self.manager.t_changes_domains.daemon = True
self.manager.t_changes_domains.start()
logs.main.debug('Launching Download Changes Thread')
self.manager.t_downloads_changes = launch_thread_download_changes(
self.manager)
self.manager.t_broom = launch_thread_broom(self.manager)
first_loop = False
logs.main.info('THREADS LAUNCHED FROM BACKGROUND THREAD')
update_table_field('engine', 'engine',
'status_all_threads', 'Starting')
while True:
sleep(0.1)
alive, dead, not_defined = self.manager.update_info_threads_engine(
)
pprint.pprint({
'alive': alive,
'dead': dead,
'not_defined': not_defined
})
if len(not_defined) == 0 and len(dead) == 0:
update_table_field('engine', 'engine',
'status_all_threads', 'Started')
self.manager.num_workers = len(
self.manager.t_workers)
self.manager.threads_started = True
break
try:
action = q.get(timeout=self.manager.TEST_HYP_FAIL_INTERVAL)
if action['type'] == 'stop':
self.manager.quit = True
logs.main.info('engine end')
except queue.Empty:
pass
except Exception as e:
logs.main.error(e)
return False
def run(self):
self.tid = get_tid()
logs.changes.info('starting thread: {} (TID {})'.format(self.name, self.tid))
logs.changes.debug('^^^^^^^^^^^^^^^^^^^ DOMAIN CHANGES THREAD ^^^^^^^^^^^^^^^^^')
ui = UiActions(self.manager)
self.r_conn = new_rethink_connection()
cursor = r.table('domains').pluck('id', 'kind', 'status', 'detail').merge({'table': 'domains'}).changes().\
union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).\
run(self.r_conn)
for c in cursor:
if self.stop is True:
break
if c.get('new_val', None) is not None:
if c['new_val']['table'] == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
else:
continue
logs.changes.debug('domain changes detected in main thread')
detail_msg_if_no_hyps_online = 'No hypervisors Online in pool'
if self.manager.check_actions_domains_enabled() is False:
if c.get('new_val', None) is not None:
if c.get('old_val', None) is not None:
if c['new_val']['status'][-3:] == 'ing':
update_domain_status(c['old_val']['status'], c['old_val']['id'], detail=detail_msg_if_no_hyps_online)
#if no hypervisor availables no check status changes
continue
new_domain = False
new_status = False
old_status = False
logs.changes.debug(pprint.pformat(c))
# action deleted
if c.get('new_val', None) is None:
pass
# action created
if c.get('old_val', None) is None:
new_domain = True
new_status = c['new_val']['status']
domain_id = c['new_val']['id']
logs.changes.debug('domain_id: {}'.format(new_domain))
# if engine is stopped/restarting or not hypervisors online
if self.manager.check_actions_domains_enabled() is False:
continue
pass
if c.get('new_val', None) is not None and c.get('old_val', None) is not None:
old_status = c['old_val']['status']
new_status = c['new_val']['status']
new_detail = c['new_val']['detail']
domain_id = c['new_val']['id']
logs.changes.debug('domain_id: {}'.format(domain_id))
# if engine is stopped/restarting or not hypervisors online
if old_status != new_status:
# print('&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.format(domain_id,old_status,new_status, new_detail))
# if new_status[-3:] == 'ing':
if 1 > 0:
date_now = datetime.now()
update_domain_history_from_id_domain(domain_id, new_status, new_detail, date_now)
else:
# print('&&&&&&&ESTADOS IGUALES OJO &&&&&&&\n&&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.
# format(domain_id,old_status,new_status,new_detail))
pass
if (new_domain is True and new_status == "CreatingDiskFromScratch") or \
(old_status == 'FailedCreatingDomain' and new_status == "CreatingDiskFromScratch"):
ui.creating_disk_from_scratch(domain_id)
if (new_domain is True and new_status == "Creating") or \
(old_status == 'FailedCreatingDomain' and new_status == "Creating"):
ui.creating_disks_from_template(domain_id)
if (new_domain is True and new_status == "CreatingAndStarting"):
update_domain_start_after_created(domain_id)
ui.creating_disks_from_template(domain_id)
# INFO TO DEVELOPER
# recoger template de la que hay que derivar
# verificar que realmente es una template
# hay que recoger ram?? cpu?? o si no hay nada copiamos de la template??
if (new_domain is True and new_status == "CreatingFromBuilder") or \
(old_status == 'FailedCreatingDomain' and new_status == "CreatingFromBuilder"):
ui.creating_disk_from_virtbuilder(domain_id)
if (old_status in ['CreatingDisk','CreatingDiskFromScratch'] and new_status == "CreatingDomain") or \
(old_status == 'RunningVirtBuilder' and new_status == "CreatingDomainFromBuilder"):
logs.changes.debug('llamo a creating_and_test_xml con domain id {}'.format(domain_id))
if new_status == "CreatingDomainFromBuilder":
ui.creating_and_test_xml_start(domain_id,
creating_from_create_dict=True,
xml_from_virt_install=True,ssl=True)
if new_status == "CreatingDomain":
ui.creating_and_test_xml_start(domain_id,
creating_from_create_dict=True,ssl=True)
if old_status == 'Stopped' and new_status == "CreatingTemplate":
ui.create_template_disks_from_domain(domain_id)
if old_status != 'Started' and new_status == "Deleting":
# or \
# old_status == 'Failed' and new_status == "Deleting" or \
# old_status == 'Downloaded' and new_status == "Deleting":
ui.deleting_disks_from_domain(domain_id)
if (old_status == 'Stopped' and new_status == "Updating") or \
(old_status == 'Downloaded' and new_status == "Updating"):
ui.updating_from_create_dict(domain_id,ssl=True)
if old_status == 'DeletingDomainDisk' and new_status == "DiskDeleted":
logs.changes.debug('disk deleted, mow remove domain form database')
remove_domain(domain_id)
if get_domain(domain_id) is None:
logs.changes.info('domain {} deleted from database'.format(domain_id))
else:
update_domain_status('Failed', domain_id,
detail='domain {} can not be deleted from database'.format(domain_id))
if old_status == 'CreatingTemplateDisk' and new_status == "TemplateDiskCreated":
# create_template_from_dict(dict_new_template)
if get_if_all_disk_template_created(domain_id):
ui.create_template_in_db(domain_id)
else:
# INFO TO DEVELOPER, este else no se si tiene mucho sentido, hay que hacer pruebas con la
# creación de una template con dos discos y ver si pasa por aquí
# waiting to create other disks
update_domain_status(status='CreatingTemplateDisk',
id_domain=domain_id,
hyp_id=False,
detail='Waiting to create more disks for template')
if (old_status == 'Stopped' and new_status == "Starting") or \
(old_status == 'Failed' and new_status == "Starting"):
ui.start_domain_from_id(id=domain_id, ssl=True)
if (old_status == 'Started' and new_status == "Stopping" ) or \
(old_status == 'Suspended' and new_status == "Stopping" ):
# INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started
# ui.stop_domain_from_id(id=domain_id)
hyp_started = get_domain_hyp_started(domain_id)
if hyp_started:
ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started)
else:
logs.main.error('domain without hyp_started can not be stopped: {}. '.format(domain_id))
if (old_status == 'Started' and new_status == "StoppingAndDeleting" ) or \
(old_status == 'Suspended' and new_status == "StoppingAndDeleting" ):
# INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started
# ui.stop_domain_from_id(id=domain_id)
hyp_started = get_domain_hyp_started(domain_id)
print(hyp_started)
ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started, delete_after_stopped=True)
logs.main.info('finalished thread domain changes')
def run(self):
self.tid = get_tid()
logs.broom.info('starting thread: {} (TID {})'.format(
self.name, self.tid))
self.polling()
def run(self):
self.tid = get_tid()
logs.workers.info('starting thread: {} (TID {})'.format(self.name, self.tid))
host, port, user = get_hyp_hostname_from_id(self.hyp_id)
port = int(port)
self.hostname = host
self.h = hyp(self.hostname, user=user, port=port)
# self.h.get_kvm_mod()
# self.h.get_hyp_info()
update_db_hyp_info(self.hyp_id, self.h.info)
hyp_id = self.hyp_id
while self.stop is not True:
try:
# do={type:'start_domain','xml':'xml','id_domain'='prova'}
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
logs.workers.debug('received action in working thread {}'.format(action['type']))
if action['type'] == 'start_paused_domain':
logs.workers.debug('xml to start paused some lines...: {}'.format(action['xml'][30:100]))
try:
self.h.conn.createXML(action['xml'], flags=VIR_DOMAIN_START_PAUSED)
# 32 is the constant for domains paused
# reference: https://libvirt.org/html/libvirt-libvirt-domain.html#VIR_CONNECT_LIST_DOMAINS_PAUSED
FLAG_LIST_DOMAINS_PAUSED = 32
list_all_domains = self.h.conn.listAllDomains(FLAG_LIST_DOMAINS_PAUSED)
list_names_domains = [d.name() for d in list_all_domains]
dict_domains = dict(zip(list_names_domains,list_all_domains))
if action['id_domain'] in list_names_domains:
# domain started in pause mode
domain = dict_domains[action['id_domain']]
domain_active = True
try:
domain.isActive()
domain.destroy()
try:
domain.isActive()
except Exception as e:
logs.workers.debug('verified domain {} is destroyed'.format(action['id_domain']))
domain_active = False
except libvirtError as e:
from pprint import pformat
error_msg = pformat(e.get_error_message())
update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id,
detail='domain {} failed when try to destroy from paused domain in hypervisor {}. creating domain operation is aborted')
logs.workers.error(
'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format(
action['id_domain'], self.hyp_id, error_msg))
continue
if domain_active is False:
# domain is destroyed, all ok
update_domain_status('CreatingDomain', action['id_domain'], hyp_id='',
detail='Domain created and test OK: Started, paused and now stopped in hyp {}'.format(self.hyp_id))
logs.workers.debug(
'domain {} creating operation finalished. Started paused and destroyed in hypervisor {}. Now status is Stopped. READY TO USE'.format(
action['id_domain'], self.hyp_id))
else:
update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id,
detail='Domain is created, started in pause mode but not destroyed,creating domain operation is aborted')
logs.workers.error(
'domain {} started paused but not destroyed in hypervisor {}, must be destroyed'.format(
action['id_domain'], self.hyp_id))
else:
update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id,
detail='XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted by unknown cause'.format(
action['id_domain'], self.hyp_id))
logs.workers.error(
'XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted, not exception, rare case, unknown cause'.format(
action['id_domain'], self.hyp_id))
except libvirtError as e:
from pprint import pformat
error_msg = pformat(e.get_error_message())
update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id,
detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted')
logs.workers.error(
'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format(
action['id_domain'], self.hyp_id, error_msg))
except Exception as e:
update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id,
detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted')
logs.workers.error(
'Exception starting paused xml for domain {} in hypervisor {}. NOT LIBVIRT EXCEPTION, RARE CASE. Exception message: {}'.format(
action['id_domain'], self.hyp_id, str(e)))
## START DOMAIN
elif action['type'] == 'start_domain':
logs.workers.debug('xml to start some lines...: {}'.format(action['xml'][30:100]))
try:
self.h.conn.createXML(action['xml'])
# wait to event started to save state in database
#update_domain_status('Started', action['id_domain'], hyp_id=self.hyp_id, detail='Domain has started in worker thread')
logs.workers.debug('STARTED domain {}: createdXML action in hypervisor {} has been sent'.format(
action['id_domain'], host))
except libvirtError as e:
update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id,
detail=("Hypervisor can not create domain with libvirt exception: " + str(e)))
logs.workers.debug('exception in starting domain {}: '.format(e))
except Exception as e:
update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=("Exception when starting domain: " + str(e)))
logs.workers.debug('exception in starting domain {}: '.format(e))
## STOP DOMAIN
elif action['type'] == 'stop_domain':
logs.workers.debug('action stop domain: {}'.format(action['id_domain'][30:100]))
try:
self.h.conn.lookupByName(action['id_domain']).destroy()
logs.workers.debug('STOPPED domain {}'.format(action['id_domain']))
check_if_delete = action.get('delete_after_stopped',False)
if check_if_delete is True:
update_domain_status('Stopped', action['id_domain'], hyp_id='')
update_domain_status('Deleting', action['id_domain'], hyp_id='')
else:
update_domain_status('Stopped', action['id_domain'], hyp_id='')
except Exception as e:
update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=str(e))
logs.workers.debug('exception in stopping domain {}: '.format(e))
elif action['type'] in ['create_disk', 'delete_disk']:
launch_action_disk(action,
self.hostname,
user,
port)
elif action['type'] in ['add_media_hot']:
pass
elif action['type'] in ['killall_curl']:
launch_killall_curl(self.hostname,
user,
port)
elif action['type'] in ['delete_media']:
final_status = action.get('final_status','Deleted')
launch_delete_media (action,
self.hostname,
user,
port,
final_status=final_status)
# ## DESTROY THREAD
# elif action['type'] == 'destroy_thread':
# list_works_in_queue = list(self.queue_actions.queue)
# if self.queue_master is not None:
# self.queue_master.put(['destroy_working_thread',self.hyp_id,list_works_in_queue])
# #INFO TO DEVELOPER, si entra aquí es porque no quedaba nada en cola, si no ya lo habrán matado antes
#
# logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id))
#
# raise 'destoyed'
elif action['type'] == 'create_disk':
pass
elif action['type'] == 'hyp_info':
self.h.get_hyp_info()
logs.workers.debug('hypervisor motherboard: {}'.format(self.h.info['motherboard_manufacturer']))
## DESTROY THREAD
elif action['type'] == 'stop_thread':
self.stop = True
else:
logs.workers.error('type action {} not supported in queue actions'.format(action['type']))
# time.sleep(0.1)
## TRY DOMAIN
except queue.Empty:
try:
self.h.conn.getLibVersion()
pass
# logs.workers.debug('hypervisor {} is alive'.format(host))
except:
logs.workers.info('trying to reconnect hypervisor {}, alive test in working thread failed'.format(host))
alive = False
for i in range(RETRIES_HYP_IS_ALIVE):
try:
time.sleep(TIMEOUT_BETWEEN_RETRIES_HYP_IS_ALIVE)
self.h.conn.getLibVersion()
alive = True
logs.workers.info('hypervisor {} is alive'.format(host))
break
except:
logs.workers.info('hypervisor {} is NOT alive'.format(host))
if alive is False:
try:
self.h.connect_to_hyp()
self.h.conn.getLibVersion()
update_hyp_status(self.hyp_id, 'Online')
except:
logs.workers.debug('hypervisor {} failed'.format(host))
logs.workers.error('fail reconnecting to hypervisor {} in working thread'.format(host))
reason = self.h.fail_connected_reason
update_hyp_status(self.hyp_id, 'Error', reason)
update_domains_started_in_hyp_to_unknown(self.hyp_id)
list_works_in_queue = list(self.queue_actions.queue)
if self.queue_master is not None:
self.queue_master.put(['error_working_thread', self.hyp_id, list_works_in_queue])
logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id))
self.active = False
break
def run(self):
self.tid = get_tid()
logs.downloads.debug(
'RUN-DOWNLOAD-THREAD-------------------------------------')
if self.stop is False:
self.r_conn = new_rethink_connection()
for c in r.table('media').get_all(r.args(['Deleting','Deleted','Downloaded','DownloadStarting', 'Downloading','Download','DownloadAborting']), index='status').\
pluck('id',
'path',
'url-isard',
'url-web',
'status'
).merge(
{'table': 'media'}).changes(include_initial=True).union(
r.table('domains').get_all(r.args(['Downloaded','DownloadStarting', 'Downloading','DownloadAborting']), index='status').\
pluck('id',
'create_dict',
'url-isard',
'url-web',
'status').merge(
{"table": "domains"}).changes(include_initial=True)).union(
r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).run(self.r_conn):
if self.stop:
break
if c.get('new_val', None) is not None:
if c['new_val'].get('table', False) == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
else:
continue
logs.downloads.debug('DOWNLOAD CHANGES DETECTED:')
logs.downloads.debug(pprint.pformat(c))
if c.get('old_val', None) is None:
if c['new_val']['status'] == 'DownloadStarting':
self.start_download(c['new_val'])
elif c.get('new_val', None) is None:
if c['old_val']['status'] in ['DownloadAborting']:
self.remove_download_thread(c['old_val'])
elif 'old_val' in c and 'new_val' in c:
if c['old_val']['status'] == 'FailedDownload' and c[
'new_val']['status'] == 'DownloadStarting':
self.start_download(c['new_val'])
elif c['old_val']['status'] == 'Downloaded' and c[
'new_val']['status'] == 'Deleting':
if c['new_val']['table'] == 'media':
self.delete_media(c['new_val'])
elif c['old_val']['status'] == 'Deleting' and c['new_val'][
'status'] == 'Deleted':
if c['new_val']['table'] == 'media':
remove_media(c['new_val']['id'])
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'FailedDownload':
pass
elif c['old_val']['status'] == 'DownloadStarting' and c[
'new_val']['status'] == 'Downloading':
pass
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'Downloaded':
pass
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'DownloadAborting':
self.abort_download(c['new_val'])
