def create_template_in_db(self, id_domain):
domain_dict = get_domain(id_domain)
template_dict = domain_dict['create_dict']['template_dict']
template_dict['status'] = 'CreatingNewTemplateInDB'
template_id = template_dict['id']
if insert_domain(template_dict)['inserted'] == 1:
hw_dict = domain_dict['hardware'].copy()
for i in range(len(hw_dict['disks'])):
hw_dict['disks'][i]['file'] = template_dict['create_dict'][
'hardware']['disks'][i]['file']
update_table_field('domains',
template_id,
'hardware',
hw_dict,
merge_dict=False)
xml_parsed = update_xml_from_dict_domain(id_domain=template_id,
xml=domain_dict['xml'])
if xml_parsed is False:
update_domain_status(
status='Failed',
id_domain=template_id,
hyp_id=False,
detail='XML Parser Error, xml is not valid')
return False
remove_disk_template_created_list_in_domain(id_domain)
remove_dict_new_template_from_domain(id_domain)
if 'parents' in domain_dict.keys():
domain_parents_chain_update = domain_dict['parents'].copy()
else:
domain_parents_chain_update = []
domain_parents_chain_update.append(template_id)
update_table_field('domains', id_domain, 'parents',
domain_parents_chain_update)
update_origin_and_parents_to_new_template(id_domain, template_id)
# update_table_field('domains', template_id, 'xml', xml_parsed, merge_dict=False)
update_domain_status(
status='Stopped',
id_domain=template_id,
hyp_id=False,
detail=
'Template created, ready to create domains from this template')
update_domain_status(
status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail=
'Template created from this domain, now domain is ready to start again'
)
else:
log.error(
'template {} can not be inserted in rethink, domain_id duplicated??'
.format(template_id))
return False
def start_domain_from_xml(self, xml, id_domain, pool_id='default'):
failed = False
if pool_id in self.manager.pools.keys():
force_hyp = get_domain_force_hyp(id_domain)
if force_hyp is not False:
hyps_in_pool = get_hypers_in_pool(pool_id, only_online=False)
if force_hyp in hyps_in_pool:
next_hyp = force_hyp
else:
log.error('force hypervisor failed for doomain {}: {} not in hypervisors pool {}'.format(id_domain,
force_hyp,
pool_id))
next_hyp = self.manager.pools[pool_id].get_next(domain_id=id_domain)
else:
next_hyp = self.manager.pools[pool_id].get_next(domain_id=id_domain)
if next_hyp is not False:
# update_domain_status(status='Starting',
# id_domain=id_domain,
# hyp_id=next_hyp,
# detail='desktop starting paused in pool {} on hypervisor {}'.format(pool_id,
# next_hyp))
if LOG_LEVEL == 'DEBUG':
print(f'%%%% DOMAIN: {id_domain} -- XML TO START IN HYPERVISOR: {next_hyp} %%%%')
print(xml)
update_table_field('domains',id_domain,'xml_to_start',xml)
print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
self.manager.q.workers[next_hyp].put({'type': 'start_domain', 'xml': xml, 'id_domain': id_domain})
else:
log.error('get next hypervisor in pool {} failed'.format(pool_id))
failed = True
else:
log.error('pool_id {} does not exists??'.format(pool_id))
failed = True
if failed is True:
update_domain_status(status='Failed',
id_domain=id_domain,
hyp_id=next_hyp,
detail='desktop not started: no hypervisors online in pool {}'.format(pool_id))
log.error('desktop not started: no hypervisors online in pool {}'.format(pool_id))
return False
else:
return next_hyp
def start_paused_domain_from_xml(self, xml, id_domain, pool_id):
#def start_paused_domain_from_xml(self, xml, id_domain, pool_id, start_after_created=False):
failed = False
if pool_id in self.manager.pools.keys():
next_hyp = self.manager.pools[pool_id].get_next(domain_id=id_domain)
log.debug('//////////////////////')
if next_hyp is not False:
log.debug('next_hyp={}'.format(next_hyp))
dict_action = {'type': 'start_paused_domain', 'xml': xml, 'id_domain': id_domain}
# if start_after_created is True:
# dict_action['start_after_created'] = True
#else:
if LOG_LEVEL == 'DEBUG':
print(f'%%%% DOMAIN CREATING:{id_domain} -- XML TO START PAUSED IN HYPERVISOR {next_hyp} %%%%')
print(xml)
update_table_field('domains', id_domain, 'xml_to_start', xml)
print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
self.manager.q.workers[next_hyp].put(
dict_action)
update_domain_status(status='CreatingDomain',
id_domain=id_domain,
hyp_id=False,
detail='Waiting to try starting paused in hypervisor {} in pool {} ({} operations in queue)'.format(
next_hyp,
pool_id,
self.manager.q.workers[next_hyp].qsize()))
else:
log.error('get next hypervisor in pool {} failed'.format(pool_id))
failed = True
else:
log.error('pool_id {} does not exists??'.format(pool_id))
failed = True
if failed is True:
update_domain_status(status='FailedCreatingDomain',
id_domain=id_domain,
hyp_id=next_hyp,
detail='desktop not started: no hypervisors online in pool {}'.format(pool_id))
log.error('desktop not started: no hypervisors online in pool {}'.format(pool_id))
return False
else:
return next_hyp
def create_template_in_db(self, id_domain):
domain_dict = get_domain(id_domain)
template_dict = domain_dict['create_dict']['template_dict']
template_dict['status'] = 'CreatingNewTemplateInDB'
template_id = template_dict['id']
if insert_domain(template_dict)['inserted'] == 1:
hw_dict = domain_dict['hardware'].copy()
for i in range(len(hw_dict['disks'])):
hw_dict['disks'][i]['file'] = template_dict['create_dict']['hardware']['disks'][i]['file']
update_table_field('domains', template_id, 'hardware', hw_dict, merge_dict=False)
xml_parsed = update_xml_from_dict_domain(id_domain=template_id, xml=domain_dict['xml'])
if xml_parsed is False:
update_domain_status(status='Failed',
id_domain=template_id,
hyp_id=False,
detail='XML Parser Error, xml is not valid')
return False
remove_disk_template_created_list_in_domain(id_domain)
remove_dict_new_template_from_domain(id_domain)
if 'parents' in domain_dict.keys():
domain_parents_chain_update = domain_dict['parents'].copy()
else:
domain_parents_chain_update = []
domain_parents_chain_update.append(template_id)
update_table_field('domains', id_domain, 'parents', domain_parents_chain_update)
update_origin_and_parents_to_new_template(id_domain,template_id)
# update_table_field('domains', template_id, 'xml', xml_parsed, merge_dict=False)
update_domain_status(status='Stopped',
id_domain=template_id,
hyp_id=False,
detail='Template created, ready to create domains from this template')
update_domain_status(status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail='Template created from this domain, now domain is ready to start again')
else:
log.error('template {} can not be inserted in rethink, domain_id duplicated??'.format(template_id))
return False
def run(self):
self.tid = get_tid()
logs.downloads.debug(
'RUN-DOWNLOAD-THREAD-------------------------------------')
pool_id = 'default'
first_loop = True
if self.stop is False:
if first_loop is True:
# if domains or media have status Downloading when engine restart
# we need to resetdownloading deleting file and
first_loop = False
# wait a hyp to downloads
next_hyp = False
while next_hyp is False:
logs.downloads.info(
'waiting an hypervisor online to launch downloading actions'
)
if pool_id in self.manager.pools.keys():
next_hyp = self.manager.pools[pool_id].get_next()
sleep(1)
for hyp_id in get_hypers_in_pool():
self.killall_curl(hyp_id)
domains_status_downloading = get_domains_with_status(
'Downloading')
medias_status_downloading = get_media_with_status(
'Downloading')
for id_domain in domains_status_downloading:
create_dict = get_domain(id_domain)['create_dict']
dict_changes = {
'id': id_domain,
'table': 'domains',
'create_dict': create_dict
}
update_domain_status('ResetDownloading', id_domain)
self.abort_download(dict_changes,
final_status='DownloadFailed')
for id_media in medias_status_downloading:
dict_media = get_media(id_media)
dict_changes = {
'id': id_media,
'table': 'media',
'path': dict_media['path'],
'hypervisors_pools': dict_media['hypervisors_pools']
}
update_status_table('media', 'ResetDownloading', id_media)
self.abort_download(dict_changes,
final_status='DownloadFailed')
self.r_conn = new_rethink_connection()
update_table_field('hypervisors_pools', pool_id,
'download_changes', 'Started')
for c in r.table('media').get_all(r.args(
['Deleting', 'Deleted', 'Downloaded', 'DownloadFailed', 'DownloadStarting', 'Downloading', 'Download',
'DownloadAborting','ResetDownloading']), index='status'). \
pluck('id',
'path',
'url-isard',
'url-web',
'status'
).merge(
{'table': 'media'}).changes(include_initial=True).union(
r.table('domains').get_all(
r.args(['Downloaded', 'DownloadFailed','DownloadStarting', 'Downloading', 'DownloadAborting','ResetDownloading']), index='status'). \
pluck('id',
'create_dict',
'url-isard',
'url-web',
'status').merge(
{"table": "domains"}).changes(include_initial=True)).union(
r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).run(
self.r_conn):
if self.stop:
break
if c.get('new_val', None) is not None:
if c['new_val'].get('table', False) == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
else:
continue
logs.downloads.debug('DOWNLOAD CHANGES DETECTED:')
logs.downloads.debug(pprint.pformat(c))
if c.get('old_val', None) is None:
if c['new_val']['status'] == 'DownloadStarting':
self.start_download(c['new_val'])
elif c.get('new_val', None) is None:
if c['old_val']['status'] in ['DownloadAborting']:
self.remove_download_thread(c['old_val'])
elif 'old_val' in c and 'new_val' in c:
if c['old_val']['status'] == 'DownloadFailed' and c[
'new_val']['status'] == 'DownloadStarting':
self.start_download(c['new_val'])
elif c['old_val']['status'] == 'Downloaded' and c[
'new_val']['status'] == 'Deleting':
if c['new_val']['table'] == 'media':
self.delete_media(c['new_val'])
elif c['old_val']['status'] == 'Deleting' and c['new_val'][
'status'] == 'Deleted':
if c['new_val']['table'] == 'media':
remove_media(c['new_val']['id'])
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'DownloadFailed':
pass
elif c['old_val']['status'] == 'DownloadStarting' and c[
'new_val']['status'] == 'Downloading':
pass
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'Downloaded':
pass
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'DownloadAborting':
self.abort_download(c['new_val'])
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'ResetDownloading':
self.abort_download(c['new_val'],
final_status='DownloadFailed')
def populate_dict_hardware_from_create_dict(id_domain):
domain = get_domain(id_domain)
create_dict = domain['create_dict']
new_hardware_dict = {}
if 'origin' in create_dict.keys():
template_origin = create_dict['origin']
template = get_domain(template_origin)
new_hardware_dict = template['hardware'].copy()
else:
# TODO domain from iso or new
pass
if 'hardware' in create_dict.keys():
if 'disks' in create_dict['hardware'].keys():
new_hardware_dict['disks'] = create_dict['hardware']['disks'].copy(
)
for media_type in ['isos', 'floppies']:
if media_type in create_dict['hardware'].keys():
new_hardware_dict[media_type] = []
for d in create_dict['hardware'][media_type]:
new_media_dict = {}
media = get_media(d['id'])
new_media_dict['path'] = media['path_downloaded']
new_hardware_dict[media_type].append(new_media_dict)
new_hardware_dict['name'] = id_domain
new_hardware_dict['uuid'] = None
# MEMORY and CPUS
new_hardware_dict['vcpus'] = create_dict['hardware']['vcpus']
new_hardware_dict['currentMemory'] = create_dict['hardware'].get(
'currentMemory',
int(create_dict['hardware']['memory'] * DEFAULT_BALLOON))
new_hardware_dict['memory'] = create_dict['hardware']['memory']
new_hardware_dict['currentMemory_unit'] = 'KiB'
new_hardware_dict['memory_unit'] = 'KiB'
# VIDEO
id_video = create_dict['hardware']['videos'][0]
new_hardware_dict['video'] = create_dict_video_from_id(id_video)
# GRAPHICS
id_graphics = create_dict['hardware']['graphics'][0]
pool_var = domain['hypervisors_pools']
id_pool = pool_var if type(pool_var) is str else pool_var[0]
new_hardware_dict['graphics'] = create_dict_graphics_from_id(
id_graphics, id_pool)
# INTERFACES
list_ids_interfaces = create_dict['hardware']['interfaces']
new_hardware_dict['interfaces'] = create_list_interfaces_from_list_ids(
list_ids_interfaces)
# BOOT MENU
if 'hardware' in create_dict.keys():
if 'boot_order' in create_dict['hardware'].keys():
new_hardware_dict['boot_order'] = create_dict['hardware'][
'boot_order']
if 'boot_menu_enable' in create_dict['hardware'].keys():
new_hardware_dict['boot_menu_enable'] = create_dict['hardware'][
'boot_menu_enable']
#import pprint
#pprint.pprint(new_hardware_dict)
#print('############### domain {}'.format(id_domain))
update_table_field('domains',
id_domain,
'hardware',
new_hardware_dict,
merge_dict=False)
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port
)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format(
action['disk_path'], action['domain'], self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status('Failed', id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'],
action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install)
update_domain_status('CreatingDomainFromBuilder', id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def creating_and_test_xml_start(self,
id_domain,
creating_from_create_dict=False,
xml_from_virt_install=False,
xml_string=None):
if creating_from_create_dict is True:
try:
populate_dict_hardware_from_create_dict(id_domain)
except Exception as e:
log.error(
'error when populate dict hardware from create dict in domain {}'
.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
domain = get_domain(id_domain)
#create_dict_hw = domain['create_dict']['hardware']
# for media in ['isos','floppies']
# if 'isos' in create_dict_hw.keys():
# for index_disk in range(len(create_dict_hw['isos'])):
# update_hw['hardware']['isos'][index_disk]['file'] = new_file
if type(xml_string) is str:
xml_from = xml_string
elif 'create_from_virt_install_xml' in domain['create_dict']:
xml_from = get_dict_from_item_in_table(
'virt_install',
domain['create_dict']['create_from_virt_install_xml'])['xml']
elif xml_from_virt_install is False:
id_template = domain['create_dict']['origin']
template = get_domain(id_template)
xml_from = template['xml']
elif xml_from_virt_install is True:
xml_from = domain['xml_virt_install']
else:
return False
update_table_field('domains', id_domain, 'xml', xml_from)
xml_raw = update_xml_from_dict_domain(id_domain)
update_domain_status(
'CreatingDomain',
id_domain,
detail=
'xml and hardware dict updated, waiting to test if domain start paused in hypervisor'
)
pool_id = get_pool_from_domain(id_domain)
if 'start_after_created' in domain.keys():
if domain['start_after_created'] is True:
update_domain_status(
'StartingDomainDisposable',
id_domain,
detail=
'xml and hardware dict updated, starting domain disposable'
)
self.start_domain_from_id(id_domain)
else:
#change viewer password, remove selinux options and recreate network interfaces
xml = recreate_xml_to_start(id_domain)
self.start_paused_domain_from_xml(xml=xml,
id_domain=id_domain,
pool_id=pool_id)
def creating_disks_from_template(self, id_new):
dict_domain = get_domain(id_new)
if 'create_dict' in dict_domain.keys():
dict_to_create = dict_domain['create_dict']
pool_var = dict_domain['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
# INFO TO DEVELOPER DEBERÍA SER UN FOR PARA CADA DISCO
# y si el disco no tiene backing_chain, crear un disco vacío
# del tamño que marcase
# d['hardware']['disks'][0]['size']
# el backing_file debería estar asociado a cada disco:
# d['hardware']['disks'][0]['backing_file']
for index_disk in range(len(dict_to_create['hardware']['disks'])):
relative_path = dict_to_create['hardware']['disks'][index_disk][
'file']
new_file, path_selected = get_path_to_disk(relative_path,
pool=pool_id)
# UPDATE PATH IN DOMAIN
dict_to_create['hardware']['disks'][index_disk]['file'] = new_file
dict_to_create['hardware']['disks'][index_disk][
'path_selected'] = path_selected
update_table_field('domains', id_new, 'create_dict', dict_to_create)
#TODO: REVISAR SI RELAMENTE ES NECESARIO o esta acción responde a versiones antiguas de nuestras funciones de creación
hardware_update = {}
hardware_update['disks'] = dict_to_create['hardware']['disks']
update_domain_dict_hardware(id_new, hardware_update)
##################
for index_disk in range(len(dict_to_create['hardware']['disks'])):
backing_file = dict_to_create['hardware']['disks'][index_disk][
'parent']
new_file = dict_to_create['hardware']['disks'][index_disk]['file']
path_selected = dict_to_create['hardware']['disks'][index_disk][
'path_selected']
hyp_to_disk_create = get_host_disk_operations_from_path(
path_selected, pool=pool_id, type_path='groups')
cmds = create_cmds_disk_from_base(path_base=backing_file,
path_new=new_file)
log.debug(
'commands to disk create to launch in disk_operations: \n{}'.
format('\n'.join(cmds)))
action = {}
action['type'] = 'create_disk'
action['disk_path'] = new_file
action['index_disk'] = index_disk
action['domain'] = id_new
action['ssh_commands'] = cmds
try:
update_domain_status(
status='CreatingDisk',
id_domain=id_new,
hyp_id=False,
detail=
'Creating disk operation is launched in hypervisor {} ({} operations in queue)'
.format(
hyp_to_disk_create, self.manager.
q_disk_operations[hyp_to_disk_create].qsize()))
self.manager.q_disk_operations[hyp_to_disk_create].put(action)
except Exception as e:
update_domain_status(
status='FailedCreatingDomain',
id_domain=id_new,
hyp_id=False,
detail=
'Creating disk operation failed when insert action in queue for disk operations'
)
log.error(
'Creating disk operation failed when insert action in queue for disk operations. Exception: {}'
.format(e))
def create_template_disks_from_domain(self, id_domain):
dict_domain = get_domain(id_domain)
create_dict = dict_domain['create_dict']
pool_var = create_dict['template_dict']['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
try:
dict_new_template = create_dict['template_dict']
except KeyError as e:
update_domain_status(
status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail=
'Action Creating Template from domain failed. No template_json in domain dictionary'
)
log.error(
'No template_dict in keys of domain dictionary, when creating template form domain {}. Exception: {}'
.format(id_domain, str(e)))
return False
disk_index_in_bus = 0
if 'disks' in dict_domain['hardware']:
list_disk_template_path_relative = [
d['file'] for d in create_dict['hardware']['disks']
]
create_disk_template_created_list_in_domain(id_domain)
for i in range(len(list_disk_template_path_relative)):
# for disk in dict_domain['hardware']['disks']:
path_domain_disk = dict_domain['hardware']['disks'][i]['file']
try:
# path_template_disk_relative = dict_new_template['create_dict']['hardware']['disks'][i]['file']
path_template_disk_relative = list_disk_template_path_relative[
i]
except KeyError as e:
update_domain_status(
status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail=
'Action Creating Template from domain failed. No disks in template_json in domain dictionary'
)
log.error(
'No disks in template_json in keys of domain dictionary, when creating template form domain {}. Exception: {}'
.format(id_domain, str(e)))
return False
if dict_new_template['kind'] == 'base':
type_path_selected = 'bases'
else:
type_path_selected = 'templates'
new_file, path_selected = get_path_to_disk(
path_template_disk_relative,
pool=pool_id,
type_path=type_path_selected)
path_absolute_template_disk = new_file = new_file.replace(
'//', '/')
dict_new_template['create_dict']['hardware']['disks'][i][
'file'] = new_file
dict_new_template['create_dict']['hardware']['disks'][i][
'path_selected'] = path_selected
update_table_field('domains', id_domain, 'create_dict',
create_dict)
action = {}
action['id_domain'] = id_domain
action['type'] = 'create_template_disk_from_domain'
action['path_template_disk'] = path_absolute_template_disk
action['path_domain_disk'] = path_domain_disk
action['disk_index'] = disk_index_in_bus
hyp_to_disk_create = get_host_disk_operations_from_path(
path_selected, pool=pool_id, type_path=type_path_selected)
# INFO TO DEVELOPER: falta terminar de ver que hacemos con el pool para crear
# discos, debería haber un disk operations por pool
try:
update_domain_status(
status='CreatingTemplateDisk',
id_domain=id_domain,
hyp_id=False,
detail=
'Creating template disk operation is launched in hostname {} ({} operations in queue)'
.format(
hyp_to_disk_create, self.manager.
q_disk_operations[hyp_to_disk_create].qsize()))
self.manager.q_disk_operations[hyp_to_disk_create].put(
action)
except Exception as e:
update_domain_status(
status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail=
'Creating template operation failed when insert action in queue for disk operations'
)
log.error(
'Creating disk operation failed when insert action in queue for disk operations in host {}. Exception: {}'
.format(hyp_to_disk_create, e))
return False
disk_index_in_bus = disk_index_in_bus + 1
return True
def creating_disks_from_template(self,
id_new):
dict_domain = get_domain(id_new)
if 'create_dict' in dict_domain.keys():
dict_to_create = dict_domain['create_dict']
pool_var = dict_domain['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
# INFO TO DEVELOPER DEBERÍA SER UN FOR PARA CADA DISCO
# y si el disco no tiene backing_chain, crear un disco vacío
# del tamño que marcase
# d['hardware']['disks'][0]['size']
# el backing_file debería estar asociado a cada disco:
# d['hardware']['disks'][0]['backing_file']
for index_disk in range(len(dict_to_create['hardware']['disks'])):
relative_path = dict_to_create['hardware']['disks'][index_disk]['file']
new_file, path_selected = get_path_to_disk(relative_path, pool=pool_id)
# UPDATE PATH IN DOMAIN
dict_to_create['hardware']['disks'][index_disk]['file'] = new_file
dict_to_create['hardware']['disks'][index_disk]['path_selected'] = path_selected
update_table_field('domains',id_new,'create_dict',dict_to_create)
#TODO: REVISAR SI RELAMENTE ES NECESARIO o esta acción responde a versiones antiguas de nuestras funciones de creación
hardware_update = {}
hardware_update['disks'] = dict_to_create['hardware']['disks']
update_domain_dict_hardware(id_new, hardware_update)
##################
for index_disk in range(len(dict_to_create['hardware']['disks'])):
backing_file = dict_to_create['hardware']['disks'][index_disk]['parent']
new_file = dict_to_create['hardware']['disks'][index_disk]['file']
path_selected = dict_to_create['hardware']['disks'][index_disk]['path_selected']
hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=pool_id, type_path='groups')
cmds = create_cmds_disk_from_base(path_base=backing_file, path_new=new_file)
log.debug('commands to disk create to launch in disk_operations: \n{}'.format('\n'.join(cmds)))
action = {}
action['type'] = 'create_disk'
action['disk_path'] = new_file
action['index_disk'] = index_disk
action['domain'] = id_new
action['ssh_commands'] = cmds
try:
update_domain_status(status='CreatingDisk',
id_domain=id_new,
hyp_id=False,
detail='Creating disk operation is launched in hypervisor {} ({} operations in queue)'.format(
hyp_to_disk_create,
self.manager.q_disk_operations[hyp_to_disk_create].qsize()))
self.manager.q_disk_operations[hyp_to_disk_create].put(action)
except Exception as e:
update_domain_status(status='FailedCreatingDomain',
id_domain=id_new,
hyp_id=False,
detail='Creating disk operation failed when insert action in queue for disk operations')
log.error(
'Creating disk operation failed when insert action in queue for disk operations. Exception: {}'.format(
e))
def create_template_disks_from_domain(self, id_domain):
dict_domain = get_domain(id_domain)
create_dict = dict_domain['create_dict']
pool_var = create_dict['template_dict']['hypervisors_pools']
pool_id = pool_var if type(pool_var) is str else pool_var[0]
try:
dict_new_template = create_dict['template_dict']
except KeyError as e:
update_domain_status(status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail='Action Creating Template from domain failed. No template_json in domain dictionary')
log.error(
'No template_dict in keys of domain dictionary, when creating template form domain {}. Exception: {}'.format(
id_domain, str(e)))
return False
disk_index_in_bus = 0
if 'disks' in dict_domain['hardware']:
list_disk_template_path_relative = [d['file'] for d in create_dict['hardware']['disks']]
create_disk_template_created_list_in_domain(id_domain)
for i in range(len(list_disk_template_path_relative)):
# for disk in dict_domain['hardware']['disks']:
path_domain_disk = dict_domain['hardware']['disks'][i]['file']
try:
# path_template_disk_relative = dict_new_template['create_dict']['hardware']['disks'][i]['file']
path_template_disk_relative = list_disk_template_path_relative[i]
except KeyError as e:
update_domain_status(status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail='Action Creating Template from domain failed. No disks in template_json in domain dictionary')
log.error(
'No disks in template_json in keys of domain dictionary, when creating template form domain {}. Exception: {}'.format(
id_domain, str(e)))
return False
if dict_new_template['kind'] == 'base':
type_path_selected = 'bases'
else:
type_path_selected = 'templates'
new_file, path_selected = get_path_to_disk(path_template_disk_relative, pool=pool_id,
type_path=type_path_selected)
path_absolute_template_disk = new_file = new_file.replace('//', '/')
dict_new_template['create_dict']['hardware']['disks'][i]['file'] = new_file
dict_new_template['create_dict']['hardware']['disks'][i]['path_selected'] = path_selected
update_table_field('domains', id_domain, 'create_dict', create_dict)
action = {}
action['id_domain'] = id_domain
action['type'] = 'create_template_disk_from_domain'
action['path_template_disk'] = path_absolute_template_disk
action['path_domain_disk'] = path_domain_disk
action['disk_index'] = disk_index_in_bus
hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=pool_id,
type_path=type_path_selected)
# INFO TO DEVELOPER: falta terminar de ver que hacemos con el pool para crear
# discos, debería haber un disk operations por pool
try:
update_domain_status(status='CreatingTemplateDisk',
id_domain=id_domain,
hyp_id=False,
detail='Creating template disk operation is launched in hostname {} ({} operations in queue)'.format(
hyp_to_disk_create,
self.manager.q_disk_operations[hyp_to_disk_create].qsize()))
self.manager.q_disk_operations[hyp_to_disk_create].put(action)
except Exception as e:
update_domain_status(status='Stopped',
id_domain=id_domain,
hyp_id=False,
detail='Creating template operation failed when insert action in queue for disk operations')
log.error(
'Creating disk operation failed when insert action in queue for disk operations in host {}. Exception: {}'.format(
hyp_to_disk_create, e))
return False
disk_index_in_bus = disk_index_in_bus + 1
return True
def run(self):
# if self.table == 'domains':
# type_path_selected = 'groups'
# elif self.table in ['isos']:
# type_path_selected = 'isos'
# else:
# type_path_selected = 'media'
#
# new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected)
# logs.downloads.debug("PATHS ___________________________________________________________________")
# logs.downloads.debug(new_file)
# logs.downloads.debug(path_selected)
# logs.downloads.debug(pprint.pformat(self.__dict__))
#
# hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool,
# type_path=type_path_selected)
header_template = "--header '{header_key}: {header_value}' "
headers = ''
if URL_DOWNLOAD_INSECURE_SSL == True:
insecure_option = '--insecure'
else:
insecure_option = ''
for k, v in self.dict_header.items():
headers += header_template.format(header_key=k, header_value=v)
curl_template = "curl {insecure_option} -L -o '{path}' {headers} '{url}'"
ssh_template = """ssh -oBatchMode=yes -p {port} {user}@{hostname} """ \
""" "mkdir -p '{path_dir}'; """ + curl_template + '"'
logs.downloads.debug(ssh_template)
ssh_command = ssh_template.format(port=self.port,
user=self.user,
hostname=self.hostname,
path=self.path,
path_dir=dirname(self.path),
headers=headers,
url=self.url,
insecure_option=insecure_option)
logs.downloads.debug("SSH COMMAND: {}".format(ssh_command))
p = subprocess.Popen(ssh_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid)
rc = p.poll()
update_status_table(
self.table, 'Downloading', self.id,
"downloading in hypervisor: {}".format(self.hostname))
while rc != 0:
header = p.stderr.readline().decode('utf8')
header2 = p.stderr.readline().decode('utf8')
keys = [
'total_percent', 'total', 'received_percent', 'received',
'xferd_percent', 'xferd', 'speed_download_average',
'speed_upload_average', 'time_total', 'time_spent',
'time_left', 'speed_current'
]
line = ""
while True:
c = p.stderr.read(1).decode('utf8')
if self.stop is True:
curl_cmd = curl_template.format(
path=self.path,
headers=headers,
url=self.url,
insecure_option=insecure_option)
#for pkill curl order is cleaned
curl_cmd = curl_cmd.replace("'", "")
curl_cmd = curl_cmd.replace(" ", " ")
ssh_cmd_kill_curl = """ssh -p {port} {user}@{hostname} "pkill -f \\"^{curl_cmd}\\" " """.format(
port=self.port,
user=self.user,
hostname=self.hostname,
curl_cmd=curl_cmd)
logs.downloads.info(
'download {} aborted, ready to send ssh kill to curl in hypervisor {}'
.format(self.path, self.hostname))
#destroy curl in hypervisor
p_kill_curl = subprocess.Popen(ssh_cmd_kill_curl,
shell=True)
p_kill_curl.wait(timeout=5)
#destroy ssh command
try:
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
except Exception as e:
logs.downloads.debug(
'ssh process not killed, has finalished')
if self.table == 'media':
remove_media(self.id)
if self.table == 'domains':
delete_domain(self.id)
#update_status_table(self.table, 'FailedDownload', self.id, detail="download aborted")
return False
if not c:
break
if c == '\r':
if len(line) > 60:
logs.downloads.debug(line)
values = line.split()
logs.downloads.debug(self.url)
logs.downloads.debug(line)
d_progress = dict(zip(keys, values))
d_progress['total_percent'] = int(
float(d_progress['total_percent']))
d_progress['received_percent'] = int(
float(d_progress['received_percent']))
update_download_percent(d_progress, self.table,
self.id)
line = p.stderr.read(60).decode('utf8')
else:
line = line + c
rc = p.poll()
if self.stop is True:
return False
else:
logs.downloads.info('File downloaded: {}'.format(self.path))
assert rc == 0
if self.table == 'domains':
#update_table_field(self.table, self.id, 'path_downloaded', self.path)
d_update_domain = get_domain(self.id)['create_dict']
#d_update_domain = {'hardware': {'disks': [{}]}}
d_update_domain['hardware']['disks'][0]['file'] = self.path
update_domain_dict_create_dict(self.id, d_update_domain)
self.finalished_threads.append(self.path)
update_domain_status('Downloaded',
self.id,
detail="downloaded disk")
update_domain_status('Updating',
self.id,
detail="downloaded disk")
else:
self.finalished_threads.append(self.path)
update_table_field(self.table, self.id, 'path_downloaded',
self.path)
update_status_table(self.table, 'Downloaded', self.id)
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug(
'Thread to launchdisks operations in host {} with TID: {}...'.
format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(
host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error(
'Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'
.format(action['disk_path'], action['domain'],
self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status(
'Failed',
id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info(
'Disk created from virt-builder. Domain: {} , disk: {}'
.format(action['domain'], action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain,
'xml_virt_install',
xml_virt_install)
update_domain_status(
'CreatingDomainFromBuilder',
id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(
action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'
.format(disk_path, id_domain))
log.error('\n'.join([
'cmd: {}'.format(action['ssh_commands'][i])
for i in range(len(action['ssh_commands']))
]))
update_domain_status(
'Failed',
id_domain,
detail=
'new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs'
)
def populate_dict_hardware_from_create_dict(id_domain):
domain = get_domain(id_domain)
create_dict = domain['create_dict']
new_hardware_dict = {}
if 'origin' in create_dict.keys():
template_origin = create_dict['origin']
template = get_domain(template_origin)
new_hardware_dict = template['hardware'].copy()
else:
# TODO domain from iso or new
pass
if 'hardware' in create_dict.keys():
if 'disks' in create_dict['hardware'].keys():
new_hardware_dict['disks'] = create_dict['hardware']['disks'].copy()
for media_type in ['isos','floppies']:
if media_type in create_dict['hardware'].keys():
new_hardware_dict[media_type] = []
for d in create_dict['hardware'][media_type]:
new_media_dict = {}
media = get_media(d['id'])
new_media_dict['path'] = media['path_downloaded']
new_hardware_dict[media_type].append(new_media_dict)
new_hardware_dict['name'] = id_domain
new_hardware_dict['uuid'] = None
# MEMORY and CPUS
new_hardware_dict['vcpus'] = create_dict['hardware']['vcpus']
new_hardware_dict['currentMemory'] = create_dict['hardware'].get('currentMemory',int(create_dict['hardware']['memory'] * DEFAULT_BALLOON))
new_hardware_dict['memory'] = create_dict['hardware']['memory']
new_hardware_dict['currentMemory_unit'] = 'KiB'
new_hardware_dict['memory_unit'] = 'KiB'
# VIDEO
id_video = create_dict['hardware']['videos'][0]
new_hardware_dict['video'] = create_dict_video_from_id(id_video)
# GRAPHICS
id_graphics = create_dict['hardware']['graphics'][0]
pool_var = domain['hypervisors_pools']
id_pool = pool_var if type(pool_var) is str else pool_var[0]
new_hardware_dict['graphics'] = create_dict_graphics_from_id(id_graphics, id_pool)
# INTERFACES
list_ids_interfaces = create_dict['hardware']['interfaces']
new_hardware_dict['interfaces'] = create_list_interfaces_from_list_ids(list_ids_interfaces)
# BOOT MENU
if 'hardware' in create_dict.keys():
if 'boot_order' in create_dict['hardware'].keys():
new_hardware_dict['boot_order'] = create_dict['hardware']['boot_order']
if 'boot_menu_enable' in create_dict['hardware'].keys():
new_hardware_dict['boot_menu_enable'] = create_dict['hardware']['boot_menu_enable']
#import pprint
#pprint.pprint(new_hardware_dict)
#print('############### domain {}'.format(id_domain))
update_table_field('domains',
id_domain,
'hardware',
new_hardware_dict,
merge_dict=False)
def run(self):
# if self.table == 'domains':
# type_path_selected = 'groups'
# elif self.table in ['isos']:
# type_path_selected = 'isos'
# else:
# type_path_selected = 'media'
#
# new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected)
# logs.downloads.debug("PATHS ___________________________________________________________________")
# logs.downloads.debug(new_file)
# logs.downloads.debug(path_selected)
# logs.downloads.debug(pprint.pformat(self.__dict__))
#
# hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool,
# type_path=type_path_selected)
header_template = "--header '{header_key}: {header_value}' "
headers = ''
for k, v in self.dict_header.items():
headers += header_template.format(header_key=k, header_value=v)
ssh_template = """ssh -oBatchMode=yes -p {port} {user}@{hostname} """ \
""" "mkdir -p '{path_dir}'; curl -L -o '{path}' {headers} '{url}' " """
print(ssh_template)
ssh_command = ssh_template.format(port=self.port,
user=self.user,
hostname=self.hostname,
path=self.path,
path_dir=dirname(self.path),
headers=headers,
url=self.url)
print(ssh_command)
logs.downloads.debug("SSH COMMAND: {}".format(ssh_command))
p = subprocess.Popen(ssh_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
rc = p.poll()
update_status_table(
self.table, 'Downloading', self.id,
"downloading in hypervisor: {}".format(self.hostname))
while rc != 0:
header = p.stderr.readline().decode('utf8')
header2 = p.stderr.readline().decode('utf8')
keys = [
'total_percent', 'total', 'received_percent', 'received',
'xferd_percent', 'xferd', 'speed_download_average',
'speed_upload_average', 'time_total', 'time_spent',
'time_left', 'speed_current'
]
line = ""
while True:
c = p.stderr.read(1).decode('utf8')
if self.stop is True:
update_domain_status(self.table,
'FailedDownload',
id,
detail="download aborted")
break
if not c:
break
if c == '\r':
if len(line) > 60:
logs.downloads.debug(line)
values = line.split()
print(self.url)
print(line)
d_progress = dict(zip(keys, values))
d_progress['total_percent'] = int(
float(d_progress['total_percent']))
d_progress['received_percent'] = int(
float(d_progress['received_percent']))
update_download_percent(d_progress, self.table,
self.id)
line = p.stderr.read(60).decode('utf8')
else:
line = line + c
rc = p.poll()
logs.downloads.info('File downloaded: {}'.format(self.path))
assert rc == 0
if self.table == 'domains':
#update_table_field(self.table, self.id, 'path_downloaded', self.path)
d_update_domain = get_domain(self.id)['create_dict']
#d_update_domain = {'hardware': {'disks': [{}]}}
d_update_domain['hardware']['disks'][0]['file'] = self.path
update_domain_dict_create_dict(self.id, d_update_domain)
self.finalished_threads.append(self.path)
update_domain_status('Downloaded',
self.id,
detail="downloaded disk")
update_domain_status('Updating', self.id, detail="downloaded disk")
else:
self.finalished_threads.append(self.path)
update_table_field(self.table, self.id, 'path_downloaded',
self.path)
update_status_table(self.table, 'Downloaded', self.id)
def creating_and_test_xml_start(self, id_domain, creating_from_create_dict=False,
xml_from_virt_install=False,
xml_string=None,ssl=True):
if creating_from_create_dict is True:
try:
populate_dict_hardware_from_create_dict(id_domain)
except Exception as e:
log.error('error when populate dict hardware from create dict in domain {}'.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
domain = get_domain(id_domain)
#create_dict_hw = domain['create_dict']['hardware']
# for media in ['isos','floppies']
# if 'isos' in create_dict_hw.keys():
# for index_disk in range(len(create_dict_hw['isos'])):
# update_hw['hardware']['isos'][index_disk]['file'] = new_file
if type(xml_string) is str:
xml_from = xml_string
elif 'create_from_virt_install_xml' in domain['create_dict']:
xml_from = get_dict_from_item_in_table('virt_install',domain['create_dict']['create_from_virt_install_xml'])['xml']
elif xml_from_virt_install is False:
id_template = domain['create_dict']['origin']
template = get_domain(id_template)
xml_from = template['xml']
parents_chain = template.get('parents',[]) + domain.get('parents',[])
#when creating template from domain, the domain would be inserted as a parent while template is creating
# parent_chain never can't have id_domain as parent
if id_domain in parents_chain:
for i in range(parents_chain.count('a')):
parents_chain.remove(id_domain)
update_table_field('domains', id_domain, 'parents', parents_chain)
elif xml_from_virt_install is True:
xml_from = domain['xml_virt_install']
else:
return False
update_table_field('domains', id_domain, 'xml', xml_from)
xml_raw = update_xml_from_dict_domain(id_domain)
if xml_raw is False:
update_domain_status(status='FailedCreatingDomain',
id_domain=id_domain,
detail='XML Parser Error, xml is not valid')
return False
update_domain_status('CreatingDomain', id_domain,
detail='xml and hardware dict updated, waiting to test if domain start paused in hypervisor')
pool_id = get_pool_from_domain(id_domain)
if 'start_after_created' in domain.keys():
if domain['start_after_created'] is True:
update_domain_status('StartingDomainDisposable', id_domain,
detail='xml and hardware dict updated, starting domain disposable')
self.start_domain_from_id(id_domain)
else:
#change viewer password, remove selinux options and recreate network interfaces
try:
cpu_host_model = self.manager.pools[pool_id].conf.get('cpu_host_model', DEFAULT_HOST_MODE)
xml = recreate_xml_to_start(id_domain,ssl,cpu_host_model)
except Exception as e:
log.error('recreate_xml_to_start in domain {}'.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
xml = False
if xml is False:
update_domain_status('Failed', id_domain,
detail="DomainXML can't parse and modify xml to start")
else:
self.start_paused_domain_from_xml(xml=xml,
id_domain=id_domain,
pool_id=pool_id)
def run(self):
# if self.table == 'domains':
# type_path_selected = 'groups'
# elif self.table in ['isos']:
# type_path_selected = 'isos'
# else:
# type_path_selected = 'media'
#
# new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected)
# logs.downloads.debug("PATHS ___________________________________________________________________")
# logs.downloads.debug(new_file)
# logs.downloads.debug(path_selected)
# logs.downloads.debug(pprint.pformat(self.__dict__))
#
# hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool,
# type_path=type_path_selected)
# hypervisor to launch download command
# wait to threads disk_operations are alive
time_elapsed = 0
path_selected = self.path_selected
while True:
if len(self.manager.t_disk_operations) > 0:
hyp_to_disk_create = get_host_disk_operations_from_path(
path_selected,
pool=self.pool_id,
type_path=self.type_path_selected)
logs.downloads.debug(
f'Thread download started to in hypervisor: {hyp_to_disk_create}'
)
if self.manager.t_disk_operations.get(hyp_to_disk_create,
False) is not False:
if self.manager.t_disk_operations[
hyp_to_disk_create].is_alive():
d = get_hyp_hostname_user_port_from_id(
hyp_to_disk_create)
self.hostname = d['hostname']
self.user = d['user']
self.port = d['port']
break
sleep(0.2)
time_elapsed += 0.2
if time_elapsed > TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD:
logs.downloads.info(
f'Timeout ({TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD} sec) waiting hypervisor online to download {url_base}'
)
if self.table == 'domains':
update_domain_status('DownloadFailed',
self.id,
detail="downloaded disk")
else:
update_status_table(self.table, 'DownloadFailed', self.id)
self.finalished_threads.append(self.path)
return False
header_template = "--header '{header_key}: {header_value}' "
headers = ''
if URL_DOWNLOAD_INSECURE_SSL == True:
insecure_option = '--insecure'
else:
insecure_option = ''
dict_header = {}
for k, v in self.dict_header.items():
headers += header_template.format(header_key=k, header_value=v)
dict_header[k] = v
# TEST IF url return an stream of data
ok, error_msg = test_url_for_download(
self.url,
url_download_insecure_ssl=URL_DOWNLOAD_INSECURE_SSL,
timeout_time_limit=TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD,
dict_header=dict_header)
if ok is False:
logs.downloads.error(f'URL check failed for url: {self.url}')
logs.downloads.error(f'Failed url check reason: {error_msg}')
update_status_table(self.table,
'DownloadFailed',
self.id,
detail=error_msg)
return False
curl_template = "curl {insecure_option} -L -o '{path}' {headers} '{url}'"
ssh_template = """ssh -oBatchMode=yes -p {port} {user}@{hostname} """ \
""" "mkdir -p '{path_dir}'; """ + curl_template + '"'
logs.downloads.debug(ssh_template)
ssh_command = ssh_template.format(port=self.port,
user=self.user,
hostname=self.hostname,
path=self.path,
path_dir=dirname(self.path),
headers=headers,
url=self.url,
insecure_option=insecure_option)
logs.downloads.debug("SSH COMMAND: {}".format(ssh_command))
p = subprocess.Popen(ssh_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid)
rc = p.poll()
update_status_table(
self.table, 'Downloading', self.id,
"downloading in hypervisor: {}".format(self.hostname))
while rc != 0:
header = p.stderr.readline().decode('utf8')
header2 = p.stderr.readline().decode('utf8')
keys = [
'total_percent', 'total', 'received_percent', 'received',
'xferd_percent', 'xferd', 'speed_download_average',
'speed_upload_average', 'time_total', 'time_spent',
'time_left', 'speed_current'
]
line = ""
while True:
c = p.stderr.read(1).decode('utf8')
if self.stop is True:
curl_cmd = curl_template.format(
path=self.path,
headers=headers,
url=self.url,
insecure_option=insecure_option)
# for pkill curl order is cleaned
curl_cmd = curl_cmd.replace("'", "")
curl_cmd = curl_cmd.replace(" ", " ")
ssh_cmd_kill_curl = """ssh -p {port} {user}@{hostname} "pkill -f \\"^{curl_cmd}\\" " """.format(
port=self.port,
user=self.user,
hostname=self.hostname,
curl_cmd=curl_cmd)
logs.downloads.info(
'download {} aborted, ready to send ssh kill to curl in hypervisor {}'
.format(self.path, self.hostname))
# destroy curl in hypervisor
p_kill_curl = subprocess.Popen(ssh_cmd_kill_curl,
shell=True)
p_kill_curl.wait(timeout=5)
# destroy ssh command
try:
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
except Exception as e:
logs.downloads.debug(
'ssh process not killed, has finalished')
if self.table == 'media':
remove_media(self.id)
if self.table == 'domains':
delete_domain(self.id)
# update_status_table(self.table, 'DownloadFailed', self.id, detail="download aborted")
return False
if not c:
break
if c == '\r':
if len(line) > 60:
values = line.split()
logs.downloads.debug(self.url)
logs.downloads.debug(line)
d_progress = dict(zip(keys, values))
try:
d_progress['total_percent'] = int(
float(d_progress['total_percent']))
d_progress['received_percent'] = int(
float(d_progress['received_percent']))
if d_progress['received_percent'] > 1:
pass
except:
d_progress['total_percent'] = 0
d_progress['received_percent'] = 0
update_download_percent(d_progress, self.table,
self.id)
line = p.stderr.read(60).decode('utf8')
else:
line = line + c
rc = p.poll()
if self.stop is True:
return False
else:
logs.downloads.info('File downloaded: {}'.format(self.path))
assert rc == 0
if self.table == 'domains':
# update_table_field(self.table, self.id, 'path_downloaded', self.path)
d_update_domain = get_domain(self.id)['create_dict']
# d_update_domain = {'hardware': {'disks': [{}]}}
d_update_domain['hardware']['disks'][0]['file'] = self.path
update_domain_dict_create_dict(self.id, d_update_domain)
self.finalished_threads.append(self.path)
update_domain_status('Downloaded',
self.id,
detail="downloaded disk")
update_domain_status('Updating',
self.id,
detail="downloaded disk")
else:
self.finalished_threads.append(self.path)
update_table_field(self.table, self.id, 'path_downloaded',
self.path)
update_status_table(self.table, 'Downloaded', self.id)
def run(self):
self.tid = get_tid()
logs.workers.info('starting thread: {} (TID {})'.format(self.name, self.tid))
host, port, user = get_hyp_hostname_from_id(self.hyp_id)
port = int(port)
self.hostname = host
self.h = hyp(self.hostname, user=user, port=port)
self.h.get_hyp_info()
if self.h.info['virtualization_bios_enabled'] is True:
update_db_hyp_info(self.hyp_id, self.h.info)
hyp_id = self.hyp_id
while self.stop is not True:
try:
# do={type:'start_domain','xml':'xml','id_domain'='prova'}
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
logs.workers.debug('received action in working thread {}'.format(action['type']))
if action['type'] == 'start_paused_domain':
logs.workers.debug('xml to start some lines...: {}'.format(action['xml'][30:100]))
try:
self.h.conn.createXML(action['xml'], flags=VIR_DOMAIN_START_PAUSED)
# 32 is the constant for domains paused
# reference: https://libvirt.org/html/libvirt-libvirt-domain.html#VIR_CONNECT_LIST_DOMAINS_PAUSED
FLAG_LIST_DOMAINS_PAUSED = 32
list_all_domains = self.h.conn.listAllDomains(FLAG_LIST_DOMAINS_PAUSED)
list_names_domains = [d.name() for d in list_all_domains]
dict_domains = dict(zip(list_names_domains,list_all_domains))
if action['id_domain'] in list_names_domains:
# domain started in pause mode
domain = dict_domains[action['id_domain']]
domain_active = True
try:
domain.isActive()
domain.destroy()
try:
domain.isActive()
except Exception as e:
logs.workers.debug('verified domain {} is destroyed'.format(action['id_domain']))
domain_active = False
except libvirtError as e:
from pprint import pformat
error_msg = pformat(e.get_error_message())
update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id,
detail='domain {} failed when try to destroy from paused domain in hypervisor {}. creating domain operation is aborted')
logs.workers.error(
'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format(
action['id_domain'], self.hyp_id, error_msg))
continue
if domain_active is False:
# domain is destroyed, all ok
update_domain_status('CreatingDomain', action['id_domain'], hyp_id='',
detail='Domain created and test OK: Started, paused and now stopped in hyp {}'.format(self.hyp_id))
logs.workers.debug(
'domain {} creating operation finalished. Started paused and destroyed in hypervisor {}. Now status is Stopped. READY TO USE'.format(
action['id_domain'], self.hyp_id))
else:
update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id,
detail='Domain is created, started in pause mode but not destroyed,creating domain operation is aborted')
logs.workers.error(
'domain {} started paused but not destroyed in hypervisor {}, must be destroyed'.format(
action['id_domain'], self.hyp_id))
else:
update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id,
detail='XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted by unknown cause'.format(
action['id_domain'], self.hyp_id))
logs.workers.error(
'XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted, not exception, rare case, unknown cause'.format(
action['id_domain'], self.hyp_id))
except libvirtError as e:
from pprint import pformat
error_msg = pformat(e.get_error_message())
update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id,
detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted')
logs.workers.error(
'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format(
action['id_domain'], self.hyp_id, error_msg))
except Exception as e:
update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id,
detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted')
logs.workers.error(
'Exception starting paused xml for domain {} in hypervisor {}. NOT LIBVIRT EXCEPTION, RARE CASE. Exception message: {}'.format(
action['id_domain'], self.hyp_id, str(e)))
## START DOMAIN
elif action['type'] == 'start_domain':
logs.workers.debug('xml to start some lines...: {}'.format(action['xml'][30:100]))
try:
self.h.conn.createXML(action['xml'])
# wait to event started to save state in database
#update_domain_status('Started', action['id_domain'], hyp_id=self.hyp_id, detail='Domain has started in worker thread')
logs.workers.debug('STARTED domain {}: createdXML action in hypervisor {} has been sent'.format(
action['id_domain'], host))
except libvirtError as e:
update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id,
detail=("Hypervisor can not create domain with libvirt exception: " + str(e)))
logs.workers.debug('exception in starting domain {}: '.format(e))
except Exception as e:
update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=("Exception when starting domain: " + str(e)))
logs.workers.debug('exception in starting domain {}: '.format(e))
## STOP DOMAIN
elif action['type'] == 'stop_domain':
logs.workers.debug('action stop domain: {}'.format(action['id_domain'][30:100]))
try:
self.h.conn.lookupByName(action['id_domain']).destroy()
logs.workers.debug('STOPPED domain {}'.format(action['id_domain']))
check_if_delete = action.get('delete_after_stopped',False)
if check_if_delete is True:
update_domain_status('Stopped', action['id_domain'], hyp_id='')
update_domain_status('Deleting', action['id_domain'], hyp_id='')
else:
update_domain_status('Stopped', action['id_domain'], hyp_id='')
except Exception as e:
update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=str(e))
logs.workers.debug('exception in stopping domain {}: '.format(e))
elif action['type'] in ['create_disk', 'delete_disk']:
launch_action_disk(action,
self.hostname,
user,
port)
elif action['type'] in ['add_media_hot']:
pass
elif action['type'] in ['killall_curl']:
launch_killall_curl(self.hostname,
user,
port)
elif action['type'] in ['delete_media']:
final_status = action.get('final_status','Deleted')
launch_delete_media (action,
self.hostname,
user,
port,
final_status=final_status)
# ## DESTROY THREAD
# elif action['type'] == 'destroy_thread':
# list_works_in_queue = list(self.queue_actions.queue)
# if self.queue_master is not None:
# self.queue_master.put(['destroy_working_thread',self.hyp_id,list_works_in_queue])
# #INFO TO DEVELOPER, si entra aquí es porque no quedaba nada en cola, si no ya lo habrán matado antes
#
# logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id))
#
# raise 'destoyed'
elif action['type'] == 'create_disk':
pass
elif action['type'] == 'hyp_info':
self.h.get_hyp_info()
logs.workers.debug('hypervisor motherboard: {}'.format(self.h.info['motherboard_manufacturer']))
## DESTROY THREAD
elif action['type'] == 'stop_thread':
self.stop = True
else:
logs.workers.error('type action {} not supported in queue actions'.format(action['type']))
# time.sleep(0.1)
## TRY DOMAIN
except queue.Empty:
try:
self.h.conn.getLibVersion()
pass
# logs.workers.debug('hypervisor {} is alive'.format(host))
except:
logs.workers.info('trying to reconnect hypervisor {}, alive test in working thread failed'.format(host))
alive = False
for i in range(RETRIES_HYP_IS_ALIVE):
try:
time.sleep(TIMEOUT_BETWEEN_RETRIES_HYP_IS_ALIVE)
self.h.conn.getLibVersion()
alive = True
logs.workers.info('hypervisor {} is alive'.format(host))
break
except:
logs.workers.info('hypervisor {} is NOT alive'.format(host))
if alive is False:
try:
self.h.connect_to_hyp()
self.h.conn.getLibVersion()
update_hyp_status(self.hyp_id, 'Online')
except:
logs.workers.debug('hypervisor {} failed'.format(host))
logs.workers.error('fail reconnecting to hypervisor {} in working thread'.format(host))
reason = self.h.fail_connected_reason
update_hyp_status(self.hyp_id, 'Error', reason)
update_domains_started_in_hyp_to_unknown(self.hyp_id)
list_works_in_queue = list(self.queue_actions.queue)
if self.queue_master is not None:
self.queue_master.put(['error_working_thread', self.hyp_id, list_works_in_queue])
logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id))
self.active = False
break
else:
update_hyp_status(self.hyp_id, 'Error','bios vmx or svm virtualization capabilities not activated')
update_table_field('hypervisors',self.hyp_id,'enabled',False)
logs.workers.error('hypervisor {} disabled: bios vmx or svm virtualization capabilities not activated')
#restart when engine is started (not starting)
timeout = 10
i = 0.0
while i < timeout:
if get_engine()['status_all_threads'] == 'Started':
engine_restart()
break
else:
i + 0.2
sleep(0.2)
def creating_and_test_xml_start(self,
id_domain,
creating_from_create_dict=False,
xml_from_virt_install=False,
xml_string=None):
if creating_from_create_dict is True:
try:
populate_dict_hardware_from_create_dict(id_domain)
except Exception as e:
log.error(
'error when populate dict hardware from create dict in domain {}'
.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
domain = get_domain(id_domain)
#create_dict_hw = domain['create_dict']['hardware']
# for media in ['isos','floppies']
# if 'isos' in create_dict_hw.keys():
# for index_disk in range(len(create_dict_hw['isos'])):
# update_hw['hardware']['isos'][index_disk]['file'] = new_file
if type(xml_string) is str:
xml_from = xml_string
elif 'create_from_virt_install_xml' in domain['create_dict']:
xml_from = get_dict_from_item_in_table(
'virt_install',
domain['create_dict']['create_from_virt_install_xml'])['xml']
elif xml_from_virt_install is False:
id_template = domain['create_dict']['origin']
template = get_domain(id_template)
xml_from = template['xml']
parents_chain = template.get('parents', []) + domain.get(
'parents', [])
#when creating template from domain, the domain would be inserted as a parent while template is creating
# parent_chain never can't have id_domain as parent
if id_domain in parents_chain:
for i in range(parents_chain.count('a')):
parents_chain.remove(id_domain)
update_table_field('domains', id_domain, 'parents', parents_chain)
elif xml_from_virt_install is True:
xml_from = domain['xml_virt_install']
else:
return False
update_table_field('domains', id_domain, 'xml', xml_from)
xml_raw = update_xml_from_dict_domain(id_domain)
if xml_raw is False:
update_domain_status(status='FailedCreatingDomain',
id_domain=id_domain,
detail='XML Parser Error, xml is not valid')
return False
update_domain_status(
'CreatingDomain',
id_domain,
detail=
'xml and hardware dict updated, waiting to test if domain start paused in hypervisor'
)
pool_id = get_pool_from_domain(id_domain)
if 'start_after_created' in domain.keys():
if domain['start_after_created'] is True:
update_domain_status(
'StartingDomainDisposable',
id_domain,
detail=
'xml and hardware dict updated, starting domain disposable'
)
self.start_domain_from_id(id_domain)
else:
#change viewer password, remove selinux options and recreate network interfaces
try:
xml = recreate_xml_to_start(id_domain)
except Exception as e:
log.error(
'recreate_xml_to_start in domain {}'.format(id_domain))
log.error('Traceback: \n .{}'.format(traceback.format_exc()))
log.error('Exception message: {}'.format(e))
xml = False
if xml is False:
update_domain_status(
'Failed',
id_domain,
detail="DomainXML can't parse and modify xml to start")
else:
self.start_paused_domain_from_xml(xml=xml,
id_domain=id_domain,
pool_id=pool_id)
