def create_template_in_db(self, id_domain): domain_dict = get_domain(id_domain) template_dict = domain_dict['create_dict']['template_dict'] template_dict['status'] = 'CreatingNewTemplateInDB' template_id = template_dict['id'] if insert_domain(template_dict)['inserted'] == 1: hw_dict = domain_dict['hardware'].copy() for i in range(len(hw_dict['disks'])): hw_dict['disks'][i]['file'] = template_dict['create_dict'][ 'hardware']['disks'][i]['file'] update_table_field('domains', template_id, 'hardware', hw_dict, merge_dict=False) xml_parsed = update_xml_from_dict_domain(id_domain=template_id, xml=domain_dict['xml']) if xml_parsed is False: update_domain_status( status='Failed', id_domain=template_id, hyp_id=False, detail='XML Parser Error, xml is not valid') return False remove_disk_template_created_list_in_domain(id_domain) remove_dict_new_template_from_domain(id_domain) if 'parents' in domain_dict.keys(): domain_parents_chain_update = domain_dict['parents'].copy() else: domain_parents_chain_update = [] domain_parents_chain_update.append(template_id) update_table_field('domains', id_domain, 'parents', domain_parents_chain_update) update_origin_and_parents_to_new_template(id_domain, template_id) # update_table_field('domains', template_id, 'xml', xml_parsed, merge_dict=False) update_domain_status( status='Stopped', id_domain=template_id, hyp_id=False, detail= 'Template created, ready to create domains from this template') update_domain_status( status='Stopped', id_domain=id_domain, hyp_id=False, detail= 'Template created from this domain, now domain is ready to start again' ) else: log.error( 'template {} can not be inserted in rethink, domain_id duplicated??' .format(template_id)) return False
def start_domain_from_xml(self, xml, id_domain, pool_id='default'): failed = False if pool_id in self.manager.pools.keys(): force_hyp = get_domain_force_hyp(id_domain) if force_hyp is not False: hyps_in_pool = get_hypers_in_pool(pool_id, only_online=False) if force_hyp in hyps_in_pool: next_hyp = force_hyp else: log.error('force hypervisor failed for doomain {}: {} not in hypervisors pool {}'.format(id_domain, force_hyp, pool_id)) next_hyp = self.manager.pools[pool_id].get_next(domain_id=id_domain) else: next_hyp = self.manager.pools[pool_id].get_next(domain_id=id_domain) if next_hyp is not False: # update_domain_status(status='Starting', # id_domain=id_domain, # hyp_id=next_hyp, # detail='desktop starting paused in pool {} on hypervisor {}'.format(pool_id, # next_hyp)) if LOG_LEVEL == 'DEBUG': print(f'%%%% DOMAIN: {id_domain} -- XML TO START IN HYPERVISOR: {next_hyp} %%%%') print(xml) update_table_field('domains',id_domain,'xml_to_start',xml) print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') self.manager.q.workers[next_hyp].put({'type': 'start_domain', 'xml': xml, 'id_domain': id_domain}) else: log.error('get next hypervisor in pool {} failed'.format(pool_id)) failed = True else: log.error('pool_id {} does not exists??'.format(pool_id)) failed = True if failed is True: update_domain_status(status='Failed', id_domain=id_domain, hyp_id=next_hyp, detail='desktop not started: no hypervisors online in pool {}'.format(pool_id)) log.error('desktop not started: no hypervisors online in pool {}'.format(pool_id)) return False else: return next_hyp
def start_paused_domain_from_xml(self, xml, id_domain, pool_id): #def start_paused_domain_from_xml(self, xml, id_domain, pool_id, start_after_created=False): failed = False if pool_id in self.manager.pools.keys(): next_hyp = self.manager.pools[pool_id].get_next(domain_id=id_domain) log.debug('//////////////////////') if next_hyp is not False: log.debug('next_hyp={}'.format(next_hyp)) dict_action = {'type': 'start_paused_domain', 'xml': xml, 'id_domain': id_domain} # if start_after_created is True: # dict_action['start_after_created'] = True #else: if LOG_LEVEL == 'DEBUG': print(f'%%%% DOMAIN CREATING:{id_domain} -- XML TO START PAUSED IN HYPERVISOR {next_hyp} %%%%') print(xml) update_table_field('domains', id_domain, 'xml_to_start', xml) print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') self.manager.q.workers[next_hyp].put( dict_action) update_domain_status(status='CreatingDomain', id_domain=id_domain, hyp_id=False, detail='Waiting to try starting paused in hypervisor {} in pool {} ({} operations in queue)'.format( next_hyp, pool_id, self.manager.q.workers[next_hyp].qsize())) else: log.error('get next hypervisor in pool {} failed'.format(pool_id)) failed = True else: log.error('pool_id {} does not exists??'.format(pool_id)) failed = True if failed is True: update_domain_status(status='FailedCreatingDomain', id_domain=id_domain, hyp_id=next_hyp, detail='desktop not started: no hypervisors online in pool {}'.format(pool_id)) log.error('desktop not started: no hypervisors online in pool {}'.format(pool_id)) return False else: return next_hyp
def create_template_in_db(self, id_domain): domain_dict = get_domain(id_domain) template_dict = domain_dict['create_dict']['template_dict'] template_dict['status'] = 'CreatingNewTemplateInDB' template_id = template_dict['id'] if insert_domain(template_dict)['inserted'] == 1: hw_dict = domain_dict['hardware'].copy() for i in range(len(hw_dict['disks'])): hw_dict['disks'][i]['file'] = template_dict['create_dict']['hardware']['disks'][i]['file'] update_table_field('domains', template_id, 'hardware', hw_dict, merge_dict=False) xml_parsed = update_xml_from_dict_domain(id_domain=template_id, xml=domain_dict['xml']) if xml_parsed is False: update_domain_status(status='Failed', id_domain=template_id, hyp_id=False, detail='XML Parser Error, xml is not valid') return False remove_disk_template_created_list_in_domain(id_domain) remove_dict_new_template_from_domain(id_domain) if 'parents' in domain_dict.keys(): domain_parents_chain_update = domain_dict['parents'].copy() else: domain_parents_chain_update = [] domain_parents_chain_update.append(template_id) update_table_field('domains', id_domain, 'parents', domain_parents_chain_update) update_origin_and_parents_to_new_template(id_domain,template_id) # update_table_field('domains', template_id, 'xml', xml_parsed, merge_dict=False) update_domain_status(status='Stopped', id_domain=template_id, hyp_id=False, detail='Template created, ready to create domains from this template') update_domain_status(status='Stopped', id_domain=id_domain, hyp_id=False, detail='Template created from this domain, now domain is ready to start again') else: log.error('template {} can not be inserted in rethink, domain_id duplicated??'.format(template_id)) return False
def run(self): self.tid = get_tid() logs.downloads.debug( 'RUN-DOWNLOAD-THREAD-------------------------------------') pool_id = 'default' first_loop = True if self.stop is False: if first_loop is True: # if domains or media have status Downloading when engine restart # we need to resetdownloading deleting file and first_loop = False # wait a hyp to downloads next_hyp = False while next_hyp is False: logs.downloads.info( 'waiting an hypervisor online to launch downloading actions' ) if pool_id in self.manager.pools.keys(): next_hyp = self.manager.pools[pool_id].get_next() sleep(1) for hyp_id in get_hypers_in_pool(): self.killall_curl(hyp_id) domains_status_downloading = get_domains_with_status( 'Downloading') medias_status_downloading = get_media_with_status( 'Downloading') for id_domain in domains_status_downloading: create_dict = get_domain(id_domain)['create_dict'] dict_changes = { 'id': id_domain, 'table': 'domains', 'create_dict': create_dict } update_domain_status('ResetDownloading', id_domain) self.abort_download(dict_changes, final_status='DownloadFailed') for id_media in medias_status_downloading: dict_media = get_media(id_media) dict_changes = { 'id': id_media, 'table': 'media', 'path': dict_media['path'], 'hypervisors_pools': dict_media['hypervisors_pools'] } update_status_table('media', 'ResetDownloading', id_media) self.abort_download(dict_changes, final_status='DownloadFailed') self.r_conn = new_rethink_connection() update_table_field('hypervisors_pools', pool_id, 'download_changes', 'Started') for c in r.table('media').get_all(r.args( ['Deleting', 'Deleted', 'Downloaded', 'DownloadFailed', 'DownloadStarting', 'Downloading', 'Download', 'DownloadAborting','ResetDownloading']), index='status'). \ pluck('id', 'path', 'url-isard', 'url-web', 'status' ).merge( {'table': 'media'}).changes(include_initial=True).union( r.table('domains').get_all( r.args(['Downloaded', 'DownloadFailed','DownloadStarting', 'Downloading', 'DownloadAborting','ResetDownloading']), index='status'). \ pluck('id', 'create_dict', 'url-isard', 'url-web', 'status').merge( {"table": "domains"}).changes(include_initial=True)).union( r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).run( self.r_conn): if self.stop: break if c.get('new_val', None) is not None: if c['new_val'].get('table', False) == 'engine': if c['new_val']['status_all_threads'] == 'Stopping': break else: continue logs.downloads.debug('DOWNLOAD CHANGES DETECTED:') logs.downloads.debug(pprint.pformat(c)) if c.get('old_val', None) is None: if c['new_val']['status'] == 'DownloadStarting': self.start_download(c['new_val']) elif c.get('new_val', None) is None: if c['old_val']['status'] in ['DownloadAborting']: self.remove_download_thread(c['old_val']) elif 'old_val' in c and 'new_val' in c: if c['old_val']['status'] == 'DownloadFailed' and c[ 'new_val']['status'] == 'DownloadStarting': self.start_download(c['new_val']) elif c['old_val']['status'] == 'Downloaded' and c[ 'new_val']['status'] == 'Deleting': if c['new_val']['table'] == 'media': self.delete_media(c['new_val']) elif c['old_val']['status'] == 'Deleting' and c['new_val'][ 'status'] == 'Deleted': if c['new_val']['table'] == 'media': remove_media(c['new_val']['id']) elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'DownloadFailed': pass elif c['old_val']['status'] == 'DownloadStarting' and c[ 'new_val']['status'] == 'Downloading': pass elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'Downloaded': pass elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'DownloadAborting': self.abort_download(c['new_val']) elif c['old_val']['status'] == 'Downloading' and c[ 'new_val']['status'] == 'ResetDownloading': self.abort_download(c['new_val'], final_status='DownloadFailed')
def populate_dict_hardware_from_create_dict(id_domain): domain = get_domain(id_domain) create_dict = domain['create_dict'] new_hardware_dict = {} if 'origin' in create_dict.keys(): template_origin = create_dict['origin'] template = get_domain(template_origin) new_hardware_dict = template['hardware'].copy() else: # TODO domain from iso or new pass if 'hardware' in create_dict.keys(): if 'disks' in create_dict['hardware'].keys(): new_hardware_dict['disks'] = create_dict['hardware']['disks'].copy( ) for media_type in ['isos', 'floppies']: if media_type in create_dict['hardware'].keys(): new_hardware_dict[media_type] = [] for d in create_dict['hardware'][media_type]: new_media_dict = {} media = get_media(d['id']) new_media_dict['path'] = media['path_downloaded'] new_hardware_dict[media_type].append(new_media_dict) new_hardware_dict['name'] = id_domain new_hardware_dict['uuid'] = None # MEMORY and CPUS new_hardware_dict['vcpus'] = create_dict['hardware']['vcpus'] new_hardware_dict['currentMemory'] = create_dict['hardware'].get( 'currentMemory', int(create_dict['hardware']['memory'] * DEFAULT_BALLOON)) new_hardware_dict['memory'] = create_dict['hardware']['memory'] new_hardware_dict['currentMemory_unit'] = 'KiB' new_hardware_dict['memory_unit'] = 'KiB' # VIDEO id_video = create_dict['hardware']['videos'][0] new_hardware_dict['video'] = create_dict_video_from_id(id_video) # GRAPHICS id_graphics = create_dict['hardware']['graphics'][0] pool_var = domain['hypervisors_pools'] id_pool = pool_var if type(pool_var) is str else pool_var[0] new_hardware_dict['graphics'] = create_dict_graphics_from_id( id_graphics, id_pool) # INTERFACES list_ids_interfaces = create_dict['hardware']['interfaces'] new_hardware_dict['interfaces'] = create_list_interfaces_from_list_ids( list_ids_interfaces) # BOOT MENU if 'hardware' in create_dict.keys(): if 'boot_order' in create_dict['hardware'].keys(): new_hardware_dict['boot_order'] = create_dict['hardware'][ 'boot_order'] if 'boot_menu_enable' in create_dict['hardware'].keys(): new_hardware_dict['boot_menu_enable'] = create_dict['hardware'][ 'boot_menu_enable'] #import pprint #pprint.pprint(new_hardware_dict) #print('############### domain {}'.format(id_domain)) update_table_field('domains', id_domain, 'hardware', new_hardware_dict, merge_dict=False)
def long_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands id_domain = action['domain'] if action['type'] in ['create_disk_virt_builder']: cmds_done = execute_commands(host=self.hostname, ssh_commands=action['ssh_commands'], dict_mode=True, user=self.user, port=self.port ) if len([d for d in cmds_done if len(d['err']) > 0]) > 1: log.error('some error in virt builder operations') log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format( action['disk_path'], action['domain'], self.hyp_id)) log.debug('print cmds_done:') log.debug(pprint.pprint(cmds_done)) log.debug('print ssh_commands:') log.debug(pprint.pprint(action['ssh_commands'])) update_domain_status('Failed', id_domain, detail='Virt Builder Failed creating disk file') else: log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'], action['disk_path'])) xml_virt_install = cmds_done[-1]['out'] update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install) update_domain_status('CreatingDomainFromBuilder', id_domain, detail='disk created from virt-builder') elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format(action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format( disk_path, id_domain)) log.error('\n'.join( ['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))])) update_domain_status('Failed', id_domain, detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def creating_and_test_xml_start(self, id_domain, creating_from_create_dict=False, xml_from_virt_install=False, xml_string=None): if creating_from_create_dict is True: try: populate_dict_hardware_from_create_dict(id_domain) except Exception as e: log.error( 'error when populate dict hardware from create dict in domain {}' .format(id_domain)) log.error('Traceback: \n .{}'.format(traceback.format_exc())) log.error('Exception message: {}'.format(e)) domain = get_domain(id_domain) #create_dict_hw = domain['create_dict']['hardware'] # for media in ['isos','floppies'] # if 'isos' in create_dict_hw.keys(): # for index_disk in range(len(create_dict_hw['isos'])): # update_hw['hardware']['isos'][index_disk]['file'] = new_file if type(xml_string) is str: xml_from = xml_string elif 'create_from_virt_install_xml' in domain['create_dict']: xml_from = get_dict_from_item_in_table( 'virt_install', domain['create_dict']['create_from_virt_install_xml'])['xml'] elif xml_from_virt_install is False: id_template = domain['create_dict']['origin'] template = get_domain(id_template) xml_from = template['xml'] elif xml_from_virt_install is True: xml_from = domain['xml_virt_install'] else: return False update_table_field('domains', id_domain, 'xml', xml_from) xml_raw = update_xml_from_dict_domain(id_domain) update_domain_status( 'CreatingDomain', id_domain, detail= 'xml and hardware dict updated, waiting to test if domain start paused in hypervisor' ) pool_id = get_pool_from_domain(id_domain) if 'start_after_created' in domain.keys(): if domain['start_after_created'] is True: update_domain_status( 'StartingDomainDisposable', id_domain, detail= 'xml and hardware dict updated, starting domain disposable' ) self.start_domain_from_id(id_domain) else: #change viewer password, remove selinux options and recreate network interfaces xml = recreate_xml_to_start(id_domain) self.start_paused_domain_from_xml(xml=xml, id_domain=id_domain, pool_id=pool_id)
def creating_disks_from_template(self, id_new): dict_domain = get_domain(id_new) if 'create_dict' in dict_domain.keys(): dict_to_create = dict_domain['create_dict'] pool_var = dict_domain['hypervisors_pools'] pool_id = pool_var if type(pool_var) is str else pool_var[0] # INFO TO DEVELOPER DEBERÍA SER UN FOR PARA CADA DISCO # y si el disco no tiene backing_chain, crear un disco vacío # del tamño que marcase # d['hardware']['disks'][0]['size'] # el backing_file debería estar asociado a cada disco: # d['hardware']['disks'][0]['backing_file'] for index_disk in range(len(dict_to_create['hardware']['disks'])): relative_path = dict_to_create['hardware']['disks'][index_disk][ 'file'] new_file, path_selected = get_path_to_disk(relative_path, pool=pool_id) # UPDATE PATH IN DOMAIN dict_to_create['hardware']['disks'][index_disk]['file'] = new_file dict_to_create['hardware']['disks'][index_disk][ 'path_selected'] = path_selected update_table_field('domains', id_new, 'create_dict', dict_to_create) #TODO: REVISAR SI RELAMENTE ES NECESARIO o esta acción responde a versiones antiguas de nuestras funciones de creación hardware_update = {} hardware_update['disks'] = dict_to_create['hardware']['disks'] update_domain_dict_hardware(id_new, hardware_update) ################## for index_disk in range(len(dict_to_create['hardware']['disks'])): backing_file = dict_to_create['hardware']['disks'][index_disk][ 'parent'] new_file = dict_to_create['hardware']['disks'][index_disk]['file'] path_selected = dict_to_create['hardware']['disks'][index_disk][ 'path_selected'] hyp_to_disk_create = get_host_disk_operations_from_path( path_selected, pool=pool_id, type_path='groups') cmds = create_cmds_disk_from_base(path_base=backing_file, path_new=new_file) log.debug( 'commands to disk create to launch in disk_operations: \n{}'. format('\n'.join(cmds))) action = {} action['type'] = 'create_disk' action['disk_path'] = new_file action['index_disk'] = index_disk action['domain'] = id_new action['ssh_commands'] = cmds try: update_domain_status( status='CreatingDisk', id_domain=id_new, hyp_id=False, detail= 'Creating disk operation is launched in hypervisor {} ({} operations in queue)' .format( hyp_to_disk_create, self.manager. q_disk_operations[hyp_to_disk_create].qsize())) self.manager.q_disk_operations[hyp_to_disk_create].put(action) except Exception as e: update_domain_status( status='FailedCreatingDomain', id_domain=id_new, hyp_id=False, detail= 'Creating disk operation failed when insert action in queue for disk operations' ) log.error( 'Creating disk operation failed when insert action in queue for disk operations. Exception: {}' .format(e))
def create_template_disks_from_domain(self, id_domain): dict_domain = get_domain(id_domain) create_dict = dict_domain['create_dict'] pool_var = create_dict['template_dict']['hypervisors_pools'] pool_id = pool_var if type(pool_var) is str else pool_var[0] try: dict_new_template = create_dict['template_dict'] except KeyError as e: update_domain_status( status='Stopped', id_domain=id_domain, hyp_id=False, detail= 'Action Creating Template from domain failed. No template_json in domain dictionary' ) log.error( 'No template_dict in keys of domain dictionary, when creating template form domain {}. Exception: {}' .format(id_domain, str(e))) return False disk_index_in_bus = 0 if 'disks' in dict_domain['hardware']: list_disk_template_path_relative = [ d['file'] for d in create_dict['hardware']['disks'] ] create_disk_template_created_list_in_domain(id_domain) for i in range(len(list_disk_template_path_relative)): # for disk in dict_domain['hardware']['disks']: path_domain_disk = dict_domain['hardware']['disks'][i]['file'] try: # path_template_disk_relative = dict_new_template['create_dict']['hardware']['disks'][i]['file'] path_template_disk_relative = list_disk_template_path_relative[ i] except KeyError as e: update_domain_status( status='Stopped', id_domain=id_domain, hyp_id=False, detail= 'Action Creating Template from domain failed. No disks in template_json in domain dictionary' ) log.error( 'No disks in template_json in keys of domain dictionary, when creating template form domain {}. Exception: {}' .format(id_domain, str(e))) return False if dict_new_template['kind'] == 'base': type_path_selected = 'bases' else: type_path_selected = 'templates' new_file, path_selected = get_path_to_disk( path_template_disk_relative, pool=pool_id, type_path=type_path_selected) path_absolute_template_disk = new_file = new_file.replace( '//', '/') dict_new_template['create_dict']['hardware']['disks'][i][ 'file'] = new_file dict_new_template['create_dict']['hardware']['disks'][i][ 'path_selected'] = path_selected update_table_field('domains', id_domain, 'create_dict', create_dict) action = {} action['id_domain'] = id_domain action['type'] = 'create_template_disk_from_domain' action['path_template_disk'] = path_absolute_template_disk action['path_domain_disk'] = path_domain_disk action['disk_index'] = disk_index_in_bus hyp_to_disk_create = get_host_disk_operations_from_path( path_selected, pool=pool_id, type_path=type_path_selected) # INFO TO DEVELOPER: falta terminar de ver que hacemos con el pool para crear # discos, debería haber un disk operations por pool try: update_domain_status( status='CreatingTemplateDisk', id_domain=id_domain, hyp_id=False, detail= 'Creating template disk operation is launched in hostname {} ({} operations in queue)' .format( hyp_to_disk_create, self.manager. q_disk_operations[hyp_to_disk_create].qsize())) self.manager.q_disk_operations[hyp_to_disk_create].put( action) except Exception as e: update_domain_status( status='Stopped', id_domain=id_domain, hyp_id=False, detail= 'Creating template operation failed when insert action in queue for disk operations' ) log.error( 'Creating disk operation failed when insert action in queue for disk operations in host {}. Exception: {}' .format(hyp_to_disk_create, e)) return False disk_index_in_bus = disk_index_in_bus + 1 return True
def creating_disks_from_template(self, id_new): dict_domain = get_domain(id_new) if 'create_dict' in dict_domain.keys(): dict_to_create = dict_domain['create_dict'] pool_var = dict_domain['hypervisors_pools'] pool_id = pool_var if type(pool_var) is str else pool_var[0] # INFO TO DEVELOPER DEBERÍA SER UN FOR PARA CADA DISCO # y si el disco no tiene backing_chain, crear un disco vacío # del tamño que marcase # d['hardware']['disks'][0]['size'] # el backing_file debería estar asociado a cada disco: # d['hardware']['disks'][0]['backing_file'] for index_disk in range(len(dict_to_create['hardware']['disks'])): relative_path = dict_to_create['hardware']['disks'][index_disk]['file'] new_file, path_selected = get_path_to_disk(relative_path, pool=pool_id) # UPDATE PATH IN DOMAIN dict_to_create['hardware']['disks'][index_disk]['file'] = new_file dict_to_create['hardware']['disks'][index_disk]['path_selected'] = path_selected update_table_field('domains',id_new,'create_dict',dict_to_create) #TODO: REVISAR SI RELAMENTE ES NECESARIO o esta acción responde a versiones antiguas de nuestras funciones de creación hardware_update = {} hardware_update['disks'] = dict_to_create['hardware']['disks'] update_domain_dict_hardware(id_new, hardware_update) ################## for index_disk in range(len(dict_to_create['hardware']['disks'])): backing_file = dict_to_create['hardware']['disks'][index_disk]['parent'] new_file = dict_to_create['hardware']['disks'][index_disk]['file'] path_selected = dict_to_create['hardware']['disks'][index_disk]['path_selected'] hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=pool_id, type_path='groups') cmds = create_cmds_disk_from_base(path_base=backing_file, path_new=new_file) log.debug('commands to disk create to launch in disk_operations: \n{}'.format('\n'.join(cmds))) action = {} action['type'] = 'create_disk' action['disk_path'] = new_file action['index_disk'] = index_disk action['domain'] = id_new action['ssh_commands'] = cmds try: update_domain_status(status='CreatingDisk', id_domain=id_new, hyp_id=False, detail='Creating disk operation is launched in hypervisor {} ({} operations in queue)'.format( hyp_to_disk_create, self.manager.q_disk_operations[hyp_to_disk_create].qsize())) self.manager.q_disk_operations[hyp_to_disk_create].put(action) except Exception as e: update_domain_status(status='FailedCreatingDomain', id_domain=id_new, hyp_id=False, detail='Creating disk operation failed when insert action in queue for disk operations') log.error( 'Creating disk operation failed when insert action in queue for disk operations. Exception: {}'.format( e))
def create_template_disks_from_domain(self, id_domain): dict_domain = get_domain(id_domain) create_dict = dict_domain['create_dict'] pool_var = create_dict['template_dict']['hypervisors_pools'] pool_id = pool_var if type(pool_var) is str else pool_var[0] try: dict_new_template = create_dict['template_dict'] except KeyError as e: update_domain_status(status='Stopped', id_domain=id_domain, hyp_id=False, detail='Action Creating Template from domain failed. No template_json in domain dictionary') log.error( 'No template_dict in keys of domain dictionary, when creating template form domain {}. Exception: {}'.format( id_domain, str(e))) return False disk_index_in_bus = 0 if 'disks' in dict_domain['hardware']: list_disk_template_path_relative = [d['file'] for d in create_dict['hardware']['disks']] create_disk_template_created_list_in_domain(id_domain) for i in range(len(list_disk_template_path_relative)): # for disk in dict_domain['hardware']['disks']: path_domain_disk = dict_domain['hardware']['disks'][i]['file'] try: # path_template_disk_relative = dict_new_template['create_dict']['hardware']['disks'][i]['file'] path_template_disk_relative = list_disk_template_path_relative[i] except KeyError as e: update_domain_status(status='Stopped', id_domain=id_domain, hyp_id=False, detail='Action Creating Template from domain failed. No disks in template_json in domain dictionary') log.error( 'No disks in template_json in keys of domain dictionary, when creating template form domain {}. Exception: {}'.format( id_domain, str(e))) return False if dict_new_template['kind'] == 'base': type_path_selected = 'bases' else: type_path_selected = 'templates' new_file, path_selected = get_path_to_disk(path_template_disk_relative, pool=pool_id, type_path=type_path_selected) path_absolute_template_disk = new_file = new_file.replace('//', '/') dict_new_template['create_dict']['hardware']['disks'][i]['file'] = new_file dict_new_template['create_dict']['hardware']['disks'][i]['path_selected'] = path_selected update_table_field('domains', id_domain, 'create_dict', create_dict) action = {} action['id_domain'] = id_domain action['type'] = 'create_template_disk_from_domain' action['path_template_disk'] = path_absolute_template_disk action['path_domain_disk'] = path_domain_disk action['disk_index'] = disk_index_in_bus hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=pool_id, type_path=type_path_selected) # INFO TO DEVELOPER: falta terminar de ver que hacemos con el pool para crear # discos, debería haber un disk operations por pool try: update_domain_status(status='CreatingTemplateDisk', id_domain=id_domain, hyp_id=False, detail='Creating template disk operation is launched in hostname {} ({} operations in queue)'.format( hyp_to_disk_create, self.manager.q_disk_operations[hyp_to_disk_create].qsize())) self.manager.q_disk_operations[hyp_to_disk_create].put(action) except Exception as e: update_domain_status(status='Stopped', id_domain=id_domain, hyp_id=False, detail='Creating template operation failed when insert action in queue for disk operations') log.error( 'Creating disk operation failed when insert action in queue for disk operations in host {}. Exception: {}'.format( hyp_to_disk_create, e)) return False disk_index_in_bus = disk_index_in_bus + 1 return True
def run(self): # if self.table == 'domains': # type_path_selected = 'groups' # elif self.table in ['isos']: # type_path_selected = 'isos' # else: # type_path_selected = 'media' # # new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected) # logs.downloads.debug("PATHS ___________________________________________________________________") # logs.downloads.debug(new_file) # logs.downloads.debug(path_selected) # logs.downloads.debug(pprint.pformat(self.__dict__)) # # hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool, # type_path=type_path_selected) header_template = "--header '{header_key}: {header_value}' " headers = '' if URL_DOWNLOAD_INSECURE_SSL == True: insecure_option = '--insecure' else: insecure_option = '' for k, v in self.dict_header.items(): headers += header_template.format(header_key=k, header_value=v) curl_template = "curl {insecure_option} -L -o '{path}' {headers} '{url}'" ssh_template = """ssh -oBatchMode=yes -p {port} {user}@{hostname} """ \ """ "mkdir -p '{path_dir}'; """ + curl_template + '"' logs.downloads.debug(ssh_template) ssh_command = ssh_template.format(port=self.port, user=self.user, hostname=self.hostname, path=self.path, path_dir=dirname(self.path), headers=headers, url=self.url, insecure_option=insecure_option) logs.downloads.debug("SSH COMMAND: {}".format(ssh_command)) p = subprocess.Popen(ssh_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) rc = p.poll() update_status_table( self.table, 'Downloading', self.id, "downloading in hypervisor: {}".format(self.hostname)) while rc != 0: header = p.stderr.readline().decode('utf8') header2 = p.stderr.readline().decode('utf8') keys = [ 'total_percent', 'total', 'received_percent', 'received', 'xferd_percent', 'xferd', 'speed_download_average', 'speed_upload_average', 'time_total', 'time_spent', 'time_left', 'speed_current' ] line = "" while True: c = p.stderr.read(1).decode('utf8') if self.stop is True: curl_cmd = curl_template.format( path=self.path, headers=headers, url=self.url, insecure_option=insecure_option) #for pkill curl order is cleaned curl_cmd = curl_cmd.replace("'", "") curl_cmd = curl_cmd.replace(" ", " ") ssh_cmd_kill_curl = """ssh -p {port} {user}@{hostname} "pkill -f \\"^{curl_cmd}\\" " """.format( port=self.port, user=self.user, hostname=self.hostname, curl_cmd=curl_cmd) logs.downloads.info( 'download {} aborted, ready to send ssh kill to curl in hypervisor {}' .format(self.path, self.hostname)) #destroy curl in hypervisor p_kill_curl = subprocess.Popen(ssh_cmd_kill_curl, shell=True) p_kill_curl.wait(timeout=5) #destroy ssh command try: os.killpg(os.getpgid(p.pid), signal.SIGTERM) except Exception as e: logs.downloads.debug( 'ssh process not killed, has finalished') if self.table == 'media': remove_media(self.id) if self.table == 'domains': delete_domain(self.id) #update_status_table(self.table, 'FailedDownload', self.id, detail="download aborted") return False if not c: break if c == '\r': if len(line) > 60: logs.downloads.debug(line) values = line.split() logs.downloads.debug(self.url) logs.downloads.debug(line) d_progress = dict(zip(keys, values)) d_progress['total_percent'] = int( float(d_progress['total_percent'])) d_progress['received_percent'] = int( float(d_progress['received_percent'])) update_download_percent(d_progress, self.table, self.id) line = p.stderr.read(60).decode('utf8') else: line = line + c rc = p.poll() if self.stop is True: return False else: logs.downloads.info('File downloaded: {}'.format(self.path)) assert rc == 0 if self.table == 'domains': #update_table_field(self.table, self.id, 'path_downloaded', self.path) d_update_domain = get_domain(self.id)['create_dict'] #d_update_domain = {'hardware': {'disks': [{}]}} d_update_domain['hardware']['disks'][0]['file'] = self.path update_domain_dict_create_dict(self.id, d_update_domain) self.finalished_threads.append(self.path) update_domain_status('Downloaded', self.id, detail="downloaded disk") update_domain_status('Updating', self.id, detail="downloaded disk") else: self.finalished_threads.append(self.path) update_table_field(self.table, self.id, 'path_downloaded', self.path) update_status_table(self.table, 'Downloaded', self.id)
def long_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug( 'Thread to launchdisks operations in host {} with TID: {}...'. format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands id_domain = action['domain'] if action['type'] in ['create_disk_virt_builder']: cmds_done = execute_commands( host=self.hostname, ssh_commands=action['ssh_commands'], dict_mode=True, user=self.user, port=self.port) if len([d for d in cmds_done if len(d['err']) > 0]) > 1: log.error('some error in virt builder operations') log.error( 'Virt Builder Failed creating disk file {} in domain {} in hypervisor {}' .format(action['disk_path'], action['domain'], self.hyp_id)) log.debug('print cmds_done:') log.debug(pprint.pprint(cmds_done)) log.debug('print ssh_commands:') log.debug(pprint.pprint(action['ssh_commands'])) update_domain_status( 'Failed', id_domain, detail='Virt Builder Failed creating disk file') else: log.info( 'Disk created from virt-builder. Domain: {} , disk: {}' .format(action['domain'], action['disk_path'])) xml_virt_install = cmds_done[-1]['out'] update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install) update_domain_status( 'CreatingDomainFromBuilder', id_domain, detail='disk created from virt-builder') elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format( action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}' .format(disk_path, id_domain)) log.error('\n'.join([ 'cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands'])) ])) update_domain_status( 'Failed', id_domain, detail= 'new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs' )
def populate_dict_hardware_from_create_dict(id_domain): domain = get_domain(id_domain) create_dict = domain['create_dict'] new_hardware_dict = {} if 'origin' in create_dict.keys(): template_origin = create_dict['origin'] template = get_domain(template_origin) new_hardware_dict = template['hardware'].copy() else: # TODO domain from iso or new pass if 'hardware' in create_dict.keys(): if 'disks' in create_dict['hardware'].keys(): new_hardware_dict['disks'] = create_dict['hardware']['disks'].copy() for media_type in ['isos','floppies']: if media_type in create_dict['hardware'].keys(): new_hardware_dict[media_type] = [] for d in create_dict['hardware'][media_type]: new_media_dict = {} media = get_media(d['id']) new_media_dict['path'] = media['path_downloaded'] new_hardware_dict[media_type].append(new_media_dict) new_hardware_dict['name'] = id_domain new_hardware_dict['uuid'] = None # MEMORY and CPUS new_hardware_dict['vcpus'] = create_dict['hardware']['vcpus'] new_hardware_dict['currentMemory'] = create_dict['hardware'].get('currentMemory',int(create_dict['hardware']['memory'] * DEFAULT_BALLOON)) new_hardware_dict['memory'] = create_dict['hardware']['memory'] new_hardware_dict['currentMemory_unit'] = 'KiB' new_hardware_dict['memory_unit'] = 'KiB' # VIDEO id_video = create_dict['hardware']['videos'][0] new_hardware_dict['video'] = create_dict_video_from_id(id_video) # GRAPHICS id_graphics = create_dict['hardware']['graphics'][0] pool_var = domain['hypervisors_pools'] id_pool = pool_var if type(pool_var) is str else pool_var[0] new_hardware_dict['graphics'] = create_dict_graphics_from_id(id_graphics, id_pool) # INTERFACES list_ids_interfaces = create_dict['hardware']['interfaces'] new_hardware_dict['interfaces'] = create_list_interfaces_from_list_ids(list_ids_interfaces) # BOOT MENU if 'hardware' in create_dict.keys(): if 'boot_order' in create_dict['hardware'].keys(): new_hardware_dict['boot_order'] = create_dict['hardware']['boot_order'] if 'boot_menu_enable' in create_dict['hardware'].keys(): new_hardware_dict['boot_menu_enable'] = create_dict['hardware']['boot_menu_enable'] #import pprint #pprint.pprint(new_hardware_dict) #print('############### domain {}'.format(id_domain)) update_table_field('domains', id_domain, 'hardware', new_hardware_dict, merge_dict=False)
def run(self): # if self.table == 'domains': # type_path_selected = 'groups' # elif self.table in ['isos']: # type_path_selected = 'isos' # else: # type_path_selected = 'media' # # new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected) # logs.downloads.debug("PATHS ___________________________________________________________________") # logs.downloads.debug(new_file) # logs.downloads.debug(path_selected) # logs.downloads.debug(pprint.pformat(self.__dict__)) # # hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool, # type_path=type_path_selected) header_template = "--header '{header_key}: {header_value}' " headers = '' for k, v in self.dict_header.items(): headers += header_template.format(header_key=k, header_value=v) ssh_template = """ssh -oBatchMode=yes -p {port} {user}@{hostname} """ \ """ "mkdir -p '{path_dir}'; curl -L -o '{path}' {headers} '{url}' " """ print(ssh_template) ssh_command = ssh_template.format(port=self.port, user=self.user, hostname=self.hostname, path=self.path, path_dir=dirname(self.path), headers=headers, url=self.url) print(ssh_command) logs.downloads.debug("SSH COMMAND: {}".format(ssh_command)) p = subprocess.Popen(ssh_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) rc = p.poll() update_status_table( self.table, 'Downloading', self.id, "downloading in hypervisor: {}".format(self.hostname)) while rc != 0: header = p.stderr.readline().decode('utf8') header2 = p.stderr.readline().decode('utf8') keys = [ 'total_percent', 'total', 'received_percent', 'received', 'xferd_percent', 'xferd', 'speed_download_average', 'speed_upload_average', 'time_total', 'time_spent', 'time_left', 'speed_current' ] line = "" while True: c = p.stderr.read(1).decode('utf8') if self.stop is True: update_domain_status(self.table, 'FailedDownload', id, detail="download aborted") break if not c: break if c == '\r': if len(line) > 60: logs.downloads.debug(line) values = line.split() print(self.url) print(line) d_progress = dict(zip(keys, values)) d_progress['total_percent'] = int( float(d_progress['total_percent'])) d_progress['received_percent'] = int( float(d_progress['received_percent'])) update_download_percent(d_progress, self.table, self.id) line = p.stderr.read(60).decode('utf8') else: line = line + c rc = p.poll() logs.downloads.info('File downloaded: {}'.format(self.path)) assert rc == 0 if self.table == 'domains': #update_table_field(self.table, self.id, 'path_downloaded', self.path) d_update_domain = get_domain(self.id)['create_dict'] #d_update_domain = {'hardware': {'disks': [{}]}} d_update_domain['hardware']['disks'][0]['file'] = self.path update_domain_dict_create_dict(self.id, d_update_domain) self.finalished_threads.append(self.path) update_domain_status('Downloaded', self.id, detail="downloaded disk") update_domain_status('Updating', self.id, detail="downloaded disk") else: self.finalished_threads.append(self.path) update_table_field(self.table, self.id, 'path_downloaded', self.path) update_status_table(self.table, 'Downloaded', self.id)
def creating_and_test_xml_start(self, id_domain, creating_from_create_dict=False, xml_from_virt_install=False, xml_string=None,ssl=True): if creating_from_create_dict is True: try: populate_dict_hardware_from_create_dict(id_domain) except Exception as e: log.error('error when populate dict hardware from create dict in domain {}'.format(id_domain)) log.error('Traceback: \n .{}'.format(traceback.format_exc())) log.error('Exception message: {}'.format(e)) domain = get_domain(id_domain) #create_dict_hw = domain['create_dict']['hardware'] # for media in ['isos','floppies'] # if 'isos' in create_dict_hw.keys(): # for index_disk in range(len(create_dict_hw['isos'])): # update_hw['hardware']['isos'][index_disk]['file'] = new_file if type(xml_string) is str: xml_from = xml_string elif 'create_from_virt_install_xml' in domain['create_dict']: xml_from = get_dict_from_item_in_table('virt_install',domain['create_dict']['create_from_virt_install_xml'])['xml'] elif xml_from_virt_install is False: id_template = domain['create_dict']['origin'] template = get_domain(id_template) xml_from = template['xml'] parents_chain = template.get('parents',[]) + domain.get('parents',[]) #when creating template from domain, the domain would be inserted as a parent while template is creating # parent_chain never can't have id_domain as parent if id_domain in parents_chain: for i in range(parents_chain.count('a')): parents_chain.remove(id_domain) update_table_field('domains', id_domain, 'parents', parents_chain) elif xml_from_virt_install is True: xml_from = domain['xml_virt_install'] else: return False update_table_field('domains', id_domain, 'xml', xml_from) xml_raw = update_xml_from_dict_domain(id_domain) if xml_raw is False: update_domain_status(status='FailedCreatingDomain', id_domain=id_domain, detail='XML Parser Error, xml is not valid') return False update_domain_status('CreatingDomain', id_domain, detail='xml and hardware dict updated, waiting to test if domain start paused in hypervisor') pool_id = get_pool_from_domain(id_domain) if 'start_after_created' in domain.keys(): if domain['start_after_created'] is True: update_domain_status('StartingDomainDisposable', id_domain, detail='xml and hardware dict updated, starting domain disposable') self.start_domain_from_id(id_domain) else: #change viewer password, remove selinux options and recreate network interfaces try: cpu_host_model = self.manager.pools[pool_id].conf.get('cpu_host_model', DEFAULT_HOST_MODE) xml = recreate_xml_to_start(id_domain,ssl,cpu_host_model) except Exception as e: log.error('recreate_xml_to_start in domain {}'.format(id_domain)) log.error('Traceback: \n .{}'.format(traceback.format_exc())) log.error('Exception message: {}'.format(e)) xml = False if xml is False: update_domain_status('Failed', id_domain, detail="DomainXML can't parse and modify xml to start") else: self.start_paused_domain_from_xml(xml=xml, id_domain=id_domain, pool_id=pool_id)
def run(self): # if self.table == 'domains': # type_path_selected = 'groups' # elif self.table in ['isos']: # type_path_selected = 'isos' # else: # type_path_selected = 'media' # # new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected) # logs.downloads.debug("PATHS ___________________________________________________________________") # logs.downloads.debug(new_file) # logs.downloads.debug(path_selected) # logs.downloads.debug(pprint.pformat(self.__dict__)) # # hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool, # type_path=type_path_selected) # hypervisor to launch download command # wait to threads disk_operations are alive time_elapsed = 0 path_selected = self.path_selected while True: if len(self.manager.t_disk_operations) > 0: hyp_to_disk_create = get_host_disk_operations_from_path( path_selected, pool=self.pool_id, type_path=self.type_path_selected) logs.downloads.debug( f'Thread download started to in hypervisor: {hyp_to_disk_create}' ) if self.manager.t_disk_operations.get(hyp_to_disk_create, False) is not False: if self.manager.t_disk_operations[ hyp_to_disk_create].is_alive(): d = get_hyp_hostname_user_port_from_id( hyp_to_disk_create) self.hostname = d['hostname'] self.user = d['user'] self.port = d['port'] break sleep(0.2) time_elapsed += 0.2 if time_elapsed > TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD: logs.downloads.info( f'Timeout ({TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD} sec) waiting hypervisor online to download {url_base}' ) if self.table == 'domains': update_domain_status('DownloadFailed', self.id, detail="downloaded disk") else: update_status_table(self.table, 'DownloadFailed', self.id) self.finalished_threads.append(self.path) return False header_template = "--header '{header_key}: {header_value}' " headers = '' if URL_DOWNLOAD_INSECURE_SSL == True: insecure_option = '--insecure' else: insecure_option = '' dict_header = {} for k, v in self.dict_header.items(): headers += header_template.format(header_key=k, header_value=v) dict_header[k] = v # TEST IF url return an stream of data ok, error_msg = test_url_for_download( self.url, url_download_insecure_ssl=URL_DOWNLOAD_INSECURE_SSL, timeout_time_limit=TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD, dict_header=dict_header) if ok is False: logs.downloads.error(f'URL check failed for url: {self.url}') logs.downloads.error(f'Failed url check reason: {error_msg}') update_status_table(self.table, 'DownloadFailed', self.id, detail=error_msg) return False curl_template = "curl {insecure_option} -L -o '{path}' {headers} '{url}'" ssh_template = """ssh -oBatchMode=yes -p {port} {user}@{hostname} """ \ """ "mkdir -p '{path_dir}'; """ + curl_template + '"' logs.downloads.debug(ssh_template) ssh_command = ssh_template.format(port=self.port, user=self.user, hostname=self.hostname, path=self.path, path_dir=dirname(self.path), headers=headers, url=self.url, insecure_option=insecure_option) logs.downloads.debug("SSH COMMAND: {}".format(ssh_command)) p = subprocess.Popen(ssh_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) rc = p.poll() update_status_table( self.table, 'Downloading', self.id, "downloading in hypervisor: {}".format(self.hostname)) while rc != 0: header = p.stderr.readline().decode('utf8') header2 = p.stderr.readline().decode('utf8') keys = [ 'total_percent', 'total', 'received_percent', 'received', 'xferd_percent', 'xferd', 'speed_download_average', 'speed_upload_average', 'time_total', 'time_spent', 'time_left', 'speed_current' ] line = "" while True: c = p.stderr.read(1).decode('utf8') if self.stop is True: curl_cmd = curl_template.format( path=self.path, headers=headers, url=self.url, insecure_option=insecure_option) # for pkill curl order is cleaned curl_cmd = curl_cmd.replace("'", "") curl_cmd = curl_cmd.replace(" ", " ") ssh_cmd_kill_curl = """ssh -p {port} {user}@{hostname} "pkill -f \\"^{curl_cmd}\\" " """.format( port=self.port, user=self.user, hostname=self.hostname, curl_cmd=curl_cmd) logs.downloads.info( 'download {} aborted, ready to send ssh kill to curl in hypervisor {}' .format(self.path, self.hostname)) # destroy curl in hypervisor p_kill_curl = subprocess.Popen(ssh_cmd_kill_curl, shell=True) p_kill_curl.wait(timeout=5) # destroy ssh command try: os.killpg(os.getpgid(p.pid), signal.SIGTERM) except Exception as e: logs.downloads.debug( 'ssh process not killed, has finalished') if self.table == 'media': remove_media(self.id) if self.table == 'domains': delete_domain(self.id) # update_status_table(self.table, 'DownloadFailed', self.id, detail="download aborted") return False if not c: break if c == '\r': if len(line) > 60: values = line.split() logs.downloads.debug(self.url) logs.downloads.debug(line) d_progress = dict(zip(keys, values)) try: d_progress['total_percent'] = int( float(d_progress['total_percent'])) d_progress['received_percent'] = int( float(d_progress['received_percent'])) if d_progress['received_percent'] > 1: pass except: d_progress['total_percent'] = 0 d_progress['received_percent'] = 0 update_download_percent(d_progress, self.table, self.id) line = p.stderr.read(60).decode('utf8') else: line = line + c rc = p.poll() if self.stop is True: return False else: logs.downloads.info('File downloaded: {}'.format(self.path)) assert rc == 0 if self.table == 'domains': # update_table_field(self.table, self.id, 'path_downloaded', self.path) d_update_domain = get_domain(self.id)['create_dict'] # d_update_domain = {'hardware': {'disks': [{}]}} d_update_domain['hardware']['disks'][0]['file'] = self.path update_domain_dict_create_dict(self.id, d_update_domain) self.finalished_threads.append(self.path) update_domain_status('Downloaded', self.id, detail="downloaded disk") update_domain_status('Updating', self.id, detail="downloaded disk") else: self.finalished_threads.append(self.path) update_table_field(self.table, self.id, 'path_downloaded', self.path) update_status_table(self.table, 'Downloaded', self.id)
def run(self): self.tid = get_tid() logs.workers.info('starting thread: {} (TID {})'.format(self.name, self.tid)) host, port, user = get_hyp_hostname_from_id(self.hyp_id) port = int(port) self.hostname = host self.h = hyp(self.hostname, user=user, port=port) self.h.get_hyp_info() if self.h.info['virtualization_bios_enabled'] is True: update_db_hyp_info(self.hyp_id, self.h.info) hyp_id = self.hyp_id while self.stop is not True: try: # do={type:'start_domain','xml':'xml','id_domain'='prova'} action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) logs.workers.debug('received action in working thread {}'.format(action['type'])) if action['type'] == 'start_paused_domain': logs.workers.debug('xml to start some lines...: {}'.format(action['xml'][30:100])) try: self.h.conn.createXML(action['xml'], flags=VIR_DOMAIN_START_PAUSED) # 32 is the constant for domains paused # reference: https://libvirt.org/html/libvirt-libvirt-domain.html#VIR_CONNECT_LIST_DOMAINS_PAUSED FLAG_LIST_DOMAINS_PAUSED = 32 list_all_domains = self.h.conn.listAllDomains(FLAG_LIST_DOMAINS_PAUSED) list_names_domains = [d.name() for d in list_all_domains] dict_domains = dict(zip(list_names_domains,list_all_domains)) if action['id_domain'] in list_names_domains: # domain started in pause mode domain = dict_domains[action['id_domain']] domain_active = True try: domain.isActive() domain.destroy() try: domain.isActive() except Exception as e: logs.workers.debug('verified domain {} is destroyed'.format(action['id_domain'])) domain_active = False except libvirtError as e: from pprint import pformat error_msg = pformat(e.get_error_message()) update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id, detail='domain {} failed when try to destroy from paused domain in hypervisor {}. creating domain operation is aborted') logs.workers.error( 'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format( action['id_domain'], self.hyp_id, error_msg)) continue if domain_active is False: # domain is destroyed, all ok update_domain_status('CreatingDomain', action['id_domain'], hyp_id='', detail='Domain created and test OK: Started, paused and now stopped in hyp {}'.format(self.hyp_id)) logs.workers.debug( 'domain {} creating operation finalished. Started paused and destroyed in hypervisor {}. Now status is Stopped. READY TO USE'.format( action['id_domain'], self.hyp_id)) else: update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id, detail='Domain is created, started in pause mode but not destroyed,creating domain operation is aborted') logs.workers.error( 'domain {} started paused but not destroyed in hypervisor {}, must be destroyed'.format( action['id_domain'], self.hyp_id)) else: update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id, detail='XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted by unknown cause'.format( action['id_domain'], self.hyp_id)) logs.workers.error( 'XML for domain {} can not start in pause mode in hypervisor {}, creating domain operation is aborted, not exception, rare case, unknown cause'.format( action['id_domain'], self.hyp_id)) except libvirtError as e: from pprint import pformat error_msg = pformat(e.get_error_message()) update_domain_status('FailedCreatingDomain', action['id_domain'], hyp_id=self.hyp_id, detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted') logs.workers.error( 'Exception in libvirt starting paused xml for domain {} in hypervisor {}. Exception message: {} '.format( action['id_domain'], self.hyp_id, error_msg)) except Exception as e: update_domain_status('Crashed', action['id_domain'], hyp_id=self.hyp_id, detail='domain {} failed when try to start in pause mode in hypervisor {}. creating domain operation is aborted') logs.workers.error( 'Exception starting paused xml for domain {} in hypervisor {}. NOT LIBVIRT EXCEPTION, RARE CASE. Exception message: {}'.format( action['id_domain'], self.hyp_id, str(e))) ## START DOMAIN elif action['type'] == 'start_domain': logs.workers.debug('xml to start some lines...: {}'.format(action['xml'][30:100])) try: self.h.conn.createXML(action['xml']) # wait to event started to save state in database #update_domain_status('Started', action['id_domain'], hyp_id=self.hyp_id, detail='Domain has started in worker thread') logs.workers.debug('STARTED domain {}: createdXML action in hypervisor {} has been sent'.format( action['id_domain'], host)) except libvirtError as e: update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=("Hypervisor can not create domain with libvirt exception: " + str(e))) logs.workers.debug('exception in starting domain {}: '.format(e)) except Exception as e: update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=("Exception when starting domain: " + str(e))) logs.workers.debug('exception in starting domain {}: '.format(e)) ## STOP DOMAIN elif action['type'] == 'stop_domain': logs.workers.debug('action stop domain: {}'.format(action['id_domain'][30:100])) try: self.h.conn.lookupByName(action['id_domain']).destroy() logs.workers.debug('STOPPED domain {}'.format(action['id_domain'])) check_if_delete = action.get('delete_after_stopped',False) if check_if_delete is True: update_domain_status('Stopped', action['id_domain'], hyp_id='') update_domain_status('Deleting', action['id_domain'], hyp_id='') else: update_domain_status('Stopped', action['id_domain'], hyp_id='') except Exception as e: update_domain_status('Failed', action['id_domain'], hyp_id=self.hyp_id, detail=str(e)) logs.workers.debug('exception in stopping domain {}: '.format(e)) elif action['type'] in ['create_disk', 'delete_disk']: launch_action_disk(action, self.hostname, user, port) elif action['type'] in ['add_media_hot']: pass elif action['type'] in ['killall_curl']: launch_killall_curl(self.hostname, user, port) elif action['type'] in ['delete_media']: final_status = action.get('final_status','Deleted') launch_delete_media (action, self.hostname, user, port, final_status=final_status) # ## DESTROY THREAD # elif action['type'] == 'destroy_thread': # list_works_in_queue = list(self.queue_actions.queue) # if self.queue_master is not None: # self.queue_master.put(['destroy_working_thread',self.hyp_id,list_works_in_queue]) # #INFO TO DEVELOPER, si entra aquí es porque no quedaba nada en cola, si no ya lo habrán matado antes # # logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id)) # # raise 'destoyed' elif action['type'] == 'create_disk': pass elif action['type'] == 'hyp_info': self.h.get_hyp_info() logs.workers.debug('hypervisor motherboard: {}'.format(self.h.info['motherboard_manufacturer'])) ## DESTROY THREAD elif action['type'] == 'stop_thread': self.stop = True else: logs.workers.error('type action {} not supported in queue actions'.format(action['type'])) # time.sleep(0.1) ## TRY DOMAIN except queue.Empty: try: self.h.conn.getLibVersion() pass # logs.workers.debug('hypervisor {} is alive'.format(host)) except: logs.workers.info('trying to reconnect hypervisor {}, alive test in working thread failed'.format(host)) alive = False for i in range(RETRIES_HYP_IS_ALIVE): try: time.sleep(TIMEOUT_BETWEEN_RETRIES_HYP_IS_ALIVE) self.h.conn.getLibVersion() alive = True logs.workers.info('hypervisor {} is alive'.format(host)) break except: logs.workers.info('hypervisor {} is NOT alive'.format(host)) if alive is False: try: self.h.connect_to_hyp() self.h.conn.getLibVersion() update_hyp_status(self.hyp_id, 'Online') except: logs.workers.debug('hypervisor {} failed'.format(host)) logs.workers.error('fail reconnecting to hypervisor {} in working thread'.format(host)) reason = self.h.fail_connected_reason update_hyp_status(self.hyp_id, 'Error', reason) update_domains_started_in_hyp_to_unknown(self.hyp_id) list_works_in_queue = list(self.queue_actions.queue) if self.queue_master is not None: self.queue_master.put(['error_working_thread', self.hyp_id, list_works_in_queue]) logs.workers.error('thread worker from hypervisor {} exit from error status'.format(hyp_id)) self.active = False break else: update_hyp_status(self.hyp_id, 'Error','bios vmx or svm virtualization capabilities not activated') update_table_field('hypervisors',self.hyp_id,'enabled',False) logs.workers.error('hypervisor {} disabled: bios vmx or svm virtualization capabilities not activated') #restart when engine is started (not starting) timeout = 10 i = 0.0 while i < timeout: if get_engine()['status_all_threads'] == 'Started': engine_restart() break else: i + 0.2 sleep(0.2)
def creating_and_test_xml_start(self, id_domain, creating_from_create_dict=False, xml_from_virt_install=False, xml_string=None): if creating_from_create_dict is True: try: populate_dict_hardware_from_create_dict(id_domain) except Exception as e: log.error( 'error when populate dict hardware from create dict in domain {}' .format(id_domain)) log.error('Traceback: \n .{}'.format(traceback.format_exc())) log.error('Exception message: {}'.format(e)) domain = get_domain(id_domain) #create_dict_hw = domain['create_dict']['hardware'] # for media in ['isos','floppies'] # if 'isos' in create_dict_hw.keys(): # for index_disk in range(len(create_dict_hw['isos'])): # update_hw['hardware']['isos'][index_disk]['file'] = new_file if type(xml_string) is str: xml_from = xml_string elif 'create_from_virt_install_xml' in domain['create_dict']: xml_from = get_dict_from_item_in_table( 'virt_install', domain['create_dict']['create_from_virt_install_xml'])['xml'] elif xml_from_virt_install is False: id_template = domain['create_dict']['origin'] template = get_domain(id_template) xml_from = template['xml'] parents_chain = template.get('parents', []) + domain.get( 'parents', []) #when creating template from domain, the domain would be inserted as a parent while template is creating # parent_chain never can't have id_domain as parent if id_domain in parents_chain: for i in range(parents_chain.count('a')): parents_chain.remove(id_domain) update_table_field('domains', id_domain, 'parents', parents_chain) elif xml_from_virt_install is True: xml_from = domain['xml_virt_install'] else: return False update_table_field('domains', id_domain, 'xml', xml_from) xml_raw = update_xml_from_dict_domain(id_domain) if xml_raw is False: update_domain_status(status='FailedCreatingDomain', id_domain=id_domain, detail='XML Parser Error, xml is not valid') return False update_domain_status( 'CreatingDomain', id_domain, detail= 'xml and hardware dict updated, waiting to test if domain start paused in hypervisor' ) pool_id = get_pool_from_domain(id_domain) if 'start_after_created' in domain.keys(): if domain['start_after_created'] is True: update_domain_status( 'StartingDomainDisposable', id_domain, detail= 'xml and hardware dict updated, starting domain disposable' ) self.start_domain_from_id(id_domain) else: #change viewer password, remove selinux options and recreate network interfaces try: xml = recreate_xml_to_start(id_domain) except Exception as e: log.error( 'recreate_xml_to_start in domain {}'.format(id_domain)) log.error('Traceback: \n .{}'.format(traceback.format_exc())) log.error('Exception message: {}'.format(e)) xml = False if xml is False: update_domain_status( 'Failed', id_domain, detail="DomainXML can't parse and modify xml to start") else: self.start_paused_domain_from_xml(xml=xml, id_domain=id_domain, pool_id=pool_id)