def test_hypervisor_conn(uri): """ test hypervisor connecton, if fail an error message in log """ try: handle = libvirt.open(uri) return handle except: log.error(sys.exc_info()[1]) return False
def insert_event_in_db(self, dict_event): log.debug(pformat(dict_event)) r_conn = new_rethink_connection() try: r.table('hypervisors_events').insert(dict_event).run(r_conn) close_rethink_connection(r_conn) except Exception as e: log.error('rethink insert hyp event fail: {}'.format(e))
def get_hyp_hostname_from_id(id): r_conn = new_rethink_connection() l = r.table('hypervisors').get(id).pluck('hostname', 'port', 'user').run(r_conn) close_rethink_connection(r_conn) if len(l) > 0: if l.__contains__('user') and l.__contains__('port'): return l['hostname'], l['port'], l['user'] else: log.error('hypervisor {} does not contain user or port in database'.format(id)) return False else: return False
def get_id_hyp_from_uri(uri): r_conn = new_rethink_connection() rtable = r.table('hypervisors') l = list(rtable. \ filter({'uri': uri}). \ pluck('id'). \ run(r_conn)) close_rethink_connection(r_conn) if len(l) > 0: return l[0]['id'] else: log.error('function: {} uri {} not defined in hypervisors table'.format(str(__name__), uri))
def new_template_from_domain(name, description, original_domain_id, user, group, category, kind='public_template'): id = '_' + user + '_' + name dir_disk = category + '/' + group + '/' + name disk_filename = name + '.qcow2' relative_dir = dir_disk + '/' + disk_filename old_domain = get_domain(original_domain_id) new_backing_chain = [relative_dir] new_backing_chain.append(old_domain['hardware']['disks'][0]['file']) new_hardware_dict = old_domain['hardware'] new_hardware_dict['disks'][0]['file'] = relative_dir new_hardware_dict['disks'][0]['backing_chain'] = new_backing_chain new_hardware_dict['name'] = id new_hardware_dict['uuid'] = None template_field = { 'id': id, 'kind': kind, 'user': user, 'status': 'Creating', 'detail': None, 'category': category, 'group': group, 'server': False, 'hardware': new_hardware_dict, 'hypervisors_pools': old_domain['hypervisors_pools'], 'xml': None, 'name': name, 'description': description, 'icon': old_domain['icon'], 'os': old_domain['icon'] } result = update_domain_createing_template(original_domain_id, template_field) if result['errors'] > 0: log.error( 'error when inserting new domain {} in db, duplicate id??'.format( id)) return False else: return id
def new_domain_from_template(name, description, template_origin, user, group, category): id = '_' + user + '_' + name dir_disk = category + '/' + group + '/' + name disk_filename = name + '.qcow2' relative_path = dir_disk + '/' + disk_filename template = get_domain(template_origin) # new_backing_chain = [relative_dir] # new_backing_chain.append(template['hardware']['disks'][0]['file']) base_disk_path = template['hardware']['disks'][0]['file'] new_hardware_dict = template['hardware'].copy() new_hardware_dict['disks'][0]['file'] = relative_path new_hardware_dict['disks'][0]['parent'] = base_disk_path new_hardware_dict['name'] = id new_hardware_dict['uuid'] = None domain = { 'id': id, 'kind': 'desktop', 'user': user, 'status': 'Creating', 'detail': None, 'category': category, 'group': group, 'server': False, 'hardware': new_hardware_dict, 'hypervisors_pools': template['hypervisors_pools'], 'xml': template['xml'], 'name': name, 'description': description, 'icon': template['icon'], 'os': template['icon'] } result = insert_domain(domain) if result['errors'] > 0: log.error( 'error when inserting new domain {} in db, duplicate id??'.format( id)) return False else: return id
def new_template_from_domain(name, description, original_domain_id, user, group, category, kind='public_template'): id = '_' + user + '_' + name dir_disk = category + '/' + group + '/' + name disk_filename = name + '.qcow2' relative_dir = dir_disk + '/' + disk_filename old_domain = get_domain(original_domain_id) new_backing_chain = [relative_dir] new_backing_chain.append(old_domain['hardware']['disks'][0]['file']) new_hardware_dict = old_domain['hardware'] new_hardware_dict['disks'][0]['file'] = relative_dir new_hardware_dict['disks'][0]['backing_chain'] = new_backing_chain new_hardware_dict['name'] = id new_hardware_dict['uuid'] = None template_field = {'id': id, 'kind': kind, 'user': user, 'status': 'Creating', 'detail': None, 'category': category, 'group': group, 'server': False, 'hardware': new_hardware_dict, 'hypervisors_pools': old_domain['hypervisors_pools'], 'xml': None, 'name': name, 'description': description, 'icon': old_domain['icon'], 'os': old_domain['icon']} result = update_domain_createing_template(original_domain_id, template_field) if result['errors'] > 0: log.error('error when inserting new domain {} in db, duplicate id??'.format(id)) return False else: return id
def new_domain_from_template(name, description, template_origin, user, group, category): id = '_' + user + '_' + name dir_disk = category + '/' + group + '/' + name disk_filename = name + '.qcow2' relative_path = dir_disk + '/' + disk_filename template = get_domain(template_origin) # new_backing_chain = [relative_dir] # new_backing_chain.append(template['hardware']['disks'][0]['file']) base_disk_path = template['hardware']['disks'][0]['file'] new_hardware_dict = template['hardware'].copy() new_hardware_dict['disks'][0]['file'] = relative_path new_hardware_dict['disks'][0]['parent'] = base_disk_path new_hardware_dict['name'] = id new_hardware_dict['uuid'] = None domain = {'id': id, 'kind': 'desktop', 'user': user, 'status': 'Creating', 'detail': None, 'category': category, 'group': group, 'server': False, 'hardware': new_hardware_dict, 'hypervisors_pools': template['hypervisors_pools'], 'xml': template['xml'], 'name': name, 'description': description, 'icon': template['icon'], 'os': template['icon']} result = insert_domain(domain) if result['errors'] > 0: log.error('error when inserting new domain {} in db, duplicate id??'.format(id)) return False else: return id
def try_socket(hostname, port, timeout): try: ip = socket.gethostbyname(hostname) addr = (hostname, port) sock = socket.socket(2, socket.SOCK_STREAM) sock.settimeout(timeout) try: sock.connect(addr) sock.close() return True except socket.error as e: log.error('trying socket has error: {}'.format(e)) return False except Exception as e: log.error(e) return False except socket.error as e: log.error(e) log.error('not resolves ip from hostname: {}'.format(hostname)) return False
def disk_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands if action['type'] in ['create_disk']: launch_action_disk(action, self.hostname, self.user, self.port) elif action['type'] in ['create_disk_from_scratch']: launch_action_disk(action, self.hostname, self.user, self.port, from_scratch=True) elif action['type'] in ['delete_disk']: launch_delete_disk_action(action, self.hostname, self.user, self.port) elif action['type'] in ['create_template_disk_from_domain']: launch_action_create_template_disk(action, self.hostname, self.user, self.port) elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format(action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format( disk_path, id_domain)) log.error('\n'.join( ['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))])) update_domain_status('Failed', id_domain, detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def try_ssh(hostname, port, user, timeout): if try_socket(hostname, port, timeout) is True: ip = socket.gethostbyname(hostname) ssh = paramiko.SSHClient() # ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ##INFO TO DEVELOPER TERMINAR DE ENTENDER POR QUE A VECES VA Y A VECES NO ssh.load_system_host_keys() # ssh.load_host_keys('/home/vimet/.ssh/known_hosts') time.sleep(1) try: # timelimit(3,test_hypervisor_conn,hostname, # username=user, # port= port, # timeout=CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection']) log.debug('@@@@@@@@@@@@@@@@@@@@') log.debug('@@@@@@@@@@@@@@@@@@@@') log.debug('@@@@@@@@@@@@@@@@@@@@') ssh.connect(hostname, username=user, port=port, timeout=timeout, banner_timeout=timeout) log.debug("host {} with ip {} can connect with ssh without password with paramiko".format(hostname, ip)) log.debug('############################################') log.debug('############################################') log.debug('############################################') log.debug('############################################') ssh.close() # # cmd = 'timeout 20 ssh -p {} {}@{} pwd'.format(port,user,hostname) # # proc = subprocess.Popen(["bash", "-c", cmd],stdout=subprocess.PIPE,stderr=subprocess.STDOUT) # proc = subprocess.Popen(shlex.split(cmd),stdout=subprocess.PIPE,stderr=subprocess.STDOUT) # proc = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=True) # stdout,stderr=proc.communicate() # # if len(stdout) > 0 # log.debug(stdout) # log.debug(stderr) # if return True except Exception as e: try: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname, username=user, port=port, timeout=timeout, banner_timeout=timeout) log.debug("The authenticity of host '{} ({})' can't be established".format(hostname, ip)) log.debug( "host {} with ip {} can connect with ssh without password but the key fingerprint must be incorporated in ~/.ssh/known_hosts".format( hostname, ip)) ssh.close() return False except: log.error( "host {} with ip {} can't connect with ssh without password. Reasons? timeout, ssh authentication with keys is needed, port is correct?".format( hostname, ip)) log.error('reason: {}'.format(e)) return False else: log.error('socket error, try if ssh is listen in hostname {} and port {}'.format(hostname, port)) return False
def long_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands id_domain = action['domain'] if action['type'] in ['create_disk_virt_builder']: cmds_done = execute_commands(host=self.hostname, ssh_commands=action['ssh_commands'], dict_mode=True, user=self.user, port=self.port ) if len([d for d in cmds_done if len(d['err']) > 0]) > 1: log.error('some error in virt builder operations') log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format( action['disk_path'], action['domain'], self.hyp_id)) log.debug('print cmds_done:') log.debug(pprint.pprint(cmds_done)) log.debug('print ssh_commands:') log.debug(pprint.pprint(action['ssh_commands'])) update_domain_status('Failed', id_domain, detail='Virt Builder Failed creating disk file') else: log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'], action['disk_path'])) xml_virt_install = cmds_done[-1]['out'] update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install) update_domain_status('CreatingDomainFromBuilder', id_domain, detail='disk created from virt-builder') elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format(action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format( disk_path, id_domain)) log.error('\n'.join( ['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))])) update_domain_status('Failed', id_domain, detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def update_hyp_status(id, status, detail='', uri=''): # INFO TO DEVELOPER: TODO debería pillar el estado anterior del hypervisor y ponerlo en un campo, # o mejor aún, guardar un histórico con los tiempos de cambios en un diccionario que # en python puede ser internamente una cola de X elementos (número de elementos de configuración) # como una especie de log de cuando cambio de estado # INFO TO DEVELOPER: pasarlo a una función en functions defined_status = ['Offline', 'TryConnection', 'ReadyToStart', 'StartingThreads', 'Error', 'Online', 'Blocked', 'DestroyingDomains', 'StoppingThreads'] if status in defined_status: r_conn = new_rethink_connection() rtable = r.table('hypervisors') if len(uri) > 0: dict_update = {'status': status, 'uri': uri} else: dict_update = {'status': status} d = rtable.get(id).pluck('status', 'status_time', 'prev_status', 'detail').run(r_conn) if 'status' in d.keys(): if 'prev_status' not in d.keys(): dict_update['prev_status'] = [] else: if type(d['prev_status']) is list: dict_update['prev_status'] = d['prev_status'] else: dict_update['prev_status'] = [] d_old_status = {} d_old_status['status'] = d['status'] if 'detail' in d.keys(): d_old_status['detail'] = d['detail'] else: d_old_status['detail'] = '' if 'status_time' in d.keys(): d_old_status['status_time'] = d['status_time'] dict_update['prev_status'].insert(0, d_old_status) dict_update['prev_status'] = dict_update['prev_status'][:MAX_LEN_PREV_STATUS_HYP] now = time.time() dict_update['status_time'] = now # if len(detail) == 0: # rtable.filter({'id':id}).\ # update(dict_update).\ # run(r_conn) # # rtable.filter({'id':id}).\ # # replace(r.row.without('detail')).\ # # run(r_conn) # close_rethink_connection(r_conn) # # else: dict_update['detail'] = str(detail) rtable.filter({'id': id}). \ update(dict_update). \ run(r_conn) close_rethink_connection(r_conn) else: log.error('hypervisor status {} is not defined'.format(status)) return False
def try_ssh(hostname, port, user, timeout): if try_socket(hostname, port, timeout) is True: ip = socket.gethostbyname(hostname) ssh = paramiko.SSHClient() # ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ##INFO TO DEVELOPER TERMINAR DE ENTENDER POR QUE A VECES VA Y A VECES NO ssh.load_system_host_keys() # ssh.load_host_keys('/home/vimet/.ssh/known_hosts') time.sleep(1) try: # timelimit(3,test_hypervisor_conn,hostname, # username=user, # port= port, # timeout=CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection']) log.debug('@@@@@@@@@@@@@@@@@@@@') log.debug('@@@@@@@@@@@@@@@@@@@@') log.debug('@@@@@@@@@@@@@@@@@@@@') ssh.connect(hostname, username=user, port=port, timeout=timeout, banner_timeout=timeout) log.debug( "host {} with ip {} can connect with ssh without password with paramiko" .format(hostname, ip)) log.debug('############################################') log.debug('############################################') log.debug('############################################') log.debug('############################################') ssh.close() # # cmd = 'timeout 20 ssh -p {} {}@{} pwd'.format(port,user,hostname) # # proc = subprocess.Popen(["bash", "-c", cmd],stdout=subprocess.PIPE,stderr=subprocess.STDOUT) # proc = subprocess.Popen(shlex.split(cmd),stdout=subprocess.PIPE,stderr=subprocess.STDOUT) # proc = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=True) # stdout,stderr=proc.communicate() # # if len(stdout) > 0 # log.debug(stdout) # log.debug(stderr) # if return True except Exception as e: try: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname, username=user, port=port, timeout=timeout, banner_timeout=timeout) log.debug( "The authenticity of host '{} ({})' can't be established". format(hostname, ip)) log.debug( "host {} with ip {} can connect with ssh without password but the key fingerprint must be incorporated in ~/.ssh/known_hosts" .format(hostname, ip)) ssh.close() return False except: log.error( "host {} with ip {} can't connect with ssh without password. Reasons? timeout, ssh authentication with keys is needed, port is correct?" .format(hostname, ip)) log.error('reason: {}'.format(e)) return False else: log.error( 'socket error, try if ssh is listen in hostname {} and port {}'. format(hostname, port)) return False
def long_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug( 'Thread to launchdisks operations in host {} with TID: {}...'. format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands id_domain = action['domain'] if action['type'] in ['create_disk_virt_builder']: cmds_done = execute_commands( host=self.hostname, ssh_commands=action['ssh_commands'], dict_mode=True, user=self.user, port=self.port) if len([d for d in cmds_done if len(d['err']) > 0]) > 1: log.error('some error in virt builder operations') log.error( 'Virt Builder Failed creating disk file {} in domain {} in hypervisor {}' .format(action['disk_path'], action['domain'], self.hyp_id)) log.debug('print cmds_done:') log.debug(pprint.pprint(cmds_done)) log.debug('print ssh_commands:') log.debug(pprint.pprint(action['ssh_commands'])) update_domain_status( 'Failed', id_domain, detail='Virt Builder Failed creating disk file') else: log.info( 'Disk created from virt-builder. Domain: {} , disk: {}' .format(action['domain'], action['disk_path'])) xml_virt_install = cmds_done[-1]['out'] update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install) update_domain_status( 'CreatingDomainFromBuilder', id_domain, detail='disk created from virt-builder') elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format( action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}' .format(disk_path, id_domain)) log.error('\n'.join([ 'cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands'])) ])) update_domain_status( 'Failed', id_domain, detail= 'new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs' )