def update_output(self, out): self.out_process = out # log.debug(out) tmp = out[:str(out).find('%')] percent = tmp[tmp.rfind(' ') + 1:] self.percent = percent log.debug('percent: ' + percent + '% \r')
def analize_backing_chains_outputs(array_out_err=[], path_to_write_json=None, path_to_read_json=None): if path_to_write_json is not None: f = open(path_to_write_json, 'w') json.dump(array_out_err, f) f.close() if path_to_read_json is not None: f = open(path_to_read_json, 'r') array_out_err = json.load(f) log.debug(len(array_out_err)) f.close() domains_ok = 0 domains_err = 0 for d in array_out_err: id = d['title'] if len(d['err']) > 0: domains_err += 1 log.info(d['err']) update_domain_status('Failed', id, detail=d['err']) else: log.debug(id) domains_ok += 1 if type(d['out']) is not str: out = out.decode('utf-8') else: out = d['out'] l = json.loads(out) from pprint import pprint pprint(l) update_disk_backing_chain(id, 0, l[0]['filename'], l) return ({'ok': domains_ok, 'err': domains_err})
def exec_remote_updating_progress(command, hostname, progress=[], username='******', port=22, sudo=False, id_domain=None): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(hostname, port=port, username=username) stdin, stdout, stderr = client.exec_command(command) progress.append(0) # INFO TO DEVELOPER, AQUÍ FALTA ACTUALIZAR EL PROGRESO A LA BASE DE DATOS # A PARTIR DE LA LISTA PROGRESS, COGIENDO EL ÚLTIMO PARÁMETRO # SE PODRÍA IR MIRANDO, AUNQUE LO SUYO ES QUE SE ACTUALIZASE AQUÍ?? log.debug('jujuju {} '.format(type(stdout))) while True: # out = stdout.readline(64).decode('utf-8') out = stdout.readline(64) if out.find('%') >= 0: tmp = out[:str(out).find('%')] percent = tmp[tmp.rfind(' ') + 1:] if len(percent) > 0: if percent.isdigit(): percent = int(percent) if progress[-1] < percent: progress.append(percent) if id_domain is not None: update_domain_progress(id_domain, percent) log.debug(out) if out == '': break out = stdout.read() err = stderr.read() client.close() return {'out': out, 'err': err}
def exec_remote_list_of_cmds_dict(hostname, list_dict_commands, username='******', port=22, ssh_key_str='', sudo=False): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if len(ssh_key_str) > 0: # TODO: make ssh_key login pass else: client.connect(hostname, port=port, username=username) returned_array = list_dict_commands.copy() i = 0 for command in list_dict_commands: log.debug('command to launch in ssh in {}: {}'.format( hostname, command['cmd'])) stdin, stdout, stderr = client.exec_command(command['cmd']) returned_array[i]['out'] = stdout.read().decode('utf-8') returned_array[i]['err'] = stderr.read().decode('utf-8') log.debug('commnad launched / out: {} / error: {}'.format( returned_array[i]['out'], returned_array[i]['err'])) i = i + 1 client.close() return returned_array
def insert_event_in_db(self, dict_event): log.debug(pformat(dict_event)) r_conn = new_rethink_connection() try: r.table('hypervisors_events').insert(dict_event).run(r_conn) close_rethink_connection(r_conn) except Exception as e: log.error('rethink insert hyp event fail: {}'.format(e))
def disk_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands if action['type'] in ['create_disk']: launch_action_disk(action, self.hostname, self.user, self.port) elif action['type'] in ['create_disk_from_scratch']: launch_action_disk(action, self.hostname, self.user, self.port, from_scratch=True) elif action['type'] in ['delete_disk']: launch_delete_disk_action(action, self.hostname, self.user, self.port) elif action['type'] in ['create_template_disk_from_domain']: launch_action_create_template_disk(action, self.hostname, self.user, self.port) elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format(action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format( disk_path, id_domain)) log.error('\n'.join( ['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))])) update_domain_status('Failed', id_domain, detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def run(self): self.tid = get_tid() log.info('starting thread: {} (TID {})'.format(self.name, self.tid)) self.hyp_obj, self.ok = try_hyp_connection(self.hyp_id, self.hostname, self.port, self.user) log.debug('Exiting from thread {} try_hyp {}'.format( self.name, self.hostname)) return self.ok
def get_hyp_hostnames_online(): r_conn = new_rethink_connection() rtable = r.table('hypervisors') l = list(rtable. \ filter({'enabled': True, 'status': 'Online'}). \ pluck('id', 'hostname'). \ run(r_conn)) close_rethink_connection(r_conn) log.debug(l) if len(l) > 0: hyps_hostnames = {d['id']: d['hostname'] for d in l} return hyps_hostnames else: return dict()
def exec_remote_list_of_cmds(hostname, commands, username='******', port=22, sudo=False): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(hostname, port=port, username=username) returned_array = [] for command in commands: log.debug('command to launch in ssh in {}: {}'.format(hostname, command)) stdin, stdout, stderr = client.exec_command(command) out = stdout.read().decode('utf-8') err = stderr.read().decode('utf-8') returned_array.append({'out': out, 'err': err}) log.debug('commnad launched / out: {} / error: {}'.format(out, err)) client.close() return returned_array
def __init__(self, src, dst, verbose=True, show_progress=True, bwlimit=''): cmd = 'rsync -a' if verbose: cmd += 'v' if len(bwlimit) > 0: cmd += ' --bwlimit={}'.format(bwlimit) if show_progress: cmd += ' --progress' cmd += ' ' + src + ' ' cmd += '' + dst + '' self.dst = dst self.src = src self.cmd = cmd log.debug('rsync command: ' + self.cmd) self.proc = False self.status = 0
def exec_remote_list_of_cmds(hostname, commands, username='******', port=22, sudo=False): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(hostname, port=port, username=username) returned_array = [] for command in commands: log.debug('command to launch in ssh in {}: {}'.format( hostname, command)) stdin, stdout, stderr = client.exec_command(command) out = stdout.read().decode('utf-8') err = stderr.read().decode('utf-8') returned_array.append({'out': out, 'err': err}) log.debug('commnad launched / out: {} / error: {}'.format(out, err)) client.close() return returned_array
def exec_remote_list_of_cmds_dict(hostname, list_dict_commands, username='******', port=22, ssh_key_str='', sudo=False): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if len(ssh_key_str) > 0: # TODO: make ssh_key login pass else: client.connect(hostname, port=port, username=username) returned_array = list_dict_commands.copy() i = 0 for command in list_dict_commands: log.debug('command to launch in ssh in {}: {}'.format(hostname, command['cmd'])) stdin, stdout, stderr = client.exec_command(command['cmd']) returned_array[i]['out'] = stdout.read().decode('utf-8') returned_array[i]['err'] = stderr.read().decode('utf-8') log.debug('commnad launched / out: {} / error: {}'.format(returned_array[i]['out'], returned_array[i]['err'])) i = i + 1 client.close() return returned_array
def get_process_stdout(self): self.stdout = '' while True: out = self.proc.stdout.readline(64).decode('utf-8') if out == '' and self.proc.poll() != None: break self.update_output(out) log.debug(out) log.debug(type(out)) self.stdout += out self.stdout = self.stdout.replace('\r', '\n') self.status = 1 log.debug('rsync finished: ' + self.stdout)
def remove_dst(self): if os.path.isdir(self.dst): if self.dst.endswith('/'): path = self.dst + self.src[self.src.rfind('/') + 1:] else: path = self.dst + self.src[self.src.rfind('/'):] else: path = self.dst if os.path.isfile(path): try: os.remove(path) log.debug('file removed: ' + path) except: log.debug('remove fail') else: log.debug('destination don\'t exist')
def start_and_wait_silence(self): try: self.start_process() self.wait_communicate() except: log.debug('call to subprocess rsync failed: ' + self.cmd)
def wait_communicate(self): self.stdout, self.stderr = self.proc.communicate() self.status = 1 log.debug('rsync finished: ' + self.stdout)
def long_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands id_domain = action['domain'] if action['type'] in ['create_disk_virt_builder']: cmds_done = execute_commands(host=self.hostname, ssh_commands=action['ssh_commands'], dict_mode=True, user=self.user, port=self.port ) if len([d for d in cmds_done if len(d['err']) > 0]) > 1: log.error('some error in virt builder operations') log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format( action['disk_path'], action['domain'], self.hyp_id)) log.debug('print cmds_done:') log.debug(pprint.pprint(cmds_done)) log.debug('print ssh_commands:') log.debug(pprint.pprint(action['ssh_commands'])) update_domain_status('Failed', id_domain, detail='Virt Builder Failed creating disk file') else: log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'], action['disk_path'])) xml_virt_install = cmds_done[-1]['out'] update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install) update_domain_status('CreatingDomainFromBuilder', id_domain, detail='disk created from virt-builder') elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format(action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format( disk_path, id_domain)) log.error('\n'.join( ['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))])) update_domain_status('Failed', id_domain, detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def start_and_wait_verbose(self): try: self.start_process() self.get_process_stdo except: log.debug('call to subprocess rsync failed: ' + self.cmd)
def try_ssh(hostname, port, user, timeout): if try_socket(hostname, port, timeout) is True: ip = socket.gethostbyname(hostname) ssh = paramiko.SSHClient() # ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ##INFO TO DEVELOPER TERMINAR DE ENTENDER POR QUE A VECES VA Y A VECES NO ssh.load_system_host_keys() # ssh.load_host_keys('/home/vimet/.ssh/known_hosts') time.sleep(1) try: # timelimit(3,test_hypervisor_conn,hostname, # username=user, # port= port, # timeout=CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection']) log.debug('@@@@@@@@@@@@@@@@@@@@') log.debug('@@@@@@@@@@@@@@@@@@@@') log.debug('@@@@@@@@@@@@@@@@@@@@') ssh.connect(hostname, username=user, port=port, timeout=timeout, banner_timeout=timeout) log.debug("host {} with ip {} can connect with ssh without password with paramiko".format(hostname, ip)) log.debug('############################################') log.debug('############################################') log.debug('############################################') log.debug('############################################') ssh.close() # # cmd = 'timeout 20 ssh -p {} {}@{} pwd'.format(port,user,hostname) # # proc = subprocess.Popen(["bash", "-c", cmd],stdout=subprocess.PIPE,stderr=subprocess.STDOUT) # proc = subprocess.Popen(shlex.split(cmd),stdout=subprocess.PIPE,stderr=subprocess.STDOUT) # proc = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=True) # stdout,stderr=proc.communicate() # # if len(stdout) > 0 # log.debug(stdout) # log.debug(stderr) # if return True except Exception as e: try: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname, username=user, port=port, timeout=timeout, banner_timeout=timeout) log.debug("The authenticity of host '{} ({})' can't be established".format(hostname, ip)) log.debug( "host {} with ip {} can connect with ssh without password but the key fingerprint must be incorporated in ~/.ssh/known_hosts".format( hostname, ip)) ssh.close() return False except: log.error( "host {} with ip {} can't connect with ssh without password. Reasons? timeout, ssh authentication with keys is needed, port is correct?".format( hostname, ip)) log.error('reason: {}'.format(e)) return False else: log.error('socket error, try if ssh is listen in hostname {} and port {}'.format(hostname, port)) return False
def start_and_continue(self): try: self.start_process() self.thread() except: log.debug('call to subprocess rsync failed')
def try_ssh(hostname, port, user, timeout): if try_socket(hostname, port, timeout) is True: ip = socket.gethostbyname(hostname) ssh = paramiko.SSHClient() # ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ##INFO TO DEVELOPER TERMINAR DE ENTENDER POR QUE A VECES VA Y A VECES NO ssh.load_system_host_keys() # ssh.load_host_keys('/home/vimet/.ssh/known_hosts') time.sleep(1) try: # timelimit(3,test_hypervisor_conn,hostname, # username=user, # port= port, # timeout=CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection']) log.debug('@@@@@@@@@@@@@@@@@@@@') log.debug('@@@@@@@@@@@@@@@@@@@@') log.debug('@@@@@@@@@@@@@@@@@@@@') ssh.connect(hostname, username=user, port=port, timeout=timeout, banner_timeout=timeout) log.debug( "host {} with ip {} can connect with ssh without password with paramiko" .format(hostname, ip)) log.debug('############################################') log.debug('############################################') log.debug('############################################') log.debug('############################################') ssh.close() # # cmd = 'timeout 20 ssh -p {} {}@{} pwd'.format(port,user,hostname) # # proc = subprocess.Popen(["bash", "-c", cmd],stdout=subprocess.PIPE,stderr=subprocess.STDOUT) # proc = subprocess.Popen(shlex.split(cmd),stdout=subprocess.PIPE,stderr=subprocess.STDOUT) # proc = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=True) # stdout,stderr=proc.communicate() # # if len(stdout) > 0 # log.debug(stdout) # log.debug(stderr) # if return True except Exception as e: try: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname, username=user, port=port, timeout=timeout, banner_timeout=timeout) log.debug( "The authenticity of host '{} ({})' can't be established". format(hostname, ip)) log.debug( "host {} with ip {} can connect with ssh without password but the key fingerprint must be incorporated in ~/.ssh/known_hosts" .format(hostname, ip)) ssh.close() return False except: log.error( "host {} with ip {} can't connect with ssh without password. Reasons? timeout, ssh authentication with keys is needed, port is correct?" .format(hostname, ip)) log.error('reason: {}'.format(e)) return False else: log.error( 'socket error, try if ssh is listen in hostname {} and port {}'. format(hostname, port)) return False
def long_operations_thread(self): host = self.hostname self.tid = get_tid() log.debug( 'Thread to launchdisks operations in host {} with TID: {}...'. format(host, self.tid)) while self.stop is not True: try: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) # for ssh commands id_domain = action['domain'] if action['type'] in ['create_disk_virt_builder']: cmds_done = execute_commands( host=self.hostname, ssh_commands=action['ssh_commands'], dict_mode=True, user=self.user, port=self.port) if len([d for d in cmds_done if len(d['err']) > 0]) > 1: log.error('some error in virt builder operations') log.error( 'Virt Builder Failed creating disk file {} in domain {} in hypervisor {}' .format(action['disk_path'], action['domain'], self.hyp_id)) log.debug('print cmds_done:') log.debug(pprint.pprint(cmds_done)) log.debug('print ssh_commands:') log.debug(pprint.pprint(action['ssh_commands'])) update_domain_status( 'Failed', id_domain, detail='Virt Builder Failed creating disk file') else: log.info( 'Disk created from virt-builder. Domain: {} , disk: {}' .format(action['domain'], action['disk_path'])) xml_virt_install = cmds_done[-1]['out'] update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install) update_domain_status( 'CreatingDomainFromBuilder', id_domain, detail='disk created from virt-builder') elif action['type'] == 'stop_thread': self.stop = True else: log.error('type action {} not supported'.format( action['type'])) except queue.Empty: pass except Exception as e: log.error('Exception when creating disk: {}'.format(e)) log.error('Action: {}'.format(pprint.pformat(action))) log.error('Traceback: {}'.format(traceback.format_exc())) return False if self.stop is True: while self.queue_actions.empty() is not True: action = self.queue_actions.get(timeout=TIMEOUT_QUEUES) if action['type'] == 'create_disk': disk_path = action['disk_path'] id_domain = action['domain'] log.error( 'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}' .format(disk_path, id_domain)) log.error('\n'.join([ 'cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands'])) ])) update_domain_status( 'Failed', id_domain, detail= 'new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs' )