def run_convert_if_need(): def do_run(): self.check_docker() save_pid() ret = shell.run(echo_pid_cmd) new_task.current_process_return_code = ret return ret pid = linux.read_file(v2v_pid_path) if not pid: return do_run() pid = int(pid.strip()) process_completed = os.path.exists(v2v_cmd_ret_path) process_has_been_killed = not os.path.exists( v2v_cmd_ret_path) and not os.path.exists('/proc/%d' % pid) process_still_running = not os.path.exists( v2v_cmd_ret_path) and os.path.exists('/proc/%d' % pid) if process_has_been_killed: return do_run() if process_still_running: linux.wait_callback_success(os.path.exists, v2v_cmd_ret_path, timeout=259200, interval=60) ret = linux.read_file(v2v_cmd_ret_path) return int(ret.strip() if ret else 126)
def cancel_sftp_download(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = AgentResponse() def check(): return shell.run("rbd ls %s | grep -q %s" % (pool, image_name)) != 0 def remove(target_name): return shell.run("rbd info {0}/{1} || rbd rm {0}/{1}".format( pool, target_name)) == 0 pool, image_name = self._parse_install_path( cmd.primaryStorageInstallPath) tmp_image_name = 'tmp-%s' % image_name if check(): return jsonobject.dumps(rsp) for image in (tmp_image_name, image_name): shell.run("pkill -9 -f '%s'" % image) linux.wait_callback_success(remove, image, timeout=30) if not check(): rsp.set_err("remove image %s/%s fail" % (pool, image_name)) return jsonobject.dumps(rsp)
def test(): ''' Test Description: Will check if test VM's mac/ip assignment will be removed in VR's DNS config, after VM is shutdown. And if VM's mac/ip will be added back after it restart. Resource required: Need support 2 VMs. ''' global test_obj_dict test_util.test_dsc('Create test vm and check. VR only has DNS and DHCP services') vm = test_stub.create_vlan_vm() test_obj_dict.add_vm(vm) vm.check() vm.stop() if linux.wait_callback_success(check_dhcp_remove, vm.vm, 5, 0.2): test_util.test_logger('[vm:] %s mac/ip was removed in VR /etc/hosts.dhcp' % vm.vm.uuid) else: test_util.test_logger('check dhcp remove failure, will try to print VR DHCP tables') test_lib.lib_print_vr_dhcp_tables(test_lib.lib_find_vr_by_vm(vm.vm)[0]) test_util.test_fail('[vm:] %s mac was not removed in VR /etc/hosts.dhcp, after vm was stopped.' % vm.vm.uuid) if linux.wait_callback_success(check_dhcp_release, vm.vm, 5, 0.2): test_util.test_logger('[vm:] %s mac/ip was removed in VR /etc/hosts.leases' % vm.vm.uuid) else: test_util.test_logger('check dhcp lease failure, will try to print VR DHCP tables') test_lib.lib_print_vr_dhcp_tables(test_lib.lib_find_vr_by_vm(vm.vm)[0]) test_util.test_fail('[vm:] %s mac was not removed in VR /etc/hosts.leases, after vm was stopped.' % vm.vm.uuid) vm.start() vm.check() vm.destroy() vm.check() test_util.test_pass('Successfully check VM DHCP release when VM stopped')
def save_pid(): linux.wait_callback_success(os.path.exists, v2v_pid_path) with open(v2v_pid_path, 'r') as fd: new_task.current_pid = fd.read().strip() new_task.current_process_cmd = echo_pid_cmd new_task.current_process_name = "virt_v2v_cmd" logger.debug("longjob[uuid:%s] saved process[pid:%s, name:%s]" % (cmd.longJobUuid, new_task.current_pid, new_task.current_process_name))
def stop(self, graceful=True, timeout=5, undefine=True): def cleanup_addons(): for chan in self.domain_xmlobject.devices.get_child_node_as_list( 'channel'): if chan.type_ == 'unix': path = chan.source.path_ shell.call('rm -f %s' % path) def loop_shutdown(_): try: self.domain.shutdown() except: #domain has been shutdown pass return self.wait_for_state_change(self.VM_STATE_SHUTDOWN) def loop_undefine(_): if not undefine: return True if not self.is_alive(): return True try: self.domain.undefineFlags( libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA) except libvirt.libvirtError: self.domain.undefine() return self.wait_for_state_change(None) def loop_destroy(_): try: self.domain.destroy() except: #domain has been destroyed pass return self.wait_for_state_change(self.VM_STATE_SHUTDOWN) do_destroy = True if graceful: if linux.wait_callback_success(loop_shutdown, None, timeout=60): do_destroy = False if do_destroy: if not linux.wait_callback_success(loop_destroy, None, timeout=60): raise kvmagent.KvmError( 'failed to destroy vm, timeout after 60 secs') cleanup_addons() if not linux.wait_callback_success(loop_undefine, None, timeout=60): raise kvmagent.KvmError( 'failed to undefine vm, timeout after 60 secs')
def stop(self, graceful=True, timeout=5, undefine=True): def cleanup_addons(): for chan in self.domain_xmlobject.devices.get_child_node_as_list('channel'): if chan.type_ == 'unix': path = chan.source.path_ shell.call('rm -f %s' % path) def loop_shutdown(_): try: self.domain.shutdown() except: #domain has been shutdown pass return self.wait_for_state_change(self.VM_STATE_SHUTDOWN) def loop_undefine(_): if not undefine: return True if not self.is_alive(): return True try: self.domain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE|libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA) except libvirt.libvirtError: self.domain.undefine() return self.wait_for_state_change(None) def loop_destroy(_): try: self.domain.destroy() except: #domain has been destroyed pass return self.wait_for_state_change(self.VM_STATE_SHUTDOWN) do_destroy = True if graceful: if linux.wait_callback_success(loop_shutdown, None, timeout=60): do_destroy = False if do_destroy: if not linux.wait_callback_success(loop_destroy, None, timeout=60): raise kvmagent.KvmError('failed to destroy vm, timeout after 60 secs') cleanup_addons() if not linux.wait_callback_success(loop_undefine, None, timeout=60): raise kvmagent.KvmError('failed to undefine vm, timeout after 60 secs')
def test(): global host if test_lib.lib_get_active_host_number() < 2: test_util.test_fail('Not available host to do maintenance, since there are not 2 hosts') vm_cpu = 1 vm_memory = 1073741824 #1G cond = res_ops.gen_query_conditions('name', '=', 'ttylinux') image_uuid = res_ops.query_resource(res_ops.IMAGE, cond)[0].uuid l3_network_uuid = res_ops.query_resource(res_ops.L3_NETWORK)[0].uuid vm = test_stub.create_mini_vm([l3_network_uuid], image_uuid, cpu_num = vm_cpu, memory_size = vm_memory) test_obj_dict.add_vm(vm) host_uuid = test_lib.lib_get_vm_host(vm.vm).uuid host_ops.change_host_state(host_uuid, 'maintain') #need to update vm's inventory, since they will be changed by maintenace mode vm.update() vm.set_state(vm_header.STOPPED) vm.check() host_ops.change_host_state(host_uuid, 'enable') if not linux.wait_callback_success(is_host_connected, host_uuid, 120): test_util.test_fail('host status is not changed to connected, after changing its state to Enable') vm.start() vm.check() vm.destroy() test_obj_dict.rm_vm(vm) test_util.test_pass('Maintain Host Test Success')
def _restart_dnsmasq(self): self._do_dnsmasq_restart() def check_start(_): dnsmasq_pid = linux.get_pid_by_process_name('dnsmasq') return dnsmasq_pid is not None if not linux.wait_callback_success(check_start, None, 5, 0.5): logger.debug('dnsmasq is not running, former start failed, try to start it again ...') cmd = self._do_dnsmasq_start() if cmd.return_code != 0: raise virtualrouter.VirtualRouterError('dnsmasq in virtual router is not running, we try to start it but fail, error is %s' % cmd.stdout) if not linux.wait_callback_success(check_start, None, 5, 0.5): raise virtualrouter.VirtualRouterError('dnsmasq in virtual router is not running, "/etc/init.d/dnsmasq start" returns success, but the process is not running after 5 seconds')
def migrate(self, destHostIp): destUrl = "qemu+tcp://{0}/system".format(destHostIp) tcpUri = "tcp://{0}".format(destHostIp) try: self.domain.migrateToURI2( destUrl, tcpUri, None, libvirt.VIR_MIGRATE_LIVE | libvirt.VIR_MIGRATE_PEER2PEER | libvirt.VIR_MIGRATE_UNDEFINE_SOURCE | libvirt.VIR_MIGRATE_PERSIST_DEST | libvirt.VIR_MIGRATE_TUNNELLED, None, 0) except libvirt.libvirtError as ex: logger.warn(linux.get_exception_stacktrace()) raise kvmagent.KvmError('unable to migrate vm[uuid:%s] to %s, %s' % (self.uuid, destUrl, str(ex))) try: if not linux.wait_callback_success(self.wait_for_state_change, callback_data=None, timeout=300): raise kvmagent.KvmError('timeout after 300 seconds') except kvmagent.KvmError as ke: raise ke except Exception as e: logger.debug(linux.get_exception_stacktrace()) logger.debug( 'successfully migrated vm[uuid:{0}] to dest url[{1}]'.format( self.uuid, destUrl))
def check_sg_rule_exist(test_vm, protocol, direction, extra_str, exp_result): if linux.wait_callback_success(do_check_sg_rule_exist, (test_vm, direction, extra_str, exp_result), 5, 0.2): test_result = exp_result if exp_result: test_util.test_logger( 'Expected result: [Security Group] find %s %s rule for [vm:] %s. ' % (protocol, direction, test_vm.uuid)) else: test_util.test_logger( 'Expected result: [Security Group] FAIL to find %s %s rule for [vm:] %s in 5 seconds. ' % (protocol, direction, test_vm.uuid)) else: test_result = not exp_result if exp_result: test_util.test_logger( 'Unexpected result: [Security Group] Not find %s %s rule for [vm:] %s. ' % (protocol, direction, test_vm.uuid)) else: test_util.test_logger( 'Unexpected result: [Security Group] find %s %s rule for [vm:] %s in 5 seconds. ' % (protocol, direction, test_vm.uuid)) return test_result
def rebase_all_to_active_file(): self.domain.blockRebase(disk_name, None, 0, 0) def wait_job(_): return not self._wait_for_block_job(disk_name, abort_on_error=True) if not linux.wait_callback_success(wait_job, timeout=300): raise kvmagent.KvmError('live full snapshot merge failed')
def _start_multi_nodes(self, restart = False): nodes = [] threads = [] for node in self.nodes: #The reserved node is used by test cases. if not restart and node.reserve__: continue if not node.dockerImage__: print 'Deploy node in hosts' #consider some zstack-server is running in vm, the server # startup speed is slow. Increase timeout to 180s. cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % (node.ip_, node.ip_) thread = threading.Thread(target=shell_cmd_thread, args=(cmd, True, )) threads.append(thread) else: print 'Deploy node in docker' docker_node = DockerNode(self) docker_node.set_docker_image(node.dockerImage__) docker_node.set_node_ip(node.ip__) docker_node.prepare_node() nodes.append(docker_node) thread = threading.Thread(target=docker_node.start_node) threads.append(thread) for thread in threads: thread.start() self._wait_for_thread_completion('start management node', 200) if node_exception: print 'node start meets exception:' info1 = node_exception[0][1] info2 = node_exception[0][2] raise info1, None, info2 current_time = time.time() #largest timeout time for multi nodes startup is 300s timeout_time = current_time + 300 for node in self.nodes: #The reserved node is used by test cases. if node.reserve__: continue new_time = time.time() if new_time >= timeout_time: new_timeout = 1 else: new_timeout = timeout_time - new_time if not linux.wait_callback_success(\ node_ops.is_management_node_start, \ node.ip_, timeout=new_timeout, interval=0.5): raise ActionError('multi node does not startup on host: %s' \ % node.ip_) zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home shell.call(cmd)
def _start_multi_nodes(self, restart = False): nodes = [] threads = [] for node in self.nodes: #The reserved node is used by test cases. if not restart and node.reserve__: continue if not node.dockerImage__: print 'Deploy node in hosts' #consider some zstack-server is running in vm, the server # startup speed is slow. Increase timeout to 180s. cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % (node.ip_, node.ip_) thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) threads.append(thread) else: print 'Deploy node in docker' docker_node = DockerNode(self) docker_node.set_docker_image(node.dockerImage__) docker_node.set_node_ip(node.ip__) docker_node.prepare_node() nodes.append(docker_node) thread = threading.Thread(target=docker_node.start_node) threads.append(thread) for thread in threads: thread.start() self._wait_for_thread_completion('start management node', 200) if node_exception: print 'node start meets exception:' info1 = node_exception[0][1] info2 = node_exception[0][2] raise info1, None, info2 current_time = time.time() #largest timeout time for multi nodes startup is 300s timeout_time = current_time + 300 for node in self.nodes: #The reserved node is used by test cases. if node.reserve__: continue new_time = time.time() if new_time >= timeout_time: new_timeout = 1 else: new_timeout = timeout_time - new_time if not linux.wait_callback_success(\ node_ops.is_management_node_start, \ node.ip_, timeout=new_timeout, interval=0.5): raise ActionError('multi node does not startup on host: %s' \ % node.ip_) zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home shell.call(cmd)
def wait_for_management_server_start(wait_start_timeout=120, \ node_uuid=None): if not linux.wait_callback_success(is_management_node_ready, \ node_uuid, \ wait_start_timeout, 1, True): raise Exception('waiting for management server start up time out\ after %s seconds' % wait_start_timeout) else: print('management starts successfully, is running now ...')
def start(self, timeout=60): #TODO: 1. enbale hair_pin mode logger.debug('creating vm:\n%s' % self.domain_xml) conn = kvmagent.get_libvirt_connection() domain = conn.defineXML(self.domain_xml) self.domain = domain self.domain.createWithFlags(0) if not linux.wait_callback_success(self.wait_for_state_change, self.VM_STATE_RUNNING, timeout=timeout): raise kvmagent.KvmError('unable to start vm[uuid:%s, name:%s], vm state is not changing to running after %s seconds' % (self.uuid, self.get_name(), timeout))
def wait_for_node_start(self, timeout=120): if not linux.wait_callback_success(node_ops.is_management_node_start, \ self.node_option.get_management_ip(), timeout=timeout, \ interval=0.5): test_util.test_logger('multi node does not startup on host: %s' \ % self.node_option.get_management_ip()) return False test_util.test_logger('multi node startup on host: %s' \ % self.node_option.get_management_ip()) return True
def delete(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) pool, image_name = self._parse_install_path(cmd.installPath) def delete_image(_): # in case image is deleted, we don't have to wait for timeout img = "%s/%s" % (pool, image_name) shell.check_run('rbd info %s && rbd rm %s' % (img, img)) return True # 'rbd rm' might fail due to client crash. We wait for 30 seconds as suggested by 'rbd'. # # rbd: error: image still has watchers # This means the image is still open or the client using it crashed. Try again after # closing/unmapping it or waiting 30s for the crashed client to timeout. linux.wait_callback_success(delete_image, interval=5, timeout=30, ignore_exception_in_callback=True) rsp = AgentResponse() self._set_capacity_to_response(rsp) return jsonobject.dumps(rsp)
def _start_multi_nodes(self, restart=False): nodes = [] threads = [] for node in self.nodes: #The reserved node is used by test cases. if not restart and node.reserve__: continue if not node.dockerImage__: print 'Deploy node in hosts' cmd = 'zstack-ctl stop_node --remote=%s ; zstack-ctl start_node --remote=%s' % ( node.ip_, node.ip_) thread = threading.Thread(target=shell_cmd_thread, args=(cmd, )) threads.append(thread) else: print 'Deploy node in docker' docker_node = DockerNode(self) docker_node.set_docker_image(node.dockerImage__) docker_node.set_node_ip(node.ip__) docker_node.prepare_node() nodes.append(docker_node) thread = threading.Thread(target=docker_node.start_node) threads.append(thread) for thread in threads: thread.start() self._wait_for_thread_completion('start management node', 200) if node_exception: print 'node start meets exception:' info1 = node_exception[0][1] info2 = node_exception[0][2] raise info1, None, info2 current_time = time.time() #largest timeout time for multi nodes startup is 300s timeout_time = current_time + 300 for node in self.nodes: #The reserved node is used by test cases. if node.reserve__: continue new_time = time.time() if new_time >= timeout_time: new_timeout = 1 else: new_timeout = timeout_time - new_time if not linux.wait_callback_success(\ node_ops.is_management_node_start, \ node.ip_, timeout=new_timeout, interval=0.5): raise ActionError('multi node does not startup on host: %s' \ % node.ip_)
def check_sanlock_renewal_failure(lockspace): last_record = linux.wait_callback_success(get_sanlock_renewal, lockspace, 10, 1, True) if last_record is False: logger.warn("unable find correct sanlock renewal record, may be rotated") return True, "" if "next_errors=" not in last_record: return True, "" errors = int(last_record.split("next_errors=")[-1]) if errors > 2: return False, "sanlock renew lease of lockspace %s failed for %s times, storage may failed" % ( lockspace, errors) return True, ""
def do_commit(base, top, flags=0): self.domain.blockCommit(disk_name, base, top, 0, flags) logger.debug('start block commit %s --> %s' % (top, base)) def wait_job(_): logger.debug('merging snapshot chain is waiting for blockCommit job completion') return not self._wait_for_block_job(disk_name, abort_on_error=True) if not linux.wait_callback_success(wait_job, timeout=300): raise kvmagent.KvmError('live merging snapshot chain failed, timeout after 300s') logger.debug('end block commit %s --> %s' % (top, base))
def run_convert_if_need(): def do_run(): save_pid() ret = shell.run(echo_pid_cmd) new_task.current_process_return_code = ret retry_if_needed(ret) return ret def retry_if_needed(ret): if ret == 0: return if retry_counter[0] != max_retry_times and shell.run("grep -q 'guestfs_launch failed' %s" % log_path) == 0: retry_counter[0] += 1 raise RetryException( "launch guestfs failed, rerun v2v longjob %s" % cmd.longJobUuid) pid = linux.read_file(v2v_pid_path) log_path = "%s/virt_v2v_log" % storage_dir if not pid: return do_run() pid = int(pid.strip()) process_completed = os.path.exists(v2v_cmd_ret_path) process_has_been_killed = not os.path.exists(v2v_cmd_ret_path) and not os.path.exists('/proc/%d' % pid) process_still_running = not os.path.exists(v2v_cmd_ret_path) and os.path.exists('/proc/%d' % pid) if process_has_been_killed: return do_run() if process_still_running: linux.wait_callback_success(os.path.exists, v2v_cmd_ret_path, timeout=259200, interval=60) # delete password file passwd_file = os.path.join(storage_dir, "passwd") if os.path.exists(passwd_file): os.remove(passwd_file) ret = linux.read_file(v2v_cmd_ret_path) retry_if_needed(ret) return int(ret.strip() if ret else 126)
def _start_multi_nodes(self, restart = False): nodes = [] threads = [] for node in self.nodes: #The reserved node is used by test cases. if not restart and node.reserve__: continue if not node.dockerImage__: print 'Deploy node in hosts' cmd = 'zstack-ctl stop_node --remote=%s ; zstack-ctl start_node --remote=%s' % (node.ip_, node.ip_) thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) threads.append(thread) else: print 'Deploy node in docker' docker_node = DockerNode(self) docker_node.set_docker_image(node.dockerImage__) docker_node.set_node_ip(node.ip__) docker_node.prepare_node() nodes.append(docker_node) thread = threading.Thread(target=docker_node.start_node) threads.append(thread) for thread in threads: thread.start() self._wait_for_thread_completion('start management node', 200) if node_exception: print 'node start meets exception:' info1 = node_exception[0][1] info2 = node_exception[0][2] raise info1, None, info2 current_time = time.time() #largest timeout time for multi nodes startup is 300s timeout_time = current_time + 300 for node in self.nodes: #The reserved node is used by test cases. if node.reserve__: continue new_time = time.time() if new_time >= timeout_time: new_timeout = 1 else: new_timeout = timeout_time - new_time if not linux.wait_callback_success(\ node_ops.is_management_node_start, \ node.ip_, timeout=new_timeout, interval=0.5): raise ActionError('multi node does not startup on host: %s' \ % node.ip_)
def take_full_snapshot(): logger.debug('start rebasing to make a full snapshot') self.domain.blockRebase(disk_name, None, 0, 0) logger.debug('rebasing full snapshot is in processing') def wait_job(_): logger.debug('full snapshot is waiting for blockRebase job completion') return not self._wait_for_block_job(disk_name, abort_on_error=True) if not linux.wait_callback_success(wait_job, timeout=300): raise kvmagent.KvmError('live full snapshot failed') return take_delta_snapshot()
def test(): test_util.test_dsc('Test Host Start/Stop function') vm_cpu = 1 vm_memory = 1073741824 #1G cond = res_ops.gen_query_conditions('name', '=', 'ttylinux') image_uuid = res_ops.query_resource(res_ops.IMAGE, cond)[0].uuid l3_network_uuid = res_ops.query_resource(res_ops.L3_NETWORK)[0].uuid vm = test_stub.create_mini_vm([l3_network_uuid], image_uuid, cpu_num = vm_cpu, memory_size = vm_memory) test_obj_dict.add_vm(vm) host_uuid = test_lib.lib_get_vm_host(vm.vm).uuid host_ops.change_host_state(host_uuid, 'disable') if not linux.wait_callback_success(is_host_disabled, host_uuid, 120): test_util.test_fail('host state is not changed to disabled') host_ops.change_host_state(host_uuid, 'enable') if not linux.wait_callback_success(is_host_enabled, host_uuid, 120): test_util.test_fail('host state is not changed to enabled') vm.destroy() test_obj_dict.rm_vm(vm) test_util.test_pass('Stop/Start Host Test Pass')
def start(self, timeout=60): #TODO: 1. enbale hair_pin mode logger.debug('creating vm:\n%s' % self.domain_xml) conn = kvmagent.get_libvirt_connection() domain = conn.defineXML(self.domain_xml) self.domain = domain self.domain.createWithFlags(0) if not linux.wait_callback_success(self.wait_for_state_change, self.VM_STATE_RUNNING, timeout=timeout): raise kvmagent.KvmError( 'unable to start vm[uuid:%s, name:%s], vm state is not changing to running after %s seconds' % (self.uuid, self.get_name(), timeout))
def _apply_userdata_restart_httpd(self, to): conf_folder = os.path.join(self.USERDATA_ROOT, to.namespaceName) conf_path = os.path.join(conf_folder, 'lighttpd.conf') pid = linux.find_process_by_cmdline([conf_path]) if not pid: shell.call('ip netns exec %s lighttpd -f %s' % (to.namespaceName, conf_path)) def check(_): pid = linux.find_process_by_cmdline([conf_path]) return pid is not None if not linux.wait_callback_success(check, None, 5): raise Exception('lighttpd[conf-file:%s] is not running after being started %s seconds' % (conf_path, 5))
def run_convert_if_need(): def do_run(): save_pid() ret = shell.run(echo_pid_cmd) new_task.current_process_return_code = ret return ret pid = linux.read_file(v2v_pid_path) if not pid: return do_run() pid = int(pid.strip()) process_completed = os.path.exists(v2v_cmd_ret_path) process_has_been_killed = not os.path.exists(v2v_cmd_ret_path) and not os.path.exists('/proc/%d' % pid) process_still_running = not os.path.exists(v2v_cmd_ret_path) and os.path.exists('/proc/%d' % pid) if process_has_been_killed: return do_run() if process_still_running: linux.wait_callback_success(os.path.exists, v2v_cmd_ret_path, timeout=259200, interval=60) ret = linux.read_file(v2v_cmd_ret_path) return int(ret.strip() if ret else 126)
def check_sanlock_renewal_failure(lockspace): last_record = linux.wait_callback_success(get_sanlock_renewal, lockspace, 10, 1, True) if last_record is False: logger.warn( "unable find correct sanlock renewal record, may be rotated") return True if "next_errors=" not in last_record: return True, "" errors = int(last_record.split("next_errors=")[-1]) if errors > 2: return False, "sanlock renew lease of lockspace %s failed for %s times, storage may failed" % ( lockspace, errors) return True, ""
def cancel_sftp_download(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = AgentResponse() def check(): return shell.run("rbd ls %s | grep -q %s" % (pool, image_name)) != 0 def remove(target_name): return shell.run("rbd info {0}/{1} || rbd rm {0}/{1}".format(pool, target_name)) == 0 pool, image_name = self._parse_install_path(cmd.primaryStorageInstallPath) tmp_image_name = 'tmp-%s' % image_name if check(): return jsonobject.dumps(rsp) for image in (tmp_image_name, image_name): shell.run("pkill -9 -f '%s'" % image) linux.wait_callback_success(remove, image, timeout=30) if not check(): rsp.set_err("remove image %s/%s fail" % (pool, image_name)) return jsonobject.dumps(rsp)
def check_sg_rule_exist(test_vm, protocol, direction, extra_str, exp_result): if linux.wait_callback_success(do_check_sg_rule_exist, (test_vm, direction, extra_str, exp_result), 5, 0.2): test_result = exp_result if exp_result: test_util.test_logger('Expected result: [Security Group] find %s %s rule for [vm:] %s. ' % (protocol, direction, test_vm.uuid)) else: test_util.test_logger('Expected result: [Security Group] FAIL to find %s %s rule for [vm:] %s in 5 seconds. ' % (protocol, direction, test_vm.uuid)) else: test_result = not exp_result if exp_result: test_util.test_logger('Unexpected result: [Security Group] Not find %s %s rule for [vm:] %s. ' % (protocol, direction, test_vm.uuid)) else: test_util.test_logger('Unexpected result: [Security Group] find %s %s rule for [vm:] %s in 5 seconds. ' % (protocol, direction, test_vm.uuid)) return test_result
def json_post(uri, body=None, headers={}, method='POST', fail_soon=False): ret = [] def post(_): try: pool = urllib3.PoolManager(timeout=120.0, retries=urllib3.util.retry.Retry(15)) header = { 'Content-Type': 'application/json', 'Connection': 'close' } for k in headers.keys(): header[k] = headers[k] if body is not None: assert isinstance(body, types.StringType) header['Content-Length'] = str(len(body)) content = pool.urlopen(method, uri, headers=header, body=str(body)).data #(resp, content) = http_obj.request(uri, 'POST', body='%s' % body, headers=header) else: header['Content-Length'] = '0' #(resp, content) = http_obj.request(uri, 'POST', headers=header) content = pool.urlopen(method, uri, headers=header).data pool.clear() ret.append(content) return True except Exception as e: if fail_soon: raise e logger.warn('[WARN]: %s' % linux.get_exception_stacktrace()) return False if fail_soon: post(None) else: if not linux.wait_callback_success(post, ignore_exception_in_callback=True): raise Exception( 'unable to post to %s, body: %s, see before error' % (uri, body)) return ret[0]
def take_full_snapshot(): logger.debug('start rebasing to make a full snapshot') self.domain.blockRebase(disk_name, None, 0, 0) logger.debug('rebasing full snapshot is in processing') def wait_job(_): logger.debug( 'full snapshot is waiting for blockRebase job completion') return not self._wait_for_block_job(disk_name, abort_on_error=True) if not linux.wait_callback_success(wait_job, timeout=300): raise kvmagent.KvmError('live full snapshot failed') return take_delta_snapshot()
def _restart_dnsmasq(self, ns_name, conf_file_path): pid = linux.find_process_by_cmdline([conf_file_path]) if pid: linux.kill_process(pid) NS_NAME = ns_name CONF_FILE = conf_file_path DNSMASQ = bash_errorout('which dnsmasq').strip(' \t\r\n') bash_errorout('ip netns exec {{NS_NAME}} {{DNSMASQ}} --conf-file={{CONF_FILE}} ') def check(_): pid = linux.find_process_by_cmdline([conf_file_path]) return pid is not None if not linux.wait_callback_success(check, None, 5): raise Exception('dnsmasq[conf-file:%s] is not running after being started %s seconds' % (conf_file_path, 5))
def detach(error_out): try: self.domain.detachDeviceFlags(xmlstr, libvirt.VIR_DOMAIN_AFFECT_LIVE | libvirt.VIR_DOMAIN_AFFECT_CONFIG) except libvirt.libvirtError as e: if error_out: raise e return True def wait_for_detach(_): me = get_vm_by_uuid(self.uuid) for disk in me.domain_xmlobject.devices.get_child_node_as_list('disk'): if disk.source.file_ == volume.installPath: logger.debug('volume[%s] is still in process of detaching, wait it' % volume.installPath) return False return True return linux.wait_callback_success(wait_for_detach, None, 5, 1)
def do_commit(base, top, flags=0): self.domain.blockCommit(disk_name, base, top, 0, flags) logger.debug('start block commit %s --> %s' % (top, base)) def wait_job(_): logger.debug( 'merging snapshot chain is waiting for blockCommit job completion' ) return not self._wait_for_block_job(disk_name, abort_on_error=True) if not linux.wait_callback_success(wait_job, timeout=300): raise kvmagent.KvmError( 'live merging snapshot chain failed, timeout after 300s') logger.debug('end block commit %s --> %s' % (top, base))
def _restart_dnsmasq(self, ns_name, conf_file_path): pid = linux.find_process_by_cmdline([conf_file_path]) if pid: linux.kill_process(pid) cmd = '''\ ip netns exec {{ns_name}} /sbin/dnsmasq --conf-file={{conf_file}} || ip netns exec {{ns_name}} /usr/sbin/dnsmasq --conf-file={{conf_file}} ''' tmpt = Template(cmd) cmd = tmpt.render({'ns_name': ns_name, 'conf_file': conf_file_path}) shell.call(cmd) def check(_): pid = linux.find_process_by_cmdline([conf_file_path]) return pid is not None if not linux.wait_callback_success(check, None, 5): raise Exception('dnsmasq[conf-file:%s] is not running after being started %s seconds' % (conf_file_path, 5))
def start_node(self): def _wait(data): try: shell.ShellCmd('docker ps')() print('docker service is ready') return True except: print('docker service is still starting ...') return False try: if not linux.wait_callback_success(_wait, None, \ docker_service_start_timeout, 0.1): raise ActionError('waiting for docker start up time out: %s' % \ docker_service_start_timeout) shell.ShellCmd('cd %s ; docker build --tag="%s" .' % \ (self.docker_folder, self.node_ip))() #run docker image shell.ShellCmd("docker run -d %s " % self.node_ip)() print 'docker container has been created.' except Exception as e: node_exception.append(sys.exc_info())
def start_node(self): def _wait(data): try: shell.ShellCmd('docker ps')() print('docker service is ready') return True except: print ('docker service is still starting ...') return False try: if not linux.wait_callback_success(_wait, None, \ docker_service_start_timeout, 0.1): raise ActionError('waiting for docker start up time out: %s' % \ docker_service_start_timeout) shell.ShellCmd('cd %s ; docker build --tag="%s" .' % \ (self.docker_folder, self.node_ip))() #run docker image shell.ShellCmd("docker run -d %s " % self.node_ip)() print 'docker container has been created.' except Exception as e: node_exception.append(sys.exc_info())
def detach(error_out): try: self.domain.detachDeviceFlags( xmlstr, libvirt.VIR_DOMAIN_AFFECT_LIVE | libvirt.VIR_DOMAIN_AFFECT_CONFIG) except libvirt.libvirtError as e: if error_out: raise e return True def wait_for_detach(_): me = get_vm_by_uuid(self.uuid) for disk in me.domain_xmlobject.devices.get_child_node_as_list( 'disk'): if disk.source.file_ == volume.installPath: logger.debug( 'volume[%s] is still in process of detaching, wait it' % volume.installPath) return False return True return linux.wait_callback_success(wait_for_detach, None, 5, 1)
def json_post(uri, body=None, headers={}, method='POST', fail_soon=False): ret = [] def post(_): try: pool = urllib3.PoolManager(timeout=120.0, retries=urllib3.util.retry.Retry(15)) header = {'Content-Type': 'application/json', 'Connection': 'close'} for k in headers.keys(): header[k] = headers[k] if body is not None: assert isinstance(body, types.StringType) header['Content-Length'] = str(len(body)) content = pool.urlopen(method, uri, headers=header, body=str(body)).data #(resp, content) = http_obj.request(uri, 'POST', body='%s' % body, headers=header) else: header['Content-Length'] = '0' #(resp, content) = http_obj.request(uri, 'POST', headers=header) content = pool.urlopen(method, uri, headers=header).data #logger.debug('post to %s, with content: %s, with header: %s' % (uri, body, header)) pool.clear() ret.append(content) return True except Exception as e: if fail_soon: raise e logger.warn(linux.get_exception_stacktrace()) return False if fail_soon: post(None) else: if not linux.wait_callback_success(post, ignore_exception_in_callback=True): raise Exception('unable to post to %s, body: %s, see before error' % (uri, body)) return ret[0]
def migrate(self, destHostIp): destUrl = "qemu+tcp://{0}/system".format(destHostIp) tcpUri = "tcp://{0}".format(destHostIp) try: self.domain.migrateToURI2(destUrl, tcpUri, None, libvirt.VIR_MIGRATE_LIVE| libvirt.VIR_MIGRATE_PEER2PEER| libvirt.VIR_MIGRATE_UNDEFINE_SOURCE| libvirt.VIR_MIGRATE_PERSIST_DEST | libvirt.VIR_MIGRATE_TUNNELLED, None, 0) except libvirt.libvirtError as ex: logger.warn(linux.get_exception_stacktrace()) raise kvmagent.KvmError('unable to migrate vm[uuid:%s] to %s, %s' % (self.uuid, destUrl, str(ex))) try: if not linux.wait_callback_success(self.wait_for_state_change, callback_data=None, timeout=300): raise kvmagent.KvmError('timeout after 300 seconds') except kvmagent.KvmError as ke: raise ke except Exception as e: logger.debug(linux.get_exception_stacktrace()) logger.debug('successfully migrated vm[uuid:{0}] to dest url[{1}]'.format(self.uuid, destUrl))
def deploy_test_agent(self, target=None): print('Deploy test agent\n') if not self.test_agent_path: print('Not find test_agent. Stop deploying test agent.\n') return testagentdir = None try: def untar_test_agent(): tmpdir = tempfile.mkdtemp() shell.call('tar jxf %s -C %s' % (self.test_agent_path, tmpdir)) shell.call('cd %s/zstacktestagent/; tar jcf pypi.tar.bz pypi' \ % tmpdir) return '%s/zstacktestagent' % tmpdir def _wait_echo(target_ip): try: rspstr = http.json_dump_post(testagent.build_http_path(target_ip, host_plugin.ECHO_PATH)) except: print('zstack-testagent does not startup, will try again ...') return False return True testagentdir = untar_test_agent() ansible.check_and_install_ansible() lib_files = ['testagent/zstacktestagent-1.0.0.tar.gz', \ 'testagent/zstacklib-1.1.tar.gz' ] if not target: #default will deploy all test hosts. exc_info = [] for h in self.test_agent_hosts: print('Deploy test agent in host: [%s] \n' % h.managementIp_) ansible_cmd_args = "host=%s \ pkg_testagent=zstacktestagent-1.0.0.tar.gz \ pkg_zstacklib=zstacklib-1.1.tar.gz \ pypi_source_tar=pypi.tar.bz" % \ h.managementIp_ if ENV_HTTP_PROXY: ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \ (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY) ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args thread = threading.Thread(target=ansible.execute_ansible,\ args=(h.managementIp_, h.username_, h.password_,\ testagentdir, ansible_cmd, lib_files, exc_info)) # Wrap up old zstack logs in /var/log/zstack/ print('archive test log on host: [%s] \n' % h.managementIp_) try: log.cleanup_log(h.managementIp_, h.username_, h.password_) except Exception as e: print "clean up old testing logs meet execption on management node: %s" % h.managementIp_ raise e thread.start() #if localhost is not in hosts, should do log archive for zstack log.cleanup_local_log() self._wait_for_thread_completion('install test agent', 200) for h in self.test_agent_hosts: if not linux.wait_callback_success(_wait_echo, h.managementIp_, 5, 0.2, True): raise ActionError('testagent is not start up in 5s on %s, after it is deployed by ansible.' % h.managementIp_) else: print('Deploy test agent in host: %s \n' % target.managementIp) ansible_cmd_args = "host=%s \ pkg_testagent=zstacktestagent-1.0.0.tar.gz \ pkg_zstacklib=zstacklib-1.1.tar.gz \ pypi_source_tar=pypi.tar.bz" % \ target.managementIp if ENV_HTTP_PROXY: ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \ (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY) ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args ansible.execute_ansible(target.managementIp, target.username, \ target.password, testagentdir, ansible_cmd, lib_files) if not linux.wait_callback_success(_wait_echo, target.managementIp, 5, 0.2): raise ActionError('testagent is not start up in 5s on %s, after it is deployed by ansible.' % target.managementIp) finally: if testagentdir: shell.call('rm -rf %s' % testagentdir)
def test(): vm1 = test_stub.create_vr_vm('maintain_host_vm1', 'imageName_s', 'l3VlanNetwork2') test_obj_dict.add_vm(vm1) vm2 = test_stub.create_vr_vm('maintain_host_vm2', 'imageName_s', 'l3VlanNetwork2') test_obj_dict.add_vm(vm2) vm1.check() vm2.check() if not test_lib.lib_check_vm_live_migration_cap( vm1.vm) or not test_lib.lib_check_vm_live_migration_cap(vm2.vm): test_util.test_skip('skip migrate if live migrate not supported') current_host1 = test_lib.lib_get_vm_host(vm1.vm) current_host2 = test_lib.lib_get_vm_host(vm2.vm) conditions = res_ops.gen_query_conditions('clusterUuid', '=', vm1.vm.clusterUuid) conditions = res_ops.gen_query_conditions('state', '=', host_header.ENABLED, conditions) conditions = res_ops.gen_query_conditions('status', '=', host_header.CONNECTED, conditions) all_hosts = res_ops.query_resource(res_ops.HOST, conditions) if len(all_hosts) <= 1: test_util.test_fail( 'Not available host to do maintenance, since there is only %s host' % len(all_hosts)) vr = test_lib.lib_get_all_vrs() if len(vr) == 0: test_util.test_skip('Skip test if not using vr') vr_uuid = vr[0].uuid vr_host_uuid = test_lib.lib_get_vm_host(vr[0]).uuid for host_n in all_hosts: print 'host_n%s' % (host_n.uuid) if host_n.uuid != current_host1.uuid: if host_n.uuid != current_host2.uuid: if host_n.uuid != vr_host_uuid: target_host = host_n print 'target_host_uuid%s' % (target_host.uuid) vm1.migrate(target_host.uuid) vm2.migrate(target_host.uuid) break else: test_util.test_skip('can not find a host to migrate two host') new_host = test_lib.lib_get_vm_host(vm1.vm) if new_host.uuid != target_host.uuid: test_util.test_fail( 'VM did not migrate to target [host:] %s, but to [host:] %s' % (target_host.uuid, new_host.uuid)) new_host1 = test_lib.lib_get_vm_host(vm2.vm) if new_host1.uuid != target_host.uuid: test_util.test_fail( 'VM did not migrate to target [host:] %s, but to [host:] %s' % (target_host.uuid, new_host1.uuid)) host = test_kvm_host.ZstackTestKvmHost() host.set_host(target_host) host.maintain() #need to update vm's inventory, since they will be changed by maintenace mode vm1.update() vm2.update() ps = test_lib.lib_get_primary_storage_by_vm(vm1.get_vm()) if ps.type == inventory.LOCAL_STORAGE_TYPE: vm1.set_state(vm_header.STOPPED) vm2.set_state(vm_header.STOPPED) vm1.check() vm2.check() host.change_state(test_kvm_host.ENABLE_EVENT) if not linux.wait_callback_success(is_host_connected, host.get_host().uuid, 120): test_util.test_fail( 'host status is not changed to connected or host state is not changed to Enabled within 120s' ) if ps.type == inventory.LOCAL_STORAGE_TYPE: vm1.start() vm2.start() vm1.set_state(vm_header.RUNNING) vm2.set_state(vm_header.RUNNING) vm1.check() vm2.check() post_host1 = test_lib.lib_get_vm_host(vm1.vm) post_host2 = test_lib.lib_get_vm_host(vm2.vm) if post_host1.uuid != current_host1.uuid: vm1.migrate(current_host1.uuid) if post_host2.uuid != current_host2.uuid: vm2.migrate(current_host2.uuid) vm1.check() vm2.check() vm1.destroy() test_obj_dict.rm_vm(vm1) vm2.destroy() test_obj_dict.rm_vm(vm2) test_util.test_pass('Maintain Host Test Success')
def _apply_userdata(self, to): # set VIP NS_NAME = to.namespaceName DHCP_IP = to.dhcpServerIp INNER_DEV = bash_errorout("ip netns exec {{NS_NAME}} ip addr | grep -w {{DHCP_IP}} | awk '{print $NF}'").strip(' \t\r\n') if not INNER_DEV: raise Exception('cannot find device for the DHCP IP[%s]' % DHCP_IP) ret = bash_r('ip netns exec {{NS_NAME}} ip addr | grep 169.254.169.254 > /dev/null') if ret != 0: bash_errorout('ip netns exec {{NS_NAME}} ip addr add 169.254.169.254 dev {{INNER_DEV}}') # set ebtables BR_NAME = to.bridgeName # BR_NAME is "br_%s_%s" ETH_NAME = BR_NAME.replace('br_', '', 1).replace('_', '.', 1) MAC = bash_errorout("ip netns exec {{NS_NAME}} ip link show {{INNER_DEV}} | grep -w ether | awk '{print $2}'").strip(' \t\r\n') CHAIN_NAME="USERDATA-%s" % BR_NAME ret = bash_r('ebtables -t nat -L {{CHAIN_NAME}} >/dev/null 2>&1') if ret != 0: bash_errorout('ebtables -t nat -N {{CHAIN_NAME}}') if bash_r('ebtables -t nat -L PREROUTING | grep -- "--logical-in {{BR_NAME}} -j {{CHAIN_NAME}}"') != 0: bash_errorout('ebtables -t nat -I PREROUTING --logical-in {{BR_NAME}} -j {{CHAIN_NAME}}') # ebtables has a bug that will eliminate 0 in MAC, for example, aa:bb:0c will become aa:bb:c RULE = "-p IPv4 --ip-dst 169.254.169.254 -j dnat --to-dst %s --dnat-target ACCEPT" % MAC.replace(":0", ":") ret = bash_r('ebtables -t nat -L {{CHAIN_NAME}} | grep -- "{{RULE}}" > /dev/null') if ret != 0: bash_errorout('ebtables -t nat -I {{CHAIN_NAME}} {{RULE}}') ret = bash_r('ebtables -t nat -L {{CHAIN_NAME}} | grep -- "-j RETURN" > /dev/null') if ret != 0: bash_errorout('ebtables -t nat -A {{CHAIN_NAME}} -j RETURN') ret = bash_r('ebtables -L {{CHAIN_NAME}} >/dev/null 2>&1') if ret != 0: bash_errorout('ebtables -N {{CHAIN_NAME}}') ret = bash_r('ebtables -L FORWARD | grep -- "-p ARP --arp-ip-dst 169.254.169.254 -j {{CHAIN_NAME}}" > /dev/null') if ret != 0: bash_errorout('ebtables -I FORWARD -p ARP --arp-ip-dst 169.254.169.254 -j {{CHAIN_NAME}}') ret = bash_r('ebtables -L {{CHAIN_NAME}} | grep -- "-i {{ETH_NAME}} -j DROP" > /dev/null') if ret != 0: bash_errorout('ebtables -I {{CHAIN_NAME}} -i {{ETH_NAME}} -j DROP') ret = bash_r('ebtables -L {{CHAIN_NAME}} | grep -- "-o {{ETH_NAME}} -j DROP" > /dev/null') if ret != 0: bash_errorout('ebtables -I {{CHAIN_NAME}} -o {{ETH_NAME}} -j DROP') ret = bash_r("ebtables-save | grep '\-A {{CHAIN_NAME}} -j RETURN'") if ret != 0: bash_errorout('ebtables -A {{CHAIN_NAME}} -j RETURN') # DNAT port 80 PORT = to.port PORT_CHAIN_NAME = "UD-PORT-%s" % PORT # delete old chains not matching our port OLD_CHAIN = bash_errorout("iptables-save | awk '/^:UD-PORT-/{print substr($1,2)}'").strip(' \n\r\t') if OLD_CHAIN and OLD_CHAIN != CHAIN_NAME: ret = bash_r('iptables-save -t nat | grep -- "-j {{OLD_CHAIN}}"') if ret == 0: bash_r('iptables -t nat -D PREROUTING -j {{OLD_CHAIN}}') bash_errorout('iptables -t nat -F {{OLD_CHAIN}}') bash_errorout('iptables -t nat -X {{OLD_CHAIN}}') ret = bash_r('iptables-save | grep -w ":{{PORT_CHAIN_NAME}}" > /dev/null') if ret != 0: bash_errorout('iptables -t nat -N {{PORT_CHAIN_NAME}}') ret = bash_r('iptables -t nat -L PREROUTING | grep -- "-j {{PORT_CHAIN_NAME}}"') if ret != 0: bash_errorout('iptables -t nat -I PREROUTING -j {{PORT_CHAIN_NAME}}') ret = bash_r('iptables-save -t nat | grep -- "{{PORT_CHAIN_NAME}} -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :{{PORT}}"') if ret != 0: bash_errorout('iptables -t nat -A {{PORT_CHAIN_NAME}} -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :{{PORT}}') conf_folder = os.path.join(self.USERDATA_ROOT, to.namespaceName) if not os.path.exists(conf_folder): shell.call('mkdir -p %s' % conf_folder) conf_path = os.path.join(conf_folder, 'lighttpd.conf') http_root = os.path.join(conf_folder, 'html') conf = '''\ server.document-root = "{{http_root}}" server.port = {{port}} server.bind = "169.254.169.254" dir-listing.activate = "enable" index-file.names = ( "index.html" ) server.modules += ( "mod_rewrite" ) $HTTP["remoteip"] =~ "^(.*)$" { url.rewrite-once = ( "^/.*/meta-data/(.+)$" => "../%1/meta-data/$1", "^/.*/meta-data$" => "../%1/meta-data", "^/.*/meta-data/$" => "../%1/meta-data/", "^/.*/user-data$" => "../%1/user-data" ) } mimetype.assign = ( ".html" => "text/html", ".txt" => "text/plain", ".jpg" => "image/jpeg", ".png" => "image/png" )''' tmpt = Template(conf) conf = tmpt.render({ 'http_root': http_root, 'dhcp_server_ip': to.dhcpServerIp, 'port': to.port }) if not os.path.exists(conf_path): with open(conf_path, 'w') as fd: fd.write(conf) else: with open(conf_path, 'r') as fd: current_conf = fd.read() if current_conf != conf: with open(conf_path, 'w') as fd: fd.write(conf) root = os.path.join(http_root, to.vmIp) meta_root = os.path.join(root, 'meta-data') if not os.path.exists(meta_root): shell.call('mkdir -p %s' % meta_root) index_file_path = os.path.join(meta_root, 'index.html') with open(index_file_path, 'w') as fd: fd.write('instance-id') instance_id_file_path = os.path.join(meta_root, 'instance-id') with open(instance_id_file_path, 'w') as fd: fd.write(to.metadata.vmUuid) if to.userdata: userdata_file_path = os.path.join(root, 'user-data') with open(userdata_file_path, 'w') as fd: fd.write(to.userdata) pid = linux.find_process_by_cmdline([conf_path]) if not pid: shell.call('ip netns exec %s lighttpd -f %s' % (to.namespaceName, conf_path)) def check(_): pid = linux.find_process_by_cmdline([conf_path]) return pid is not None if not linux.wait_callback_success(check, None, 5): raise Exception('lighttpd[conf-file:%s] is not running after being started %s seconds' % (conf_path, 5))
def _apply_userdata(self, to): # set VIP NS_NAME = to.namespaceName DHCP_IP = to.dhcpServerIp INNER_DEV = bash_errorout("ip netns exec {{NS_NAME}} ip addr | grep -w {{DHCP_IP}} | awk '{print $NF}'").strip(' \t\r\n') if not INNER_DEV: raise Exception('cannot find device for the DHCP IP[%s]' % DHCP_IP) ret = bash_r('ip netns exec {{NS_NAME}} ip addr | grep 169.254.169.254 > /dev/null') if ret != 0: bash_errorout('ip netns exec {{NS_NAME}} ip addr add 169.254.169.254 dev {{INNER_DEV}}') # set ebtables BR_NAME = to.bridgeName # BR_NAME is "br_%s_%s" ETH_NAME = BR_NAME.replace('br_', '', 1).replace('_', '.', 1) MAC = bash_errorout("ip netns exec {{NS_NAME}} ip link show {{INNER_DEV}} | grep -w ether | awk '{print $2}'").strip(' \t\r\n') CHAIN_NAME="USERDATA-%s" % BR_NAME ret = bash_r(EBTABLES_CMD + ' -t nat -L {{CHAIN_NAME}} >/dev/null 2>&1') if ret != 0: bash_errorout(EBTABLES_CMD + ' -t nat -N {{CHAIN_NAME}}') if bash_r(EBTABLES_CMD + ' -t nat -L PREROUTING | grep -- "--logical-in {{BR_NAME}} -j {{CHAIN_NAME}}"') != 0: bash_errorout(EBTABLES_CMD + ' -t nat -I PREROUTING --logical-in {{BR_NAME}} -j {{CHAIN_NAME}}') # ebtables has a bug that will eliminate 0 in MAC, for example, aa:bb:0c will become aa:bb:c RULE = "-p IPv4 --ip-dst 169.254.169.254 -j dnat --to-dst %s --dnat-target ACCEPT" % MAC.replace(":0", ":") ret = bash_r(EBTABLES_CMD + ' -t nat -L {{CHAIN_NAME}} | grep -- "{{RULE}}" > /dev/null') if ret != 0: bash_errorout(EBTABLES_CMD + ' -t nat -I {{CHAIN_NAME}} {{RULE}}') ret = bash_r(EBTABLES_CMD + ' -t nat -L {{CHAIN_NAME}} | grep -- "-j RETURN" > /dev/null') if ret != 0: bash_errorout(EBTABLES_CMD + ' -t nat -A {{CHAIN_NAME}} -j RETURN') ret = bash_r(EBTABLES_CMD + ' -L {{CHAIN_NAME}} >/dev/null 2>&1') if ret != 0: bash_errorout(EBTABLES_CMD + ' -N {{CHAIN_NAME}}') ret = bash_r(EBTABLES_CMD + ' -L FORWARD | grep -- "-p ARP --arp-ip-dst 169.254.169.254 -j {{CHAIN_NAME}}" > /dev/null') if ret != 0: bash_errorout(EBTABLES_CMD + ' -I FORWARD -p ARP --arp-ip-dst 169.254.169.254 -j {{CHAIN_NAME}}') ret = bash_r(EBTABLES_CMD + ' -L {{CHAIN_NAME}} | grep -- "-i {{ETH_NAME}} -j DROP" > /dev/null') if ret != 0: bash_errorout(EBTABLES_CMD + ' -I {{CHAIN_NAME}} -i {{ETH_NAME}} -j DROP') ret = bash_r(EBTABLES_CMD + ' -L {{CHAIN_NAME}} | grep -- "-o {{ETH_NAME}} -j DROP" > /dev/null') if ret != 0: bash_errorout(EBTABLES_CMD + ' -I {{CHAIN_NAME}} -o {{ETH_NAME}} -j DROP') ret = bash_r("ebtables-save | grep '\-A {{CHAIN_NAME}} -j RETURN'") if ret != 0: bash_errorout(EBTABLES_CMD + ' -A {{CHAIN_NAME}} -j RETURN') # DNAT port 80 PORT = to.port PORT_CHAIN_NAME = "UD-PORT-%s" % PORT # delete old chains not matching our port OLD_CHAIN = bash_errorout("iptables-save | awk '/^:UD-PORT-/{print substr($1,2)}'").strip(' \n\r\t') if OLD_CHAIN and OLD_CHAIN != CHAIN_NAME: ret = bash_r('iptables-save -t nat | grep -- "-j {{OLD_CHAIN}}"') if ret == 0: bash_r('iptables -t nat -D PREROUTING -j {{OLD_CHAIN}}') bash_errorout('iptables -t nat -F {{OLD_CHAIN}}') bash_errorout('iptables -t nat -X {{OLD_CHAIN}}') ret = bash_r('iptables-save | grep -w ":{{PORT_CHAIN_NAME}}" > /dev/null') if ret != 0: bash_errorout('iptables -t nat -N {{PORT_CHAIN_NAME}}') ret = bash_r('iptables -t nat -L PREROUTING | grep -- "-j {{PORT_CHAIN_NAME}}"') if ret != 0: bash_errorout('iptables -t nat -I PREROUTING -j {{PORT_CHAIN_NAME}}') ret = bash_r('iptables-save -t nat | grep -- "{{PORT_CHAIN_NAME}} -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :{{PORT}}"') if ret != 0: bash_errorout('iptables -t nat -A {{PORT_CHAIN_NAME}} -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :{{PORT}}') conf_folder = os.path.join(self.USERDATA_ROOT, to.namespaceName) if not os.path.exists(conf_folder): shell.call('mkdir -p %s' % conf_folder) conf_path = os.path.join(conf_folder, 'lighttpd.conf') http_root = os.path.join(conf_folder, 'html') conf = '''\ server.document-root = "{{http_root}}" server.port = {{port}} server.bind = "169.254.169.254" dir-listing.activate = "enable" index-file.names = ( "index.html" ) server.modules += ( "mod_rewrite" ) $HTTP["remoteip"] =~ "^(.*)$" { url.rewrite-once = ( "^/.*/meta-data/(.+)$" => "../%1/meta-data/$1", "^/.*/meta-data$" => "../%1/meta-data", "^/.*/meta-data/$" => "../%1/meta-data/", "^/.*/user-data$" => "../%1/user-data" ) } mimetype.assign = ( ".html" => "text/html", ".txt" => "text/plain", ".jpg" => "image/jpeg", ".png" => "image/png" )''' tmpt = Template(conf) conf = tmpt.render({ 'http_root': http_root, 'dhcp_server_ip': to.dhcpServerIp, 'port': to.port }) if not os.path.exists(conf_path): with open(conf_path, 'w') as fd: fd.write(conf) else: with open(conf_path, 'r') as fd: current_conf = fd.read() if current_conf != conf: with open(conf_path, 'w') as fd: fd.write(conf) root = os.path.join(http_root, to.vmIp) meta_root = os.path.join(root, 'meta-data') if not os.path.exists(meta_root): shell.call('mkdir -p %s' % meta_root) index_file_path = os.path.join(meta_root, 'index.html') with open(index_file_path, 'w') as fd: fd.write('instance-id') instance_id_file_path = os.path.join(meta_root, 'instance-id') with open(instance_id_file_path, 'w') as fd: fd.write(to.metadata.vmUuid) if to.userdata: userdata_file_path = os.path.join(root, 'user-data') with open(userdata_file_path, 'w') as fd: fd.write(to.userdata) pid = linux.find_process_by_cmdline([conf_path]) if not pid: shell.call('ip netns exec %s lighttpd -f %s' % (to.namespaceName, conf_path)) def check(_): pid = linux.find_process_by_cmdline([conf_path]) return pid is not None if not linux.wait_callback_success(check, None, 5): raise Exception('lighttpd[conf-file:%s] is not running after being started %s seconds' % (conf_path, 5))
def deploy_test_agent(self, target=None): print('Deploy test agent\n') if not self.test_agent_path: print('Not find test_agent. Stop deploying test agent.\n') return testagentdir = None try: def untar_test_agent(): tmpdir = tempfile.mkdtemp() shell.call('tar jxf %s -C %s' % (self.test_agent_path, tmpdir)) shell.call('cd %s/zstacktestagent/; tar jcf pypi.tar.bz pypi' \ % tmpdir) return '%s/zstacktestagent' % tmpdir def _wait_echo(target_ip): try: rspstr = http.json_dump_post( testagent.build_http_path(target_ip, host_plugin.ECHO_PATH)) except: print( 'zstack-testagent does not startup, will try again ...' ) return False return True def find_zstacklib(search_path): match = glob.glob( os.path.join(search_path, 'zstacklib-*.tar.gz')) try: return os.path.basename(match[0]) except: return 'zstacklib-1.6.tar.gz' testagentdir = untar_test_agent() pkg_zstacklib = find_zstacklib(testagentdir) ansible.check_and_install_ansible() lib_files = ['testagent/zstacktestagent-1.0.0.tar.gz', \ 'testagent/%s' % (pkg_zstacklib) ] if not target: #default will deploy all test hosts. exc_info = [] for h in self.test_agent_hosts: print('Deploy test agent in host: [%s] \n' % h.managementIp_) if h.username_ != 'root': ansible_become_args = "ansible_become=yes become_user=root ansible_become_pass=%s" % ( h.password_) else: ansible_become_args = "" if hasattr(h, 'port_'): ansible_port_args = "ansible_ssh_port=%s" % (h.port_) else: ansible_port_args = "" ansible_cmd_args = "host=%s \ ansible_ssh_user=%s \ ansible_ssh_pass=%s \ %s \ %s \ pkg_testagent=zstacktestagent-1.0.0.tar.gz \ pkg_zstacklib=%s \ pypi_source_tar=pypi.tar.bz" % \ (h.managementIp_, h.username_, h.password_, ansible_become_args, ansible_port_args, pkg_zstacklib) if ENV_HTTP_PROXY: ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \ (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY) ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args if hasattr(h, 'port_'): thread = threading.Thread(target=ansible.execute_ansible,\ args=(h.managementIp_, h.username_, h.password_,\ testagentdir, ansible_cmd, lib_files, exc_info, h.port_)) else: thread = threading.Thread(target=ansible.execute_ansible,\ args=(h.managementIp_, h.username_, h.password_,\ testagentdir, ansible_cmd, lib_files, exc_info)) # Wrap up old zstack logs in /var/log/zstack/ #print('archive test log on host: [%s] \n' % h.managementIp_) #try: # if hasattr(h, 'port_'): # log.cleanup_log(h.managementIp_, h.username_, h.password_, h.port_) # else: # log.cleanup_log(h.managementIp_, h.username_, h.password_) #except Exception as e: # print "clean up old testing logs meet execption on management node: %s" % h.managementIp_ # raise e thread.start() #if localhost is not in hosts, should do log archive for zstack log.cleanup_local_log() self._wait_for_thread_completion('install test agent', 200) for h in self.test_agent_hosts: if not linux.wait_callback_success( _wait_echo, h.managementIp_, 5, 0.2, True): raise ActionError( 'testagent is not start up in 5s on %s, after it is deployed by ansible.' % h.managementIp_) else: print('Deploy test agent in host: %s \n' % target.managementIp) ansible_cmd_args = "host=%s \ pkg_testagent=zstacktestagent-1.0.0.tar.gz \ pkg_zstacklib=%s \ pypi_source_tar=pypi.tar.bz" % \ (target.managementIp, pkg_zstacklib) if ENV_HTTP_PROXY: ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \ (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY) ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args ansible.execute_ansible(target.managementIp, target.username, \ target.password, testagentdir, ansible_cmd, lib_files) if not linux.wait_callback_success( _wait_echo, target.managementIp, 5, 0.2): raise ActionError( 'testagent is not start up in 5s on %s, after it is deployed by ansible.' % target.managementIp) finally: if testagentdir: shell.call('rm -rf %s' % testagentdir)
def _apply_userdata(self, to): set_vip_cmd = ''' NS_NAME={{ns_name}} DHCP_IP={{dhcp_ip}} NS="ip netns exec $NS_NAME" exit_on_error() { if [ $? -ne 0 ]; then echo "error on line $1" exit 1 fi } eval $NS ip addr | grep 169.254.169.254 > /dev/null if [ $? -eq 0 ]; then exit 0 fi eth=`eval $NS ip addr | grep -w $DHCP_IP | awk '{print $NF}'` exit_on_error $LINENO if [ x$eth == "x" ]; then echo "cannot find device for the DHCP IP $DHCP_IP" exit 1 fi eval $NS ip addr add 169.254.169.254 dev $eth exit_on_error $LINENO exit 0 ''' tmpt = Template(set_vip_cmd) set_vip_cmd = tmpt.render({ 'ns_name': to.bridgeName, 'dhcp_ip': to.dhcpServerIp, }) shell.call(set_vip_cmd) set_ebtables = ''' BR_NAME={{br_name}} NS_NAME={{ns_name}} NS="ip netns exec $NS_NAME" exit_on_error() { if [ $? -ne 0 ]; then echo "error on line $1" exit 1 fi } eth=`eval $NS ip addr | grep 169.254.169.254 | awk '{print $NF}'` mac=`eval $NS ip link show $eth | grep -w ether | awk '{print $2}'` exit_on_error $LINENO CHAIN_NAME="USERDATA-$BR_NAME" ebtables-save | grep -w ":$CHAIN_NAME" > /dev/null if [ $? -ne 0 ]; then ebtables -t nat -N $CHAIN_NAME exit_on_error $LINENO ebtables -t nat -I PREROUTING --logical-in $BR_NAME -j $CHAIN_NAME exit_on_error $LINENO fi rule="$CHAIN_NAME -p IPv4 --ip-dst 169.254.169.254 -j dnat --to-dst $mac --dnat-target ACCEPT" ebtables-save | grep -- "$rule" > /dev/null if [ $? -ne 0 ]; then ebtables -t nat -I $rule exit_on_error $LINENO fi exit 0 ''' tmpt = Template(set_ebtables) set_ebtables = tmpt.render({ 'br_name': to.bridgeName, 'ns_name': to.bridgeName, }) shell.call(set_ebtables) dnat_port_cmd = ''' PORT={{port}} CHAIN_NAME="UD-PORT-$PORT" exit_on_error() { if [ $? -ne 0 ]; then echo "error on line $1" exit 1 fi } # delete old chains not matching our port old_chain=`iptables-save | awk '/^:UD-PORT-/{print substr($1,2)}'` if [ x"$old_chain" != "x" -a $CHAIN_NAME != $old_chain ]; then iptables -t nat -F $old_chain exit_on_error $LINENO iptables -t nat -X $old_chain exit_on_error $LINENO fi iptables-save | grep -w ":$CHAIN_NAME" > /dev/null if [ $? -ne 0 ]; then iptables -t nat -N $CHAIN_NAME exit_on_error $LINENO iptables -t nat -I PREROUTING -j $CHAIN_NAME exit_on_error $LINENO fi iptables-save -t nat | grep -- "$CHAIN_NAME -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :$PORT" > /dev/null || iptables -t nat -A $CHAIN_NAME -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :$PORT exit_on_error $LINENO exit 0 ''' tmpt = Template(dnat_port_cmd) dnat_port_cmd = tmpt.render({ 'port': to.port }) shell.call(dnat_port_cmd) conf_folder = os.path.join(self.USERDATA_ROOT, to.bridgeName) if not os.path.exists(conf_folder): shell.call('mkdir -p %s' % conf_folder) conf_path = os.path.join(conf_folder, 'lighttpd.conf') http_root = os.path.join(conf_folder, 'html') conf = '''\ server.document-root = "{{http_root}}" server.port = {{port}} server.bind = "169.254.169.254" dir-listing.activate = "enable" index-file.names = ( "index.html" ) server.modules += ( "mod_rewrite" ) $HTTP["remoteip"] =~ "^(.*)$" { url.rewrite-once = ( "^/.*/meta-data/(.+)$" => "../%1/meta-data/$1", "^/.*/meta-data$" => "../%1/meta-data", "^/.*/meta-data/$" => "../%1/meta-data/", "^/.*/user-data$" => "../%1/user-data" ) } mimetype.assign = ( ".html" => "text/html", ".txt" => "text/plain", ".jpg" => "image/jpeg", ".png" => "image/png" )''' tmpt = Template(conf) conf = tmpt.render({ 'http_root': http_root, 'dhcp_server_ip': to.dhcpServerIp, 'port': to.port }) if not os.path.exists(conf_path): with open(conf_path, 'w') as fd: fd.write(conf) else: with open(conf_path, 'r') as fd: current_conf = fd.read() if current_conf != conf: with open(conf_path, 'w') as fd: fd.write(conf) root = os.path.join(http_root, to.vmIp) meta_root = os.path.join(root, 'meta-data') if not os.path.exists(meta_root): shell.call('mkdir -p %s' % meta_root) index_file_path = os.path.join(meta_root, 'index.html') with open(index_file_path, 'w') as fd: fd.write('instance-id') instance_id_file_path = os.path.join(meta_root, 'instance-id') with open(instance_id_file_path, 'w') as fd: fd.write(to.metadata.vmUuid) if to.userdata: userdata_file_path = os.path.join(root, 'user-data') with open(userdata_file_path, 'w') as fd: fd.write(to.userdata) pid = linux.find_process_by_cmdline([conf_path]) if not pid: shell.call('ip netns exec %s lighttpd -f %s' % (to.bridgeName, conf_path)) def check(_): pid = linux.find_process_by_cmdline([conf_path]) return pid is not None if not linux.wait_callback_success(check, None, 5): raise Exception('lighttpd[conf-file:%s] is not running after being started %s seconds' % (conf_path, 5))
def test(): vm1 = test_stub.create_vr_vm('maintain_host_vm1', 'imageName_s', 'l3VlanNetwork2') test_obj_dict.add_vm(vm1) vm2 = test_stub.create_vr_vm('maintain_host_vm2', 'imageName_s', 'l3VlanNetwork2') test_obj_dict.add_vm(vm2) vm1.check() vm2.check() if not test_lib.lib_check_vm_live_migration_cap(vm1.vm) or not test_lib.lib_check_vm_live_migration_cap(vm2.vm): test_util.test_skip('skip migrate if live migrate not supported') test_util.test_dsc('Create volume and check') disk_offering = test_lib.lib_get_disk_offering_by_name(os.environ.get('smallDiskOfferingName')) volume_creation_option = test_util.VolumeOption() volume_creation_option.set_disk_offering_uuid(disk_offering.uuid) volume = test_stub.create_volume(volume_creation_option) test_obj_dict.add_volume(volume) test_util.test_dsc('Attach volume and check') volume.attach(vm1) volume.check() current_host1 = test_lib.lib_get_vm_host(vm1.vm) conditions = res_ops.gen_query_conditions('clusterUuid', '=', vm1.vm.clusterUuid) conditions = res_ops.gen_query_conditions('state', '=', host_header.ENABLED, conditions) conditions = res_ops.gen_query_conditions('status', '=', host_header.CONNECTED, conditions) all_hosts = res_ops.query_resource(res_ops.HOST, conditions) if len(all_hosts) <= 1: test_util.test_fail('Not available host to do maintenance, since there is only %s host' % len(all_hosts)) target_host = random.choice(all_hosts) if current_host1.uuid != target_host.uuid: vm1.migrate(target_host.uuid) current_host2 = test_lib.lib_get_vm_host(vm2.vm) if current_host2.uuid != target_host.uuid: vm2.migrate(target_host.uuid) new_host = test_lib.lib_get_vm_host(vm1.vm) if new_host.uuid != target_host.uuid: test_util.test_fail('VM did not migrate to target [host:] %s, but to [host:] %s' % (target_host.uuid, new_host.uuid)) volume.check() host = test_kvm_host.ZstackTestKvmHost() host.set_host(target_host) host.maintain() #need to update vm's inventory, since they will be changed by maintenace mode vm1.update() vm2.update() vm1.check() vm2.check() volume.check() host.change_state(test_kvm_host.ENABLE_EVENT) if not linux.wait_callback_success(is_host_connected, host.get_host().uuid, 120): test_util.test_fail('host status is not changed to connected, after changing its state to Enable') vm1.migrate(target_host.uuid) vm2.migrate(target_host.uuid) vm1.check() vm2.check() volume.check() vm1.destroy() test_obj_dict.rm_vm(vm1) vm2.destroy() test_obj_dict.rm_vm(vm2) volume.delete() test_obj_dict.rm_volume(volume) test_util.test_pass('Maintain Host Test Success')
def test(): global session_uuid session_uuid = acc_ops.login_as_admin() l3_1_name = os.environ.get('l3VlanNetworkName1') l3_2_name = os.environ.get('l3VlanDNATNetworkName') l3_3_name = os.environ.get('l3VlanNetworkName3') #l3_4_name = os.environ.get('l3VlanNetworkName5') l3_1 = test_lib.lib_get_l3_by_name(l3_1_name) l3_2 = test_lib.lib_get_l3_by_name(l3_2_name) l3_3 = test_lib.lib_get_l3_by_name(l3_3_name) #l3_4 = test_lib.lib_get_l3_by_name(l3_4_name) #create 4 VRs. vrs = test_lib.lib_find_vr_by_l3_uuid(l3_1.uuid) if not vrs: vm = test_stub.create_vlan_vm(l3_name=l3_1_name) vm.destroy() vr1 = test_lib.lib_find_vr_by_l3_uuid(l3_1.uuid)[0] else: vr1 = vrs[0] vrs = test_lib.lib_find_vr_by_l3_uuid(l3_2.uuid) if not vrs: vm = test_stub.create_vlan_vm(l3_name=l3_2_name) vm.destroy() vr2 = test_lib.lib_find_vr_by_l3_uuid(l3_2.uuid)[0] else: vr2 = vrs[0] vrs = test_lib.lib_find_vr_by_l3_uuid(l3_3.uuid) if not vrs: vm = test_stub.create_vlan_vm(l3_name=l3_3_name) vm.destroy() vr3 = test_lib.lib_find_vr_by_l3_uuid(l3_3.uuid)[0] else: vr3 = vrs[0] #vrs = test_lib.lib_find_vr_by_l3_uuid(l3_4.uuid) #if not vrs: # vm = test_stub.create_vlan_vm(l3_name=l3_4_name) # vm.destroy() # vr4 = test_lib.lib_find_vr_by_l3_uuid(l3_4.uuid)[0] #else: # vr4 = vrs[0] vrs = [vr1, vr2, vr3] #vrs = [vr1, vr2, vr3, vr4] for vr in vrs: thread = threading.Thread(target=stop_vm, args=(vr.uuid,)) thread.start() while threading.activeCount() > 1: check_exception() time.sleep(0.1) for vr in vrs: if not linux.wait_callback_success(check_status, (vr.uuid, 'Stopped'), 10): test_util.test_fail('VM: %s is not stopped, after waiting for extra 10s' % vr.uuid) check_exception() for vr in vrs: thread = threading.Thread(target=start_vm, args=(vr.uuid,)) thread.start() time.sleep(1) acc_ops.logout(session_uuid) while threading.activeCount() > 1: check_exception() time.sleep(0.1) check_exception() test_util.test_pass('Test start VRs simultaneously success')
def deploy_test_agent(self, target=None): print('Deploy test agent\n') if not self.test_agent_path: print('Not find test_agent. Stop deploying test agent.\n') return testagentdir = None try: def untar_test_agent(): tmpdir = tempfile.mkdtemp() shell.call('tar xzf %s -C %s' % (self.test_agent_path, tmpdir)) return '%s/zstacktestagent' % tmpdir def _wait_echo(target_ip): try: rspstr = http.json_dump_post( testagent.build_http_path(target_ip, host_plugin.ECHO_PATH)) except: print( 'zstack-testagent does not startup, will try again ...' ) return False return True testagentdir = untar_test_agent() ansible.check_and_install_ansible() lib_files = ['testagent/zstacktestagent-0.1.0.tar.gz', \ 'testagent/zstacklib-0.1.0.tar.gz' ] if not target: #default will deploy all test hosts. exc_info = [] for h in self.test_agent_hosts: print('Deploy test agent in host: [%s] \n' % h.managementIp_) ansible_cmd_args = "host=%s \ pkg_testagent=zstacktestagent-0.1.0.tar.gz \ pkg_zstacklib=zstacklib-0.6.tar.gz \ pypi_url=%s" % \ (h.managementIp_, ENV_PYPI_URL) if ENV_HTTP_PROXY: ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \ (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY) ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args thread = threading.Thread(target=ansible.execute_ansible,\ args=(h.managementIp_, h.username_, h.password_,\ testagentdir, ansible_cmd, lib_files, exc_info)) # Wrap up old zstack logs in /var/log/zstack/ print('archive test log on host: [%s] \n' % h.managementIp_) try: log.cleanup_log(h.managementIp_, h.username_, h.password_) except Exception as e: print "clean up old testing logs meet execption on management node: %s" % h.managementIp_ raise e thread.start() #if localhost is not in hosts, should do log archive for zstack log.cleanup_local_log() self._wait_for_thread_completion('install test agent', 200) for h in self.test_agent_hosts: if not linux.wait_callback_success( _wait_echo, h.managementIp_, 5, 0.2, True): raise ActionError( 'testagent is not start up in 5s on %s, after it is deployed by ansible.' % h.managementIp_) else: print('Deploy test agent in host: %s \n' % target.managementIp) ansible_cmd_args = "host=%s \ pkg_testagent=zstacktestagent-0.1.0.tar.gz \ pkg_zstacklib=zstacklib-0.6.tar.gz \ pypi_url=%s" % \ (target.managementIp, ENV_PYPI_URL) if ENV_HTTP_PROXY: ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \ (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY) ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args ansible.execute_ansible(target.managementIp, target.username, \ target.password, testagentdir, ansible_cmd, lib_files) if not linux.wait_callback_success( _wait_echo, target.managementIp, 5, 0.2): raise ActionError( 'testagent is not start up in 5s on %s, after it is deployed by ansible.' % target.managementIp) finally: if testagentdir: shell.call('rm -rf %s' % testagentdir)