Example #1
0
        def run_convert_if_need():
            def do_run():
                self.check_docker()
                save_pid()
                ret = shell.run(echo_pid_cmd)
                new_task.current_process_return_code = ret
                return ret

            pid = linux.read_file(v2v_pid_path)
            if not pid:
                return do_run()

            pid = int(pid.strip())
            process_completed = os.path.exists(v2v_cmd_ret_path)
            process_has_been_killed = not os.path.exists(
                v2v_cmd_ret_path) and not os.path.exists('/proc/%d' % pid)
            process_still_running = not os.path.exists(
                v2v_cmd_ret_path) and os.path.exists('/proc/%d' % pid)
            if process_has_been_killed:
                return do_run()

            if process_still_running:
                linux.wait_callback_success(os.path.exists,
                                            v2v_cmd_ret_path,
                                            timeout=259200,
                                            interval=60)

            ret = linux.read_file(v2v_cmd_ret_path)
            return int(ret.strip() if ret else 126)
Example #2
0
    def cancel_sftp_download(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = AgentResponse()

        def check():
            return shell.run("rbd ls %s | grep -q %s" %
                             (pool, image_name)) != 0

        def remove(target_name):
            return shell.run("rbd info {0}/{1} || rbd rm {0}/{1}".format(
                pool, target_name)) == 0

        pool, image_name = self._parse_install_path(
            cmd.primaryStorageInstallPath)
        tmp_image_name = 'tmp-%s' % image_name

        if check():
            return jsonobject.dumps(rsp)

        for image in (tmp_image_name, image_name):
            shell.run("pkill -9 -f '%s'" % image)
            linux.wait_callback_success(remove, image, timeout=30)

        if not check():
            rsp.set_err("remove image %s/%s fail" % (pool, image_name))

        return jsonobject.dumps(rsp)
def test():
    '''
    Test Description:
        Will check if test VM's mac/ip assignment will be removed in VR's DNS config, after VM is shutdown. And if VM's mac/ip will be added back after it restart. 
    Resource required:
        Need support 2 VMs. 
    '''
    global test_obj_dict
    test_util.test_dsc('Create test vm and check. VR only has DNS and DHCP services')
    vm = test_stub.create_vlan_vm()
    test_obj_dict.add_vm(vm)

    vm.check()
    vm.stop()
    if linux.wait_callback_success(check_dhcp_remove, vm.vm, 5, 0.2):
        test_util.test_logger('[vm:] %s mac/ip was removed in VR /etc/hosts.dhcp' % vm.vm.uuid)
    else:
        test_util.test_logger('check dhcp remove failure, will try to print VR DHCP tables')
        test_lib.lib_print_vr_dhcp_tables(test_lib.lib_find_vr_by_vm(vm.vm)[0])
        test_util.test_fail('[vm:] %s mac was not removed in VR /etc/hosts.dhcp, after vm was stopped.' % vm.vm.uuid)
    if linux.wait_callback_success(check_dhcp_release, vm.vm, 5, 0.2):
        test_util.test_logger('[vm:] %s mac/ip was removed in VR /etc/hosts.leases' % vm.vm.uuid)
    else:
        test_util.test_logger('check dhcp lease failure, will try to print VR DHCP tables')
        test_lib.lib_print_vr_dhcp_tables(test_lib.lib_find_vr_by_vm(vm.vm)[0])
        test_util.test_fail('[vm:] %s mac was not removed in VR /etc/hosts.leases, after vm was stopped.' % vm.vm.uuid)

    vm.start()
    vm.check()
    vm.destroy()
    vm.check()
    test_util.test_pass('Successfully check VM DHCP release when VM stopped')
Example #4
0
 def save_pid():
     linux.wait_callback_success(os.path.exists, v2v_pid_path)
     with open(v2v_pid_path, 'r') as fd:
         new_task.current_pid = fd.read().strip()
     new_task.current_process_cmd = echo_pid_cmd
     new_task.current_process_name = "virt_v2v_cmd"
     logger.debug("longjob[uuid:%s] saved process[pid:%s, name:%s]" %
                  (cmd.longJobUuid, new_task.current_pid, new_task.current_process_name))
 def save_pid():
     linux.wait_callback_success(os.path.exists, v2v_pid_path)
     with open(v2v_pid_path, 'r') as fd:
         new_task.current_pid = fd.read().strip()
     new_task.current_process_cmd = echo_pid_cmd
     new_task.current_process_name = "virt_v2v_cmd"
     logger.debug("longjob[uuid:%s] saved process[pid:%s, name:%s]" %
                  (cmd.longJobUuid, new_task.current_pid, new_task.current_process_name))
Example #6
0
    def stop(self, graceful=True, timeout=5, undefine=True):
        def cleanup_addons():
            for chan in self.domain_xmlobject.devices.get_child_node_as_list(
                    'channel'):
                if chan.type_ == 'unix':
                    path = chan.source.path_
                    shell.call('rm -f %s' % path)

        def loop_shutdown(_):
            try:
                self.domain.shutdown()
            except:
                #domain has been shutdown
                pass

            return self.wait_for_state_change(self.VM_STATE_SHUTDOWN)

        def loop_undefine(_):
            if not undefine:
                return True

            if not self.is_alive():
                return True

            try:
                self.domain.undefineFlags(
                    libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE
                    | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA)
            except libvirt.libvirtError:
                self.domain.undefine()

            return self.wait_for_state_change(None)

        def loop_destroy(_):
            try:
                self.domain.destroy()
            except:
                #domain has been destroyed
                pass

            return self.wait_for_state_change(self.VM_STATE_SHUTDOWN)

        do_destroy = True
        if graceful:
            if linux.wait_callback_success(loop_shutdown, None, timeout=60):
                do_destroy = False

        if do_destroy:
            if not linux.wait_callback_success(loop_destroy, None, timeout=60):
                raise kvmagent.KvmError(
                    'failed to destroy vm, timeout after 60 secs')

        cleanup_addons()
        if not linux.wait_callback_success(loop_undefine, None, timeout=60):
            raise kvmagent.KvmError(
                'failed to undefine vm, timeout after 60 secs')
Example #7
0
    def stop(self, graceful=True, timeout=5, undefine=True):
        def cleanup_addons():
            for chan in self.domain_xmlobject.devices.get_child_node_as_list('channel'):
                if chan.type_ == 'unix':
                    path = chan.source.path_
                    shell.call('rm -f %s' % path)

        def loop_shutdown(_):
            try:
                self.domain.shutdown()
            except:
                #domain has been shutdown
                pass

            return self.wait_for_state_change(self.VM_STATE_SHUTDOWN)
        
        def loop_undefine(_):
            if not undefine:
                return True

            if not self.is_alive():
                return True

            try:
                self.domain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE|libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA)
            except libvirt.libvirtError:
                self.domain.undefine()

            return self.wait_for_state_change(None)

        def loop_destroy(_):
            try:
                self.domain.destroy()
            except:
                #domain has been destroyed
                pass

            return self.wait_for_state_change(self.VM_STATE_SHUTDOWN)

        do_destroy = True
        if graceful:
            if linux.wait_callback_success(loop_shutdown, None, timeout=60):
                do_destroy = False

        if do_destroy:
            if not linux.wait_callback_success(loop_destroy, None, timeout=60):
                raise kvmagent.KvmError('failed to destroy vm, timeout after 60 secs')

        cleanup_addons()
        if not linux.wait_callback_success(loop_undefine, None, timeout=60):
            raise kvmagent.KvmError('failed to undefine vm, timeout after 60 secs')
def test():
    global host
    if test_lib.lib_get_active_host_number() < 2:
        test_util.test_fail('Not available host to do maintenance, since there are not 2 hosts')

    vm_cpu = 1
    vm_memory = 1073741824 #1G
    cond = res_ops.gen_query_conditions('name', '=', 'ttylinux')
    image_uuid = res_ops.query_resource(res_ops.IMAGE, cond)[0].uuid
    l3_network_uuid = res_ops.query_resource(res_ops.L3_NETWORK)[0].uuid
    vm = test_stub.create_mini_vm([l3_network_uuid], image_uuid, cpu_num = vm_cpu, memory_size = vm_memory)
    test_obj_dict.add_vm(vm)

    host_uuid = test_lib.lib_get_vm_host(vm.vm).uuid
    host_ops.change_host_state(host_uuid, 'maintain')

    #need to update vm's inventory, since they will be changed by maintenace mode
    vm.update()
    vm.set_state(vm_header.STOPPED)

    vm.check()

    host_ops.change_host_state(host_uuid, 'enable')
    if not linux.wait_callback_success(is_host_connected, host_uuid, 120):
        test_util.test_fail('host status is not changed to connected, after changing its state to Enable')

    vm.start()
    vm.check()
    vm.destroy()
    test_obj_dict.rm_vm(vm)
    test_util.test_pass('Maintain Host Test Success')
Example #9
0
    def _restart_dnsmasq(self):
        self._do_dnsmasq_restart()
        
        def check_start(_):
            dnsmasq_pid = linux.get_pid_by_process_name('dnsmasq')
            return dnsmasq_pid is not None 

        if not linux.wait_callback_success(check_start, None, 5, 0.5):
            logger.debug('dnsmasq is not running, former start failed, try to start it again ...')
            cmd = self._do_dnsmasq_start()
            
            if cmd.return_code != 0:
                raise virtualrouter.VirtualRouterError('dnsmasq in virtual router is not running, we try to start it but fail, error is %s' % cmd.stdout)

            if not linux.wait_callback_success(check_start, None, 5, 0.5):
                raise virtualrouter.VirtualRouterError('dnsmasq in virtual router is not running, "/etc/init.d/dnsmasq start" returns success, but the process is not running after 5 seconds')
Example #10
0
    def migrate(self, destHostIp):
        destUrl = "qemu+tcp://{0}/system".format(destHostIp)
        tcpUri = "tcp://{0}".format(destHostIp)
        try:
            self.domain.migrateToURI2(
                destUrl, tcpUri, None,
                libvirt.VIR_MIGRATE_LIVE | libvirt.VIR_MIGRATE_PEER2PEER
                | libvirt.VIR_MIGRATE_UNDEFINE_SOURCE
                | libvirt.VIR_MIGRATE_PERSIST_DEST
                | libvirt.VIR_MIGRATE_TUNNELLED, None, 0)
        except libvirt.libvirtError as ex:
            logger.warn(linux.get_exception_stacktrace())
            raise kvmagent.KvmError('unable to migrate vm[uuid:%s] to %s, %s' %
                                    (self.uuid, destUrl, str(ex)))

        try:
            if not linux.wait_callback_success(self.wait_for_state_change,
                                               callback_data=None,
                                               timeout=300):
                raise kvmagent.KvmError('timeout after 300 seconds')
        except kvmagent.KvmError as ke:
            raise ke
        except Exception as e:
            logger.debug(linux.get_exception_stacktrace())

        logger.debug(
            'successfully migrated vm[uuid:{0}] to dest url[{1}]'.format(
                self.uuid, destUrl))
Example #11
0
def check_sg_rule_exist(test_vm, protocol, direction, extra_str, exp_result):
    if linux.wait_callback_success(do_check_sg_rule_exist,
                                   (test_vm, direction, extra_str, exp_result),
                                   5, 0.2):
        test_result = exp_result
        if exp_result:
            test_util.test_logger(
                'Expected result: [Security Group] find %s %s rule for [vm:] %s. '
                % (protocol, direction, test_vm.uuid))
        else:
            test_util.test_logger(
                'Expected result: [Security Group] FAIL to find %s %s rule for [vm:] %s in 5 seconds. '
                % (protocol, direction, test_vm.uuid))
    else:
        test_result = not exp_result
        if exp_result:
            test_util.test_logger(
                'Unexpected result: [Security Group] Not find %s %s rule for [vm:] %s. '
                % (protocol, direction, test_vm.uuid))
        else:
            test_util.test_logger(
                'Unexpected result: [Security Group]  find %s %s rule for [vm:] %s in 5 seconds. '
                % (protocol, direction, test_vm.uuid))

    return test_result
Example #12
0
    def _restart_dnsmasq(self):
        self._do_dnsmasq_restart()
        
        def check_start(_):
            dnsmasq_pid = linux.get_pid_by_process_name('dnsmasq')
            return dnsmasq_pid is not None 

        if not linux.wait_callback_success(check_start, None, 5, 0.5):
            logger.debug('dnsmasq is not running, former start failed, try to start it again ...')
            cmd = self._do_dnsmasq_start()
            
            if cmd.return_code != 0:
                raise virtualrouter.VirtualRouterError('dnsmasq in virtual router is not running, we try to start it but fail, error is %s' % cmd.stdout)

            if not linux.wait_callback_success(check_start, None, 5, 0.5):
                raise virtualrouter.VirtualRouterError('dnsmasq in virtual router is not running, "/etc/init.d/dnsmasq start" returns success, but the process is not running after 5 seconds')
Example #13
0
        def rebase_all_to_active_file():
            self.domain.blockRebase(disk_name, None, 0, 0)

            def wait_job(_):
                return not self._wait_for_block_job(disk_name, abort_on_error=True)

            if not linux.wait_callback_success(wait_job, timeout=300):
                raise kvmagent.KvmError('live full snapshot merge failed')
    def _start_multi_nodes(self, restart = False):
        nodes = []
        threads = []
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if not restart and node.reserve__:
                continue

            if not node.dockerImage__:
                print 'Deploy node in hosts'
                #consider some zstack-server is running in vm, the server 
                # startup speed is slow. Increase timeout to 180s.
                cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % (node.ip_, node.ip_)
                thread = threading.Thread(target=shell_cmd_thread, args=(cmd, True, ))
                threads.append(thread)
            else:
                print 'Deploy node in docker'
                docker_node = DockerNode(self)
                docker_node.set_docker_image(node.dockerImage__)
                docker_node.set_node_ip(node.ip__)
                docker_node.prepare_node()
                nodes.append(docker_node)
                thread = threading.Thread(target=docker_node.start_node)
                threads.append(thread)

        for thread in threads:
            thread.start()

        self._wait_for_thread_completion('start management node', 200)

        if node_exception:
            print 'node start meets exception:'
            info1 = node_exception[0][1]
            info2 = node_exception[0][2]
            raise info1, None, info2
                
        current_time = time.time()
        #largest timeout time for multi nodes startup is 300s
        timeout_time = current_time + 300
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if node.reserve__:
                continue
            new_time = time.time() 
            if new_time >= timeout_time:
                new_timeout = 1
            else:
                new_timeout = timeout_time - new_time

            if not linux.wait_callback_success(\
                    node_ops.is_management_node_start, \
                    node.ip_, timeout=new_timeout, interval=0.5):
                raise ActionError('multi node does not startup on host: %s' \
                        % node.ip_)

        zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path
        cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home
        shell.call(cmd)
Example #15
0
    def _start_multi_nodes(self, restart = False):
        nodes = []
        threads = []
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if not restart and node.reserve__:
                continue

            if not node.dockerImage__:
                print 'Deploy node in hosts'
                #consider some zstack-server is running in vm, the server 
                # startup speed is slow. Increase timeout to 180s.
                cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % (node.ip_, node.ip_)
                thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
                threads.append(thread)
            else:
                print 'Deploy node in docker'
                docker_node = DockerNode(self)
                docker_node.set_docker_image(node.dockerImage__)
                docker_node.set_node_ip(node.ip__)
                docker_node.prepare_node()
                nodes.append(docker_node)
                thread = threading.Thread(target=docker_node.start_node)
                threads.append(thread)

        for thread in threads:
            thread.start()

        self._wait_for_thread_completion('start management node', 200)

        if node_exception:
            print 'node start meets exception:'
            info1 = node_exception[0][1]
            info2 = node_exception[0][2]
            raise info1, None, info2
                
        current_time = time.time()
        #largest timeout time for multi nodes startup is 300s
        timeout_time = current_time + 300
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if node.reserve__:
                continue
            new_time = time.time() 
            if new_time >= timeout_time:
                new_timeout = 1
            else:
                new_timeout = timeout_time - new_time

            if not linux.wait_callback_success(\
                    node_ops.is_management_node_start, \
                    node.ip_, timeout=new_timeout, interval=0.5):
                raise ActionError('multi node does not startup on host: %s' \
                        % node.ip_)

        zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path
        cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home
        shell.call(cmd)
Example #16
0
def wait_for_management_server_start(wait_start_timeout=120, \
        node_uuid=None):
    if not linux.wait_callback_success(is_management_node_ready, \
            node_uuid, \
            wait_start_timeout, 1, True):
        raise Exception('waiting for management server start up time out\
                after %s seconds' % wait_start_timeout)
    else:
        print('management starts successfully, is running now ...')
def wait_for_management_server_start(wait_start_timeout=120, \
        node_uuid=None):
    if not linux.wait_callback_success(is_management_node_ready, \
            node_uuid, \
            wait_start_timeout, 1, True):
        raise Exception('waiting for management server start up time out\
                after %s seconds' % wait_start_timeout)
    else:
        print('management starts successfully, is running now ...')
Example #18
0
 def start(self, timeout=60):
     #TODO: 1. enbale hair_pin mode
     logger.debug('creating vm:\n%s' % self.domain_xml)
     conn = kvmagent.get_libvirt_connection()
     domain = conn.defineXML(self.domain_xml)
     self.domain = domain
     self.domain.createWithFlags(0)    
     if not linux.wait_callback_success(self.wait_for_state_change, self.VM_STATE_RUNNING, timeout=timeout):
         raise kvmagent.KvmError('unable to start vm[uuid:%s, name:%s], vm state is not changing to running after %s seconds' % (self.uuid, self.get_name(), timeout))
Example #19
0
        def rebase_all_to_active_file():
            self.domain.blockRebase(disk_name, None, 0, 0)

            def wait_job(_):
                return not self._wait_for_block_job(disk_name,
                                                    abort_on_error=True)

            if not linux.wait_callback_success(wait_job, timeout=300):
                raise kvmagent.KvmError('live full snapshot merge failed')
Example #20
0
 def wait_for_node_start(self, timeout=120):
     if not linux.wait_callback_success(node_ops.is_management_node_start, \
             self.node_option.get_management_ip(), timeout=timeout, \
             interval=0.5):
         test_util.test_logger('multi node does not startup on host: %s' \
                 % self.node_option.get_management_ip())
         return False
     test_util.test_logger('multi node startup on host: %s' \
             % self.node_option.get_management_ip())
     return True
Example #21
0
    def delete(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        pool, image_name = self._parse_install_path(cmd.installPath)

        def delete_image(_):
            # in case image is deleted, we don't have to wait for timeout
            img = "%s/%s" % (pool, image_name)
            shell.check_run('rbd info %s && rbd rm %s' % (img, img))
            return True

        # 'rbd rm' might fail due to client crash. We wait for 30 seconds as suggested by 'rbd'.
        #
        # rbd: error: image still has watchers
        # This means the image is still open or the client using it crashed. Try again after
        # closing/unmapping it or waiting 30s for the crashed client to timeout.
        linux.wait_callback_success(delete_image, interval=5, timeout=30, ignore_exception_in_callback=True)

        rsp = AgentResponse()
        self._set_capacity_to_response(rsp)
        return jsonobject.dumps(rsp)
Example #22
0
    def delete(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        pool, image_name = self._parse_install_path(cmd.installPath)

        def delete_image(_):
            # in case image is deleted, we don't have to wait for timeout
            img = "%s/%s" % (pool, image_name)
            shell.check_run('rbd info %s && rbd rm %s' % (img, img))
            return True

        # 'rbd rm' might fail due to client crash. We wait for 30 seconds as suggested by 'rbd'.
        #
        # rbd: error: image still has watchers
        # This means the image is still open or the client using it crashed. Try again after
        # closing/unmapping it or waiting 30s for the crashed client to timeout.
        linux.wait_callback_success(delete_image, interval=5, timeout=30, ignore_exception_in_callback=True)

        rsp = AgentResponse()
        self._set_capacity_to_response(rsp)
        return jsonobject.dumps(rsp)
    def _start_multi_nodes(self, restart=False):
        nodes = []
        threads = []
        for node in self.nodes:
            #The reserved node is used by test cases.
            if not restart and node.reserve__:
                continue

            if not node.dockerImage__:
                print 'Deploy node in hosts'
                cmd = 'zstack-ctl stop_node --remote=%s ; zstack-ctl start_node --remote=%s' % (
                    node.ip_, node.ip_)
                thread = threading.Thread(target=shell_cmd_thread,
                                          args=(cmd, ))
                threads.append(thread)
            else:
                print 'Deploy node in docker'
                docker_node = DockerNode(self)
                docker_node.set_docker_image(node.dockerImage__)
                docker_node.set_node_ip(node.ip__)
                docker_node.prepare_node()
                nodes.append(docker_node)
                thread = threading.Thread(target=docker_node.start_node)
                threads.append(thread)

        for thread in threads:
            thread.start()

        self._wait_for_thread_completion('start management node', 200)

        if node_exception:
            print 'node start meets exception:'
            info1 = node_exception[0][1]
            info2 = node_exception[0][2]
            raise info1, None, info2

        current_time = time.time()
        #largest timeout time for multi nodes startup is 300s
        timeout_time = current_time + 300
        for node in self.nodes:
            #The reserved node is used by test cases.
            if node.reserve__:
                continue
            new_time = time.time()
            if new_time >= timeout_time:
                new_timeout = 1
            else:
                new_timeout = timeout_time - new_time

            if not linux.wait_callback_success(\
                    node_ops.is_management_node_start, \
                    node.ip_, timeout=new_timeout, interval=0.5):
                raise ActionError('multi node does not startup on host: %s' \
                        % node.ip_)
Example #24
0
def check_sanlock_renewal_failure(lockspace):
    last_record = linux.wait_callback_success(get_sanlock_renewal, lockspace, 10, 1, True)
    if last_record is False:
        logger.warn("unable find correct sanlock renewal record, may be rotated")
        return True, ""
    if "next_errors=" not in last_record:
        return True, ""
    errors = int(last_record.split("next_errors=")[-1])
    if errors > 2:
        return False, "sanlock renew lease of lockspace %s failed for %s times, storage may failed" % (
        lockspace, errors)
    return True, ""
Example #25
0
        def do_commit(base, top, flags=0):
            self.domain.blockCommit(disk_name, base, top, 0, flags)

            logger.debug('start block commit %s --> %s' % (top, base))
            def wait_job(_):
                logger.debug('merging snapshot chain is waiting for blockCommit job completion')
                return not self._wait_for_block_job(disk_name, abort_on_error=True)

            if not linux.wait_callback_success(wait_job, timeout=300):
                raise kvmagent.KvmError('live merging snapshot chain failed, timeout after 300s')

            logger.debug('end block commit %s --> %s' % (top, base))
Example #26
0
        def run_convert_if_need():
            def do_run():
                save_pid()
                ret = shell.run(echo_pid_cmd)
                new_task.current_process_return_code = ret
                retry_if_needed(ret)
                return ret

            def retry_if_needed(ret):
                if ret == 0:
                    return

                if retry_counter[0] != max_retry_times and shell.run("grep -q 'guestfs_launch failed' %s" % log_path) == 0:
                    retry_counter[0] += 1
                    raise RetryException(
                        "launch guestfs failed, rerun v2v longjob %s" % cmd.longJobUuid)

            pid = linux.read_file(v2v_pid_path)
            log_path = "%s/virt_v2v_log" % storage_dir
            if not pid:
                return do_run()

            pid = int(pid.strip())
            process_completed = os.path.exists(v2v_cmd_ret_path)
            process_has_been_killed = not os.path.exists(v2v_cmd_ret_path) and not os.path.exists('/proc/%d' % pid)
            process_still_running = not os.path.exists(v2v_cmd_ret_path) and os.path.exists('/proc/%d' % pid)
            if process_has_been_killed:
                return do_run()

            if process_still_running:
                linux.wait_callback_success(os.path.exists, v2v_cmd_ret_path, timeout=259200, interval=60)

            # delete password file
            passwd_file = os.path.join(storage_dir, "passwd")
            if os.path.exists(passwd_file):
                os.remove(passwd_file)

            ret = linux.read_file(v2v_cmd_ret_path)
            retry_if_needed(ret)
            return int(ret.strip() if ret else 126)
    def _start_multi_nodes(self, restart = False):
        nodes = []
        threads = []
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if not restart and node.reserve__:
                continue

            if not node.dockerImage__:
                print 'Deploy node in hosts'
                cmd = 'zstack-ctl stop_node --remote=%s ; zstack-ctl start_node --remote=%s' % (node.ip_, node.ip_)
                thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
                threads.append(thread)
            else:
                print 'Deploy node in docker'
                docker_node = DockerNode(self)
                docker_node.set_docker_image(node.dockerImage__)
                docker_node.set_node_ip(node.ip__)
                docker_node.prepare_node()
                nodes.append(docker_node)
                thread = threading.Thread(target=docker_node.start_node)
                threads.append(thread)

        for thread in threads:
            thread.start()

        self._wait_for_thread_completion('start management node', 200)

        if node_exception:
            print 'node start meets exception:'
            info1 = node_exception[0][1]
            info2 = node_exception[0][2]
            raise info1, None, info2
                
        current_time = time.time()
        #largest timeout time for multi nodes startup is 300s
        timeout_time = current_time + 300
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if node.reserve__:
                continue
            new_time = time.time() 
            if new_time >= timeout_time:
                new_timeout = 1
            else:
                new_timeout = timeout_time - new_time

            if not linux.wait_callback_success(\
                    node_ops.is_management_node_start, \
                    node.ip_, timeout=new_timeout, interval=0.5):
                raise ActionError('multi node does not startup on host: %s' \
                        % node.ip_)
Example #28
0
        def take_full_snapshot():
            logger.debug('start rebasing to make a full snapshot')
            self.domain.blockRebase(disk_name, None, 0, 0)

            logger.debug('rebasing full snapshot is in processing')
            def wait_job(_):
                logger.debug('full snapshot is waiting for blockRebase job completion')
                return not self._wait_for_block_job(disk_name, abort_on_error=True)

            if not linux.wait_callback_success(wait_job, timeout=300):
                raise kvmagent.KvmError('live full snapshot failed')

            return take_delta_snapshot()
def test():
    test_util.test_dsc('Test Host Start/Stop function')
    vm_cpu = 1
    vm_memory = 1073741824 #1G
    cond = res_ops.gen_query_conditions('name', '=', 'ttylinux')
    image_uuid = res_ops.query_resource(res_ops.IMAGE, cond)[0].uuid
    l3_network_uuid = res_ops.query_resource(res_ops.L3_NETWORK)[0].uuid
    vm = test_stub.create_mini_vm([l3_network_uuid], image_uuid, cpu_num = vm_cpu, memory_size = vm_memory)
    test_obj_dict.add_vm(vm)

    host_uuid = test_lib.lib_get_vm_host(vm.vm).uuid
    host_ops.change_host_state(host_uuid, 'disable')
    if not linux.wait_callback_success(is_host_disabled, host_uuid, 120):
        test_util.test_fail('host state is not changed to disabled')

    host_ops.change_host_state(host_uuid, 'enable')
    if not linux.wait_callback_success(is_host_enabled, host_uuid, 120):
        test_util.test_fail('host state is not changed to enabled')

    vm.destroy()
    test_obj_dict.rm_vm(vm)
    test_util.test_pass('Stop/Start Host Test Pass')
Example #30
0
 def start(self, timeout=60):
     #TODO: 1. enbale hair_pin mode
     logger.debug('creating vm:\n%s' % self.domain_xml)
     conn = kvmagent.get_libvirt_connection()
     domain = conn.defineXML(self.domain_xml)
     self.domain = domain
     self.domain.createWithFlags(0)
     if not linux.wait_callback_success(self.wait_for_state_change,
                                        self.VM_STATE_RUNNING,
                                        timeout=timeout):
         raise kvmagent.KvmError(
             'unable to start vm[uuid:%s, name:%s], vm state is not changing to running after %s seconds'
             % (self.uuid, self.get_name(), timeout))
Example #31
0
    def _apply_userdata_restart_httpd(self, to):
        conf_folder = os.path.join(self.USERDATA_ROOT, to.namespaceName)
        conf_path = os.path.join(conf_folder, 'lighttpd.conf')
        pid = linux.find_process_by_cmdline([conf_path])
        if not pid:
            shell.call('ip netns exec %s lighttpd -f %s' % (to.namespaceName, conf_path))

            def check(_):
                pid = linux.find_process_by_cmdline([conf_path])
                return pid is not None

            if not linux.wait_callback_success(check, None, 5):
                raise Exception('lighttpd[conf-file:%s] is not running after being started %s seconds' % (conf_path, 5))
        def run_convert_if_need():
            def do_run():
                save_pid()
                ret = shell.run(echo_pid_cmd)
                new_task.current_process_return_code = ret
                return ret

            pid = linux.read_file(v2v_pid_path)
            if not pid:
                return do_run()

            pid = int(pid.strip())
            process_completed = os.path.exists(v2v_cmd_ret_path)
            process_has_been_killed = not os.path.exists(v2v_cmd_ret_path) and not os.path.exists('/proc/%d' % pid)
            process_still_running = not os.path.exists(v2v_cmd_ret_path) and os.path.exists('/proc/%d' % pid)
            if process_has_been_killed:
                return do_run()

            if process_still_running:
                linux.wait_callback_success(os.path.exists, v2v_cmd_ret_path, timeout=259200, interval=60)

            ret = linux.read_file(v2v_cmd_ret_path)
            return int(ret.strip() if ret else 126)
Example #33
0
def check_sanlock_renewal_failure(lockspace):
    last_record = linux.wait_callback_success(get_sanlock_renewal, lockspace,
                                              10, 1, True)
    if last_record is False:
        logger.warn(
            "unable find correct sanlock renewal record, may be rotated")
        return True
    if "next_errors=" not in last_record:
        return True, ""
    errors = int(last_record.split("next_errors=")[-1])
    if errors > 2:
        return False, "sanlock renew lease of lockspace %s failed for %s times, storage may failed" % (
            lockspace, errors)
    return True, ""
Example #34
0
    def cancel_sftp_download(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = AgentResponse()

        def check():
            return shell.run("rbd ls %s | grep -q %s" % (pool, image_name)) != 0

        def remove(target_name):
            return shell.run("rbd info {0}/{1} || rbd rm {0}/{1}".format(pool, target_name)) == 0

        pool, image_name = self._parse_install_path(cmd.primaryStorageInstallPath)
        tmp_image_name = 'tmp-%s' % image_name

        if check():
            return jsonobject.dumps(rsp)

        for image in (tmp_image_name, image_name):
            shell.run("pkill -9 -f '%s'" % image)
            linux.wait_callback_success(remove, image, timeout=30)

        if not check():
            rsp.set_err("remove image %s/%s fail" % (pool, image_name))

        return jsonobject.dumps(rsp)
def check_sg_rule_exist(test_vm, protocol, direction, extra_str, exp_result):
    if linux.wait_callback_success(do_check_sg_rule_exist, (test_vm, direction, extra_str, exp_result), 5, 0.2):
        test_result = exp_result
        if exp_result:
            test_util.test_logger('Expected result: [Security Group] find %s %s rule for [vm:] %s. ' % (protocol, direction, test_vm.uuid))
        else:
            test_util.test_logger('Expected result: [Security Group] FAIL to find %s %s rule for [vm:] %s in 5 seconds. ' % (protocol, direction, test_vm.uuid))
    else:
        test_result = not exp_result
        if exp_result:
            test_util.test_logger('Unexpected result: [Security Group] Not find %s %s rule for [vm:] %s. ' % (protocol, direction, test_vm.uuid))
        else:
            test_util.test_logger('Unexpected result: [Security Group]  find %s %s rule for [vm:] %s in 5 seconds. ' % (protocol, direction, test_vm.uuid))

    return test_result
Example #36
0
def json_post(uri, body=None, headers={}, method='POST', fail_soon=False):
    ret = []

    def post(_):
        try:
            pool = urllib3.PoolManager(timeout=120.0,
                                       retries=urllib3.util.retry.Retry(15))
            header = {
                'Content-Type': 'application/json',
                'Connection': 'close'
            }
            for k in headers.keys():
                header[k] = headers[k]

            if body is not None:
                assert isinstance(body, types.StringType)
                header['Content-Length'] = str(len(body))
                content = pool.urlopen(method,
                                       uri,
                                       headers=header,
                                       body=str(body)).data

                #(resp, content) = http_obj.request(uri, 'POST', body='%s' % body, headers=header)
            else:
                header['Content-Length'] = '0'
                #(resp, content) = http_obj.request(uri, 'POST', headers=header)
                content = pool.urlopen(method, uri, headers=header).data

            pool.clear()
            ret.append(content)
            return True
        except Exception as e:
            if fail_soon:
                raise e

            logger.warn('[WARN]: %s' % linux.get_exception_stacktrace())
            return False

    if fail_soon:
        post(None)
    else:
        if not linux.wait_callback_success(post,
                                           ignore_exception_in_callback=True):
            raise Exception(
                'unable to post to %s, body: %s, see before error' %
                (uri, body))

    return ret[0]
Example #37
0
        def take_full_snapshot():
            logger.debug('start rebasing to make a full snapshot')
            self.domain.blockRebase(disk_name, None, 0, 0)

            logger.debug('rebasing full snapshot is in processing')

            def wait_job(_):
                logger.debug(
                    'full snapshot is waiting for blockRebase job completion')
                return not self._wait_for_block_job(disk_name,
                                                    abort_on_error=True)

            if not linux.wait_callback_success(wait_job, timeout=300):
                raise kvmagent.KvmError('live full snapshot failed')

            return take_delta_snapshot()
Example #38
0
    def _restart_dnsmasq(self, ns_name, conf_file_path):
        pid = linux.find_process_by_cmdline([conf_file_path])
        if pid:
            linux.kill_process(pid)

        NS_NAME = ns_name
        CONF_FILE = conf_file_path
        DNSMASQ = bash_errorout('which dnsmasq').strip(' \t\r\n')
        bash_errorout('ip netns exec {{NS_NAME}} {{DNSMASQ}} --conf-file={{CONF_FILE}} ')

        def check(_):
            pid = linux.find_process_by_cmdline([conf_file_path])
            return pid is not None

        if not linux.wait_callback_success(check, None, 5):
            raise Exception('dnsmasq[conf-file:%s] is not running after being started %s seconds' % (conf_file_path, 5))
Example #39
0
    def _restart_dnsmasq(self, ns_name, conf_file_path):
        pid = linux.find_process_by_cmdline([conf_file_path])
        if pid:
            linux.kill_process(pid)

        NS_NAME = ns_name
        CONF_FILE = conf_file_path
        DNSMASQ = bash_errorout('which dnsmasq').strip(' \t\r\n')
        bash_errorout('ip netns exec {{NS_NAME}} {{DNSMASQ}} --conf-file={{CONF_FILE}} ')

        def check(_):
            pid = linux.find_process_by_cmdline([conf_file_path])
            return pid is not None

        if not linux.wait_callback_success(check, None, 5):
            raise Exception('dnsmasq[conf-file:%s] is not running after being started %s seconds' % (conf_file_path, 5))
Example #40
0
            def detach(error_out):
                try:
                    self.domain.detachDeviceFlags(xmlstr, libvirt.VIR_DOMAIN_AFFECT_LIVE | libvirt.VIR_DOMAIN_AFFECT_CONFIG)
                except libvirt.libvirtError as e:
                    if error_out:
                        raise e
                    return True

                def wait_for_detach(_):
                    me = get_vm_by_uuid(self.uuid)
                    for disk in me.domain_xmlobject.devices.get_child_node_as_list('disk'):
                        if disk.source.file_ == volume.installPath:
                            logger.debug('volume[%s] is still in process of detaching, wait it' % volume.installPath)
                            return False
                    return True

                return linux.wait_callback_success(wait_for_detach, None, 5, 1)
Example #41
0
        def do_commit(base, top, flags=0):
            self.domain.blockCommit(disk_name, base, top, 0, flags)

            logger.debug('start block commit %s --> %s' % (top, base))

            def wait_job(_):
                logger.debug(
                    'merging snapshot chain is waiting for blockCommit job completion'
                )
                return not self._wait_for_block_job(disk_name,
                                                    abort_on_error=True)

            if not linux.wait_callback_success(wait_job, timeout=300):
                raise kvmagent.KvmError(
                    'live merging snapshot chain failed, timeout after 300s')

            logger.debug('end block commit %s --> %s' % (top, base))
Example #42
0
    def _restart_dnsmasq(self, ns_name, conf_file_path):
        pid = linux.find_process_by_cmdline([conf_file_path])
        if pid:
            linux.kill_process(pid)

        cmd = '''\
ip netns exec {{ns_name}} /sbin/dnsmasq --conf-file={{conf_file}} || ip netns exec {{ns_name}} /usr/sbin/dnsmasq --conf-file={{conf_file}}
'''
        tmpt = Template(cmd)
        cmd = tmpt.render({'ns_name': ns_name, 'conf_file': conf_file_path})
        shell.call(cmd)

        def check(_):
            pid = linux.find_process_by_cmdline([conf_file_path])
            return pid is not None

        if not linux.wait_callback_success(check, None, 5):
            raise Exception('dnsmasq[conf-file:%s] is not running after being started %s seconds' % (conf_file_path, 5))
Example #43
0
    def start_node(self):
        def _wait(data):
            try:
                shell.ShellCmd('docker ps')()
                print('docker service is ready')
                return True
            except:
                print('docker service is still starting ...')
                return False

        try:
            if not linux.wait_callback_success(_wait, None, \
                    docker_service_start_timeout, 0.1):
                raise ActionError('waiting for docker start up time out: %s' % \
                    docker_service_start_timeout)
            shell.ShellCmd('cd %s ; docker build --tag="%s" .' % \
                    (self.docker_folder, self.node_ip))()
            #run docker image
            shell.ShellCmd("docker run -d %s " % self.node_ip)()
            print 'docker container has been created.'
        except Exception as e:
            node_exception.append(sys.exc_info())
 def start_node(self):
     def _wait(data):
         try:
             shell.ShellCmd('docker ps')()
             print('docker service is ready')
             return True
         except:
             print ('docker service is still starting ...')
             return False
     
     try:
         if not linux.wait_callback_success(_wait, None, \
                 docker_service_start_timeout, 0.1):
             raise ActionError('waiting for docker start up time out: %s' % \
                 docker_service_start_timeout)
         shell.ShellCmd('cd %s ; docker build --tag="%s" .' % \
                 (self.docker_folder, self.node_ip))()
         #run docker image
         shell.ShellCmd("docker run -d %s " % self.node_ip)()
         print 'docker container has been created.'
     except Exception as e:
         node_exception.append(sys.exc_info())
Example #45
0
            def detach(error_out):
                try:
                    self.domain.detachDeviceFlags(
                        xmlstr, libvirt.VIR_DOMAIN_AFFECT_LIVE
                        | libvirt.VIR_DOMAIN_AFFECT_CONFIG)
                except libvirt.libvirtError as e:
                    if error_out:
                        raise e
                    return True

                def wait_for_detach(_):
                    me = get_vm_by_uuid(self.uuid)
                    for disk in me.domain_xmlobject.devices.get_child_node_as_list(
                            'disk'):
                        if disk.source.file_ == volume.installPath:
                            logger.debug(
                                'volume[%s] is still in process of detaching, wait it'
                                % volume.installPath)
                            return False
                    return True

                return linux.wait_callback_success(wait_for_detach, None, 5, 1)
Example #46
0
def json_post(uri, body=None, headers={}, method='POST', fail_soon=False):
    ret = []
    def post(_):
        try:
            pool = urllib3.PoolManager(timeout=120.0, retries=urllib3.util.retry.Retry(15))
            header = {'Content-Type': 'application/json', 'Connection': 'close'}
            for k in headers.keys():
                header[k] = headers[k]

            if body is not None:
                assert isinstance(body, types.StringType)
                header['Content-Length'] = str(len(body))
                content = pool.urlopen(method, uri, headers=header, body=str(body)).data

                #(resp, content) = http_obj.request(uri, 'POST', body='%s' % body, headers=header)
            else:
                header['Content-Length'] = '0'
                #(resp, content) = http_obj.request(uri, 'POST', headers=header)
                content = pool.urlopen(method, uri, headers=header).data

            #logger.debug('post to %s, with content: %s, with header: %s' % (uri, body, header))
            pool.clear()
            ret.append(content)
            return True
        except Exception as e:
            if fail_soon:
                raise e

            logger.warn(linux.get_exception_stacktrace())
            return False

    if fail_soon:
        post(None)
    else:
        if not linux.wait_callback_success(post, ignore_exception_in_callback=True):
            raise Exception('unable to post to %s, body: %s, see before error' % (uri, body))

    return ret[0]
Example #47
0
    def migrate(self, destHostIp):
        destUrl = "qemu+tcp://{0}/system".format(destHostIp)
        tcpUri = "tcp://{0}".format(destHostIp)
        try:
            self.domain.migrateToURI2(destUrl, tcpUri, None,
                                      libvirt.VIR_MIGRATE_LIVE|
                                      libvirt.VIR_MIGRATE_PEER2PEER|
                                      libvirt.VIR_MIGRATE_UNDEFINE_SOURCE|
                                      libvirt.VIR_MIGRATE_PERSIST_DEST |
                                      libvirt.VIR_MIGRATE_TUNNELLED,
                                      None, 0)
        except libvirt.libvirtError as ex:
            logger.warn(linux.get_exception_stacktrace())
            raise kvmagent.KvmError('unable to migrate vm[uuid:%s] to %s, %s' % (self.uuid, destUrl, str(ex)))

        try:
            if not linux.wait_callback_success(self.wait_for_state_change, callback_data=None, timeout=300):
                raise kvmagent.KvmError('timeout after 300 seconds')
        except kvmagent.KvmError as ke:
            raise ke
        except Exception as e:
            logger.debug(linux.get_exception_stacktrace())

        logger.debug('successfully migrated vm[uuid:{0}] to dest url[{1}]'.format(self.uuid, destUrl))
    def deploy_test_agent(self, target=None):
        print('Deploy test agent\n')
        if not self.test_agent_path:
            print('Not find test_agent. Stop deploying test agent.\n')
            return
        
        testagentdir = None
        try:
            def untar_test_agent():
                tmpdir = tempfile.mkdtemp()
                shell.call('tar jxf %s -C %s' % (self.test_agent_path, tmpdir))
                shell.call('cd %s/zstacktestagent/; tar jcf pypi.tar.bz pypi' \
                        % tmpdir)
                return '%s/zstacktestagent' % tmpdir

            def _wait_echo(target_ip):
                try:
                    rspstr = http.json_dump_post(testagent.build_http_path(target_ip, host_plugin.ECHO_PATH))
                except:
                    print('zstack-testagent does not startup, will try again ...')
                    return False
                return True

            testagentdir = untar_test_agent()
            ansible.check_and_install_ansible()

            lib_files = ['testagent/zstacktestagent-1.0.0.tar.gz', \
                    'testagent/zstacklib-1.1.tar.gz' ]

            if not target:
                #default will deploy all test hosts.
                exc_info = []
                for h in self.test_agent_hosts:
                    print('Deploy test agent in host: [%s] \n' % h.managementIp_)
                    ansible_cmd_args = "host=%s \
                            pkg_testagent=zstacktestagent-1.0.0.tar.gz \
                            pkg_zstacklib=zstacklib-1.1.tar.gz \
                            pypi_source_tar=pypi.tar.bz" % \
                            h.managementIp_

                    if ENV_HTTP_PROXY:
                        ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \
                            (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY)

                    ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args

                    thread = threading.Thread(target=ansible.execute_ansible,\
                        args=(h.managementIp_, h.username_, h.password_,\
                        testagentdir, ansible_cmd, lib_files, exc_info))

                    # Wrap up old zstack logs in /var/log/zstack/
                    print('archive test log on host: [%s] \n' % h.managementIp_)
                    try:
                        log.cleanup_log(h.managementIp_, h.username_, h.password_)
                    except Exception as e:
                        print "clean up old testing logs meet execption on management node: %s" % h.managementIp_
                        raise e

                    thread.start()
                #if localhost is not in hosts, should do log archive for zstack
                log.cleanup_local_log()

                self._wait_for_thread_completion('install test agent', 200)

                for h in self.test_agent_hosts:
                    if not linux.wait_callback_success(_wait_echo, h.managementIp_, 5, 0.2, True):
                        raise ActionError('testagent is not start up in 5s on %s, after it is deployed by ansible.' % h.managementIp_)

            else:
                print('Deploy test agent in host: %s \n' % target.managementIp)
                ansible_cmd_args = "host=%s \
                        pkg_testagent=zstacktestagent-1.0.0.tar.gz \
                        pkg_zstacklib=zstacklib-1.1.tar.gz \
                        pypi_source_tar=pypi.tar.bz" % \
                        target.managementIp
                if ENV_HTTP_PROXY:
                    ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \
                        (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY)

                ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args
                ansible.execute_ansible(target.managementIp, target.username, \
                        target.password, testagentdir, ansible_cmd, lib_files)
                if not linux.wait_callback_success(_wait_echo, target.managementIp, 5, 0.2):
                    raise ActionError('testagent is not start up in 5s on %s, after it is deployed by ansible.' % target.managementIp)
                
        finally:
            if testagentdir:
                shell.call('rm -rf %s' % testagentdir)
def test():
    vm1 = test_stub.create_vr_vm('maintain_host_vm1', 'imageName_s',
                                 'l3VlanNetwork2')
    test_obj_dict.add_vm(vm1)

    vm2 = test_stub.create_vr_vm('maintain_host_vm2', 'imageName_s',
                                 'l3VlanNetwork2')
    test_obj_dict.add_vm(vm2)

    vm1.check()
    vm2.check()
    if not test_lib.lib_check_vm_live_migration_cap(
            vm1.vm) or not test_lib.lib_check_vm_live_migration_cap(vm2.vm):
        test_util.test_skip('skip migrate if live migrate not supported')

    current_host1 = test_lib.lib_get_vm_host(vm1.vm)
    current_host2 = test_lib.lib_get_vm_host(vm2.vm)
    conditions = res_ops.gen_query_conditions('clusterUuid', '=',
                                              vm1.vm.clusterUuid)
    conditions = res_ops.gen_query_conditions('state', '=',
                                              host_header.ENABLED, conditions)
    conditions = res_ops.gen_query_conditions('status', '=',
                                              host_header.CONNECTED,
                                              conditions)
    all_hosts = res_ops.query_resource(res_ops.HOST, conditions)
    if len(all_hosts) <= 1:
        test_util.test_fail(
            'Not available host to do maintenance, since there is only %s host'
            % len(all_hosts))
    vr = test_lib.lib_get_all_vrs()
    if len(vr) == 0:
        test_util.test_skip('Skip test if not using vr')
    vr_uuid = vr[0].uuid
    vr_host_uuid = test_lib.lib_get_vm_host(vr[0]).uuid

    for host_n in all_hosts:
        print 'host_n%s' % (host_n.uuid)
        if host_n.uuid != current_host1.uuid:
            if host_n.uuid != current_host2.uuid:
                if host_n.uuid != vr_host_uuid:
                    target_host = host_n
                    print 'target_host_uuid%s' % (target_host.uuid)
                    vm1.migrate(target_host.uuid)
                    vm2.migrate(target_host.uuid)
                    break
    else:
        test_util.test_skip('can not find a host to migrate two host')
    new_host = test_lib.lib_get_vm_host(vm1.vm)
    if new_host.uuid != target_host.uuid:
        test_util.test_fail(
            'VM did not migrate to target [host:] %s, but to [host:] %s' %
            (target_host.uuid, new_host.uuid))

    new_host1 = test_lib.lib_get_vm_host(vm2.vm)
    if new_host1.uuid != target_host.uuid:
        test_util.test_fail(
            'VM did not migrate to target [host:] %s, but to [host:] %s' %
            (target_host.uuid, new_host1.uuid))

    host = test_kvm_host.ZstackTestKvmHost()
    host.set_host(target_host)
    host.maintain()
    #need to update vm's inventory, since they will be changed by maintenace mode
    vm1.update()
    vm2.update()

    ps = test_lib.lib_get_primary_storage_by_vm(vm1.get_vm())
    if ps.type == inventory.LOCAL_STORAGE_TYPE:
        vm1.set_state(vm_header.STOPPED)
        vm2.set_state(vm_header.STOPPED)
    vm1.check()
    vm2.check()
    host.change_state(test_kvm_host.ENABLE_EVENT)
    if not linux.wait_callback_success(is_host_connected,
                                       host.get_host().uuid, 120):
        test_util.test_fail(
            'host status is not changed to connected or host state is not changed to Enabled within 120s'
        )

    if ps.type == inventory.LOCAL_STORAGE_TYPE:
        vm1.start()
        vm2.start()

    vm1.set_state(vm_header.RUNNING)
    vm2.set_state(vm_header.RUNNING)
    vm1.check()
    vm2.check()

    post_host1 = test_lib.lib_get_vm_host(vm1.vm)
    post_host2 = test_lib.lib_get_vm_host(vm2.vm)

    if post_host1.uuid != current_host1.uuid:
        vm1.migrate(current_host1.uuid)

    if post_host2.uuid != current_host2.uuid:
        vm2.migrate(current_host2.uuid)

    vm1.check()
    vm2.check()

    vm1.destroy()
    test_obj_dict.rm_vm(vm1)
    vm2.destroy()
    test_obj_dict.rm_vm(vm2)
    test_util.test_pass('Maintain Host Test Success')
Example #50
0
    def _apply_userdata(self, to):
        # set VIP
        NS_NAME = to.namespaceName
        DHCP_IP = to.dhcpServerIp
        INNER_DEV = bash_errorout("ip netns exec {{NS_NAME}} ip addr | grep -w {{DHCP_IP}} | awk '{print $NF}'").strip(' \t\r\n')
        if not INNER_DEV:
            raise Exception('cannot find device for the DHCP IP[%s]' % DHCP_IP)

        ret = bash_r('ip netns exec {{NS_NAME}} ip addr | grep 169.254.169.254 > /dev/null')
        if ret != 0:
            bash_errorout('ip netns exec {{NS_NAME}} ip addr add 169.254.169.254 dev {{INNER_DEV}}')

        # set ebtables
        BR_NAME = to.bridgeName
        # BR_NAME is "br_%s_%s"
        ETH_NAME = BR_NAME.replace('br_', '', 1).replace('_', '.', 1)
        MAC = bash_errorout("ip netns exec {{NS_NAME}} ip link show {{INNER_DEV}} | grep -w ether | awk '{print $2}'").strip(' \t\r\n')
        CHAIN_NAME="USERDATA-%s" % BR_NAME

        ret = bash_r('ebtables -t nat -L {{CHAIN_NAME}} >/dev/null 2>&1')
        if ret != 0:
            bash_errorout('ebtables -t nat -N {{CHAIN_NAME}}')

        if bash_r('ebtables -t nat -L PREROUTING | grep -- "--logical-in {{BR_NAME}} -j {{CHAIN_NAME}}"') != 0:
            bash_errorout('ebtables -t nat -I PREROUTING --logical-in {{BR_NAME}} -j {{CHAIN_NAME}}')

        # ebtables has a bug that will eliminate 0 in MAC, for example, aa:bb:0c will become aa:bb:c
        RULE = "-p IPv4 --ip-dst 169.254.169.254 -j dnat --to-dst %s --dnat-target ACCEPT" % MAC.replace(":0", ":")
        ret = bash_r('ebtables -t nat -L {{CHAIN_NAME}} | grep -- "{{RULE}}" > /dev/null')
        if ret != 0:
            bash_errorout('ebtables -t nat -I {{CHAIN_NAME}} {{RULE}}')

        ret = bash_r('ebtables -t nat -L {{CHAIN_NAME}} | grep -- "-j RETURN" > /dev/null')
        if ret != 0:
            bash_errorout('ebtables -t nat -A {{CHAIN_NAME}} -j RETURN')

        ret = bash_r('ebtables -L {{CHAIN_NAME}} >/dev/null 2>&1')
        if ret != 0:
            bash_errorout('ebtables -N {{CHAIN_NAME}}')

        ret = bash_r('ebtables -L FORWARD | grep -- "-p ARP --arp-ip-dst 169.254.169.254 -j {{CHAIN_NAME}}" > /dev/null')
        if ret != 0:
            bash_errorout('ebtables -I FORWARD -p ARP --arp-ip-dst 169.254.169.254 -j {{CHAIN_NAME}}')

        ret = bash_r('ebtables -L {{CHAIN_NAME}} | grep -- "-i {{ETH_NAME}} -j DROP" > /dev/null')
        if ret != 0:
            bash_errorout('ebtables -I {{CHAIN_NAME}} -i {{ETH_NAME}} -j DROP')

        ret = bash_r('ebtables -L {{CHAIN_NAME}} | grep -- "-o {{ETH_NAME}} -j DROP" > /dev/null')
        if ret != 0:
            bash_errorout('ebtables -I {{CHAIN_NAME}} -o {{ETH_NAME}} -j DROP')

        ret = bash_r("ebtables-save | grep '\-A {{CHAIN_NAME}} -j RETURN'")
        if ret != 0:
            bash_errorout('ebtables -A {{CHAIN_NAME}} -j RETURN')


        # DNAT port 80
        PORT = to.port
        PORT_CHAIN_NAME = "UD-PORT-%s" % PORT
        # delete old chains not matching our port
        OLD_CHAIN = bash_errorout("iptables-save | awk '/^:UD-PORT-/{print substr($1,2)}'").strip(' \n\r\t')
        if OLD_CHAIN and OLD_CHAIN != CHAIN_NAME:
            ret = bash_r('iptables-save -t nat | grep -- "-j {{OLD_CHAIN}}"')
            if ret == 0:
                bash_r('iptables -t nat -D PREROUTING -j {{OLD_CHAIN}}')

            bash_errorout('iptables -t nat -F {{OLD_CHAIN}}')
            bash_errorout('iptables -t nat -X {{OLD_CHAIN}}')

        ret = bash_r('iptables-save | grep -w ":{{PORT_CHAIN_NAME}}" > /dev/null')
        if ret != 0:
            bash_errorout('iptables -t nat -N {{PORT_CHAIN_NAME}}')

        ret = bash_r('iptables -t nat -L PREROUTING | grep -- "-j {{PORT_CHAIN_NAME}}"')
        if ret != 0:
            bash_errorout('iptables -t nat -I PREROUTING -j {{PORT_CHAIN_NAME}}')

        ret = bash_r('iptables-save -t nat | grep -- "{{PORT_CHAIN_NAME}} -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :{{PORT}}"')
        if ret != 0:
            bash_errorout('iptables -t nat -A {{PORT_CHAIN_NAME}} -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :{{PORT}}')

        conf_folder = os.path.join(self.USERDATA_ROOT, to.namespaceName)
        if not os.path.exists(conf_folder):
            shell.call('mkdir -p %s' % conf_folder)

        conf_path = os.path.join(conf_folder, 'lighttpd.conf')
        http_root = os.path.join(conf_folder, 'html')

        conf = '''\
server.document-root = "{{http_root}}"

server.port = {{port}}
server.bind = "169.254.169.254"
dir-listing.activate = "enable"
index-file.names = ( "index.html" )

server.modules += ( "mod_rewrite" )

$HTTP["remoteip"] =~ "^(.*)$" {
    url.rewrite-once = (
        "^/.*/meta-data/(.+)$" => "../%1/meta-data/$1",
        "^/.*/meta-data$" => "../%1/meta-data",
        "^/.*/meta-data/$" => "../%1/meta-data/",
        "^/.*/user-data$" => "../%1/user-data"
    )
}

mimetype.assign = (
  ".html" => "text/html",
  ".txt" => "text/plain",
  ".jpg" => "image/jpeg",
  ".png" => "image/png"
)'''

        tmpt = Template(conf)
        conf = tmpt.render({
            'http_root': http_root,
            'dhcp_server_ip': to.dhcpServerIp,
            'port': to.port
        })

        if not os.path.exists(conf_path):
            with open(conf_path, 'w') as fd:
                fd.write(conf)
        else:
            with open(conf_path, 'r') as fd:
                current_conf = fd.read()

            if current_conf != conf:
                with open(conf_path, 'w') as fd:
                    fd.write(conf)

        root = os.path.join(http_root, to.vmIp)
        meta_root = os.path.join(root, 'meta-data')
        if not os.path.exists(meta_root):
            shell.call('mkdir -p %s' % meta_root)

        index_file_path = os.path.join(meta_root, 'index.html')
        with open(index_file_path, 'w') as fd:
            fd.write('instance-id')

        instance_id_file_path = os.path.join(meta_root, 'instance-id')
        with open(instance_id_file_path, 'w') as fd:
            fd.write(to.metadata.vmUuid)

        if to.userdata:
            userdata_file_path = os.path.join(root, 'user-data')
            with open(userdata_file_path, 'w') as fd:
                fd.write(to.userdata)

        pid = linux.find_process_by_cmdline([conf_path])
        if not pid:
            shell.call('ip netns exec %s lighttpd -f %s' % (to.namespaceName, conf_path))

            def check(_):
                pid = linux.find_process_by_cmdline([conf_path])
                return pid is not None

            if not linux.wait_callback_success(check, None, 5):
                raise Exception('lighttpd[conf-file:%s] is not running after being started %s seconds' % (conf_path, 5))
Example #51
0
    def _apply_userdata(self, to):
        # set VIP
        NS_NAME = to.namespaceName
        DHCP_IP = to.dhcpServerIp
        INNER_DEV = bash_errorout("ip netns exec {{NS_NAME}} ip addr | grep -w {{DHCP_IP}} | awk '{print $NF}'").strip(' \t\r\n')
        if not INNER_DEV:
            raise Exception('cannot find device for the DHCP IP[%s]' % DHCP_IP)

        ret = bash_r('ip netns exec {{NS_NAME}} ip addr | grep 169.254.169.254 > /dev/null')
        if ret != 0:
            bash_errorout('ip netns exec {{NS_NAME}} ip addr add 169.254.169.254 dev {{INNER_DEV}}')

        # set ebtables
        BR_NAME = to.bridgeName
        # BR_NAME is "br_%s_%s"
        ETH_NAME = BR_NAME.replace('br_', '', 1).replace('_', '.', 1)
        MAC = bash_errorout("ip netns exec {{NS_NAME}} ip link show {{INNER_DEV}} | grep -w ether | awk '{print $2}'").strip(' \t\r\n')
        CHAIN_NAME="USERDATA-%s" % BR_NAME

        ret = bash_r(EBTABLES_CMD + ' -t nat -L {{CHAIN_NAME}} >/dev/null 2>&1')
        if ret != 0:
            bash_errorout(EBTABLES_CMD + ' -t nat -N {{CHAIN_NAME}}')

        if bash_r(EBTABLES_CMD + ' -t nat -L PREROUTING | grep -- "--logical-in {{BR_NAME}} -j {{CHAIN_NAME}}"') != 0:
            bash_errorout(EBTABLES_CMD + ' -t nat -I PREROUTING --logical-in {{BR_NAME}} -j {{CHAIN_NAME}}')

        # ebtables has a bug that will eliminate 0 in MAC, for example, aa:bb:0c will become aa:bb:c
        RULE = "-p IPv4 --ip-dst 169.254.169.254 -j dnat --to-dst %s --dnat-target ACCEPT" % MAC.replace(":0", ":")
        ret = bash_r(EBTABLES_CMD + ' -t nat -L {{CHAIN_NAME}} | grep -- "{{RULE}}" > /dev/null')
        if ret != 0:
            bash_errorout(EBTABLES_CMD + ' -t nat -I {{CHAIN_NAME}} {{RULE}}')

        ret = bash_r(EBTABLES_CMD + ' -t nat -L {{CHAIN_NAME}} | grep -- "-j RETURN" > /dev/null')
        if ret != 0:
            bash_errorout(EBTABLES_CMD + ' -t nat -A {{CHAIN_NAME}} -j RETURN')

        ret = bash_r(EBTABLES_CMD + ' -L {{CHAIN_NAME}} >/dev/null 2>&1')
        if ret != 0:
            bash_errorout(EBTABLES_CMD + ' -N {{CHAIN_NAME}}')

        ret = bash_r(EBTABLES_CMD + ' -L FORWARD | grep -- "-p ARP --arp-ip-dst 169.254.169.254 -j {{CHAIN_NAME}}" > /dev/null')
        if ret != 0:
            bash_errorout(EBTABLES_CMD + ' -I FORWARD -p ARP --arp-ip-dst 169.254.169.254 -j {{CHAIN_NAME}}')

        ret = bash_r(EBTABLES_CMD + ' -L {{CHAIN_NAME}} | grep -- "-i {{ETH_NAME}} -j DROP" > /dev/null')
        if ret != 0:
            bash_errorout(EBTABLES_CMD + ' -I {{CHAIN_NAME}} -i {{ETH_NAME}} -j DROP')

        ret = bash_r(EBTABLES_CMD + ' -L {{CHAIN_NAME}} | grep -- "-o {{ETH_NAME}} -j DROP" > /dev/null')
        if ret != 0:
            bash_errorout(EBTABLES_CMD + ' -I {{CHAIN_NAME}} -o {{ETH_NAME}} -j DROP')

        ret = bash_r("ebtables-save | grep '\-A {{CHAIN_NAME}} -j RETURN'")
        if ret != 0:
            bash_errorout(EBTABLES_CMD + ' -A {{CHAIN_NAME}} -j RETURN')


        # DNAT port 80
        PORT = to.port
        PORT_CHAIN_NAME = "UD-PORT-%s" % PORT
        # delete old chains not matching our port
        OLD_CHAIN = bash_errorout("iptables-save | awk '/^:UD-PORT-/{print substr($1,2)}'").strip(' \n\r\t')
        if OLD_CHAIN and OLD_CHAIN != CHAIN_NAME:
            ret = bash_r('iptables-save -t nat | grep -- "-j {{OLD_CHAIN}}"')
            if ret == 0:
                bash_r('iptables -t nat -D PREROUTING -j {{OLD_CHAIN}}')

            bash_errorout('iptables -t nat -F {{OLD_CHAIN}}')
            bash_errorout('iptables -t nat -X {{OLD_CHAIN}}')

        ret = bash_r('iptables-save | grep -w ":{{PORT_CHAIN_NAME}}" > /dev/null')
        if ret != 0:
            bash_errorout('iptables -t nat -N {{PORT_CHAIN_NAME}}')

        ret = bash_r('iptables -t nat -L PREROUTING | grep -- "-j {{PORT_CHAIN_NAME}}"')
        if ret != 0:
            bash_errorout('iptables -t nat -I PREROUTING -j {{PORT_CHAIN_NAME}}')

        ret = bash_r('iptables-save -t nat | grep -- "{{PORT_CHAIN_NAME}} -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :{{PORT}}"')
        if ret != 0:
            bash_errorout('iptables -t nat -A {{PORT_CHAIN_NAME}} -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :{{PORT}}')

        conf_folder = os.path.join(self.USERDATA_ROOT, to.namespaceName)
        if not os.path.exists(conf_folder):
            shell.call('mkdir -p %s' % conf_folder)

        conf_path = os.path.join(conf_folder, 'lighttpd.conf')
        http_root = os.path.join(conf_folder, 'html')

        conf = '''\
server.document-root = "{{http_root}}"

server.port = {{port}}
server.bind = "169.254.169.254"
dir-listing.activate = "enable"
index-file.names = ( "index.html" )

server.modules += ( "mod_rewrite" )

$HTTP["remoteip"] =~ "^(.*)$" {
    url.rewrite-once = (
        "^/.*/meta-data/(.+)$" => "../%1/meta-data/$1",
        "^/.*/meta-data$" => "../%1/meta-data",
        "^/.*/meta-data/$" => "../%1/meta-data/",
        "^/.*/user-data$" => "../%1/user-data"
    )
}

mimetype.assign = (
  ".html" => "text/html",
  ".txt" => "text/plain",
  ".jpg" => "image/jpeg",
  ".png" => "image/png"
)'''

        tmpt = Template(conf)
        conf = tmpt.render({
            'http_root': http_root,
            'dhcp_server_ip': to.dhcpServerIp,
            'port': to.port
        })

        if not os.path.exists(conf_path):
            with open(conf_path, 'w') as fd:
                fd.write(conf)
        else:
            with open(conf_path, 'r') as fd:
                current_conf = fd.read()

            if current_conf != conf:
                with open(conf_path, 'w') as fd:
                    fd.write(conf)

        root = os.path.join(http_root, to.vmIp)
        meta_root = os.path.join(root, 'meta-data')
        if not os.path.exists(meta_root):
            shell.call('mkdir -p %s' % meta_root)

        index_file_path = os.path.join(meta_root, 'index.html')
        with open(index_file_path, 'w') as fd:
            fd.write('instance-id')

        instance_id_file_path = os.path.join(meta_root, 'instance-id')
        with open(instance_id_file_path, 'w') as fd:
            fd.write(to.metadata.vmUuid)

        if to.userdata:
            userdata_file_path = os.path.join(root, 'user-data')
            with open(userdata_file_path, 'w') as fd:
                fd.write(to.userdata)

        pid = linux.find_process_by_cmdline([conf_path])
        if not pid:
            shell.call('ip netns exec %s lighttpd -f %s' % (to.namespaceName, conf_path))

            def check(_):
                pid = linux.find_process_by_cmdline([conf_path])
                return pid is not None

            if not linux.wait_callback_success(check, None, 5):
                raise Exception('lighttpd[conf-file:%s] is not running after being started %s seconds' % (conf_path, 5))
Example #52
0
    def deploy_test_agent(self, target=None):
        print('Deploy test agent\n')
        if not self.test_agent_path:
            print('Not find test_agent. Stop deploying test agent.\n')
            return

        testagentdir = None
        try:

            def untar_test_agent():
                tmpdir = tempfile.mkdtemp()
                shell.call('tar jxf %s -C %s' % (self.test_agent_path, tmpdir))
                shell.call('cd %s/zstacktestagent/; tar jcf pypi.tar.bz pypi' \
                        % tmpdir)
                return '%s/zstacktestagent' % tmpdir

            def _wait_echo(target_ip):
                try:
                    rspstr = http.json_dump_post(
                        testagent.build_http_path(target_ip,
                                                  host_plugin.ECHO_PATH))
                except:
                    print(
                        'zstack-testagent does not startup, will try again ...'
                    )
                    return False
                return True

            def find_zstacklib(search_path):
                match = glob.glob(
                    os.path.join(search_path, 'zstacklib-*.tar.gz'))
                try:
                    return os.path.basename(match[0])
                except:
                    return 'zstacklib-1.6.tar.gz'

            testagentdir = untar_test_agent()
            pkg_zstacklib = find_zstacklib(testagentdir)
            ansible.check_and_install_ansible()

            lib_files = ['testagent/zstacktestagent-1.0.0.tar.gz', \
                    'testagent/%s' % (pkg_zstacklib) ]

            if not target:
                #default will deploy all test hosts.
                exc_info = []
                for h in self.test_agent_hosts:
                    print('Deploy test agent in host: [%s] \n' %
                          h.managementIp_)
                    if h.username_ != 'root':
                        ansible_become_args = "ansible_become=yes become_user=root ansible_become_pass=%s" % (
                            h.password_)
                    else:
                        ansible_become_args = ""

                    if hasattr(h, 'port_'):
                        ansible_port_args = "ansible_ssh_port=%s" % (h.port_)
                    else:
                        ansible_port_args = ""

                    ansible_cmd_args = "host=%s \
                            ansible_ssh_user=%s \
                            ansible_ssh_pass=%s \
                            %s \
                            %s \
                            pkg_testagent=zstacktestagent-1.0.0.tar.gz \
                            pkg_zstacklib=%s \
                            pypi_source_tar=pypi.tar.bz"                                                         % \
                            (h.managementIp_, h.username_, h.password_, ansible_become_args, ansible_port_args, pkg_zstacklib)

                    if ENV_HTTP_PROXY:
                        ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \
                            (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY)

                    ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args

                    if hasattr(h, 'port_'):
                        thread = threading.Thread(target=ansible.execute_ansible,\
                            args=(h.managementIp_, h.username_, h.password_,\
                            testagentdir, ansible_cmd, lib_files, exc_info, h.port_))
                    else:
                        thread = threading.Thread(target=ansible.execute_ansible,\
                            args=(h.managementIp_, h.username_, h.password_,\
                            testagentdir, ansible_cmd, lib_files, exc_info))

                    # Wrap up old zstack logs in /var/log/zstack/
                    #print('archive test log on host: [%s] \n' % h.managementIp_)
                    #try:
                    #    if hasattr(h, 'port_'):
                    #        log.cleanup_log(h.managementIp_, h.username_, h.password_, h.port_)
                    #    else:
                    #        log.cleanup_log(h.managementIp_, h.username_, h.password_)
                    #except Exception as e:
                    #    print "clean up old testing logs meet execption on management node: %s" % h.managementIp_
                    #    raise e

                    thread.start()
                #if localhost is not in hosts, should do log archive for zstack
                log.cleanup_local_log()

                self._wait_for_thread_completion('install test agent', 200)

                for h in self.test_agent_hosts:
                    if not linux.wait_callback_success(
                            _wait_echo, h.managementIp_, 5, 0.2, True):
                        raise ActionError(
                            'testagent is not start up in 5s on %s, after it is deployed by ansible.'
                            % h.managementIp_)

            else:
                print('Deploy test agent in host: %s \n' % target.managementIp)
                ansible_cmd_args = "host=%s \
                        pkg_testagent=zstacktestagent-1.0.0.tar.gz \
                        pkg_zstacklib=%s \
                        pypi_source_tar=pypi.tar.bz"                                                     % \
                        (target.managementIp, pkg_zstacklib)
                if ENV_HTTP_PROXY:
                    ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \
                        (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY)

                ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args
                ansible.execute_ansible(target.managementIp, target.username, \
                        target.password, testagentdir, ansible_cmd, lib_files)
                if not linux.wait_callback_success(
                        _wait_echo, target.managementIp, 5, 0.2):
                    raise ActionError(
                        'testagent is not start up in 5s on %s, after it is deployed by ansible.'
                        % target.managementIp)

        finally:
            if testagentdir:
                shell.call('rm -rf %s' % testagentdir)
Example #53
0
    def _apply_userdata(self, to):
        set_vip_cmd = '''
NS_NAME={{ns_name}}
DHCP_IP={{dhcp_ip}}

NS="ip netns exec $NS_NAME"

exit_on_error() {
    if [ $? -ne 0 ]; then
        echo "error on line $1"
        exit 1
    fi
}

eval $NS ip addr | grep 169.254.169.254 > /dev/null
if [ $? -eq 0 ]; then
    exit 0
fi

eth=`eval $NS ip addr | grep -w $DHCP_IP | awk '{print $NF}'`
exit_on_error $LINENO
if [ x$eth == "x" ]; then
    echo "cannot find device for the DHCP IP $DHCP_IP"
    exit 1
fi

eval $NS ip addr add 169.254.169.254 dev $eth
exit_on_error $LINENO

exit 0
'''
        tmpt = Template(set_vip_cmd)
        set_vip_cmd = tmpt.render({
            'ns_name': to.bridgeName,
            'dhcp_ip': to.dhcpServerIp,
        })
        shell.call(set_vip_cmd)

        set_ebtables = '''
BR_NAME={{br_name}}
NS_NAME={{ns_name}}

NS="ip netns exec $NS_NAME"

exit_on_error() {
    if [ $? -ne 0 ]; then
        echo "error on line $1"
        exit 1
    fi
}

eth=`eval $NS ip addr | grep 169.254.169.254 | awk '{print $NF}'`
mac=`eval $NS ip link show $eth | grep -w ether | awk '{print $2}'`
exit_on_error $LINENO

CHAIN_NAME="USERDATA-$BR_NAME"

ebtables-save | grep -w ":$CHAIN_NAME" > /dev/null
if [ $? -ne 0 ]; then
    ebtables -t nat -N $CHAIN_NAME
    exit_on_error $LINENO
    ebtables -t nat -I PREROUTING --logical-in $BR_NAME -j $CHAIN_NAME
    exit_on_error $LINENO
fi

rule="$CHAIN_NAME -p IPv4 --ip-dst 169.254.169.254 -j dnat --to-dst $mac --dnat-target ACCEPT"
ebtables-save | grep -- "$rule" > /dev/null
if [ $? -ne 0 ]; then
    ebtables -t nat -I $rule
    exit_on_error $LINENO
fi

exit 0
'''
        tmpt = Template(set_ebtables)
        set_ebtables = tmpt.render({
            'br_name': to.bridgeName,
            'ns_name': to.bridgeName,
        })
        shell.call(set_ebtables)

        dnat_port_cmd = '''
PORT={{port}}
CHAIN_NAME="UD-PORT-$PORT"

exit_on_error() {
    if [ $? -ne 0 ]; then
        echo "error on line $1"
        exit 1
    fi
}

# delete old chains not matching our port
old_chain=`iptables-save | awk '/^:UD-PORT-/{print substr($1,2)}'`
if [ x"$old_chain" != "x" -a $CHAIN_NAME != $old_chain ]; then
    iptables -t nat -F $old_chain
    exit_on_error $LINENO
    iptables -t nat -X $old_chain
    exit_on_error $LINENO
fi

iptables-save | grep -w ":$CHAIN_NAME" > /dev/null
if [ $? -ne 0 ]; then
   iptables -t nat -N $CHAIN_NAME
   exit_on_error $LINENO
   iptables -t nat -I PREROUTING -j $CHAIN_NAME
   exit_on_error $LINENO
fi

iptables-save -t nat | grep -- "$CHAIN_NAME -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :$PORT" > /dev/null || iptables -t nat -A $CHAIN_NAME -d 169.254.169.254/32 -p tcp -j DNAT --to-destination :$PORT
exit_on_error $LINENO

exit 0
'''

        tmpt = Template(dnat_port_cmd)
        dnat_port_cmd = tmpt.render({
            'port': to.port
        })
        shell.call(dnat_port_cmd)

        conf_folder = os.path.join(self.USERDATA_ROOT, to.bridgeName)
        if not os.path.exists(conf_folder):
            shell.call('mkdir -p %s' % conf_folder)

        conf_path = os.path.join(conf_folder, 'lighttpd.conf')
        http_root = os.path.join(conf_folder, 'html')

        conf = '''\
server.document-root = "{{http_root}}"

server.port = {{port}}
server.bind = "169.254.169.254"
dir-listing.activate = "enable"
index-file.names = ( "index.html" )

server.modules += ( "mod_rewrite" )

$HTTP["remoteip"] =~ "^(.*)$" {
    url.rewrite-once = (
        "^/.*/meta-data/(.+)$" => "../%1/meta-data/$1",
        "^/.*/meta-data$" => "../%1/meta-data",
        "^/.*/meta-data/$" => "../%1/meta-data/",
        "^/.*/user-data$" => "../%1/user-data"
    )
}

mimetype.assign = (
  ".html" => "text/html",
  ".txt" => "text/plain",
  ".jpg" => "image/jpeg",
  ".png" => "image/png"
)'''

        tmpt = Template(conf)
        conf = tmpt.render({
            'http_root': http_root,
            'dhcp_server_ip': to.dhcpServerIp,
            'port': to.port
        })

        if not os.path.exists(conf_path):
            with open(conf_path, 'w') as fd:
                fd.write(conf)
        else:
            with open(conf_path, 'r') as fd:
                current_conf = fd.read()

            if current_conf != conf:
                with open(conf_path, 'w') as fd:
                    fd.write(conf)

        root = os.path.join(http_root, to.vmIp)
        meta_root = os.path.join(root, 'meta-data')
        if not os.path.exists(meta_root):
            shell.call('mkdir -p %s' % meta_root)

        index_file_path = os.path.join(meta_root, 'index.html')
        with open(index_file_path, 'w') as fd:
            fd.write('instance-id')

        instance_id_file_path = os.path.join(meta_root, 'instance-id')
        with open(instance_id_file_path, 'w') as fd:
            fd.write(to.metadata.vmUuid)

        if to.userdata:
            userdata_file_path = os.path.join(root, 'user-data')
            with open(userdata_file_path, 'w') as fd:
                fd.write(to.userdata)

        pid = linux.find_process_by_cmdline([conf_path])
        if not pid:
            shell.call('ip netns exec %s lighttpd -f %s' % (to.bridgeName, conf_path))

            def check(_):
                pid = linux.find_process_by_cmdline([conf_path])
                return pid is not None

            if not linux.wait_callback_success(check, None, 5):
                raise Exception('lighttpd[conf-file:%s] is not running after being started %s seconds' % (conf_path, 5))
def test():
    vm1 = test_stub.create_vr_vm('maintain_host_vm1', 'imageName_s', 'l3VlanNetwork2')
    test_obj_dict.add_vm(vm1)

    vm2 = test_stub.create_vr_vm('maintain_host_vm2', 'imageName_s', 'l3VlanNetwork2')
    test_obj_dict.add_vm(vm2)

    vm1.check()
    vm2.check()
    if not test_lib.lib_check_vm_live_migration_cap(vm1.vm) or not test_lib.lib_check_vm_live_migration_cap(vm2.vm):
        test_util.test_skip('skip migrate if live migrate not supported')

    test_util.test_dsc('Create volume and check')
    disk_offering = test_lib.lib_get_disk_offering_by_name(os.environ.get('smallDiskOfferingName'))
    volume_creation_option = test_util.VolumeOption()
    volume_creation_option.set_disk_offering_uuid(disk_offering.uuid)

    volume = test_stub.create_volume(volume_creation_option)
    test_obj_dict.add_volume(volume)

    test_util.test_dsc('Attach volume and check')
    volume.attach(vm1)
    volume.check()

    current_host1 = test_lib.lib_get_vm_host(vm1.vm)
    conditions = res_ops.gen_query_conditions('clusterUuid', '=', vm1.vm.clusterUuid)
    conditions = res_ops.gen_query_conditions('state', '=', host_header.ENABLED, conditions)
    conditions = res_ops.gen_query_conditions('status', '=', host_header.CONNECTED, conditions)
    all_hosts = res_ops.query_resource(res_ops.HOST, conditions)
    if len(all_hosts) <= 1:
        test_util.test_fail('Not available host to do maintenance, since there is only %s host' % len(all_hosts))

    target_host = random.choice(all_hosts)
    if current_host1.uuid != target_host.uuid:
        vm1.migrate(target_host.uuid)

    current_host2 = test_lib.lib_get_vm_host(vm2.vm)
    if current_host2.uuid != target_host.uuid:
        vm2.migrate(target_host.uuid)

    new_host = test_lib.lib_get_vm_host(vm1.vm)
    if new_host.uuid != target_host.uuid:
        test_util.test_fail('VM did not migrate to target [host:] %s, but to [host:] %s' % (target_host.uuid, new_host.uuid))

    volume.check()

    host = test_kvm_host.ZstackTestKvmHost()
    host.set_host(target_host)

    host.maintain()

    #need to update vm's inventory, since they will be changed by maintenace mode
    vm1.update()
    vm2.update()

    vm1.check()
    vm2.check()
    volume.check()

    host.change_state(test_kvm_host.ENABLE_EVENT)
    if not linux.wait_callback_success(is_host_connected, host.get_host().uuid, 120):
        test_util.test_fail('host status is not changed to connected, after changing its state to Enable')

    vm1.migrate(target_host.uuid)
    vm2.migrate(target_host.uuid)

    vm1.check()
    vm2.check()
    volume.check()

    vm1.destroy()
    test_obj_dict.rm_vm(vm1)
    vm2.destroy()
    test_obj_dict.rm_vm(vm2)
    volume.delete()
    test_obj_dict.rm_volume(volume)
    test_util.test_pass('Maintain Host Test Success')
def test():
    global session_uuid
    session_uuid = acc_ops.login_as_admin()
    l3_1_name = os.environ.get('l3VlanNetworkName1')
    l3_2_name = os.environ.get('l3VlanDNATNetworkName')
    l3_3_name = os.environ.get('l3VlanNetworkName3')
    #l3_4_name = os.environ.get('l3VlanNetworkName5')
    l3_1 = test_lib.lib_get_l3_by_name(l3_1_name)
    l3_2 = test_lib.lib_get_l3_by_name(l3_2_name)
    l3_3 = test_lib.lib_get_l3_by_name(l3_3_name)
    #l3_4 = test_lib.lib_get_l3_by_name(l3_4_name)

    #create 4 VRs.
    vrs = test_lib.lib_find_vr_by_l3_uuid(l3_1.uuid)
    if not vrs:
        vm = test_stub.create_vlan_vm(l3_name=l3_1_name)
        vm.destroy()
        vr1 = test_lib.lib_find_vr_by_l3_uuid(l3_1.uuid)[0]
    else:
        vr1 = vrs[0]

    vrs = test_lib.lib_find_vr_by_l3_uuid(l3_2.uuid)
    if not vrs:
        vm = test_stub.create_vlan_vm(l3_name=l3_2_name)
        vm.destroy()
        vr2 = test_lib.lib_find_vr_by_l3_uuid(l3_2.uuid)[0]
    else:
        vr2 = vrs[0]

    vrs = test_lib.lib_find_vr_by_l3_uuid(l3_3.uuid)
    if not vrs:
        vm = test_stub.create_vlan_vm(l3_name=l3_3_name)
        vm.destroy()
        vr3 = test_lib.lib_find_vr_by_l3_uuid(l3_3.uuid)[0]
    else:
        vr3 = vrs[0]

    #vrs = test_lib.lib_find_vr_by_l3_uuid(l3_4.uuid)
    #if not vrs:
    #    vm = test_stub.create_vlan_vm(l3_name=l3_4_name)
    #    vm.destroy()
    #    vr4 = test_lib.lib_find_vr_by_l3_uuid(l3_4.uuid)[0]
    #else:
    #    vr4 = vrs[0]

    vrs = [vr1, vr2, vr3]
    #vrs = [vr1, vr2, vr3, vr4]
    for vr in vrs:
        thread = threading.Thread(target=stop_vm, args=(vr.uuid,))
        thread.start()

    while threading.activeCount() > 1:
        check_exception()
        time.sleep(0.1)

    for vr in vrs:
        if not linux.wait_callback_success(check_status, (vr.uuid, 'Stopped'), 10):
            test_util.test_fail('VM: %s is not stopped, after waiting for extra 10s' % vr.uuid)

    check_exception()
    for vr in vrs:
        thread = threading.Thread(target=start_vm, args=(vr.uuid,))
        thread.start()

    time.sleep(1)
    acc_ops.logout(session_uuid)
    while threading.activeCount() > 1:
        check_exception()
        time.sleep(0.1)

    check_exception()
    test_util.test_pass('Test start VRs simultaneously success')
    def deploy_test_agent(self, target=None):
        print('Deploy test agent\n')
        if not self.test_agent_path:
            print('Not find test_agent. Stop deploying test agent.\n')
            return

        testagentdir = None
        try:

            def untar_test_agent():
                tmpdir = tempfile.mkdtemp()
                shell.call('tar xzf %s -C %s' % (self.test_agent_path, tmpdir))
                return '%s/zstacktestagent' % tmpdir

            def _wait_echo(target_ip):
                try:
                    rspstr = http.json_dump_post(
                        testagent.build_http_path(target_ip,
                                                  host_plugin.ECHO_PATH))
                except:
                    print(
                        'zstack-testagent does not startup, will try again ...'
                    )
                    return False
                return True

            testagentdir = untar_test_agent()
            ansible.check_and_install_ansible()

            lib_files = ['testagent/zstacktestagent-0.1.0.tar.gz', \
                    'testagent/zstacklib-0.1.0.tar.gz' ]

            if not target:
                #default will deploy all test hosts.
                exc_info = []
                for h in self.test_agent_hosts:
                    print('Deploy test agent in host: [%s] \n' %
                          h.managementIp_)
                    ansible_cmd_args = "host=%s \
                            pkg_testagent=zstacktestagent-0.1.0.tar.gz \
                            pkg_zstacklib=zstacklib-0.6.tar.gz \
                            pypi_url=%s"                                         % \
                            (h.managementIp_, ENV_PYPI_URL)
                    if ENV_HTTP_PROXY:
                        ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \
                            (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY)

                    ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args

                    thread = threading.Thread(target=ansible.execute_ansible,\
                        args=(h.managementIp_, h.username_, h.password_,\
                        testagentdir, ansible_cmd, lib_files, exc_info))

                    # Wrap up old zstack logs in /var/log/zstack/
                    print('archive test log on host: [%s] \n' %
                          h.managementIp_)
                    try:
                        log.cleanup_log(h.managementIp_, h.username_,
                                        h.password_)
                    except Exception as e:
                        print "clean up old testing logs meet execption on management node: %s" % h.managementIp_
                        raise e

                    thread.start()
                #if localhost is not in hosts, should do log archive for zstack
                log.cleanup_local_log()

                self._wait_for_thread_completion('install test agent', 200)

                for h in self.test_agent_hosts:
                    if not linux.wait_callback_success(
                            _wait_echo, h.managementIp_, 5, 0.2, True):
                        raise ActionError(
                            'testagent is not start up in 5s on %s, after it is deployed by ansible.'
                            % h.managementIp_)

            else:
                print('Deploy test agent in host: %s \n' % target.managementIp)
                ansible_cmd_args = "host=%s \
                        pkg_testagent=zstacktestagent-0.1.0.tar.gz \
                        pkg_zstacklib=zstacklib-0.6.tar.gz \
                        pypi_url=%s"                                     % \
                        (target.managementIp, ENV_PYPI_URL)
                if ENV_HTTP_PROXY:
                    ansible_cmd_args = "%s http_proxy=%s https_proxy=%s" % \
                        (ansible_cmd_args, ENV_HTTP_PROXY, ENV_HTTPS_PROXY)

                ansible_cmd = "testagent.yaml -e '%s'" % ansible_cmd_args
                ansible.execute_ansible(target.managementIp, target.username, \
                        target.password, testagentdir, ansible_cmd, lib_files)
                if not linux.wait_callback_success(
                        _wait_echo, target.managementIp, 5, 0.2):
                    raise ActionError(
                        'testagent is not start up in 5s on %s, after it is deployed by ansible.'
                        % target.managementIp)

        finally:
            if testagentdir:
                shell.call('rm -rf %s' % testagentdir)