def _copy_sshkey_from_node(self): node = self.nodes[0] if not node.dockerImage__: print 'Copy sshkey from mn node' #consider some zstack-server is running in vm, the server # startup speed is slow. Increase timeout to 180s. private_key_path = '%s/webapps/zstack/WEB-INF/classes/ansible/rsaKeys/id_rsa' % self.catalina_home public_key_path = '%s/webapps/zstack/WEB-INF/classes/ansible/rsaKeys/id_rsa.pub' % self.catalina_home if linux.is_ip_existing(node.ip_): cmd = 'scp %s /root/.ssh/id_rsa' % (private_key_path) shell.call(cmd) cmd = 'scp %s /root/.ssh/id_rsa.pub' % (public_key_path) shell.call(cmd) elif not linux.is_ip_existing(node.ip_): import zstackwoodpecker.test_lib as test_lib if test_lib.lib_wait_target_up(node.ip_, '22', 120): node_ip = node.ip_ else: node_ip = os.environ['zstackHaVip'] ssh.scp_file(private_key_path, "/root/.ssh/id_rsa", node_ip, node.username_, node.password_) ssh.scp_file(public_key_path, "/root/.ssh/id_rsa.pub", node_ip, node.username_, node.password_) cmd = 'scp %s /root/.ssh/id_rsa' % (private_key_path) shell.call(cmd) cmd = 'scp %s /root/.ssh/id_rsa.pub' % (public_key_path) shell.call(cmd) else: cmd = 'scp %s /root/.ssh/id_rsa' % (private_key_path) shell.call(cmd) cmd = 'scp %s /root/.ssh/id_rsa.pub' % (public_key_path) shell.call(cmd)
def _upgrade_management_nodes(self): for node in self.nodes: if not linux.is_ip_existing(node.ip_): cmd = 'zstack-ctl upgrade_management_node --host=%s' % node.ip_ thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) thread.start() self._wait_for_thread_completion('upgrade remote management node', 600)
def _change_node_ip(self): for node in self.nodes: cmd = 'zstack-ctl change_ip --ip=%s --mysql_root_password=%s' % (node.ip_, os.environ.get('DBAdminPassword')) if not linux.is_ip_existing(node.ip_): ssh.execute(cmd, node.ip_, node.username_, node.password_) else: thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) thread.start() self._wait_for_thread_completion('change management node ip', 120) cmd = 'zstack-ctl configure InfluxDB.startupTimeout=1200' if not linux.is_ip_existing(node.ip_): ssh.execute(cmd, node.ip_, node.username_, node.password_) else: thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) thread.start() self._wait_for_thread_completion('InfluxDB startup time setting to 10 mins', 120)
def _change_node_ip(self): for node in self.nodes: cmd = 'zstack-ctl change_ip --ip=%s --mysql_root_password=%s' % (node.ip_, os.environ.get('DBAdminPassword')) if not linux.is_ip_existing(node.ip_): ssh.execute(cmd, node.ip_, node.username_, node.password_) else: thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) thread.start() self._wait_for_thread_completion('change management node ip', 60)
def _change_node_ip(self): for node in self.nodes: cmd = 'zstack-ctl change_ip --ip=%s' % (node.ip_) if not linux.is_ip_existing(node.ip_): ssh.execute(cmd, node.ip_, node.username_, node.password_) else: thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) thread.start() self._wait_for_thread_completion('change management node ip', 60)
def _install_management_nodes(self): for node in self.nodes: if not linux.is_ip_existing(node.ip_): cmd = 'zstack-ctl install_management_node --force-reinstall \ --host=%s' % node.ip_ thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) thread.start() else: print "node: %s has been installed zstack" % node.ip_ self._wait_for_thread_completion('install remote management node', 600)
def _install_management_nodes(self): for node in self.nodes: if not linux.is_ip_existing(node.ip_) and os.environ.get('ZSTACK_ALREADY_INSTALLED') != "yes": cmd = 'zstack-ctl install_management_node --force-reinstall \ --host=%s' % node.ip_ thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) thread.start() else: print "node: %s has been installed zstack" % node.ip_ self._wait_for_thread_completion('install remote management node', 600)
def _set_extra_node_config(self): node1 = self.nodes[0] for node in self.nodes: cmd = 'zstack-ctl configure --duplicate-to-remote=%s; zstack-ctl configure --host=%s management.server.ip=%s' % \ (node.ip_, node.ip_, node.ip_) if not linux.is_ip_existing(node.ip_): ssh.execute(cmd, node1.ip_, node.username_, node.password_) else: thread = threading.Thread(target=shell_cmd_thread, args=(cmd,)) thread.start() self._wait_for_thread_completion('set extra management node config', 60)
def _start_multi_nodes(self, restart=False): nodes = [] threads = [] for node in self.nodes: #The reserved node is used by test cases. if not restart and node.reserve__: continue if not node.dockerImage__: print 'Deploy node in hosts' #consider some zstack-server is running in vm, the server # startup speed is slow. Increase timeout to 180s. if linux.is_ip_existing(node.ip_): cmd = 'zstack-ctl stop; zstack-ctl start' else: cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % ( node.ip_, node.ip_) thread = threading.Thread(target=shell_cmd_thread, args=( cmd, True, )) threads.append(thread) else: print 'Deploy node in docker' docker_node = DockerNode(self) docker_node.set_docker_image(node.dockerImage__) docker_node.set_node_ip(node.ip__) docker_node.prepare_node() nodes.append(docker_node) thread = threading.Thread(target=docker_node.start_node) threads.append(thread) for thread in threads: thread.start() self._wait_for_thread_completion('start management node', 200) if node_exception: print 'node start meets exception:' info1 = node_exception[0][1] info2 = node_exception[0][2] raise info1, None, info2 current_time = time.time() #largest timeout time for multi nodes startup is 300s timeout_time = current_time + 300 for node in self.nodes: #The reserved node is used by test cases. if node.reserve__: continue new_time = time.time() if new_time >= timeout_time: new_timeout = 1 else: new_timeout = timeout_time - new_time if not linux.wait_callback_success(\ node_ops.is_management_node_start, \ node.ip_, timeout=new_timeout, interval=0.5): raise ActionError('multi node does not startup on host: %s' \ % node.ip_) zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home shell.call(cmd)
def _set_and_validate_config(self): basic_config = self.config.basicConfig deploy_config = self.config.deployerConfig self.zstack_pkg = self._full_path(basic_config.zstackPkg.text_) self.zstack_install_script = \ self._full_path(basic_config.zstackInstallScript.text_) if not os.path.exists(self.zstack_pkg): raise ActionError('unable to find %s for ZStack binary' \ % self.zstack_pkg) if basic_config.hasattr('zstackInstallPath'): self.install_path = \ self._full_path(basic_config.zstackInstallPath.text_) else: raise ActionError(\ 'need to set config.deployerConfig.zstackInstallPath in : %s' % self.deploy_config_path) #set ZSTACK_HOME, which will be used by zstack-ctl os.environ['ZSTACK_HOME'] = '%s/apache-tomcat/webapps/zstack/' % \ self.install_path if basic_config.hasattr('testAgent'): self.test_agent_path = self._full_path( basic_config.testAgent.text_) linux.error_if_path_missing(self.test_agent_path) for zone in deploy_config.zones.get_child_node_as_list('zone'): for cluster in zone.clusters.get_child_node_as_list('cluster'): if cluster.hypervisorType_ == inventory.KVM_HYPERVISOR_TYPE: for h in cluster.hosts.get_child_node_as_list('host'): h.managementIp_ h.username_ h.password_ # will raise exception if one of above not specified in xml filea. self.test_agent_hosts.append(h) else: if xmlobject.has_element(basic_config, 'testAgentHost'): raise ActionError( '<tesgAgent> is missing while <testAgentHost> presents') self.catalina_home = self.install_path + '/apache-tomcat' self.wait_for_start_timeout = basic_config.get( 'managementServerStartTimeout') if not self.wait_for_start_timeout: self.wait_for_start_timeout = 120 else: self.wait_for_start_timeout = int(self.wait_for_start_timeout) if hasattr(basic_config, 'rabbitmq'): self.rabbitmq_server = basic_config.rabbitmq.get( 'server', 'localhost') self.rabbitmq_server_root_passwd = basic_config.rabbitmq.get( 'password', '') if not self.rabbitmq_server_root_passwd: print('!!!WARN! Rabbitmq server root password are not set!') else: raise ActionError( 'need to set config.basicConfig.rabbitmq.server in: %s' % self.deploy_config_path) if hasattr(basic_config, 'db'): self.need_deploy_db = True self.db_server = basic_config.db.get('server', 'localhost') self.db_username = basic_config.db.get('username', 'zstack') self.db_password = basic_config.db.get('password', '') self.db_admin_username = basic_config.db.get('admin', 'root') self.db_admin_password = basic_config.db.get('adminPassword', '') self.db_server_root_password = basic_config.db.get( 'server_root_password', '') if not self.db_server_root_password: print('!!!WARN! Database server root password are not set!') self.db_port = basic_config.db.get('port', '3306') if basic_config.has_element('zstackProperties'): if basic_config.zstackProperties.text_: self.zstack_properties = self._full_path( basic_config.zstackProperties.text_) if not os.path.exists(self.zstack_properties): print( 'unable to find zstackProperties at %s, use \ default one' % self.zstack_properties) self.zstack_properties = None if basic_config.has_element('zstackHaVip'): self.zstack_ha_vip = basic_config.zstackHaVip.text_ else: self.zstack_ha_vip = None if basic_config.has_element('zstackManagementIp'): self.zstack_management_ip = basic_config.zstackManagementIp.text_ else: self.zstack_management_ip = None os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = '' if deploy_config.has_element('nodes') \ and deploy_config.nodes.has_element('node'): for node in deploy_config.nodes.get_child_node_as_list('node'): node.ip_ self.nodes.append(node) if linux.is_ip_existing(node.ip_): os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = node.ip_ elif not os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP'): os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = node.ip_ else: raise ActionError( 'deploy.xml setting error. No deployerConfig.nodes.node is found. ' ) if self.zstack_ha_vip != None: os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = self.zstack_ha_vip if self.zstack_management_ip != None: os.environ[ 'ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = self.zstack_management_ip if not os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP'): raise ActionError( 'deploy.xml setting error. No deployerConfig.nodes.node.ip is defined. ' )
def _start_multi_nodes(self, restart=False): nodes = [] threads = [] node1 = self.nodes[0] for node in self.nodes: #The reserved node is used by test cases. if not restart and node.reserve__: continue if not node.dockerImage__: print 'Deploy node in hosts' #consider some zstack-server is running in vm, the server # startup speed is slow. Increase timeout to 180s. if linux.is_ip_existing(node.ip_): if os.environ.get('ZSTACK_SIMULATOR') == "yes": cmd = 'zstack-ctl stop; zstack-ctl configure unitTestOn=true; nohup zstack-ctl start_node --simulator -DredeployDB=true' else: cmd = 'zstack-ctl stop; nohup zstack-ctl start' thread = threading.Thread(target=shell_cmd_thread, args=( cmd, True, )) elif not linux.is_ip_existing(node1.ip_): # when first node1 ip is not local, it usualy means woodpecker is running on hosts other than MN cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % ( node.ip_, node.ip_) thread = threading.Thread(target=ssh.execute, args=( cmd, node1.ip_, node1.username_, node1.password_, )) else: cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % ( node.ip_, node.ip_) thread = threading.Thread(target=shell_cmd_thread, args=( cmd, True, )) threads.append(thread) else: print 'Deploy node in docker' docker_node = DockerNode(self) docker_node.set_docker_image(node.dockerImage__) docker_node.set_node_ip(node.ip__) docker_node.prepare_node() nodes.append(docker_node) thread = threading.Thread(target=docker_node.start_node) threads.append(thread) for thread in threads: thread.start() self._wait_for_thread_completion('start management node', 200) time.sleep(10) if node_exception: print 'node start meets exception:' info1 = node_exception[0][1] info2 = node_exception[0][2] raise info1, None, info2 current_time = time.time() #largest timeout time for multi nodes startup is 300s timeout_time = current_time + 300 for node in self.nodes: #The reserved node is used by test cases. if node.reserve__: continue new_time = time.time() if new_time >= timeout_time: new_timeout = 1 else: new_timeout = timeout_time - new_time if not linux.wait_callback_success(\ node_ops.is_management_node_start, \ node.ip_, timeout=new_timeout, interval=0.5): raise ActionError('multi node does not startup on host: %s' \ % node.ip_) zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home shell.call(cmd)
def _set_and_validate_config(self): basic_config = self.config.basicConfig deploy_config = self.config.deployerConfig self.zstack_pkg = self._full_path(basic_config.zstackPkg.text_) self.zstack_install_script = \ self._full_path(basic_config.zstackInstallScript.text_) if not os.path.exists(self.zstack_pkg): raise ActionError('unable to find %s for ZStack binary' \ % self.zstack_pkg) if basic_config.hasattr('zstackInstallPath'): self.install_path = \ self._full_path(basic_config.zstackInstallPath.text_) else: raise ActionError(\ 'need to set config.deployerConfig.zstackInstallPath in : %s' % self.deploy_config_path) #set ZSTACK_HOME, which will be used by zstack-ctl os.environ['ZSTACK_HOME'] = '%s/apache-tomcat/webapps/zstack/' % \ self.install_path if basic_config.hasattr('testAgent'): self.test_agent_path = self._full_path(basic_config.testAgent.text_) linux.error_if_path_missing(self.test_agent_path) for zone in deploy_config.zones.get_child_node_as_list('zone'): for cluster in zone.clusters.get_child_node_as_list('cluster'): if cluster.hypervisorType_ == inventory.KVM_HYPERVISOR_TYPE: for h in cluster.hosts.get_child_node_as_list('host'): h.managementIp_ h.username_ h.password_ # will raise exception if one of above not specified in xml filea. self.test_agent_hosts.append(h) else: if xmlobject.has_element(basic_config, 'testAgentHost'): raise ActionError('<tesgAgent> is missing while <testAgentHost> presents') self.catalina_home = self.install_path + '/apache-tomcat' self.wait_for_start_timeout = basic_config.get('managementServerStartTimeout') if not self.wait_for_start_timeout: self.wait_for_start_timeout = 120 else: self.wait_for_start_timeout = int(self.wait_for_start_timeout) if hasattr(basic_config, 'rabbitmq'): self.rabbitmq_server = basic_config.rabbitmq.get('server', 'localhost') self.rabbitmq_server_root_passwd = basic_config.rabbitmq.get('password', '') if not self.rabbitmq_server_root_passwd: print ('!!!WARN! Rabbitmq server root password are not set!') else: raise ActionError('need to set config.basicConfig.rabbitmq.server in: %s' % self.deploy_config_path) if hasattr(basic_config, 'db'): self.need_deploy_db = True self.db_server = basic_config.db.get('server', 'localhost') self.db_username = basic_config.db.get('username', 'zstack') self.db_password = basic_config.db.get('password', '') self.db_admin_username = basic_config.db.get('admin', 'root') self.db_admin_password = basic_config.db.get('adminPassword', '') self.db_server_root_password = basic_config.db.get('server_root_password', '') if not self.db_server_root_password: print ('!!!WARN! Database server root password are not set!') self.db_port = basic_config.db.get('port', '3306') if basic_config.has_element('zstackProperties'): if basic_config.zstackProperties.text_: self.zstack_properties = self._full_path(basic_config.zstackProperties.text_) if not os.path.exists(self.zstack_properties): print('unable to find zstackProperties at %s, use \ default one' % self.zstack_properties) self.zstack_properties = None if basic_config.has_element('zstackHaVip'): self.zstack_ha_vip = basic_config.zstackHaVip.text_ else: self.zstack_ha_vip = None os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = '' if deploy_config.has_element('nodes') \ and deploy_config.nodes.has_element('node'): for node in deploy_config.nodes.get_child_node_as_list('node'): node.ip_ self.nodes.append(node) if linux.is_ip_existing(node.ip_): os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = node.ip_ elif not os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP'): os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = node.ip_ else: raise ActionError('deploy.xml setting error. No deployerConfig.nodes.node is found. ') if self.zstack_ha_vip != None: os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = self.zstack_ha_vip if not os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP'): raise ActionError('deploy.xml setting error. No deployerConfig.nodes.node.ip is defined. ')
def _start_multi_nodes(self, restart = False): nodes = [] threads = [] node1 = self.nodes[0] for node in self.nodes: #The reserved node is used by test cases. if not restart and node.reserve__: continue if not node.dockerImage__: print 'Deploy node in hosts' #consider some zstack-server is running in vm, the server # startup speed is slow. Increase timeout to 180s. if linux.is_ip_existing(node.ip_): if os.environ.get('ZSTACK_SIMULATOR') == "yes": cmd = 'zstack-ctl stop_node; %s zstack-ctl configure unitTestOn=true; zstack-ctl configure ThreadFacade.maxThreadNum=1000; nohup zstack-ctl start_node --simulator -DredeployDB=true; zstack-ctl stop_ui' % (self._enable_jacoco_agent_cmd()) else: cmd = 'zstack-ctl stop_node; %s nohup zstack-ctl start_node; zstack-ctl stop_ui' % (self._enable_jacoco_agent_cmd()) thread = threading.Thread(target=shell_cmd_thread, args=(cmd, True, )) elif not linux.is_ip_existing(node1.ip_): # when first node1 ip is not local, it usualy means woodpecker is running on hosts other than MN cmd = 'zstack-ctl stop_node --host=%s ; %s zstack-ctl start_node --host=%s; zstack-ctl stop_ui --host=%s' % (node.ip_, self._enable_jacoco_agent_cmd(), node.ip_, node.ip_) thread = threading.Thread(target=ssh.execute, args=(cmd, node1.ip_, node1.username_, node1.password_, )) else: cmd = 'zstack-ctl stop_node --host=%s ; %s zstack-ctl start_node --host=%s; zstack-ctl stop_ui --host=%s' % (node.ip_, self._enable_jacoco_agent_cmd(), node.ip_, node.ip_) thread = threading.Thread(target=shell_cmd_thread, args=(cmd, True, )) threads.append(thread) else: print 'Deploy node in docker' docker_node = DockerNode(self) docker_node.set_docker_image(node.dockerImage__) docker_node.set_node_ip(node.ip__) docker_node.prepare_node() nodes.append(docker_node) thread = threading.Thread(target=docker_node.start_node) threads.append(thread) for thread in threads: thread.start() self._wait_for_thread_completion('start management node', 400) time.sleep(10) if node_exception: print 'node start meets exception:' info1 = node_exception[0][1] info2 = node_exception[0][2] raise info1, None, info2 current_time = time.time() #largest timeout time for multi nodes startup is 300s timeout_time = current_time + 300 for node in self.nodes: #The reserved node is used by test cases. if node.reserve__: continue new_time = time.time() if new_time >= timeout_time: new_timeout = 1 else: new_timeout = timeout_time - new_time if not linux.wait_callback_success(\ node_ops.is_management_node_start, \ node.ip_, timeout=new_timeout, interval=0.5): raise ActionError('multi node does not startup on host: %s' \ % node.ip_) zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home shell.call(cmd)