def _copy_sshkey_from_node(self):
node = self.nodes[0]
if not node.dockerImage__:
print 'Copy sshkey from mn node'
#consider some zstack-server is running in vm, the server
# startup speed is slow. Increase timeout to 180s.
private_key_path = '%s/webapps/zstack/WEB-INF/classes/ansible/rsaKeys/id_rsa' % self.catalina_home
public_key_path = '%s/webapps/zstack/WEB-INF/classes/ansible/rsaKeys/id_rsa.pub' % self.catalina_home
if linux.is_ip_existing(node.ip_):
cmd = 'scp %s /root/.ssh/id_rsa' % (private_key_path)
shell.call(cmd)
cmd = 'scp %s /root/.ssh/id_rsa.pub' % (public_key_path)
shell.call(cmd)
elif not linux.is_ip_existing(node.ip_):
import zstackwoodpecker.test_lib as test_lib
if test_lib.lib_wait_target_up(node.ip_, '22', 120):
node_ip = node.ip_
else:
node_ip = os.environ['zstackHaVip']
ssh.scp_file(private_key_path, "/root/.ssh/id_rsa", node_ip, node.username_, node.password_)
ssh.scp_file(public_key_path, "/root/.ssh/id_rsa.pub", node_ip, node.username_, node.password_)
cmd = 'scp %s /root/.ssh/id_rsa' % (private_key_path)
shell.call(cmd)
cmd = 'scp %s /root/.ssh/id_rsa.pub' % (public_key_path)
shell.call(cmd)
else:
cmd = 'scp %s /root/.ssh/id_rsa' % (private_key_path)
shell.call(cmd)
cmd = 'scp %s /root/.ssh/id_rsa.pub' % (public_key_path)
shell.call(cmd)
def _upgrade_management_nodes(self):
for node in self.nodes:
if not linux.is_ip_existing(node.ip_):
cmd = 'zstack-ctl upgrade_management_node --host=%s' % node.ip_
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
self._wait_for_thread_completion('upgrade remote management node', 600)
def _upgrade_management_nodes(self):
for node in self.nodes:
if not linux.is_ip_existing(node.ip_):
cmd = 'zstack-ctl upgrade_management_node --host=%s' % node.ip_
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
self._wait_for_thread_completion('upgrade remote management node', 600)
def _change_node_ip(self):
for node in self.nodes:
cmd = 'zstack-ctl change_ip --ip=%s --mysql_root_password=%s' % (node.ip_, os.environ.get('DBAdminPassword'))
if not linux.is_ip_existing(node.ip_):
ssh.execute(cmd, node.ip_, node.username_, node.password_)
else:
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
self._wait_for_thread_completion('change management node ip', 120)
cmd = 'zstack-ctl configure InfluxDB.startupTimeout=1200'
if not linux.is_ip_existing(node.ip_):
ssh.execute(cmd, node.ip_, node.username_, node.password_)
else:
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
self._wait_for_thread_completion('InfluxDB startup time setting to 10 mins', 120)
def _change_node_ip(self):
for node in self.nodes:
cmd = 'zstack-ctl change_ip --ip=%s --mysql_root_password=%s' % (node.ip_, os.environ.get('DBAdminPassword'))
if not linux.is_ip_existing(node.ip_):
ssh.execute(cmd, node.ip_, node.username_, node.password_)
else:
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
self._wait_for_thread_completion('change management node ip', 60)
def _change_node_ip(self):
for node in self.nodes:
cmd = 'zstack-ctl change_ip --ip=%s' % (node.ip_)
if not linux.is_ip_existing(node.ip_):
ssh.execute(cmd, node.ip_, node.username_, node.password_)
else:
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
self._wait_for_thread_completion('change management node ip', 60)
def _install_management_nodes(self):
for node in self.nodes:
if not linux.is_ip_existing(node.ip_):
cmd = 'zstack-ctl install_management_node --force-reinstall \
--host=%s' % node.ip_
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
else:
print "node: %s has been installed zstack" % node.ip_
self._wait_for_thread_completion('install remote management node', 600)
def _install_management_nodes(self):
for node in self.nodes:
if not linux.is_ip_existing(node.ip_) and os.environ.get('ZSTACK_ALREADY_INSTALLED') != "yes":
cmd = 'zstack-ctl install_management_node --force-reinstall \
--host=%s' % node.ip_
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
else:
print "node: %s has been installed zstack" % node.ip_
self._wait_for_thread_completion('install remote management node', 600)
def _set_extra_node_config(self):
node1 = self.nodes[0]
for node in self.nodes:
cmd = 'zstack-ctl configure --duplicate-to-remote=%s; zstack-ctl configure --host=%s management.server.ip=%s' % \
(node.ip_, node.ip_, node.ip_)
if not linux.is_ip_existing(node.ip_):
ssh.execute(cmd, node1.ip_, node.username_, node.password_)
else:
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
self._wait_for_thread_completion('set extra management node config', 60)
def _set_extra_node_config(self):
node1 = self.nodes[0]
for node in self.nodes:
cmd = 'zstack-ctl configure --duplicate-to-remote=%s; zstack-ctl configure --host=%s management.server.ip=%s' % \
(node.ip_, node.ip_, node.ip_)
if not linux.is_ip_existing(node.ip_):
ssh.execute(cmd, node1.ip_, node.username_, node.password_)
else:
thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
thread.start()
self._wait_for_thread_completion('set extra management node config', 60)
def _start_multi_nodes(self, restart=False):
nodes = []
threads = []
for node in self.nodes:
#The reserved node is used by test cases.
if not restart and node.reserve__:
continue
if not node.dockerImage__:
print 'Deploy node in hosts'
#consider some zstack-server is running in vm, the server
# startup speed is slow. Increase timeout to 180s.
if linux.is_ip_existing(node.ip_):
cmd = 'zstack-ctl stop; zstack-ctl start'
else:
cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % (
node.ip_, node.ip_)
thread = threading.Thread(target=shell_cmd_thread,
args=(
cmd,
True,
))
threads.append(thread)
else:
print 'Deploy node in docker'
docker_node = DockerNode(self)
docker_node.set_docker_image(node.dockerImage__)
docker_node.set_node_ip(node.ip__)
docker_node.prepare_node()
nodes.append(docker_node)
thread = threading.Thread(target=docker_node.start_node)
threads.append(thread)
for thread in threads:
thread.start()
self._wait_for_thread_completion('start management node', 200)
if node_exception:
print 'node start meets exception:'
info1 = node_exception[0][1]
info2 = node_exception[0][2]
raise info1, None, info2
current_time = time.time()
#largest timeout time for multi nodes startup is 300s
timeout_time = current_time + 300
for node in self.nodes:
#The reserved node is used by test cases.
if node.reserve__:
continue
new_time = time.time()
if new_time >= timeout_time:
new_timeout = 1
else:
new_timeout = timeout_time - new_time
if not linux.wait_callback_success(\
node_ops.is_management_node_start, \
node.ip_, timeout=new_timeout, interval=0.5):
raise ActionError('multi node does not startup on host: %s' \
% node.ip_)
zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path
cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home
shell.call(cmd)
def _set_and_validate_config(self):
basic_config = self.config.basicConfig
deploy_config = self.config.deployerConfig
self.zstack_pkg = self._full_path(basic_config.zstackPkg.text_)
self.zstack_install_script = \
self._full_path(basic_config.zstackInstallScript.text_)
if not os.path.exists(self.zstack_pkg):
raise ActionError('unable to find %s for ZStack binary' \
% self.zstack_pkg)
if basic_config.hasattr('zstackInstallPath'):
self.install_path = \
self._full_path(basic_config.zstackInstallPath.text_)
else:
raise ActionError(\
'need to set config.deployerConfig.zstackInstallPath in : %s' % self.deploy_config_path)
#set ZSTACK_HOME, which will be used by zstack-ctl
os.environ['ZSTACK_HOME'] = '%s/apache-tomcat/webapps/zstack/' % \
self.install_path
if basic_config.hasattr('testAgent'):
self.test_agent_path = self._full_path(
basic_config.testAgent.text_)
linux.error_if_path_missing(self.test_agent_path)
for zone in deploy_config.zones.get_child_node_as_list('zone'):
for cluster in zone.clusters.get_child_node_as_list('cluster'):
if cluster.hypervisorType_ == inventory.KVM_HYPERVISOR_TYPE:
for h in cluster.hosts.get_child_node_as_list('host'):
h.managementIp_
h.username_
h.password_
# will raise exception if one of above not specified in xml filea.
self.test_agent_hosts.append(h)
else:
if xmlobject.has_element(basic_config, 'testAgentHost'):
raise ActionError(
'<tesgAgent> is missing while <testAgentHost> presents')
self.catalina_home = self.install_path + '/apache-tomcat'
self.wait_for_start_timeout = basic_config.get(
'managementServerStartTimeout')
if not self.wait_for_start_timeout:
self.wait_for_start_timeout = 120
else:
self.wait_for_start_timeout = int(self.wait_for_start_timeout)
if hasattr(basic_config, 'rabbitmq'):
self.rabbitmq_server = basic_config.rabbitmq.get(
'server', 'localhost')
self.rabbitmq_server_root_passwd = basic_config.rabbitmq.get(
'password', '')
if not self.rabbitmq_server_root_passwd:
print('!!!WARN! Rabbitmq server root password are not set!')
else:
raise ActionError(
'need to set config.basicConfig.rabbitmq.server in: %s' %
self.deploy_config_path)
if hasattr(basic_config, 'db'):
self.need_deploy_db = True
self.db_server = basic_config.db.get('server', 'localhost')
self.db_username = basic_config.db.get('username', 'zstack')
self.db_password = basic_config.db.get('password', '')
self.db_admin_username = basic_config.db.get('admin', 'root')
self.db_admin_password = basic_config.db.get('adminPassword', '')
self.db_server_root_password = basic_config.db.get(
'server_root_password', '')
if not self.db_server_root_password:
print('!!!WARN! Database server root password are not set!')
self.db_port = basic_config.db.get('port', '3306')
if basic_config.has_element('zstackProperties'):
if basic_config.zstackProperties.text_:
self.zstack_properties = self._full_path(
basic_config.zstackProperties.text_)
if not os.path.exists(self.zstack_properties):
print(
'unable to find zstackProperties at %s, use \
default one' % self.zstack_properties)
self.zstack_properties = None
if basic_config.has_element('zstackHaVip'):
self.zstack_ha_vip = basic_config.zstackHaVip.text_
else:
self.zstack_ha_vip = None
if basic_config.has_element('zstackManagementIp'):
self.zstack_management_ip = basic_config.zstackManagementIp.text_
else:
self.zstack_management_ip = None
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = ''
if deploy_config.has_element('nodes') \
and deploy_config.nodes.has_element('node'):
for node in deploy_config.nodes.get_child_node_as_list('node'):
node.ip_
self.nodes.append(node)
if linux.is_ip_existing(node.ip_):
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = node.ip_
elif not os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP'):
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = node.ip_
else:
raise ActionError(
'deploy.xml setting error. No deployerConfig.nodes.node is found. '
)
if self.zstack_ha_vip != None:
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = self.zstack_ha_vip
if self.zstack_management_ip != None:
os.environ[
'ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = self.zstack_management_ip
if not os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP'):
raise ActionError(
'deploy.xml setting error. No deployerConfig.nodes.node.ip is defined. '
)
def _start_multi_nodes(self, restart=False):
nodes = []
threads = []
node1 = self.nodes[0]
for node in self.nodes:
#The reserved node is used by test cases.
if not restart and node.reserve__:
continue
if not node.dockerImage__:
print 'Deploy node in hosts'
#consider some zstack-server is running in vm, the server
# startup speed is slow. Increase timeout to 180s.
if linux.is_ip_existing(node.ip_):
if os.environ.get('ZSTACK_SIMULATOR') == "yes":
cmd = 'zstack-ctl stop; zstack-ctl configure unitTestOn=true; nohup zstack-ctl start_node --simulator -DredeployDB=true'
else:
cmd = 'zstack-ctl stop; nohup zstack-ctl start'
thread = threading.Thread(target=shell_cmd_thread,
args=(
cmd,
True,
))
elif not linux.is_ip_existing(node1.ip_):
# when first node1 ip is not local, it usualy means woodpecker is running on hosts other than MN
cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % (
node.ip_, node.ip_)
thread = threading.Thread(target=ssh.execute,
args=(
cmd,
node1.ip_,
node1.username_,
node1.password_,
))
else:
cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % (
node.ip_, node.ip_)
thread = threading.Thread(target=shell_cmd_thread,
args=(
cmd,
True,
))
threads.append(thread)
else:
print 'Deploy node in docker'
docker_node = DockerNode(self)
docker_node.set_docker_image(node.dockerImage__)
docker_node.set_node_ip(node.ip__)
docker_node.prepare_node()
nodes.append(docker_node)
thread = threading.Thread(target=docker_node.start_node)
threads.append(thread)
for thread in threads:
thread.start()
self._wait_for_thread_completion('start management node', 200)
time.sleep(10)
if node_exception:
print 'node start meets exception:'
info1 = node_exception[0][1]
info2 = node_exception[0][2]
raise info1, None, info2
current_time = time.time()
#largest timeout time for multi nodes startup is 300s
timeout_time = current_time + 300
for node in self.nodes:
#The reserved node is used by test cases.
if node.reserve__:
continue
new_time = time.time()
if new_time >= timeout_time:
new_timeout = 1
else:
new_timeout = timeout_time - new_time
if not linux.wait_callback_success(\
node_ops.is_management_node_start, \
node.ip_, timeout=new_timeout, interval=0.5):
raise ActionError('multi node does not startup on host: %s' \
% node.ip_)
zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path
cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home
shell.call(cmd)
def _set_and_validate_config(self):
basic_config = self.config.basicConfig
deploy_config = self.config.deployerConfig
self.zstack_pkg = self._full_path(basic_config.zstackPkg.text_)
self.zstack_install_script = \
self._full_path(basic_config.zstackInstallScript.text_)
if not os.path.exists(self.zstack_pkg):
raise ActionError('unable to find %s for ZStack binary' \
% self.zstack_pkg)
if basic_config.hasattr('zstackInstallPath'):
self.install_path = \
self._full_path(basic_config.zstackInstallPath.text_)
else:
raise ActionError(\
'need to set config.deployerConfig.zstackInstallPath in : %s' % self.deploy_config_path)
#set ZSTACK_HOME, which will be used by zstack-ctl
os.environ['ZSTACK_HOME'] = '%s/apache-tomcat/webapps/zstack/' % \
self.install_path
if basic_config.hasattr('testAgent'):
self.test_agent_path = self._full_path(basic_config.testAgent.text_)
linux.error_if_path_missing(self.test_agent_path)
for zone in deploy_config.zones.get_child_node_as_list('zone'):
for cluster in zone.clusters.get_child_node_as_list('cluster'):
if cluster.hypervisorType_ == inventory.KVM_HYPERVISOR_TYPE:
for h in cluster.hosts.get_child_node_as_list('host'):
h.managementIp_
h.username_
h.password_
# will raise exception if one of above not specified in xml filea.
self.test_agent_hosts.append(h)
else:
if xmlobject.has_element(basic_config, 'testAgentHost'):
raise ActionError('<tesgAgent> is missing while <testAgentHost> presents')
self.catalina_home = self.install_path + '/apache-tomcat'
self.wait_for_start_timeout = basic_config.get('managementServerStartTimeout')
if not self.wait_for_start_timeout:
self.wait_for_start_timeout = 120
else:
self.wait_for_start_timeout = int(self.wait_for_start_timeout)
if hasattr(basic_config, 'rabbitmq'):
self.rabbitmq_server = basic_config.rabbitmq.get('server', 'localhost')
self.rabbitmq_server_root_passwd = basic_config.rabbitmq.get('password', '')
if not self.rabbitmq_server_root_passwd:
print ('!!!WARN! Rabbitmq server root password are not set!')
else:
raise ActionError('need to set config.basicConfig.rabbitmq.server in: %s' % self.deploy_config_path)
if hasattr(basic_config, 'db'):
self.need_deploy_db = True
self.db_server = basic_config.db.get('server', 'localhost')
self.db_username = basic_config.db.get('username', 'zstack')
self.db_password = basic_config.db.get('password', '')
self.db_admin_username = basic_config.db.get('admin', 'root')
self.db_admin_password = basic_config.db.get('adminPassword', '')
self.db_server_root_password = basic_config.db.get('server_root_password', '')
if not self.db_server_root_password:
print ('!!!WARN! Database server root password are not set!')
self.db_port = basic_config.db.get('port', '3306')
if basic_config.has_element('zstackProperties'):
if basic_config.zstackProperties.text_:
self.zstack_properties = self._full_path(basic_config.zstackProperties.text_)
if not os.path.exists(self.zstack_properties):
print('unable to find zstackProperties at %s, use \
default one' % self.zstack_properties)
self.zstack_properties = None
if basic_config.has_element('zstackHaVip'):
self.zstack_ha_vip = basic_config.zstackHaVip.text_
else:
self.zstack_ha_vip = None
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = ''
if deploy_config.has_element('nodes') \
and deploy_config.nodes.has_element('node'):
for node in deploy_config.nodes.get_child_node_as_list('node'):
node.ip_
self.nodes.append(node)
if linux.is_ip_existing(node.ip_):
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = node.ip_
elif not os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP'):
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = node.ip_
else:
raise ActionError('deploy.xml setting error. No deployerConfig.nodes.node is found. ')
if self.zstack_ha_vip != None:
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = self.zstack_ha_vip
if not os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP'):
raise ActionError('deploy.xml setting error. No deployerConfig.nodes.node.ip is defined. ')
def _start_multi_nodes(self, restart = False):
nodes = []
threads = []
node1 = self.nodes[0]
for node in self.nodes:
#The reserved node is used by test cases.
if not restart and node.reserve__:
continue
if not node.dockerImage__:
print 'Deploy node in hosts'
#consider some zstack-server is running in vm, the server
# startup speed is slow. Increase timeout to 180s.
if linux.is_ip_existing(node.ip_):
if os.environ.get('ZSTACK_SIMULATOR') == "yes":
cmd = 'zstack-ctl stop_node; %s zstack-ctl configure unitTestOn=true; zstack-ctl configure ThreadFacade.maxThreadNum=1000; nohup zstack-ctl start_node --simulator -DredeployDB=true; zstack-ctl stop_ui' % (self._enable_jacoco_agent_cmd())
else:
cmd = 'zstack-ctl stop_node; %s nohup zstack-ctl start_node; zstack-ctl stop_ui' % (self._enable_jacoco_agent_cmd())
thread = threading.Thread(target=shell_cmd_thread, args=(cmd, True, ))
elif not linux.is_ip_existing(node1.ip_):
# when first node1 ip is not local, it usualy means woodpecker is running on hosts other than MN
cmd = 'zstack-ctl stop_node --host=%s ; %s zstack-ctl start_node --host=%s; zstack-ctl stop_ui --host=%s' % (node.ip_, self._enable_jacoco_agent_cmd(), node.ip_, node.ip_)
thread = threading.Thread(target=ssh.execute, args=(cmd, node1.ip_, node1.username_, node1.password_, ))
else:
cmd = 'zstack-ctl stop_node --host=%s ; %s zstack-ctl start_node --host=%s; zstack-ctl stop_ui --host=%s' % (node.ip_, self._enable_jacoco_agent_cmd(), node.ip_, node.ip_)
thread = threading.Thread(target=shell_cmd_thread, args=(cmd, True, ))
threads.append(thread)
else:
print 'Deploy node in docker'
docker_node = DockerNode(self)
docker_node.set_docker_image(node.dockerImage__)
docker_node.set_node_ip(node.ip__)
docker_node.prepare_node()
nodes.append(docker_node)
thread = threading.Thread(target=docker_node.start_node)
threads.append(thread)
for thread in threads:
thread.start()
self._wait_for_thread_completion('start management node', 400)
time.sleep(10)
if node_exception:
print 'node start meets exception:'
info1 = node_exception[0][1]
info2 = node_exception[0][2]
raise info1, None, info2
current_time = time.time()
#largest timeout time for multi nodes startup is 300s
timeout_time = current_time + 300
for node in self.nodes:
#The reserved node is used by test cases.
if node.reserve__:
continue
new_time = time.time()
if new_time >= timeout_time:
new_timeout = 1
else:
new_timeout = timeout_time - new_time
if not linux.wait_callback_success(\
node_ops.is_management_node_start, \
node.ip_, timeout=new_timeout, interval=0.5):
raise ActionError('multi node does not startup on host: %s' \
% node.ip_)
zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path
cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home
shell.call(cmd)
