def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.dummy_installer = StandalonePrestoInstaller( self, (os.path.join(prestoadmin.main_dir, 'tests', 'product', 'resources'), 'dummy-rpm.rpm')) self.real_installer = StandalonePrestoInstaller(self)
def assert_common_configs(self, host): installer = StandalonePrestoInstaller(self) installer.assert_installed(self, host) self.assert_file_content(host, '/etc/presto/jvm.config', self.default_jvm_config_) self.assert_node_config(host, self.default_node_properties_) self.assert_has_default_catalog(host)
def test_install_when_catalog_json_exists(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) cmd_output = installer.install() expected = [ 'Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'catalog configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'catalog configurations on: slave1 ', 'Using rpm_specifier as a local path', 'Fetching local presto rpm at path: .*', 'Found existing rpm at: .*' ] actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container)
def assert_common_configs(self, container): installer = StandalonePrestoInstaller(self) installer.assert_installed(self, container) self.assert_file_content(container, '/etc/presto/jvm.config', self.default_jvm_config_) self.assert_node_config(container, self.default_node_properties_) self.assert_has_default_connector(container)
def test_install_with_java_home(self): installer = StandalonePrestoInstaller(self) with relocate_jdk_directory(self.cluster, '/usr') as new_java_home: topology = { "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "java_home": new_java_home } self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) cmd_output = installer.install() expected = self.format_err_msgs_with_internal_hosts( installed_all_hosts_output) actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for host in self.cluster.all_hosts(): installer.assert_installed(self, host) self.assert_has_default_config(host) self.assert_has_default_catalog(host) self.assert_has_jmx_catalog(host)
def test_connection_to_coord_lost(self): installer = StandalonePrestoInstaller(self) down_node = self.cluster.internal_slaves[0] topology = { "coordinator": down_node, "workers": [ self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2] ] } self.upload_topology(topology=topology) self.cluster.stop_host(self.cluster.slaves[0]) actual_out = installer.install(coordinator=down_node, pa_raise_error=False) self.assertRegexpMatches(actual_out, self.down_node_connection_error(down_node)) for host in [ self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2] ]: self.assert_common_configs(host) self.assert_file_content(host, '/etc/presto/config.properties', self.default_workers_config_with_slave1_)
def test_install_when_connector_json_exists(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) cmd_output = installer.install() expected = [ 'Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: slave1 ' ] actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container)
def test_install_twice(self): installer = StandalonePrestoInstaller(self) self.upload_topology() cmd_output = installer.install() expected = self.format_err_msgs_with_internal_hosts( installed_all_hosts_output) actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) output = installer.install(pa_raise_error=False) self.default_keywords.update(installer.get_keywords()) with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') as f: expected = f.read() expected = self.escape_for_regex(self.replace_keywords(expected)) self.assertRegexpMatchesLineByLine(output.splitlines(), expected.splitlines()) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container)
def test_connector_add_lost_host(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(self.PA_ONLY_CLUSTER) self.upload_topology() installer.install() self.run_prestoadmin('connector remove tpch') self.cluster.stop_host( self.cluster.slaves[0]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(constants.CONNECTORS_DIR, 'tpch.properties'), self.cluster.master ) output = self.run_prestoadmin('connector add tpch') for host in self.cluster.all_internal_hosts(): deploying_message = 'Deploying tpch.properties connector ' \ 'configurations on: %s' self.assertTrue(deploying_message % host in output, 'expected %s \n actual %s' % (deploying_message % host, output)) self.assertRegexpMatches( output, self.down_node_connection_error(self.cluster.internal_slaves[0]) ) self.assertEqual(len(output.splitlines()), len(self.cluster.all_hosts()) + self.len_down_node_error) self.run_prestoadmin('server start') for host in [self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_has_default_connector(host) self._assert_connectors_loaded([['system'], ['tpch']])
def test_catalog_add_lost_host(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER) self.upload_topology() installer.install() self.run_prestoadmin('catalog remove tpch') self.cluster.stop_host(self.cluster.slaves[0]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master) output = self.run_prestoadmin('catalog add tpch', raise_error=False) for host in self.cluster.all_internal_hosts(): deploying_message = 'Deploying tpch.properties catalog configurations on: %s' self.assertTrue( deploying_message % host in output, 'expected %s \n actual %s' % (deploying_message % host, output)) self.assertRegexpMatches( output, self.down_node_connection_error(self.cluster.internal_slaves[0])) self.assertEqual( len(output.splitlines()), len(self.cluster.all_hosts()) + self.len_down_node_error) self.run_prestoadmin('server start', raise_error=False) for host in [ self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2] ]: self.assert_has_default_catalog(host) self._assert_catalogs_loaded([['system'], ['tpch']])
def test_install_interactive(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) additional_keywords = { 'user': self.cluster.user, 'rpm_dir': self.cluster.rpm_cache_dir, 'rpm': rpm_name } cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install %(rpm_dir)s/%(rpm)s ', **additional_keywords) actual = cmd_output.splitlines() expected = [ r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Using rpm_specifier as a local path', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties catalog ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties catalog ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*' ] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container)
def test_system_info_pa_separate_node(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology=topology) installer.install(coordinator='slave1') self.test_basic_system_info( coordinator=self.cluster.internal_slaves[0], hosts=self.cluster.slaves)
def test_start_coordinator_down(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["master", "slave2", "slave3"]} self.upload_topology(topology=topology) installer.install(coordinator='slave1') self.assert_start_coordinator_down( self.cluster.slaves[0], self.cluster.internal_slaves[0])
def test_start_stop_restart_worker_down(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(self.PA_ONLY_CLUSTER) topology = { "coordinator": "slave1", "workers": ["master", "slave2", "slave3"] } self.upload_topology(topology=topology) installer.install() self.assert_start_stop_restart_down_node( self.cluster.slaves[0], self.cluster.internal_slaves[0])
def test_system_info_pa_separate_node(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology=topology) installer.install(coordinator='slave1') self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect system_info') self._test_basic_system_info( actual, coordinator=self.cluster.internal_slaves[0], hosts=self.cluster.slaves)
def test_install_ext_host_is_pa_master(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology) cmd_output = installer.install(coordinator='slave1') expected = install_with_ext_host_pa_master_out actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) self.assert_installed_with_configs( self.cluster.slaves[0], [self.cluster.slaves[1], self.cluster.slaves[2]])
def test_install_with_wrong_topology(self): installer = StandalonePrestoInstaller(self) rpm_name = installer.copy_presto_rpm_to_master() topology = {'coordinator': 'dummy_master', 'workers': ['slave1']} self.upload_topology(topology) expected = 'u\'dummy_master\' is not a valid ip address or' \ ' host name.' \ ' More detailed information can be found in ' \ '/var/log/prestoadmin/presto-admin.log\n' self.assertRaisesRegexp(OSError, expected, self.run_prestoadmin, 'server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name)
def test_install_with_malformed_topology(self): installer = StandalonePrestoInstaller(self) rpm_name = installer.copy_presto_rpm_to_master() topology = {'coordinator': 'master', 'workers': 'slave1' 'slave2'} self.upload_topology(topology) expected = 'Workers must be of type list. Found <type \'unicode\'>.' \ ' More detailed information can be found in ' \ '/var/log/prestoadmin/presto-admin.log' self.assertRaisesRegexp(OSError, expected, self.run_prestoadmin, 'server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name)
def test_install_interactive_with_hostnames(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name) actual = cmd_output.splitlines() expected = [ r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Deploying rpm on .*\.\.\.', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.' ] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container)
def test_install_interactive_with_ips(self): installer = StandalonePrestoInstaller(self) ips = self.cluster.get_ip_address_dict() rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) additional_keywords = { 'rpm': rpm_name, 'master_ip': ips[self.cluster.master], 'slave1_ip': ips[self.cluster.slaves[0]] } cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | ' './presto-admin server install /mnt/presto-admin/%(rpm)s ', **additional_keywords).splitlines() expected = [ r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Using rpm_specifier as a local path', r'Package deployed successfully on: ' + ips[self.cluster.master], r'Package installed successfully on: ' + ips[self.cluster.master], r'Package deployed successfully on: ' + ips[self.cluster.slaves[0]], r'Package installed successfully on: ' + ips[self.cluster.slaves[0]], r'Deploying configuration on: ' + ips[self.cluster.master], r'Deploying tpch.properties connector ' r'configurations on: ' + ips[self.cluster.master] + r' ', r'Deploying configuration on: ' + ips[self.cluster.slaves[0]], r'Deploying tpch.properties connector ' r'configurations on: ' + ips[self.cluster.slaves[0]] + r' ', r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*' ] cmd_output.sort() expected.sort() for expected_regexp, actual_line in zip(expected, cmd_output): self.assertRegexpMatches(actual_line, expected_regexp) self.assert_installed_with_regex_configs(self.cluster.master, [self.cluster.slaves[0]])
def test_install_with_malformed_connector(self): installer = StandalonePrestoInstaller(self) self.upload_topology() self.cluster.write_content_to_host( 'connectr.typo:invalid', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) actual_out = installer.install(pa_raise_error=False) expected = 'Underlying exception:\n Catalog configuration ' \ 'jmx.properties does not contain connector.name' self.assertRegexpMatches(actual_out, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container)
def test_install_when_topology_has_ips(self): installer = StandalonePrestoInstaller(self) ips = self.cluster.get_ip_address_dict() topology = { "coordinator": ips[self.cluster.internal_master], "workers": [ips[self.cluster.internal_slaves[0]]] } self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) cmd_output = installer.install().splitlines() expected = [ r'Deploying rpm on %s...' % ips[self.cluster.internal_master], r'Deploying rpm on %s...' % ips[self.cluster.internal_slaves[0]], r'Package deployed successfully on: ' + ips[self.cluster.internal_master], r'Package installed successfully on: ' + ips[self.cluster.internal_master], r'Package deployed successfully on: ' + ips[self.cluster.internal_slaves[0]], r'Package installed successfully on: ' + ips[self.cluster.internal_slaves[0]], r'Deploying configuration on: ' + ips[self.cluster.internal_master], r'Deploying jmx.properties, tpch.properties ' r'catalog configurations on: ' + ips[self.cluster.internal_master] + r' ', r'Deploying configuration on: ' + ips[self.cluster.internal_slaves[0]], r'Deploying jmx.properties, tpch.properties ' r'catalog configurations on: ' + ips[self.cluster.internal_slaves[0]] + r' ', r'Using rpm_specifier as a local path', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*' ] cmd_output.sort() expected.sort() self.assertRegexpMatchesLineByLine(cmd_output, expected) self.assert_installed_with_regex_configs(self.cluster.master, [self.cluster.slaves[0]]) for host in [self.cluster.master, self.cluster.slaves[0]]: self.assert_has_jmx_catalog(host)
def test_install(self, installer=None): if installer is None: installer = StandalonePrestoInstaller(self) self.upload_topology() cmd_output = installer.install() expected = installed_all_hosts_output actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_fails_java_not_found(self): installer = StandalonePrestoInstaller(self) with relocate_jdk_directory(self.cluster, '/usr'): self.upload_topology() cmd_output = installer.install(pa_raise_error=False) actual = cmd_output.splitlines() num_failures = 0 for line in enumerate(actual): if str(line).find('Error: Required Java version' ' could not be found') != -1: num_failures += 1 self.assertEqual(4, num_failures) for container in self.cluster.all_hosts(): installer.assert_uninstalled(container)
def test_server_starts_java8_in_bin_java(self): installer = StandalonePrestoInstaller(self) for container in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host(container, 'mv /usr/java/jdk1.8.0_40 /usr') self.cluster.exec_cmd_on_host( container, 'ln -s /usr/jdk1.8.0_40/bin/java ' '/bin/java') self.upload_topology() installer.install() # starts successfully with java8_home set output = self.run_prestoadmin('server start') self.assertFalse( 'Warning: No value found for JAVA8_HOME. Default Java will be ' 'used.' in output)
def test_install_worker_is_pa_master(self): installer = StandalonePrestoInstaller(self) topology = { "coordinator": "slave1", "workers": ["master", "slave2", "slave3"] } self.upload_topology(topology) cmd_output = installer.install(coordinator='slave1') expected = install_with_worker_pa_master_out actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) self.assert_installed_with_configs(self.cluster.slaves[0], [ self.cluster.slaves[1], self.cluster.slaves[2], self.cluster.master ])
def test_install_non_root_user(self): installer = StandalonePrestoInstaller(self) self.upload_topology({ "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******" }) rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster) self.write_test_configs(self.cluster) self.run_prestoadmin( 'server install {rpm_dir}/{name} -p password'.format( rpm_dir=self.cluster.rpm_cache_dir, name=rpm_name)) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container)
def test_install_non_root_user(self): installer = StandalonePrestoInstaller(self) self.upload_topology({ "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******" }) rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster) self.write_test_configs(self.cluster) self.run_prestoadmin('server install ' + os.path.join(self.cluster.mount_dir, rpm_name) + ' -p password') for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_twice(self): installer = StandalonePrestoInstaller(self) self.test_install(installer=installer) output = installer.install(pa_raise_error=False) self.default_keywords.update(installer.get_keywords()) with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') as f: expected = f.read() expected = self.escape_for_regex(self.replace_keywords(expected)) self.assertRegexpMatchesLineByLine(output.splitlines(), expected.splitlines()) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_server_starts_java_in_bin_java(self): installer = StandalonePrestoInstaller(self) with relocate_jdk_directory(self.cluster, '/usr') as new_java_home: java_bin = os.path.join(new_java_home, 'bin', 'java') for container in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host( container, 'ln -s %s /bin/java' % (java_bin, )) self.upload_topology() installer.install() # starts successfully with java_home set output = self.run_prestoadmin('server start') self.assertFalse( 'Warning: No value found for JAVA_HOME. Default Java will be ' 'used.' in output)