def test_install_with_java_home(self): installer = StandalonePrestoInstaller(self) with relocate_jdk_directory(self.cluster, '/usr') as new_java_home: topology = { "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "java_home": new_java_home } self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) cmd_output = installer.install() expected = self.format_err_msgs_with_internal_hosts( installed_all_hosts_output) actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for host in self.cluster.all_hosts(): installer.assert_installed(self, host) self.assert_has_default_config(host) self.assert_has_default_catalog(host) self.assert_has_jmx_catalog(host)
def assert_common_configs(self, host): installer = StandalonePrestoInstaller(self) installer.assert_installed(self, host) self.assert_file_content(host, '/etc/presto/jvm.config', self.default_jvm_config_) self.assert_node_config(host, self.default_node_properties_) self.assert_has_default_connector(host)
def assert_common_configs(self, host): installer = StandalonePrestoInstaller(self) installer.assert_installed(self, host) self.assert_file_content(host, '/etc/presto/jvm.config', self.default_jvm_config_) self.assert_node_config(host, self.default_node_properties_) self.assert_has_default_catalog(host)
def test_install_twice(self): installer = StandalonePrestoInstaller(self) self.upload_topology() cmd_output = installer.install() expected = self.format_err_msgs_with_internal_hosts(installed_all_hosts_output) actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) output = installer.install(pa_raise_error=False) self.default_keywords.update(installer.get_keywords()) with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') as f: expected = f.read() expected = self.escape_for_regex( self.replace_keywords(expected)) self.assertRegexpMatchesLineByLine(output.splitlines(), expected.splitlines()) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_twice(self): installer = StandalonePrestoInstaller(self) self.upload_topology() cmd_output = installer.install() expected = self.format_err_msgs_with_internal_hosts( installed_all_hosts_output) actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) output = installer.install(pa_raise_error=False) self.default_keywords.update(installer.get_keywords()) with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') as f: expected = f.read() expected = self.escape_for_regex(self.replace_keywords(expected)) self.assertRegexpMatchesLineByLine(output.splitlines(), expected.splitlines()) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container)
def test_install_when_catalog_json_exists(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) cmd_output = installer.install() expected = [ 'Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'catalog configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'catalog configurations on: slave1 ', 'Using rpm_specifier as a local path', 'Fetching local presto rpm at path: .*', 'Found existing rpm at: .*' ] actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container)
def assert_common_configs(self, container): installer = StandalonePrestoInstaller(self) installer.assert_installed(self, container) self.assert_file_content(container, '/etc/presto/jvm.config', self.default_jvm_config_) self.assert_node_config(container, self.default_node_properties_) self.assert_has_default_connector(container)
def test_install_when_connector_json_exists(self): installer = StandalonePrestoInstaller(self, dummy=True) topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master ) cmd_output = installer.install() expected = ['Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: slave1 '] actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container)
def test_install_when_connector_json_exists(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) cmd_output = installer.install() expected = [ 'Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: slave1 ' ] actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container)
def test_install_interactive(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) additional_keywords = { 'user': self.cluster.user, 'rpm_dir': self.cluster.rpm_cache_dir, 'rpm': rpm_name } cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install %(rpm_dir)s/%(rpm)s ', **additional_keywords) actual = cmd_output.splitlines() expected = [ r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Using rpm_specifier as a local path', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties catalog ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties catalog ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*' ] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container)
def test_install_interactive_with_hostnames(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master ) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name) actual = cmd_output.splitlines() expected = [r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Using rpm_specifier as a local path', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*' ] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container)
def test_install_interactive_with_hostnames(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name) actual = cmd_output.splitlines() expected = [ r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Deploying rpm on .*\.\.\.', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.' ] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container)
def test_install_with_malformed_connector(self): installer = StandalonePrestoInstaller(self) self.upload_topology() self.cluster.write_content_to_host( 'connectr.typo:invalid', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) actual_out = installer.install(pa_raise_error=False) expected = 'Underlying exception:\n Catalog configuration ' \ 'jmx.properties does not contain connector.name' self.assertRegexpMatches(actual_out, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container)
def test_install_with_malformed_connector(self): installer = StandalonePrestoInstaller(self, dummy=True) self.upload_topology() self.cluster.write_content_to_host( 'connectr.typo:invalid', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master ) actual_out = installer.install(pa_raise_error=False) expected = 'Underlying exception:\n Catalog configuration ' \ 'jmx.properties does not contain connector.name' self.assertRegexpMatches(actual_out, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container)
def test_install(self, installer=None): if installer is None: installer = StandalonePrestoInstaller(self) self.upload_topology() cmd_output = installer.install() expected = installed_all_hosts_output actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install(self, installer=None): if installer is None: installer = StandalonePrestoInstaller(self) self.upload_topology() cmd_output = installer.install() expected = installed_all_hosts_output actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_twice(self): installer = StandalonePrestoInstaller(self) self.test_install(installer=installer) output = installer.install(pa_raise_error=False) self.default_keywords.update(installer.get_keywords()) with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') as f: expected = f.read() expected = self.escape_for_regex(self.replace_keywords(expected)) self.assertRegexpMatchesLineByLine(output.splitlines(), expected.splitlines()) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_non_root_user(self): installer = StandalonePrestoInstaller(self) self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******"} ) rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster) self.write_test_configs(self.cluster) self.run_prestoadmin( 'server install ' + os.path.join(self.cluster.mount_dir, rpm_name) + ' -p password' ) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_non_root_user(self): installer = StandalonePrestoInstaller(self) self.upload_topology({ "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******" }) rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster) self.write_test_configs(self.cluster) self.run_prestoadmin( 'server install {rpm_dir}/{name} -p password'.format( rpm_dir=self.cluster.rpm_cache_dir, name=rpm_name)) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container)
def test_install_non_root_user(self): installer = StandalonePrestoInstaller(self) self.upload_topology({ "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******" }) rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster) self.write_test_configs(self.cluster) self.run_prestoadmin('server install ' + os.path.join(self.cluster.mount_dir, rpm_name) + ' -p password') for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_interactive(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) ips = self.cluster.get_ip_address_dict() self.upload_topology({ "coordinator": ips[self.cluster.internal_master], "workers": [ ips[self.cluster.internal_slaves[0]], ips[self.cluster.internal_slaves[1]], ips[self.cluster.internal_slaves[2]] ], "username": "******" }) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster, coordinator=ips[self.cluster.internal_master]) additional_keywords = { 'user': "******", 'rpm_dir': self.cluster.rpm_cache_dir, 'rpm': rpm_name } self.run_script_from_prestoadmin_dir( 'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install %(rpm_dir)s/%(rpm)s -p password', **additional_keywords) self.assert_installed_with_regex_configs(self.cluster.master, [ self.cluster.slaves[0], self.cluster.slaves[1], self.cluster.slaves[2] ]) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container)
def test_install_with_java8_home(self): installer = StandalonePrestoInstaller(self) new_java_home = relocate_default_java(self.cluster, '/usr') topology = {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "java8_home": new_java_home} self.upload_topology(topology) cmd_output = installer.install() expected = installed_all_hosts_output actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_twice(self): installer = StandalonePrestoInstaller(self, dummy=True) self.test_install(installer=installer) output = installer.install(pa_raise_error=False) self.default_keywords.update(installer.get_keywords()) with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') \ as f: expected = f.read() expected = self.escape_for_regex( self.replace_keywords(expected)) self.assertRegexpMatchesLineByLine(output.splitlines(), expected.splitlines()) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_with_java8_home(self): installer = StandalonePrestoInstaller(self) with relocate_jdk_directory(self.cluster, '/usr') as new_java_home: topology = {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "java8_home": new_java_home} self.upload_topology(topology) cmd_output = installer.install() expected = self.format_err_msgs_with_internal_hosts(installed_all_hosts_output) actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for host in self.cluster.all_hosts(): installer.assert_installed(self, host) self.assert_has_default_config(host) self.assert_has_default_connector(host)
def test_install_with_java8_home(self): installer = StandalonePrestoInstaller(self, dummy=True) for container in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host(container, "mv /usr/java/jdk1.8.0_40 /usr/") topology = {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "java8_home": "/usr/jdk1.8.0_40/jre"} self.upload_topology(topology) cmd_output = installer.install() expected = installed_all_hosts_output actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_with_java8_home(self): installer = StandalonePrestoInstaller(self) new_java_home = relocate_default_java(self.cluster, '/usr') topology = { "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "java8_home": new_java_home } self.upload_topology(topology) cmd_output = installer.install() expected = installed_all_hosts_output actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_with_java8_home(self): installer = StandalonePrestoInstaller(self) for container in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host(container, "mv /usr/java/jdk1.8.0_40 /usr/") topology = { "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "java8_home": "/usr/jdk1.8.0_40/jre" } self.upload_topology(topology) cmd_output = installer.install() expected = installed_all_hosts_output actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
def test_install_when_connector_json_exists(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_connectors_directory(), 'jmx.properties'), self.cluster.master ) cmd_output = installer.install() expected = ['Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: slave1 ', 'Using rpm_specifier as a local path', 'Fetching local presto rpm at path: .*', 'Found existing rpm at: .*'] actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container)
class TestPackageInstall(BaseProductTestCase): def setUp(self): super(TestPackageInstall, self).setUp() self.setup_cluster(self.PA_ONLY_CLUSTER) self.upload_topology() self.installer = StandalonePrestoInstaller(self) @attr("smoketest") def test_package_install(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin("package install " "/mnt/presto-admin/%(rpm)s", rpm=rpm_name) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container, msg=output) def test_install_coord_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin("package install /mnt/presto-admin/%(rpm)s -H %(master)s", rpm=rpm_name) self.installer.assert_installed(self, self.cluster.master) for slave in self.cluster.slaves: self.installer.assert_uninstalled(slave, msg=output) def test_install_worker_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin("package install /mnt/presto-admin/%(rpm)s -H %(slave1)s", rpm=rpm_name) self.installer.assert_installed(self, self.cluster.slaves[0], msg=output) self.installer.assert_uninstalled(self.cluster.master, msg=output) self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output) def test_install_workers_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( "package install /mnt/presto-admin/" "%(rpm)s -H %(slave1)s,%(slave2)s", rpm=rpm_name ) self.installer.assert_installed(self, self.cluster.slaves[0], msg=output) self.installer.assert_installed(self, self.cluster.slaves[1], msg=output) self.installer.assert_uninstalled(self.cluster.master, msg=output) self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output) def test_install_exclude_coord(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin("package install /mnt/presto-admin/" "%(rpm)s -x %(master)s", rpm=rpm_name) self.installer.assert_uninstalled(self.cluster.master, msg=output) for slave in self.cluster.slaves: self.installer.assert_installed(self, slave, msg=output) def test_install_exclude_worker(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin("package install /mnt/presto-admin/" "%(rpm)s -x %(slave1)s", rpm=rpm_name) self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output) self.installer.assert_installed(self, self.cluster.slaves[1], msg=output) self.installer.assert_installed(self, self.cluster.master, msg=output) self.installer.assert_installed(self, self.cluster.slaves[2], msg=output) def test_install_exclude_workers(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( "package install /mnt/presto-admin/" "%(rpm)s -x %(slave1)s,%(slave2)s", rpm=rpm_name ) self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output) self.installer.assert_installed(self, self.cluster.master, msg=output) self.installer.assert_installed(self, self.cluster.slaves[2], msg=output) def test_install_invalid_path(self): rpm_name = self.installer.copy_presto_rpm_to_master() cmd_output = self.run_prestoadmin("package install /mnt/presto-admin" "/invalid-path/presto.rpm", rpm=rpm_name) error = ( "\nFatal error: [%s] error: " "/mnt/presto-admin/invalid-path/presto.rpm: open failed: " "No such file or directory\n\nAborting.\n" ) expected = "" for host in self.cluster.all_internal_hosts(): expected += error % host self.assertEqualIgnoringOrder(cmd_output, expected) def test_install_no_path_arg(self): self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin("package install", raise_error=False) self.assertEqual( output, "Incorrect number of arguments to task.\n\n" "Displaying detailed information for task " "'package install':\n\n" " Install the rpm package on the cluster\n" " \n Args:\n" " local_path: Absolute path to the rpm" " to be installed\n " "--nodeps (optional): Flag to indicate if " "rpm install\n" " should ignore c" "hecking package dependencies. Equivalent\n" " to adding --nodeps flag to rpm " "-i.\n\n", ) def test_install_already_installed(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.run_prestoadmin("package install /mnt/presto-admin/%(rpm)s -H " "%(master)s", rpm=rpm_name) self.installer.assert_installed(self, self.cluster.master) cmd_output = self.run_prestoadmin("package install /mnt/presto-admin/%(rpm)s -H %(master)s", rpm=rpm_name) expected = self.escape_for_regex( self.replace_keywords( """ Fatal error: [%(master)s] sudo() received nonzero return code 1 while \ executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:'******'sudo password:' /bin/bash -l -c "rpm -i \ /opt/prestoadmin/packages/%(rpm)s" Aborting. Deploying rpm on %(master)s... Package deployed successfully on: %(master)s [%(master)s] out: error: Failed dependencies: [%(master)s] out: python >= 2.4 is needed by %(rpm_basename)s [%(master)s] out: """, **self.installer.get_keywords(rpm_name) ) self.assertRegexpMatchesLineByLine(cmd_output.splitlines(), self.escape_for_regex(expected).splitlines()) @docker_only def test_install_rpm_with_nodeps(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.cluster.exec_cmd_on_host(self.cluster.master, "rpm -e --nodeps python-2.6.6") self.assertRaisesRegexp( OSError, "package python-2.6.6 is not installed", self.cluster.exec_cmd_on_host, self.cluster.master, "rpm -q python-2.6.6", ) cmd_output = self.run_prestoadmin( "package install /mnt/presto-admin/%(rpm)s -H %(master)s --nodeps", rpm=rpm_name ) expected = ( "Deploying rpm on %(host)s...\n" "Package deployed successfully on: %(host)s\n" "Package installed successfully on: %(host)s" % {"host": self.cluster.internal_master} ) self.assertEqualIgnoringOrder(expected, cmd_output)
class TestPackageInstall(BaseProductTestCase): def setUp(self): super(TestPackageInstall, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() self.installer = StandalonePrestoInstaller(self) def tearDown(self): self._assert_uninstall() super(TestPackageInstall, self).tearDown() def _assert_uninstall(self): output = self.run_prestoadmin( 'package uninstall presto-server-rpm --force') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) @attr('smoketest') def test_package_installer(self): rpm_name = self.installer.copy_presto_rpm_to_master() # install output = self.run_prestoadmin('package install %(rpm)s', rpm=os.path.join( self.cluster.rpm_cache_dir, rpm_name)) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container, msg=output) # uninstall output = self.run_prestoadmin('package uninstall presto-server-rpm') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) def test_install_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() # install onto master and slave2 output = self.run_prestoadmin( 'package install %(rpm)s -H %(master)s,%(slave2)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)) self.installer.assert_installed(self, self.cluster.master, msg=output) self.installer.assert_installed(self, self.cluster.slaves[1], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output) # uninstall on slave2 output = self.run_prestoadmin( 'package uninstall presto-server-rpm -H %(slave2)s') self.installer.assert_installed(self, self.cluster.master, msg=output) for container in self.cluster.slaves: self.installer.assert_uninstalled(container, msg=output) # uninstall on rest output = self.run_prestoadmin( 'package uninstall presto-server-rpm --force') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) def test_install_exclude_nodes(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install %(rpm)s -x %(master)s,%(slave2)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)) # install self.installer.assert_uninstalled(self.cluster.master, msg=output) self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output) self.installer.assert_installed(self, self.cluster.slaves[0], msg=output) self.installer.assert_installed(self, self.cluster.slaves[2], msg=output) # uninstall output = self.run_prestoadmin( 'package uninstall presto-server-rpm -x %(master)s,%(slave2)s') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) # skip this tests as it depends on OS package names @attr('quarantine') @docker_only def test_install_rpm_missing_dependency(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.cluster.exec_cmd_on_host(self.cluster.master, 'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install %(rpm)s -H %(master)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name), raise_error=False) expected = self.replace_keywords( """ Fatal error: [%(master)s] sudo() received nonzero return code 1 while \ executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:'******'quarantine') @docker_only def test_install_rpm_with_nodeps(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.cluster.exec_cmd_on_host(self.cluster.master, 'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install %(rpm)s -H %(master)s --nodeps', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)) expected = 'Deploying rpm on %(host)s...\n' \ 'Package deployed successfully on: %(host)s\n' \ 'Package installed successfully on: %(host)s' \ % {'host': self.cluster.internal_master} self.assertEqualIgnoringOrder(expected, cmd_output)
class TestServerUpgrade(BaseProductTestCase): def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(self.STANDALONE_PRESTO_CLUSTER) self.installer = StandalonePrestoInstaller(self) def start_and_assert_started(self): cmd_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(cmd_output.splitlines()) self.assert_started(process_per_host) def assert_upgraded_to_dummy_rpm(self, hosts): for container in hosts: # Still should have the same configs self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) # However, dummy_rpm.rpm removes /usr/lib/presto/lib and # /usr/lib/presto/lib/plugin self.assert_path_removed(container, '/usr/lib/presto/lib') self.assert_path_removed(container, '/usr/lib/presto/lib/plugin') # And adds /usr/lib/presto/README.txt self.assert_path_exists(container, '/usr/lib/presto/README.txt') # And modifies the text of the readme in # /usr/shared/doc/presto/README.txt self.assert_file_content_regex( container, '/usr/shared/doc/presto/README.txt', r'.*New line of text here.$') @attr('smoketest') def test_upgrade(self): self.start_and_assert_started() self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster) def upgrade_and_assert_success(self, path_on_cluster): self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts()) @docker_only def test_rolling_upgrade(self): # Test that if a node is down, and then you upgrade again, it works self.run_prestoadmin('configuration deploy') self.cluster.stop_host(self.cluster.slaves[0]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.run_prestoadmin('server upgrade ' + path_on_cluster) running_hosts = self.cluster.all_hosts()[:] running_hosts.remove(self.cluster.slaves[0]) self.assert_upgraded_to_dummy_rpm(running_hosts) self.cluster.start_host(self.cluster.slaves[0]) self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster)) def copy_upgrade_rpm_to_cluster(self): rpm_name = self.installer.copy_presto_rpm_to_master() return os.path.join(self.cluster.mount_dir, rpm_name)
class TestServerUpgrade(BaseProductTestCase): def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.dummy_installer = StandalonePrestoInstaller( self, (os.path.join(prestoadmin.main_dir, 'tests', 'product', 'resources'), 'dummy-rpm.rpm')) self.real_installer = StandalonePrestoInstaller(self) def start_and_assert_started(self): cmd_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(cmd_output.splitlines()) self.assert_started(process_per_host) # # The dummy RPM is not guaranteed to have any functionality beyond not # including any real payload and adding the random README file. It's a # hacky one-off that satisfies the requirement of having *something* to # upgrade to without downloading another copy of the real RPM. This is NOT # the place to test functionality that the presto-server-rpm normally # provides, because the dummy rpm probably doesn't provide it, or worse, # provides an old and/or broken version of it. # def assert_upgraded_to_dummy_rpm(self, hosts): for container in hosts: # Still should have the same configs self.dummy_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) # However, dummy_rpm.rpm removes /usr/lib/presto/lib and # /usr/lib/presto/lib/plugin self.assert_path_removed(container, '/usr/lib/presto/lib') self.assert_path_removed(container, '/usr/lib/presto/lib/plugin') # And adds /usr/lib/presto/README.txt self.assert_path_exists(container, '/usr/lib/presto/README.txt') # And modifies the text of the readme in # /usr/shared/doc/presto/README.txt self.assert_file_content_regex( container, '/usr/shared/doc/presto/README.txt', r'.*New line of text here.$' ) @attr('smoketest') def test_upgrade(self): self.start_and_assert_started() self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster) def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''): self.run_prestoadmin('server upgrade ' + path_on_cluster + extra_arguments) self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts()) def copy_upgrade_rpm_to_cluster(self): rpm_name = self.dummy_installer.copy_presto_rpm_to_master() return os.path.join(self.cluster.rpm_cache_dir, rpm_name) def test_upgrade_fails_given_directory(self): dir_on_cluster = '/opt/prestoadmin' self.assertRaisesRegexp( OSError, 'RPM file not found at %s.' % dir_on_cluster, self.run_prestoadmin, 'server upgrade ' + dir_on_cluster ) def test_upgrade_works_with_symlink(self): self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.upgrade_and_assert_success(symlink) def test_configuration_preserved_on_upgrade(self): book_content = 'Call me Ishmael ... FINIS' book_path = '/etc/presto/moby_dick_abridged' self.run_prestoadmin('configuration deploy') big_files = {} for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) big_file = self.cluster.exec_cmd_on_host( container, "find /usr -size +2M -ls | " "sort -nk7 | " "tail -1 | " "awk '{print $NF}'").strip() self.cluster.exec_cmd_on_host( container, "cp %s /etc/presto" % (big_file,), invoke_sudo=True) big_files[container] = os.path.join("/etc/presto", os.path.basename(big_file)) self.cluster.write_content_to_host(book_content, book_path, host=container) self.cluster.exec_cmd_on_host(container, "chown presto:games %s" % (book_path,), invoke_sudo=True) self.cluster.exec_cmd_on_host(container, "chmod 272 %s" % (book_path,), invoke_sudo=True) self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container]) self.add_dummy_properties_to_host(self.cluster.slaves[1]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_dummy_properties(self.cluster.slaves[1]) for container in self.cluster.all_hosts(): self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container]) def test_upgrade_non_root_user(self): self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******"} ) self.run_prestoadmin('configuration deploy -p password') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster, extra_arguments=' -p password') def add_dummy_properties_to_host(self, host): self.cluster.write_content_to_host( 'com.facebook.presto=INFO', '/etc/presto/log.properties', host ) self.cluster.write_content_to_host( 'dummy config file', '/etc/presto/jvm.config', host ) def assert_dummy_properties(self, host): # assert log properties file is there self.assert_file_content( host, '/etc/presto/log.properties', 'com.facebook.presto=INFO' ) # assert dummy jvm config is there too self.assert_file_content( host, '/etc/presto/jvm.config', 'dummy config file' )
class TestServerUninstall(BaseProductTestCase): def setUp(self): super(TestServerUninstall, self).setUp() self.installer = StandalonePrestoInstaller(self) @attr('smoketest') def test_uninstall(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) cmd_output = self.run_prestoadmin( 'server uninstall', raise_error=False).splitlines() self.assert_stopped(process_per_host) expected = uninstall_output + self.expected_stop()[:] self.assertRegexpMatchesLineByLine(cmd_output, expected) for container in self.cluster.all_hosts(): self.assert_uninstalled_dirs_removed(container) def assert_uninstalled_dirs_removed(self, container): self.installer.assert_uninstalled(container) self.assert_path_removed(container, '/etc/presto') self.assert_path_removed(container, '/usr/lib/presto') self.assert_path_removed(container, '/var/lib/presto') self.assert_path_removed(container, '/usr/shared/doc/presto') self.assert_path_removed(container, '/etc/init.d/presto') def test_uninstall_when_server_down(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.run_prestoadmin('server stop -H %s' % self.cluster.internal_slaves[0]) cmd_output = self.run_prestoadmin('server uninstall').splitlines() self.assert_stopped(process_per_host) expected = uninstall_output + self.expected_stop( not_running=[self.cluster.internal_slaves[0]])[:] self.assertRegexpMatchesLineByLine(cmd_output, expected) for container in self.cluster.all_hosts(): self.assert_uninstalled_dirs_removed(container) def test_uninstall_twice(self): self.test_uninstall() output = self.run_prestoadmin('server uninstall', raise_error=False) with open(os.path.join(LOCAL_RESOURCES_DIR, 'uninstall_twice.txt'), 'r') as f: expected = f.read() self.assertEqualIgnoringOrder(expected, output) def test_uninstall_lost_host(self): self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) pa_installer = PrestoadminInstaller(self) pa_installer.install() topology = {"coordinator": self.cluster.internal_slaves[0], "workers": [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]} self.upload_topology(topology) self.installer.install() start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) down_node = self.cluster.internal_slaves[0] self.cluster.stop_host( self.cluster.slaves[0]) expected = self.down_node_connection_error( self.cluster.internal_slaves[0]) cmd_output = self.run_prestoadmin('server uninstall', raise_error=False) self.assertRegexpMatches(cmd_output, expected) process_per_active_host = [] for host, pid in process_per_host: if host not in down_node: process_per_active_host.append((host, pid)) self.assert_stopped(process_per_active_host) for container in [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]: self.assert_uninstalled_dirs_removed(container) def test_uninstall_with_dir_readonly(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) self.run_script_from_prestoadmin_dir("chmod 500 -R /usr/lib/presto") self.run_prestoadmin('server uninstall', raise_error=False) # The master node was not able to be stopped or uninstalled because # the permissions of the directory were changed such that the # stop command can't run pid_to_remove = None for (host, pid) in process_per_host: if host == self.cluster.internal_master: pid_to_remove = pid process_per_host.remove((self.cluster.internal_master, pid_to_remove)) self.assert_stopped(process_per_host) uninstalled_hosts = self.cluster.all_hosts()[:] uninstalled_hosts.remove(self.cluster.master) for container in uninstalled_hosts: self.assert_uninstalled_dirs_removed(container) self.installer.assert_installed(self, container=self.cluster.master) @docker_only def test_uninstall_as_non_sudo(self): self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) self.upload_topology() self.installer.install() script = './presto-admin server uninstall -u testuser -p testpass' output = self.run_script_from_prestoadmin_dir(script) with open(os.path.join(LOCAL_RESOURCES_DIR, 'non_sudo_uninstall.txt'), 'r') as f: expected = f.read() self.assertEqualIgnoringOrder(expected, output)
class TestServerUpgrade(BaseProductTestCase): def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) self.dummy_installer = StandalonePrestoInstaller( self, (os.path.join(prestoadmin.main_dir, 'tests', 'product', 'resources'), 'dummy-rpm.rpm')) self.real_installer = StandalonePrestoInstaller(self) def start_and_assert_started(self): cmd_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(cmd_output.splitlines()) self.assert_started(process_per_host) # # The dummy RPM is not guaranteed to have any functionality beyond not # including any real payload and adding the random README file. It's a # hacky one-off that satisfies the requirement of having *something* to # upgrade to without downloading another copy of the real RPM. This is NOT # the place to test functionality that the presto-server-rpm normally # provides, because the dummy rpm probably doesn't provide it, or worse, # provides an old and/or broken version of it. # def assert_upgraded_to_dummy_rpm(self, hosts): for container in hosts: # Still should have the same configs self.dummy_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) # However, dummy_rpm.rpm removes /usr/lib/presto/lib and # /usr/lib/presto/lib/plugin self.assert_path_removed(container, '/usr/lib/presto/lib') self.assert_path_removed(container, '/usr/lib/presto/lib/plugin') # And adds /usr/lib/presto/README.txt self.assert_path_exists(container, '/usr/lib/presto/README.txt') # And modifies the text of the readme in # /usr/shared/doc/presto/README.txt self.assert_file_content_regex( container, '/usr/shared/doc/presto/README.txt', r'.*New line of text here.$' ) @attr('smoketest') def test_upgrade(self): self.start_and_assert_started() self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster) def upgrade_and_assert_success(self, path_on_cluster): self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts()) @docker_only def test_rolling_upgrade(self): # Test that if a node is down, and then you upgrade again, it works self.run_prestoadmin('configuration deploy') self.cluster.stop_host(self.cluster.slaves[0]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.run_prestoadmin('server upgrade ' + path_on_cluster, raise_error=False) running_hosts = self.cluster.all_hosts()[:] running_hosts.remove(self.cluster.slaves[0]) self.assert_upgraded_to_dummy_rpm(running_hosts) self.cluster.start_host(self.cluster.slaves[0]) self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster)) def copy_upgrade_rpm_to_cluster(self): rpm_name = self.dummy_installer.copy_presto_rpm_to_master() return os.path.join(self.cluster.mount_dir, rpm_name)
class TestServerInstall(BaseProductTestCase): default_workers_config_with_slave1_ = """coordinator=false discovery.uri=http://slave1:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_coord_config_with_slave1_ = """coordinator=true discovery-server.enabled=true discovery.uri=http://slave1:8080 http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_workers_config_regex_ = """coordinator=false discovery.uri=http:.*:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_coord_config_regex_ = """coordinator=true discovery-server.enabled=true discovery.uri=http:.*:8080 http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB\n""" def setUp(self): super(TestServerInstall, self).setUp() self.setup_cluster(self.PA_ONLY_CLUSTER) self.installer = StandalonePrestoInstaller(self) def assert_common_configs(self, container): self.installer.assert_installed(self, container) self.assert_file_content(container, '/etc/presto/jvm.config', self.default_jvm_config_) self.assert_node_config(container, self.default_node_properties_) self.assert_has_default_connector(container) def assert_installed_with_configs(self, master, slaves): self.assert_common_configs(master) self.assert_file_content(master, '/etc/presto/config.properties', self.default_coord_config_with_slave1_) for container in slaves: self.assert_common_configs(container) self.assert_file_content(container, '/etc/presto/config.properties', self.default_workers_config_with_slave1_) def assert_installed_with_regex_configs(self, master, slaves): self.assert_common_configs(master) self.assert_file_content_regex(master, '/etc/presto/config.properties', self.default_coord_config_regex_) for container in slaves: self.assert_common_configs(container) self.assert_file_content_regex(container, '/etc/presto/config.properties', self.default_workers_config_regex_) @attr('smoketest') def test_install(self, dummy=False): self.upload_topology() cmd_output = self.installer.install(dummy) expected = installed_all_hosts_output actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) def test_install_worker_is_pa_master(self): topology = {"coordinator": "slave1", "workers": ["master", "slave2", "slave3"]} self.upload_topology(topology) cmd_output = self.installer.install(dummy=True) expected = install_with_worker_pa_master_out actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) self.assert_installed_with_configs( self.cluster.slaves[0], [self.cluster.slaves[1], self.cluster.slaves[2], self.cluster.master]) def test_install_ext_host_is_pa_master(self): topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology) cmd_output = self.installer.install(dummy=True) expected = install_with_ext_host_pa_master_out actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) self.assert_installed_with_configs( self.cluster.slaves[0], [self.cluster.slaves[1], self.cluster.slaves[2]]) def test_install_when_connector_json_exists(self): topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master ) cmd_output = self.installer.install(dummy=True) expected = ['Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: slave1 '] actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in [self.cluster.master, self.cluster.slaves[0]]: self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container) def test_install_when_topology_has_ips(self): ips = self.cluster.get_ip_address_dict() topology = {"coordinator": ips[self.cluster.master], "workers": [ips[self.cluster.slaves[0]]]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master ) cmd_output = self.installer.install(dummy=True).splitlines() expected = [ r'Deploying rpm on %s...' % ips[self.cluster.master], r'Deploying rpm on %s...' % ips[self.cluster.slaves[0]], r'Package deployed successfully on: ' + ips[ self.cluster.master], r'Package installed successfully on: ' + ips[ self.cluster.master], r'Package deployed successfully on: ' + ips[self.cluster.slaves[0]], r'Package installed successfully on: ' + ips[self.cluster.slaves[0]], r'Deploying configuration on: ' + ips[self.cluster.master], r'Deploying jmx.properties, tpch.properties ' r'connector configurations on: ' + ips[self.cluster.master], r'Deploying configuration on: ' + ips[self.cluster.slaves[0]], r'Deploying jmx.properties, tpch.properties ' r'connector configurations on: ' + ips[self.cluster.slaves[0]]] cmd_output.sort() expected.sort() self.assertRegexpMatchesLineByLine(expected, cmd_output) self.assert_installed_with_regex_configs( self.cluster.master, [self.cluster.slaves[0]]) for container in [self.cluster.master, self.cluster.slaves[0]]: self.assert_has_jmx_connector(container) def test_install_interactive_with_hostnames(self): self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master ) rpm_name = self.installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name) actual = cmd_output.splitlines() expected = [r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Deploying rpm on .*\.\.\.', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.'] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container) def test_install_interactive_with_ips(self): ips = self.cluster.get_ip_address_dict() rpm_name = self.installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) additional_keywords = { 'rpm': rpm_name, 'master_ip': ips[self.cluster.master], 'slave1_ip': ips[self.cluster.slaves[0]] } cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | ' './presto-admin server install /mnt/presto-admin/%(rpm)s ', **additional_keywords).splitlines() expected = [r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Deploying rpm on .*\.\.\.', r'Package deployed successfully on: ' + ips[self.cluster.master], r'Package installed successfully on: ' + ips[self.cluster.master], r'Package deployed successfully on: ' + ips[self.cluster.slaves[0]], r'Package installed successfully on: ' + ips[self.cluster.slaves[0]], r'Deploying configuration on: ' + ips[self.cluster.master], r'Deploying tpch.properties connector ' r'configurations on: ' + ips[self.cluster.master], r'Deploying configuration on: ' + ips[self.cluster.slaves[0]], r'Deploying tpch.properties connector ' r'configurations on: ' + ips[self.cluster.slaves[0]], r'Deploying rpm on .*\.\.\.'] cmd_output.sort() expected.sort() for expected_regexp, actual_line in zip(expected, cmd_output): self.assertRegexpMatches(actual_line, expected_regexp) self.assert_installed_with_regex_configs( self.cluster.master, [self.cluster.slaves[0]]) def test_install_with_wrong_topology(self): rpm_name = self.installer.copy_presto_rpm_to_master() topology = {'coordinator': 'dummy_master', 'workers': ['slave1']} self.upload_topology(topology) expected = 'u\'dummy_master\' is not a valid ip address or' \ ' host name.' \ ' More detailed information can be found in ' \ '/var/log/prestoadmin/presto-admin.log\n' self.assertRaisesRegexp(OSError, expected, self.run_prestoadmin, 'server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name) def test_install_with_malformed_topology(self): rpm_name = self.installer.copy_presto_rpm_to_master() topology = {'coordinator': 'master', 'workers': 'slave1' 'slave2'} self.upload_topology(topology) expected = 'Workers must be of type list. Found <type \'unicode\'>.' \ ' More detailed information can be found in ' \ '/var/log/prestoadmin/presto-admin.log' self.assertRaisesRegexp(OSError, expected, self.run_prestoadmin, 'server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name) def test_install_with_malformed_connector(self): self.upload_topology() self.cluster.write_content_to_host( 'connectr.typo:invalid', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master ) actual_out = self.installer.install(dummy=True) expected = 'Underlying exception:\n Catalog configuration ' \ 'jmx.properties does not contain connector.name' self.assertRegexpMatches(actual_out, expected) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container) self.assert_has_default_config(container) def test_connection_to_coord_lost(self): down_node = self.cluster.internal_slaves[0] topology = {"coordinator": down_node, "workers": [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]} self.upload_topology(topology=topology) self.cluster.stop_host( self.cluster.slaves[0]) actual_out = self.installer.install(dummy=True) self.assertRegexpMatches( actual_out, self.down_node_connection_error(down_node) ) for container in [self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_common_configs(container) self.assert_file_content( container, '/etc/presto/config.properties', self.default_workers_config_with_slave1_.replace( down_node, self.cluster.get_down_hostname(down_node) ) ) @docker_only def test_install_with_no_perm_to_local_path(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.upload_topology() self.run_prestoadmin("configuration deploy") script = 'chmod 600 /mnt/presto-admin/%(rpm)s; su app-admin -c ' \ '"./presto-admin server install /mnt/presto-admin/%(rpm)s "' error_msg = '\nFatal error: [%(host)s] error: ' \ '/mnt/presto-admin/%(rpm)s: ' \ 'open failed: Permission denied\n\nAborting.\n' expected = '' for host in self.cluster.all_internal_hosts(): expected += error_msg % {'host': host, 'rpm': rpm_name} actual = self.run_script_from_prestoadmin_dir(script, rpm=rpm_name) self.assertEqualIgnoringOrder(actual, expected) def test_install_twice(self): self.test_install(dummy=True) output = self.installer.install(dummy=True) with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') \ as f: expected = f.read() expected = self.escape_for_regex( self.replace_keywords(expected)) self.assertRegexpMatchesLineByLine(output.splitlines(), expected.splitlines()) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
class TestServerUpgrade(BaseProductTestCase): def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.dummy_installer = StandalonePrestoInstaller( self, (os.path.join(prestoadmin.main_dir, 'tests', 'product', 'resources'), 'dummy-rpm.rpm')) self.real_installer = StandalonePrestoInstaller(self) def start_and_assert_started(self): cmd_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(cmd_output.splitlines()) self.assert_started(process_per_host) # # The dummy RPM is not guaranteed to have any functionality beyond not # including any real payload and adding the random README file. It's a # hacky one-off that satisfies the requirement of having *something* to # upgrade to without downloading another copy of the real RPM. This is NOT # the place to test functionality that the presto-server-rpm normally # provides, because the dummy rpm probably doesn't provide it, or worse, # provides an old and/or broken version of it. # def assert_upgraded_to_dummy_rpm(self, hosts): for container in hosts: # Still should have the same configs self.dummy_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) # However, dummy_rpm.rpm removes /usr/lib/presto/lib and # /usr/lib/presto/lib/plugin self.assert_path_removed(container, '/usr/lib/presto/lib') self.assert_path_removed(container, '/usr/lib/presto/lib/plugin') # And modifies the text of the readme in # /usr/shared/doc/presto/README.txt self.assert_file_content_regex( container, '/usr/shared/doc/presto/README.txt', r'.*New line of text here.$') @attr('smoketest') def test_upgrade(self): self.start_and_assert_started() self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster) def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''): self.run_prestoadmin('server upgrade ' + path_on_cluster + extra_arguments) self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts()) def copy_upgrade_rpm_to_cluster(self): rpm_name = self.dummy_installer.copy_presto_rpm_to_master() return os.path.join(self.cluster.rpm_cache_dir, rpm_name) def test_upgrade_fails_given_directory(self): dir_on_cluster = '/opt/prestoadmin' self.assertRaisesRegexp(OSError, 'RPM file not found at %s.' % dir_on_cluster, self.run_prestoadmin, 'server upgrade ' + dir_on_cluster) def test_upgrade_works_with_symlink(self): self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host( self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.upgrade_and_assert_success(symlink) def test_configuration_preserved_on_upgrade(self): book_content = 'Call me Ishmael ... FINIS' book_path = '/etc/presto/moby_dick_abridged' self.run_prestoadmin('configuration deploy') big_files = {} for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) big_file = self.cluster.exec_cmd_on_host( container, "find /usr -size +2M -ls | " "sort -nk7 | " "tail -1 | " "awk '{print $NF}'").strip() self.cluster.exec_cmd_on_host(container, "cp %s /etc/presto" % (big_file, ), invoke_sudo=True) big_files[container] = os.path.join("/etc/presto", os.path.basename(big_file)) self.cluster.write_content_to_host(book_content, book_path, host=container) self.cluster.exec_cmd_on_host(container, "chown presto:games %s" % (book_path, ), invoke_sudo=True) self.cluster.exec_cmd_on_host(container, "chmod 272 %s" % (book_path, ), invoke_sudo=True) self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container]) self.add_dummy_properties_to_host(self.cluster.slaves[1]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host( self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_dummy_properties(self.cluster.slaves[1]) for container in self.cluster.all_hosts(): self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container]) def test_upgrade_non_root_user(self): self.upload_topology({ "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******" }) self.run_prestoadmin('configuration deploy -p password') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster, extra_arguments=' -p password') def add_dummy_properties_to_host(self, host): self.cluster.write_content_to_host('io.prestosql=INFO', '/etc/presto/log.properties', host) self.cluster.write_content_to_host('dummy config file', '/etc/presto/jvm.config', host) def assert_dummy_properties(self, host): # assert log properties file is there self.assert_file_content(host, '/etc/presto/log.properties', 'io.prestosql=INFO') # assert dummy jvm config is there too self.assert_file_content(host, '/etc/presto/jvm.config', 'dummy config file')
class TestServerUninstall(BaseProductTestCase): def setUp(self): super(TestServerUninstall, self).setUp() self.installer = StandalonePrestoInstaller(self) @attr('smoketest') def test_uninstall(self): self.setup_cluster(self.STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) cmd_output = self.run_prestoadmin('server uninstall').splitlines() self.assert_stopped(process_per_host) expected = uninstall_output + self.expected_stop()[:] self.assertRegexpMatchesLineByLine(cmd_output, expected) for container in self.cluster.all_hosts(): self.assert_uninstalled_dirs_removed(container) def assert_uninstalled_dirs_removed(self, container): self.installer.assert_uninstalled(container) self.assert_path_removed(container, '/etc/presto') self.assert_path_removed(container, '/usr/lib/presto') self.assert_path_removed(container, '/var/lib/presto') self.assert_path_removed(container, '/usr/shared/doc/presto') self.assert_path_removed(container, '/etc/init.d/presto') def test_uninstall_when_server_down(self): self.setup_cluster(self.STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.run_prestoadmin('server stop -H %s' % self.cluster.internal_slaves[0]) cmd_output = self.run_prestoadmin('server uninstall').splitlines() self.assert_stopped(process_per_host) expected = uninstall_output + self.expected_stop( not_running=[self.cluster.internal_slaves[0]])[:] self.assertRegexpMatchesLineByLine(cmd_output, expected) for container in self.cluster.all_hosts(): self.assert_uninstalled_dirs_removed(container) def test_uninstall_twice(self): self.test_uninstall() output = self.run_prestoadmin('server uninstall') with open(os.path.join(LOCAL_RESOURCES_DIR, 'uninstall_twice.txt'), 'r') as f: expected = f.read() self.assertEqualIgnoringOrder(expected, output) def test_uninstall_lost_host(self): self.setup_cluster(self.PA_ONLY_CLUSTER) pa_installer = PrestoadminInstaller(self) pa_installer.install() topology = { "coordinator": self.cluster.internal_slaves[0], "workers": [ self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2] ] } self.upload_topology(topology) self.installer.install() start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) down_node = self.cluster.internal_slaves[0] self.cluster.stop_host(self.cluster.slaves[0]) expected = self.down_node_connection_error( self.cluster.internal_slaves[0]) cmd_output = self.run_prestoadmin('server uninstall') self.assertRegexpMatches(cmd_output, expected) process_per_active_host = [] for host, pid in process_per_host: if host not in down_node: process_per_active_host.append((host, pid)) self.assert_stopped(process_per_active_host) for container in [ self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2] ]: self.assert_uninstalled_dirs_removed(container) def test_uninstall_with_dir_readonly(self): self.setup_cluster(self.STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) self.run_script_from_prestoadmin_dir("chmod 500 -R /usr/lib/presto") self.run_prestoadmin('server uninstall').splitlines() # The master node was not able to be stopped or uninstalled because # the permissions of the directory were changed such that the # stop command can't run pid_to_remove = None for (host, pid) in process_per_host: if host == self.cluster.internal_master: pid_to_remove = pid process_per_host.remove((self.cluster.internal_master, pid_to_remove)) self.assert_stopped(process_per_host) uninstalled_hosts = self.cluster.all_hosts()[:] uninstalled_hosts.remove(self.cluster.master) for container in uninstalled_hosts: self.assert_uninstalled_dirs_removed(container) self.installer.assert_installed(self, container=self.cluster.master) @docker_only def test_uninstall_as_non_sudo(self): self.setup_cluster(self.PA_ONLY_CLUSTER) self.upload_topology() self.installer.install(dummy=True) script = './presto-admin server uninstall -u testuser -p testpass' output = self.run_script_from_prestoadmin_dir(script) with open(os.path.join(LOCAL_RESOURCES_DIR, 'non_sudo_uninstall.txt'), 'r') as f: expected = f.read() self.assertEqualIgnoringOrder(expected, output)
class TestServerUpgrade(BaseProductTestCase): def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.dummy_installer = StandalonePrestoInstaller( self, (os.path.join(prestoadmin.main_dir, 'tests', 'product', 'resources'), 'dummy-rpm.rpm')) self.real_installer = StandalonePrestoInstaller(self) def start_and_assert_started(self): cmd_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(cmd_output.splitlines()) self.assert_started(process_per_host) # # The dummy RPM is not guaranteed to have any functionality beyond not # including any real payload and adding the random README file. It's a # hacky one-off that satisfies the requirement of having *something* to # upgrade to without downloading another copy of the real RPM. This is NOT # the place to test functionality that the presto-server-rpm normally # provides, because the dummy rpm probably doesn't provide it, or worse, # provides an old and/or broken version of it. # def assert_upgraded_to_dummy_rpm(self, hosts): for container in hosts: # Still should have the same configs self.dummy_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) # However, dummy_rpm.rpm removes /usr/lib/presto/lib and # /usr/lib/presto/lib/plugin self.assert_path_removed(container, '/usr/lib/presto/lib') self.assert_path_removed(container, '/usr/lib/presto/lib/plugin') # And adds /usr/lib/presto/README.txt self.assert_path_exists(container, '/usr/lib/presto/README.txt') # And modifies the text of the readme in # /usr/shared/doc/presto/README.txt self.assert_file_content_regex( container, '/usr/shared/doc/presto/README.txt', r'.*New line of text here.$') @attr('smoketest') def test_upgrade(self): self.start_and_assert_started() self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster) def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''): self.run_prestoadmin('server upgrade ' + path_on_cluster + extra_arguments) self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts()) @docker_only def test_rolling_upgrade(self): # Test that if a node is down, and then you upgrade again, it works self.run_prestoadmin('configuration deploy') self.cluster.stop_host(self.cluster.slaves[0]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.run_prestoadmin('server upgrade ' + path_on_cluster, raise_error=False) running_hosts = self.cluster.all_hosts()[:] running_hosts.remove(self.cluster.slaves[0]) self.assert_upgraded_to_dummy_rpm(running_hosts) self.cluster.start_host(self.cluster.slaves[0]) self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster)) def copy_upgrade_rpm_to_cluster(self): rpm_name = self.dummy_installer.copy_presto_rpm_to_master() return os.path.join(self.cluster.mount_dir, rpm_name) def test_upgrade_fails_given_directory(self): dir_on_cluster = '/opt/prestoadmin' self.assertRaisesRegexp(OSError, 'RPM file not found at %s.' % dir_on_cluster, self.run_prestoadmin, 'server upgrade ' + dir_on_cluster) def test_upgrade_works_with_symlink(self): self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = '/opt/prestoadmin/link.rpm' self.cluster.exec_cmd_on_host( self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.upgrade_and_assert_success(symlink) def test_configuration_preserved_on_upgrade(self): self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.add_dummy_properties_to_host(self.cluster.slaves[1]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = '/opt/prestoadmin/link.rpm' self.cluster.exec_cmd_on_host( self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_dummy_properties(self.cluster.slaves[1]) def test_upgrade_non_root_user(self): self.upload_topology({ "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******" }) self.run_prestoadmin('configuration deploy -p password') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster, extra_arguments=' -p password') def add_dummy_properties_to_host(self, host): self.cluster.write_content_to_host('com.facebook.presto=INFO', '/etc/presto/log.properties', host) self.cluster.write_content_to_host('dummy config file', '/etc/presto/jvm.config', host) def assert_dummy_properties(self, host): # assert log properties file is there self.assert_file_content(host, '/etc/presto/log.properties', 'com.facebook.presto=INFO') # assert dummy jvm config is there too self.assert_file_content(host, '/etc/presto/jvm.config', 'dummy config file')
class TestPackageInstall(BaseProductTestCase): def setUp(self): super(TestPackageInstall, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() self.installer = StandalonePrestoInstaller(self) def tearDown(self): self._assert_uninstall() super(TestPackageInstall, self).tearDown() def _assert_uninstall(self): output = self.run_prestoadmin('package uninstall presto-server-rpm --force') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) @attr('smoketest') def test_package_installer(self): rpm_name = self.installer.copy_presto_rpm_to_master() # install output = self.run_prestoadmin('package install %(rpm)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container, msg=output) # uninstall output = self.run_prestoadmin('package uninstall presto-server-rpm') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) def test_install_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() # install onto master and slave2 output = self.run_prestoadmin('package install %(rpm)s -H %(master)s,%(slave2)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)) self.installer.assert_installed(self, self.cluster.master, msg=output) self.installer.assert_installed(self, self.cluster.slaves[1], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output) # uninstall on slave2 output = self.run_prestoadmin('package uninstall presto-server-rpm -H %(slave2)s') self.installer.assert_installed(self, self.cluster.master, msg=output) for container in self.cluster.slaves: self.installer.assert_uninstalled(container, msg=output) # uninstall on rest output = self.run_prestoadmin('package uninstall presto-server-rpm --force') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) def test_install_exclude_nodes(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin('package install %(rpm)s -x %(master)s,%(slave2)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)) # install self.installer.assert_uninstalled(self.cluster.master, msg=output) self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output) self.installer.assert_installed(self, self.cluster.slaves[0], msg=output) self.installer.assert_installed(self, self.cluster.slaves[2], msg=output) # uninstall output = self.run_prestoadmin('package uninstall presto-server-rpm -x %(master)s,%(slave2)s') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) # skip this tests as it depends on OS package names @attr('quarantine') @docker_only def test_install_rpm_missing_dependency(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.cluster.exec_cmd_on_host( self.cluster.master, 'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install %(rpm)s -H %(master)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name), raise_error=False) expected = self.replace_keywords(""" Fatal error: [%(master)s] sudo() received nonzero return code 1 while \ executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:'******'quarantine') @docker_only def test_install_rpm_with_nodeps(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.cluster.exec_cmd_on_host( self.cluster.master, 'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install %(rpm)s -H %(master)s --nodeps', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name) ) expected = 'Deploying rpm on %(host)s...\n' \ 'Package deployed successfully on: %(host)s\n' \ 'Package installed successfully on: %(host)s' \ % {'host': self.cluster.internal_master} self.assertEqualIgnoringOrder(expected, cmd_output)
class TestServerUpgrade(BaseProductTestCase): def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.dummy_installer = StandalonePrestoInstaller( self, (os.path.join(prestoadmin.main_dir, 'tests', 'product', 'resources'), 'dummy-rpm.rpm')) self.real_installer = StandalonePrestoInstaller(self) def start_and_assert_started(self): cmd_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(cmd_output.splitlines()) self.assert_started(process_per_host) # # The dummy RPM is not guaranteed to have any functionality beyond not # including any real payload and adding the random README file. It's a # hacky one-off that satisfies the requirement of having *something* to # upgrade to without downloading another copy of the real RPM. This is NOT # the place to test functionality that the presto-server-rpm normally # provides, because the dummy rpm probably doesn't provide it, or worse, # provides an old and/or broken version of it. # def assert_upgraded_to_dummy_rpm(self, hosts): for container in hosts: # Still should have the same configs self.dummy_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) # However, dummy_rpm.rpm removes /usr/lib/presto/lib and # /usr/lib/presto/lib/plugin self.assert_path_removed(container, '/usr/lib/presto/lib') self.assert_path_removed(container, '/usr/lib/presto/lib/plugin') # And adds /usr/lib/presto/README.txt self.assert_path_exists(container, '/usr/lib/presto/README.txt') # And modifies the text of the readme in # /usr/shared/doc/presto/README.txt self.assert_file_content_regex( container, '/usr/shared/doc/presto/README.txt', r'.*New line of text here.$' ) @attr('smoketest') def test_upgrade(self): self.start_and_assert_started() self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster) def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''): self.run_prestoadmin('server upgrade ' + path_on_cluster + extra_arguments) self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts()) @docker_only def test_rolling_upgrade(self): # Test that if a node is down, and then you upgrade again, it works self.run_prestoadmin('configuration deploy') self.cluster.stop_host(self.cluster.slaves[0]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.run_prestoadmin('server upgrade ' + path_on_cluster, raise_error=False) running_hosts = self.cluster.all_hosts()[:] running_hosts.remove(self.cluster.slaves[0]) self.assert_upgraded_to_dummy_rpm(running_hosts) self.cluster.start_host(self.cluster.slaves[0]) self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster)) def copy_upgrade_rpm_to_cluster(self): rpm_name = self.dummy_installer.copy_presto_rpm_to_master() return os.path.join(self.cluster.mount_dir, rpm_name) def test_upgrade_fails_given_directory(self): dir_on_cluster = '/opt/prestoadmin' self.assertRaisesRegexp( OSError, 'RPM file not found at %s.' % dir_on_cluster, self.run_prestoadmin, 'server upgrade ' + dir_on_cluster ) def test_upgrade_works_with_symlink(self): self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = '/opt/prestoadmin/link.rpm' self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.upgrade_and_assert_success(symlink) def test_configuration_preserved_on_upgrade(self): self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.add_dummy_properties_to_host(self.cluster.slaves[1]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = '/opt/prestoadmin/link.rpm' self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_dummy_properties(self.cluster.slaves[1]) def test_upgrade_non_root_user(self): self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******"} ) self.run_prestoadmin('configuration deploy -p password') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster, extra_arguments=' -p password') def add_dummy_properties_to_host(self, host): self.cluster.write_content_to_host( 'com.facebook.presto=INFO', '/etc/presto/log.properties', host ) self.cluster.write_content_to_host( 'dummy config file', '/etc/presto/jvm.config', host ) def assert_dummy_properties(self, host): # assert log properties file is there self.assert_file_content( host, '/etc/presto/log.properties', 'com.facebook.presto=INFO' ) # assert dummy jvm config is there too self.assert_file_content( host, '/etc/presto/jvm.config', 'dummy config file' )
class TestServerInstall(BaseProductTestCase): default_workers_config_with_slave1_ = """coordinator=false discovery.uri=http://slave1:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_coord_config_with_slave1_ = """coordinator=true discovery-server.enabled=true discovery.uri=http://slave1:8080 http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_workers_config_regex_ = """coordinator=false discovery.uri=http:.*:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_coord_config_regex_ = """coordinator=true discovery-server.enabled=true discovery.uri=http:.*:8080 http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB\n""" def setUp(self): super(TestServerInstall, self).setUp() self.setup_cluster(self.PA_ONLY_CLUSTER) self.installer = StandalonePrestoInstaller(self) def assert_common_configs(self, container): self.installer.assert_installed(self, container) self.assert_file_content(container, '/etc/presto/jvm.config', self.default_jvm_config_) self.assert_node_config(container, self.default_node_properties_) self.assert_has_default_connector(container) def assert_installed_with_configs(self, master, slaves): self.assert_common_configs(master) self.assert_file_content(master, '/etc/presto/config.properties', self.default_coord_config_with_slave1_) for container in slaves: self.assert_common_configs(container) self.assert_file_content(container, '/etc/presto/config.properties', self.default_workers_config_with_slave1_) def assert_installed_with_regex_configs(self, master, slaves): self.assert_common_configs(master) self.assert_file_content_regex(master, '/etc/presto/config.properties', self.default_coord_config_regex_) for container in slaves: self.assert_common_configs(container) self.assert_file_content_regex(container, '/etc/presto/config.properties', self.default_workers_config_regex_) @attr('smoketest') def test_install(self, dummy=False): self.upload_topology() cmd_output = self.installer.install(dummy) expected = installed_all_hosts_output actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) def test_install_worker_is_pa_master(self): topology = { "coordinator": "slave1", "workers": ["master", "slave2", "slave3"] } self.upload_topology(topology) cmd_output = self.installer.install(dummy=True) expected = install_with_worker_pa_master_out actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) self.assert_installed_with_configs(self.cluster.slaves[0], [ self.cluster.slaves[1], self.cluster.slaves[2], self.cluster.master ]) def test_install_ext_host_is_pa_master(self): topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology) cmd_output = self.installer.install(dummy=True) expected = install_with_ext_host_pa_master_out actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) self.assert_installed_with_configs( self.cluster.slaves[0], [self.cluster.slaves[1], self.cluster.slaves[2]]) def test_install_when_connector_json_exists(self): topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) cmd_output = self.installer.install(dummy=True) expected = [ 'Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: slave1 ' ] actual = cmd_output.splitlines() self.assertEqual(sorted(expected), sorted(actual)) for container in [self.cluster.master, self.cluster.slaves[0]]: self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container) def test_install_when_topology_has_ips(self): ips = self.cluster.get_ip_address_dict() topology = { "coordinator": ips[self.cluster.master], "workers": [ips[self.cluster.slaves[0]]] } self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) cmd_output = self.installer.install(dummy=True).splitlines() expected = [ r'Deploying rpm on %s...' % ips[self.cluster.master], r'Deploying rpm on %s...' % ips[self.cluster.slaves[0]], r'Package deployed successfully on: ' + ips[self.cluster.master], r'Package installed successfully on: ' + ips[self.cluster.master], r'Package deployed successfully on: ' + ips[self.cluster.slaves[0]], r'Package installed successfully on: ' + ips[self.cluster.slaves[0]], r'Deploying configuration on: ' + ips[self.cluster.master], r'Deploying jmx.properties, tpch.properties ' r'connector configurations on: ' + ips[self.cluster.master], r'Deploying configuration on: ' + ips[self.cluster.slaves[0]], r'Deploying jmx.properties, tpch.properties ' r'connector configurations on: ' + ips[self.cluster.slaves[0]] ] cmd_output.sort() expected.sort() self.assertRegexpMatchesLineByLine(expected, cmd_output) self.assert_installed_with_regex_configs(self.cluster.master, [self.cluster.slaves[0]]) for container in [self.cluster.master, self.cluster.slaves[0]]: self.assert_has_jmx_connector(container) def test_install_interactive_with_hostnames(self): self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) rpm_name = self.installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name) actual = cmd_output.splitlines() expected = [ r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Deploying rpm on .*\.\.\.', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.' ] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container) def test_install_interactive_with_ips(self): ips = self.cluster.get_ip_address_dict() rpm_name = self.installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) additional_keywords = { 'rpm': rpm_name, 'master_ip': ips[self.cluster.master], 'slave1_ip': ips[self.cluster.slaves[0]] } cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | ' './presto-admin server install /mnt/presto-admin/%(rpm)s ', **additional_keywords).splitlines() expected = [ r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Deploying rpm on .*\.\.\.', r'Package deployed successfully on: ' + ips[self.cluster.master], r'Package installed successfully on: ' + ips[self.cluster.master], r'Package deployed successfully on: ' + ips[self.cluster.slaves[0]], r'Package installed successfully on: ' + ips[self.cluster.slaves[0]], r'Deploying configuration on: ' + ips[self.cluster.master], r'Deploying tpch.properties connector ' r'configurations on: ' + ips[self.cluster.master], r'Deploying configuration on: ' + ips[self.cluster.slaves[0]], r'Deploying tpch.properties connector ' r'configurations on: ' + ips[self.cluster.slaves[0]], r'Deploying rpm on .*\.\.\.' ] cmd_output.sort() expected.sort() for expected_regexp, actual_line in zip(expected, cmd_output): self.assertRegexpMatches(actual_line, expected_regexp) self.assert_installed_with_regex_configs(self.cluster.master, [self.cluster.slaves[0]]) def test_install_with_wrong_topology(self): rpm_name = self.installer.copy_presto_rpm_to_master() topology = {'coordinator': 'dummy_master', 'workers': ['slave1']} self.upload_topology(topology) expected = 'u\'dummy_master\' is not a valid ip address or' \ ' host name.' \ ' More detailed information can be found in ' \ '/var/log/prestoadmin/presto-admin.log\n' self.assertRaisesRegexp(OSError, expected, self.run_prestoadmin, 'server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name) def test_install_with_malformed_topology(self): rpm_name = self.installer.copy_presto_rpm_to_master() topology = {'coordinator': 'master', 'workers': 'slave1' 'slave2'} self.upload_topology(topology) expected = 'Workers must be of type list. Found <type \'unicode\'>.' \ ' More detailed information can be found in ' \ '/var/log/prestoadmin/presto-admin.log' self.assertRaisesRegexp(OSError, expected, self.run_prestoadmin, 'server install /mnt/presto-admin/%(rpm)s ', rpm=rpm_name) def test_install_with_malformed_connector(self): self.upload_topology() self.cluster.write_content_to_host( 'connectr.typo:invalid', os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'), self.cluster.master) actual_out = self.installer.install(dummy=True) expected = 'Underlying exception:\n Catalog configuration ' \ 'jmx.properties does not contain connector.name' self.assertRegexpMatches(actual_out, expected) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container) self.assert_has_default_config(container) def test_connection_to_coord_lost(self): down_node = self.cluster.internal_slaves[0] topology = { "coordinator": down_node, "workers": [ self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2] ] } self.upload_topology(topology=topology) self.cluster.stop_host(self.cluster.slaves[0]) actual_out = self.installer.install(dummy=True) self.assertRegexpMatches(actual_out, self.down_node_connection_error(down_node)) for container in [ self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2] ]: self.assert_common_configs(container) self.assert_file_content( container, '/etc/presto/config.properties', self.default_workers_config_with_slave1_.replace( down_node, self.cluster.get_down_hostname(down_node))) @docker_only def test_install_with_no_perm_to_local_path(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.upload_topology() self.run_prestoadmin("configuration deploy") script = 'chmod 600 /mnt/presto-admin/%(rpm)s; su app-admin -c ' \ '"./presto-admin server install /mnt/presto-admin/%(rpm)s "' error_msg = '\nFatal error: [%(host)s] error: ' \ '/mnt/presto-admin/%(rpm)s: ' \ 'open failed: Permission denied\n\nAborting.\n' expected = '' for host in self.cluster.all_internal_hosts(): expected += error_msg % {'host': host, 'rpm': rpm_name} actual = self.run_script_from_prestoadmin_dir(script, rpm=rpm_name) self.assertEqualIgnoringOrder(actual, expected) def test_install_twice(self): self.test_install(dummy=True) output = self.installer.install(dummy=True) with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') \ as f: expected = f.read() expected = self.escape_for_regex(self.replace_keywords(expected)) self.assertRegexpMatchesLineByLine(output.splitlines(), expected.splitlines()) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container)
class TestServerUpgrade(BaseProductTestCase): def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) self.installer = StandalonePrestoInstaller(self) def start_and_assert_started(self): cmd_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(cmd_output.splitlines()) self.assert_started(process_per_host) def assert_upgraded_to_dummy_rpm(self, hosts): for container in hosts: # Still should have the same configs self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) # However, dummy_rpm.rpm removes /usr/lib/presto/lib and # /usr/lib/presto/lib/plugin self.assert_path_removed(container, '/usr/lib/presto/lib') self.assert_path_removed(container, '/usr/lib/presto/lib/plugin') # And adds /usr/lib/presto/README.txt self.assert_path_exists(container, '/usr/lib/presto/README.txt') # And modifies the text of the readme in # /usr/shared/doc/presto/README.txt self.assert_file_content_regex( container, '/usr/shared/doc/presto/README.txt', r'.*New line of text here.$' ) @attr('smoketest') def test_upgrade(self): self.start_and_assert_started() self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster) def upgrade_and_assert_success(self, path_on_cluster): self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts()) @docker_only def test_rolling_upgrade(self): # Test that if a node is down, and then you upgrade again, it works self.run_prestoadmin('configuration deploy') self.cluster.stop_host(self.cluster.slaves[0]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.run_prestoadmin('server upgrade ' + path_on_cluster, raise_error=False) running_hosts = self.cluster.all_hosts()[:] running_hosts.remove(self.cluster.slaves[0]) self.assert_upgraded_to_dummy_rpm(running_hosts) self.cluster.start_host(self.cluster.slaves[0]) self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster)) def copy_upgrade_rpm_to_cluster(self): rpm_name = self.installer.copy_presto_rpm_to_master() return os.path.join(self.cluster.mount_dir, rpm_name)
class TestPackageInstall(BaseProductTestCase): def setUp(self): super(TestPackageInstall, self).setUp() self.setup_cluster(self.PA_ONLY_CLUSTER) self.upload_topology() self.installer = StandalonePrestoInstaller(self) @attr('smoketest') def test_package_install(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install ' '/mnt/presto-admin/%(rpm)s', rpm=rpm_name) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container, msg=output) def test_install_coord_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(master)s', rpm=rpm_name) self.installer.assert_installed(self, self.cluster.master) for slave in self.cluster.slaves: self.installer.assert_uninstalled(slave, msg=output) def test_install_worker_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(slave1)s', rpm=rpm_name) self.installer.assert_installed(self, self.cluster.slaves[0], msg=output) self.installer.assert_uninstalled(self.cluster.master, msg=output) self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output) def test_install_workers_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install /mnt/presto-admin/' '%(rpm)s -H %(slave1)s,%(slave2)s', rpm=rpm_name) self.installer.assert_installed(self, self.cluster.slaves[0], msg=output) self.installer.assert_installed(self, self.cluster.slaves[1], msg=output) self.installer.assert_uninstalled(self.cluster.master, msg=output) self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output) def test_install_exclude_coord(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install /mnt/presto-admin/' '%(rpm)s -x %(master)s', rpm=rpm_name) self.installer.assert_uninstalled(self.cluster.master, msg=output) for slave in self.cluster.slaves: self.installer.assert_installed(self, slave, msg=output) def test_install_exclude_worker(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install /mnt/presto-admin/' '%(rpm)s -x %(slave1)s', rpm=rpm_name) self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output) self.installer.assert_installed(self, self.cluster.slaves[1], msg=output) self.installer.assert_installed(self, self.cluster.master, msg=output) self.installer.assert_installed(self, self.cluster.slaves[2], msg=output) def test_install_exclude_workers(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install /mnt/presto-admin/' '%(rpm)s -x %(slave1)s,%(slave2)s', rpm=rpm_name) self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output) self.installer.assert_installed(self, self.cluster.master, msg=output) self.installer.assert_installed(self, self.cluster.slaves[2], msg=output) def test_install_invalid_path(self): rpm_name = self.installer.copy_presto_rpm_to_master() cmd_output = self.run_prestoadmin( 'package install /mnt/presto-admin' '/invalid-path/presto.rpm', rpm=rpm_name) error = '\nFatal error: [%s] error: ' \ '/mnt/presto-admin/invalid-path/presto.rpm: open failed: ' \ 'No such file or directory\n\nAborting.\n' expected = '' for host in self.cluster.all_internal_hosts(): expected += error % host self.assertEqualIgnoringOrder(cmd_output, expected) def test_install_no_path_arg(self): self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin('package install', raise_error=False) self.assertEqual( output, 'Incorrect number of arguments to task.\n\n' 'Displaying detailed information for task ' '\'package install\':\n\n' ' Install the rpm package on the cluster\n' ' \n Args:\n' ' local_path: Absolute path to the rpm' ' to be installed\n ' '--nodeps (optional): Flag to indicate if ' 'rpm install\n' ' should ignore c' 'hecking package dependencies. Equivalent\n' ' to adding --nodeps flag to rpm ' '-i.\n\n') def test_install_already_installed(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H ' '%(master)s', rpm=rpm_name) self.installer.assert_installed(self, self.cluster.master) cmd_output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(master)s', rpm=rpm_name) expected = self.escape_for_regex( self.replace_keywords( """ Fatal error: [%(master)s] sudo() received nonzero return code 1 while \ executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:'******'package install ' '/etc/opt/prestoadmin/config.json') error = """ Fatal error: \[%s\] error: (/etc/opt/prestoadmin/config.json: )?not an rpm \ package Aborting. """ expected = '' for host in self.cluster.all_internal_hosts(): expected += error % host self.assertRegexpMatchesLineByLine(cmd_output.splitlines(), expected.splitlines()) @docker_only def test_install_rpm_with_missing_jdk(self): rpm_name = self.installer.copy_presto_rpm_to_master( rpm_dir=prestoadmin.main_dir, rpm_name=self.installer.presto_rpm_filename) self.cluster.exec_cmd_on_host(self.cluster.master, 'rpm -e jdk1.8.0_40-1.8.0_40-fcs') self.assertRaisesRegexp( OSError, 'package jdk1.8.0_40-1.8.0_40-fcs is not ' 'installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q jdk1.8.0_40-1.8.0_40-fcs') cmd_output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(master)s', rpm=rpm_name) self.assertRegexpMatchesLineByLine( cmd_output.splitlines(), self.jdk_not_found_error_message(rpm_name).splitlines()) def jdk_not_found_error_message(self, rpm_name): with open(os.path.join(LOCAL_RESOURCES_DIR, 'jdk_not_found.txt')) as f: jdk_not_found_error = f.read() return self.escape_for_regex( self.replace_keywords(jdk_not_found_error, **self.installer.get_keywords(rpm_name))) @docker_only def test_install_rpm_missing_dependency(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.cluster.exec_cmd_on_host(self.cluster.master, 'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(master)s', rpm=rpm_name) expected = self.replace_keywords( """ Fatal error: [%(master)s] sudo() received nonzero return code 1 while \ executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:'******'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(master)s --nodeps', rpm=rpm_name) expected = 'Deploying rpm on %(host)s...\n' \ 'Package deployed successfully on: %(host)s\n' \ 'Package installed successfully on: %(host)s' \ % {'host': self.cluster.internal_master} self.assertEqualIgnoringOrder(expected, cmd_output)
class TestPackageInstall(BaseProductTestCase): def setUp(self): super(TestPackageInstall, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) self.upload_topology() self.installer = StandalonePrestoInstaller(self) @attr('smoketest') def test_package_install(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin('package install ' '/mnt/presto-admin/%(rpm)s', rpm=rpm_name) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container, msg=output) def test_install_coord_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(master)s', rpm=rpm_name) self.installer.assert_installed(self, self.cluster.master) for slave in self.cluster.slaves: self.installer.assert_uninstalled(slave, msg=output) def test_install_worker_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(slave1)s', rpm=rpm_name) self.installer.assert_installed(self, self.cluster.slaves[0], msg=output) self.installer.assert_uninstalled(self.cluster.master, msg=output) self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output) def test_install_workers_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin('package install /mnt/presto-admin/' '%(rpm)s -H %(slave1)s,%(slave2)s', rpm=rpm_name) self.installer.assert_installed(self, self.cluster.slaves[0], msg=output) self.installer.assert_installed(self, self.cluster.slaves[1], msg=output) self.installer.assert_uninstalled(self.cluster.master, msg=output) self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output) def test_install_exclude_coord(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin('package install /mnt/presto-admin/' '%(rpm)s -x %(master)s', rpm=rpm_name) self.installer.assert_uninstalled(self.cluster.master, msg=output) for slave in self.cluster.slaves: self.installer.assert_installed(self, slave, msg=output) def test_install_exclude_worker(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin('package install /mnt/presto-admin/' '%(rpm)s -x %(slave1)s', rpm=rpm_name) self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output) self.installer.assert_installed(self, self.cluster.slaves[1], msg=output) self.installer.assert_installed(self, self.cluster.master, msg=output) self.installer.assert_installed(self, self.cluster.slaves[2], msg=output) def test_install_exclude_workers(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin('package install /mnt/presto-admin/' '%(rpm)s -x %(slave1)s,%(slave2)s', rpm=rpm_name) self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output) self.installer.assert_installed(self, self.cluster.master, msg=output) self.installer.assert_installed(self, self.cluster.slaves[2], msg=output) def test_install_invalid_path(self): rpm_name = self.installer.copy_presto_rpm_to_master() cmd_output = self.run_prestoadmin('package install /mnt/presto-admin' '/invalid-path/presto.rpm', rpm=rpm_name, raise_error=False) error = '\nFatal error: [%s] error: ' \ '/mnt/presto-admin/invalid-path/presto.rpm: open failed: ' \ 'No such file or directory\n\nAborting.\n' expected = '' for host in self.cluster.all_internal_hosts(): expected += error % host self.assertEqualIgnoringOrder(cmd_output, expected) def test_install_no_path_arg(self): self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin('package install', raise_error=False) self.assertEqual(output, 'Incorrect number of arguments to task.\n\n' 'Displaying detailed information for task ' '\'package install\':\n\n' ' Install the rpm package on the cluster\n' ' \n Args:\n' ' local_path: Absolute path to the rpm' ' to be installed\n ' '--nodeps (optional): Flag to indicate if ' 'rpm install\n' ' should ignore c' 'hecking package dependencies. Equivalent\n' ' to adding --nodeps flag to rpm ' '-i.\n\n') def test_install_already_installed(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.run_prestoadmin('package install /mnt/presto-admin/%(rpm)s -H ' '%(master)s', rpm=rpm_name) self.installer.assert_installed(self, self.cluster.master) cmd_output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(master)s', rpm=rpm_name, raise_error=False) expected = self.escape_for_regex(self.replace_keywords(""" Fatal error: [%(master)s] sudo() received nonzero return code 1 while \ executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:'******'package install ' '/etc/opt/prestoadmin/config.json', raise_error=False) error = """ Fatal error: \[%s\] error: (/etc/opt/prestoadmin/config.json: )?not an rpm \ package Aborting. """ expected = '' for host in self.cluster.all_internal_hosts(): expected += error % host self.assertRegexpMatchesLineByLine(cmd_output.splitlines(), expected.splitlines()) @docker_only def test_install_rpm_missing_dependency(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.cluster.exec_cmd_on_host( self.cluster.master, 'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(master)s', rpm=rpm_name, raise_error=False) expected = self.replace_keywords(""" Fatal error: [%(master)s] sudo() received nonzero return code 1 while \ executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:'******'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install /mnt/presto-admin/%(rpm)s -H %(master)s --nodeps', rpm=rpm_name ) expected = 'Deploying rpm on %(host)s...\n' \ 'Package deployed successfully on: %(host)s\n' \ 'Package installed successfully on: %(host)s' \ % {'host': self.cluster.internal_master} self.assertEqualIgnoringOrder(expected, cmd_output)