def setup_cluster_assert_catalogs(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') for host in self.cluster.all_hosts(): self.assert_has_default_catalog(host) self._assert_catalogs_loaded([['system'], ['tpch']])
def test_start_restart_config_file_error(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) # Remove a required config file so that the server can't start self.cluster.exec_cmd_on_host( self.cluster.master, 'mv /etc/presto/config.properties ' '/etc/presto/config.properties.bak') started_hosts = self.cluster.all_internal_hosts() started_hosts.remove(self.cluster.internal_master) expected_start = self.expected_start(start_success=started_hosts) error_msg = self.escape_for_regex( self.replace_keywords( '[%(master)s] out: Starting presto\n' '[%(master)s] out: ERROR: Config file is missing: ' '/etc/presto/config.properties\n' '[%(master)s] out:\n\n' 'Fatal error: [%(master)s] sudo() received nonzero return code 4 ' 'while executing!\n\n' 'Requested: set -m; /etc/init.d/presto start\n' 'Executed: sudo -S -p \'sudo password:\' /bin/bash -l -c ' '"set -m; /etc/init.d/presto start"\n\n' 'Aborting.\n')).splitlines() expected_start += error_msg expected_stop = self.expected_stop( not_running=[self.cluster.internal_master]) self.assert_simple_start_stop(expected_start, expected_stop, pa_raise_error=False) expected_restart = expected_stop[:] + expected_start[:] self.assert_simple_server_restart(expected_restart, pa_raise_error=False)
def test_collect_logs_nonstandard_location(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) version = self.cluster.exec_cmd_on_host( self.cluster.master, 'rpm -q --qf \"%{VERSION}\\n\" ' + PRESTO_RPM_NAME) if '127t' not in version: print 'test_collect_logs_nonstandard_location only valid for 127t' return new_log_location = '/var/presto' self._add_custom_log_location(new_log_location) self.run_prestoadmin('server start') self._collect_logs_and_unzip() collected_logs_dir = os.path.join(get_install_directory(), 'logs') self.assert_path_exists( self.cluster.master, os.path.join(collected_logs_dir, ' presto-admin.log')) for host in self.cluster.all_internal_hosts(): host_directory = os.path.join(collected_logs_dir, host) self.assert_path_exists(self.cluster.master, os.path.join(host_directory, 'server.log')) self.assert_path_exists( self.cluster.master, os.path.join(host_directory, 'launcher.log'))
def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), self.BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master)
def test_uninstall_with_dir_readonly(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) self.run_script_from_prestoadmin_dir("chmod 500 -R /usr/lib/presto") self.run_prestoadmin('server uninstall', raise_error=False) # The master node was not able to be stopped or uninstalled because # the permissions of the directory were changed such that the # stop command can't run pid_to_remove = None for (host, pid) in process_per_host: if host == self.cluster.internal_master: pid_to_remove = pid process_per_host.remove((self.cluster.internal_master, pid_to_remove)) self.assert_stopped(process_per_host) uninstalled_hosts = self.cluster.all_hosts()[:] uninstalled_hosts.remove(self.cluster.master) for container in uninstalled_hosts: self.assert_uninstalled_dirs_removed(container) self.installer.assert_installed(self, container=self.cluster.master)
def test_uninstall_lost_host(self): self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) pa_installer = PrestoadminInstaller(self) pa_installer.install() topology = {"coordinator": self.cluster.internal_slaves[0], "workers": [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]} self.upload_topology(topology) self.installer.install() start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) down_node = self.cluster.internal_slaves[0] self.cluster.stop_host( self.cluster.slaves[0]) expected = self.down_node_connection_error( self.cluster.internal_slaves[0]) cmd_output = self.run_prestoadmin('server uninstall', raise_error=False) self.assertRegexpMatches(cmd_output, expected) process_per_active_host = [] for host, pid in process_per_host: if host not in down_node: process_per_active_host.append((host, pid)) self.assert_stopped(process_per_active_host) for container in [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]: self.assert_uninstalled_dirs_removed(container)
def test_basic_connector_add_remove(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') for host in self.cluster.all_hosts(): self.assert_has_default_connector(host) self._assert_connectors_loaded([['system'], ['tpch']]) self.run_prestoadmin('connector remove tpch') self.run_prestoadmin('server restart') self.assert_path_removed( self.cluster.master, os.path.join(constants.CONNECTORS_DIR, 'tpch.properties')) self._assert_connectors_loaded([['system']]) for host in self.cluster.all_hosts(): self.assert_path_removed( host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tcph.properties')) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(constants.CONNECTORS_DIR, 'tpch.properties'), self.cluster.master) self.run_prestoadmin('connector add') self.run_prestoadmin('server restart') for host in self.cluster.all_hosts(): self.assert_has_default_connector(host) self._assert_connectors_loaded([['system'], ['tpch']])
def test_connector_add_lost_host(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) self.upload_topology() installer.install() self.run_prestoadmin('connector remove tpch') self.cluster.stop_host(self.cluster.slaves[0]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(constants.CONNECTORS_DIR, 'tpch.properties'), self.cluster.master) output = self.run_prestoadmin('connector add tpch', raise_error=False) for host in self.cluster.all_internal_hosts(): deploying_message = 'Deploying tpch.properties connector ' \ 'configurations on: %s' self.assertTrue( deploying_message % host in output, 'expected %s \n actual %s' % (deploying_message % host, output)) self.assertRegexpMatches( output, self.down_node_connection_error(self.cluster.internal_slaves[0])) self.assertEqual( len(output.splitlines()), len(self.cluster.all_hosts()) + self.len_down_node_error) self.run_prestoadmin('server start', raise_error=False) for host in [ self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2] ]: self.assert_has_default_connector(host) self._assert_connectors_loaded([['system'], ['tpch']])
def test_collect_logs_multiple_server_logs(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') self.cluster.write_content_to_host('Stuff that I logged!', '/var/log/presto/server.log-2', self.cluster.master) actual = self.run_prestoadmin('collect logs') expected = 'Downloading logs from all the nodes...\nlogs archive created: ' + DEFAULT_PATH_FOR_LOGS + '\n' self.assertLazyMessage(lambda: self.log_msg(actual, expected), self.assertEqual, actual, expected) downloaded_logs_location = path.join(TMP_PRESTO_DEBUG, 'logs') self.assert_path_exists(self.cluster.master, downloaded_logs_location) for host in self.cluster.all_internal_hosts(): host_log_location = path.join(downloaded_logs_location, host) self.assert_path_exists( self.cluster.master, os.path.join(host_log_location, 'server.log')) master_path = os.path.join( downloaded_logs_location, self.cluster.internal_master, ) self.assert_path_exists(self.cluster.master, os.path.join(master_path, 'server.log-2'))
def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.dummy_installer = StandalonePrestoInstaller( self, (os.path.join(prestoadmin.main_dir, 'tests', 'product', 'resources'), 'dummy-rpm.rpm')) self.real_installer = StandalonePrestoInstaller(self)
def setUp(self, build_or_runtime): super(BaseTestInstaller, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(build_or_runtime), STANDALONE_BARE_CLUSTER) self.centos_container = \ self.__create_and_start_single_centos_container(build_or_runtime) self.pa_installer = PrestoadminInstaller(self)
def test_server_restart_nothing_started(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) # Restart when the servers aren't started expected_output = self.expected_stop( not_running=self.cluster.all_internal_hosts())[:] +\ self.expected_start()[:] self.assert_simple_server_restart(expected_output, running_host='')
def test_server_stop_not_started(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) stop_output = self.run_prestoadmin('server stop').splitlines() not_started_hosts = self.cluster.all_internal_hosts() self.assertRegexpMatchesLineByLine( stop_output, self.expected_stop(not_running=not_started_hosts))
def test_connector_remove(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) for host in self.cluster.all_hosts(): self.assert_has_default_connector(host) missing_connector_message = """[Errno 1] Fatal error: [master] Could not remove connector '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave1] Could not remove connector '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave2] Could not remove connector '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave3] Could not remove connector '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. """ # noqa success_message = """[master] Connector removed. Restart the server \ for the change to take effect [slave1] Connector removed. Restart the server for the change to take effect [slave2] Connector removed. Restart the server for the change to take effect [slave3] Connector removed. Restart the server for the change to take effect""" # test remove connector does not exist # expect error self.assertRaisesMessageIgnoringOrder( OSError, missing_connector_message % {'name': 'jmx'}, self.run_prestoadmin, 'connector remove jmx') # test remove connector not in directory, but in presto self.cluster.exec_cmd_on_host( self.cluster.master, 'rm /etc/opt/prestoadmin/connectors/tpch.properties') output = self.run_prestoadmin('connector remove tpch') self.assertEqualIgnoringOrder(success_message, output) # test remove connector in directory but not in presto self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(constants.CONNECTORS_DIR, 'tpch.properties'), self.cluster.master) self.assertRaisesMessageIgnoringOrder( OSError, missing_connector_message % {'name': 'tpch'}, self.run_prestoadmin, 'connector remove tpch')
def test_system_info_pa_separate_node(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology=topology) installer.install(coordinator='slave1') self.test_basic_system_info( coordinator=self.cluster.internal_slaves[0], hosts=self.cluster.slaves)
def assert_service_fails_without_presto(self, service): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() # Start without Presto installed start_output = self.run_prestoadmin('server %s' % service, raise_error=False).splitlines() presto_not_installed = self.presto_not_installed_message() self.assertEqualIgnoringOrder(presto_not_installed, '\n'.join(start_output))
def test_collect_system_info_dash_x_two_workers(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') actual = self.run_prestoadmin( 'collect system_info -x %(slave2)s,%(slave3)s') self._test_basic_system_info( actual, self.cluster.internal_master, [self.cluster.master, self.cluster.slaves[0]])
def test_collect_system_info_server_stopped(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) actual = self.run_prestoadmin('collect system_info', raise_error=False) message = '\nFatal error: [%s] Unable to access node ' \ 'information. Please check that server is up with ' \ 'command: server status\n\nAborting.\n' expected = message % self.cluster.internal_master self.assertEqualIgnoringOrder(actual, expected)
def test_server_restart_coordinator_started(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) # Restart when a coordinator is started but workers aren't not_running_hosts = self.cluster.all_internal_hosts()[:] not_running_hosts.remove(self.cluster.internal_master) expected_output = self.expected_stop( not_running=not_running_hosts) + self.expected_start()[:] self.assert_simple_server_restart( expected_output, running_host=self.cluster.internal_master)
def test_started_with_presto_user(self): self.setup_cluster(NoHadoopBareImageProvider(), 'presto') start_output = self.run_prestoadmin('server start').splitlines() process_per_host = self.get_process_per_host(start_output) for host, pid in process_per_host: user_for_pid = self.run_script_from_prestoadmin_dir( 'uid=$(awk \'/^Uid:/{print $2}\' /proc/%s/status);' 'getent passwd "$uid" | awk -F: \'{print $1}\'' % pid, host) self.assertEqual(user_for_pid.strip(), 'presto')
def test_connector_add_missing_connector(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) # test add a connector that does not exist not_found_error = self.fatal_error( 'Configuration for connector tpch not found') self.run_prestoadmin('connector remove tpch') self.assertRaisesRegexp(OSError, not_found_error, self.run_prestoadmin, 'connector add tpch')
def test_server_restart_worker_started(self): self.setup_cluster(NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) # Restart when one worker is started, but nothing else not_running_hosts = self.cluster.all_internal_hosts()[:] not_running_hosts.remove(self.cluster.internal_slaves[0]) expected_output = self.expected_stop( not_running=not_running_hosts) + self.expected_start()[:] self.assert_simple_server_restart( expected_output, running_host=self.cluster.internal_slaves[0])
def test_catalog_add_no_presto_user(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) for host in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host( host, "userdel %s" % (PRESTO_STANDALONE_USER,), invoke_sudo=True) self.assertRaisesRegexp( OSError, "User presto does not exist", self.run_prestoadmin, 'catalog add tpch')
def test_start_coordinator_down(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = { "coordinator": "slave1", "workers": ["master", "slave2", "slave3"] } self.upload_topology(topology=topology) installer.install(coordinator='slave1') self.assert_start_coordinator_down(self.cluster.slaves[0], self.cluster.internal_slaves[0])
def test_status_only_coordinator(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start -H master') # don't run with retries because it won't be able to query the # coordinator because the coordinator is set to not be a worker status_output = self.run_prestoadmin('server status') self.check_status( status_output, self.single_node_up_status(self.cluster.internal_master))
def test_status_non_root_user(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.upload_topology({ "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******" }) self.run_prestoadmin('server start -p password') status_output = self._server_status_with_retries( check_connectors=True, extra_arguments=' -p password') self.check_status(status_output, self.base_status())
def test_status_port_not_8080(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() port_config = """discovery.uri=http://master:8090 http-server.http.port=8090""" self.installer.install(extra_configs=port_config) self.run_prestoadmin('server start') status_output = self._server_status_with_retries(check_connectors=True) self.check_status(status_output, self.base_status(), 8090)
def test_system_info_pa_separate_node(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology=topology) installer.install(coordinator='slave1') self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect system_info') self._test_basic_system_info( actual, coordinator=self.cluster.internal_slaves[0], hosts=self.cluster.slaves)
def test_query_info_invalid_id(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') invalid_id = '1234_invalid' actual = self.run_prestoadmin('collect query_info ' + invalid_id, raise_error=False) expected = '\nFatal error: [master] Unable to retrieve information. ' \ 'Please check that the query_id is correct, or check ' \ 'that server is up with command: server status\n\n' \ 'Aborting.\n' self.assertEqual(actual, expected)
def test_server_start_all_started(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start').splitlines() process_per_host = self.get_process_per_host(start_output) self.assert_started(process_per_host) start_output = self.run_prestoadmin('server start', raise_error=False).splitlines() self.assertRegexpMatchesLineByLine( start_output, self.expected_port_error(self.cluster.all_internal_hosts()))