def test_uninstall_as_non_sudo(self):
self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER)
self.upload_topology()
self.installer.install()
script = './presto-admin server uninstall -u testuser -p testpass'
output = self.run_script_from_prestoadmin_dir(script)
with open(os.path.join(LOCAL_RESOURCES_DIR, 'non_sudo_uninstall.txt'),
'r') as f:
expected = f.read()
self.assertEqualIgnoringOrder(expected, output)
def test_query_info_invalid_id(self):
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('server start')
invalid_id = '1234_invalid'
actual = self.run_prestoadmin('collect query_info ' + invalid_id,
raise_error=False)
expected = '\nFatal error: [master] Unable to retrieve information. ' \
'Please check that the query_id is correct, or check ' \
'that server is up with command: server status\n\n' \
'Aborting.\n'
self.assertEqual(actual, expected)
def test_status_non_root_user(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
self.upload_topology({
"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"
})
self.run_prestoadmin('server start -p password')
status_output = self._server_status_with_retries(
check_catalogs=True, extra_arguments=' -p password')
self.check_status(status_output, self.base_status())
def test_uninstall(self):
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
start_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(start_output.splitlines())
self.assert_started(process_per_host)
self.run_prestoadmin(
'server uninstall', raise_error=False).splitlines()
self.assert_stopped(process_per_host)
for container in self.cluster.all_hosts():
self.assert_uninstalled_dirs_removed(container)
def test_system_info_pa_separate_node(self):
installer = StandalonePrestoInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]}
self.upload_topology(topology=topology)
installer.install(coordinator='slave1')
self.run_prestoadmin('server start')
actual = self.run_prestoadmin('collect system_info')
self.test_basic_system_info(
actual,
coordinator=self.cluster.internal_slaves[0],
hosts=self.cluster.slaves)
def test_status_port_not_8080(self):
self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER)
self.upload_topology()
port_config = """discovery.uri=http://master:8090
http-server.http.port=8090"""
self.installer.install(extra_configs=port_config)
self.run_prestoadmin('server start')
status_output = self._server_status_with_retries(check_connectors=True)
self.check_status(status_output, self.base_status(), 8090)
def test_server_restart_coordinator_started(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
# Restart when a coordinator is started but workers aren't
not_running_hosts = self.cluster.all_internal_hosts()[:]
not_running_hosts.remove(self.cluster.internal_master)
expected_output = self.expected_stop(
not_running=not_running_hosts) + self.expected_start()[:]
self.assert_simple_server_restart(
expected_output,
running_host=self.cluster.internal_master,
pa_raise_error=False)
def test_server_restart_worker_started(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
# Restart when one worker is started, but nothing else
not_running_hosts = self.cluster.all_internal_hosts()[:]
not_running_hosts.remove(self.cluster.internal_slaves[0])
expected_output = self.expected_stop(
not_running=not_running_hosts) + self.expected_start()[:]
self.assert_simple_server_restart(
expected_output,
running_host=self.cluster.internal_slaves[0],
pa_raise_error=False)
def __create_and_start_single_centos_container(self, build_or_runtime):
cluster_type = 'installer_tester'
bare_image_provider = NoHadoopBareImageProvider(build_or_runtime)
centos_container, bare_cluster = DockerCluster.start_cluster(
bare_image_provider,
cluster_type,
'master', [],
cap_add=['NET_ADMIN'])
if bare_cluster:
centos_container.commit_images(bare_image_provider, cluster_type)
return centos_container
def test_uninstall(self):
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
start_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(start_output.splitlines())
self.assert_started(process_per_host)
cmd_output = self.run_prestoadmin(
'server uninstall', raise_error=False).splitlines()
self.assert_stopped(process_per_host)
expected = uninstall_output + self.expected_stop()[:]
self.assertRegexpMatchesLineByLine(cmd_output, expected)
for container in self.cluster.all_hosts():
self.assert_uninstalled_dirs_removed(container)
def test_connector_add_by_name(self):
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('connector remove tpch')
# test add connector by name when it exists
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(constants.CONNECTORS_DIR, 'tpch.properties'),
self.cluster.master)
self.run_prestoadmin('connector add tpch')
self.run_prestoadmin('server start')
for host in self.cluster.all_hosts():
self.assert_has_default_connector(host)
self._assert_connectors_loaded([['system'], ['tpch']])
def test_connector_name_not_found(self):
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('server start')
self.cluster.write_content_to_host(
'connector.noname=example',
os.path.join(constants.CONNECTORS_DIR, 'example.properties'),
self.cluster.master)
expected = self.fatal_error('Catalog configuration '
'example.properties does not '
'contain connector.name')
self.assertRaisesRegexp(OSError, expected, self.run_prestoadmin,
'connector add example')
def test_catalog_add_by_name(self):
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('catalog remove tpch')
# test add catalog by name when it exists
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master
)
self.run_prestoadmin('catalog add tpch')
self.run_prestoadmin('server start')
for host in self.cluster.all_hosts():
self.assert_has_default_catalog(host)
self._assert_catalogs_loaded([['system'], ['tpch']])
def test_status_only_worker(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('server start -H slave1')
status_output = self._server_status_with_retries()
self.check_status(
status_output,
self.single_node_up_status(self.cluster.internal_slaves[0]))
# Check that the slave sees that it's stopped, even though the
# discovery server is not up.
self.run_prestoadmin('server stop')
status_output = self._server_status_with_retries()
self.check_status(status_output, self.not_started_status())
def test_collect_query_info(self):
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('server start')
sql_to_run = 'SELECT * FROM system.runtime.nodes WHERE 1234 = 1234'
query_id = self.retry(lambda: self.get_query_id(sql_to_run))
actual = self.run_prestoadmin('collect query_info ' + query_id)
query_info_file_name = path.join(TMP_PRESTO_DEBUG,
'query_info_' + query_id + '.json')
expected = 'Gathered query information in file: ' + \
query_info_file_name + '\n'
self.assert_path_exists(self.cluster.master, query_info_file_name)
self.assertEqual(actual, expected)
def test_uninstall_lost_host(self):
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
self.cluster.stop_host(
self.cluster.slaves[0])
expected = self.down_node_connection_error(
self.cluster.internal_slaves[0])
cmd_output = self.run_prestoadmin('server uninstall',
raise_error=False)
self.assertRegexpMatches(cmd_output, expected)
for container in [self.cluster.internal_master,
self.cluster.internal_slaves[1],
self.cluster.internal_slaves[2]]:
self.assert_uninstalled_dirs_removed(container)
def test_uninstall_when_server_down(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
start_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(start_output.splitlines())
self.run_prestoadmin('server stop -H %s' %
self.cluster.internal_slaves[0])
cmd_output = self.run_prestoadmin('server uninstall').splitlines()
self.assert_stopped(process_per_host)
expected = uninstall_output + self.expected_stop(
not_running=[self.cluster.internal_slaves[0]])[:]
self.assertRegexpMatchesLineByLine(cmd_output, expected)
for container in self.cluster.all_hosts():
self.assert_uninstalled_dirs_removed(container)
def test_collect_non_root_user(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
self.upload_topology({
"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"
})
self.run_script_from_prestoadmin_dir(
'./presto-admin server start -p password')
self.run_script_from_prestoadmin_dir(
'./presto-admin collect logs -p password')
actual = self.run_script_from_prestoadmin_dir(
'./presto-admin collect system_info -p password')
self._test_basic_system_info(actual)
def test_connection_to_coordinator_lost(self):
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
topology = {
"coordinator": "slave1",
"workers": ["master", "slave2", "slave3"]
}
self.upload_topology(topology=topology)
self.installer.install(coordinator='slave1')
self.run_prestoadmin('server start')
self.cluster.stop_host(self.cluster.slaves[0])
topology = {
"coordinator": self.cluster.get_down_hostname("slave1"),
"workers": ["master", "slave2", "slave3"]
}
status_output = self._server_status_with_retries()
statuses = self.node_not_available_status(
topology, self.cluster.internal_slaves[0], coordinator_down=True)
self.check_status(status_output, statuses)
def test_query_info_pa_separate_node(self):
installer = StandalonePrestoInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]}
self.upload_topology(topology=topology)
installer.install(coordinator='slave1')
self.run_prestoadmin('server start')
sql_to_run = 'SELECT * FROM system.runtime.nodes WHERE 1234 = 1234'
with settings(roledefs={'coordinator': ['slave1']}):
query_id = self.retry(lambda: self.get_query_id(
sql_to_run, host=self.cluster.slaves[0]))
actual = self.run_prestoadmin('collect query_info ' + query_id)
query_info_file_name = path.join(TMP_PRESTO_DEBUG,
'query_info_' + query_id + '.json')
expected = 'Gathered query information in file: ' + query_info_file_name + '\n'
self.assert_path_exists(self.cluster.master, query_info_file_name)
self.assertEqual(actual, expected)
def test_server_start_twice(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
start_output = self.run_prestoadmin('server start').splitlines()
process_per_host = self.get_process_per_host(start_output)
self.assert_started(process_per_host)
self.run_prestoadmin('server stop -H ' +
self.cluster.internal_slaves[0])
# Start all again
start_with_error = self.run_prestoadmin(
'server start', raise_error=False).splitlines()
expected = self.expected_start(
start_success=[self.cluster.internal_slaves[0]],
already_started=[],
failed_hosts=[])
alive_hosts = self.cluster.all_internal_hosts()[:]
alive_hosts.remove(self.cluster.internal_slaves[0])
expected.extend(self.expected_port_error(alive_hosts))
self.assertRegexpMatchesLineByLine(start_with_error, expected)
def test_catalog_add_empty_dir(self):
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('catalog remove tpch')
output = self.run_prestoadmin('catalog add')
expected = [r'',
r'Warning: \[slave3\] Directory .*/.prestoadmin/catalog is empty. '
r'No catalogs will be deployed',
r'',
r'',
r'Warning: \[slave2\] Directory .*/.prestoadmin/catalog is empty. '
r'No catalogs will be deployed',
r'',
r'',
r'Warning: \[slave1\] Directory .*/.prestoadmin/catalog is empty. '
r'No catalogs will be deployed',
r'',
r'',
r'Warning: \[master\] Directory .*/.prestoadmin/catalog is empty. '
r'No catalogs will be deployed',
r'']
self.assertRegexpMatchesLineByLine(output.splitlines(), expected)
def test_connector_add_two_connectors(self):
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('connector remove tpch')
# test add connectors from directory with more than one connector
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(constants.CONNECTORS_DIR, 'tpch.properties'),
self.cluster.master)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master)
self.run_prestoadmin('connector add')
self.run_prestoadmin('server start')
for host in self.cluster.all_hosts():
self.assert_has_default_connector(host)
self.assert_file_content(host,
'/etc/presto/catalog/jmx.properties',
'connector.name=jmx')
self._assert_connectors_loaded([['system'], ['jmx'], ['tpch']])
def test_collect_logs_basic(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('server start')
actual = self.run_prestoadmin('collect logs')
expected = 'Downloading logs from all the nodes...\n' + \
'logs archive created: ' + DEFAULT_PATH_FOR_LOGS + '\n'
self.assertLazyMessage(lambda: self.log_msg(actual, expected),
self.assertEqual, actual, expected)
self.assert_path_exists(self.cluster.master, DEFAULT_PATH_FOR_LOGS)
self.assert_path_exists(self.cluster.master, TMP_PRESTO_DEBUG)
downloaded_logs_location = path.join(TMP_PRESTO_DEBUG, 'logs')
self.assert_path_exists(self.cluster.master, downloaded_logs_location)
for host in self.cluster.all_internal_hosts():
host_log_location = path.join(downloaded_logs_location, host)
self.assert_path_exists(self.cluster.master, host_log_location)
admin_log = path.join(downloaded_logs_location, PRESTOADMIN_LOG_NAME)
self.assert_path_exists(self.cluster.master, admin_log)
def test_connector_add_empty_dir(self):
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
output = self.run_prestoadmin('connector remove tpch')
output = self.run_prestoadmin('connector add')
expected = """
Warning: [slave3] Directory /etc/opt/prestoadmin/connectors is empty. \
No connectors will be deployed
Warning: [slave2] Directory /etc/opt/prestoadmin/connectors is empty. \
No connectors will be deployed
Warning: [slave1] Directory /etc/opt/prestoadmin/connectors is empty. \
No connectors will be deployed
Warning: [master] Directory /etc/opt/prestoadmin/connectors is empty. \
No connectors will be deployed
"""
self.assertEqualIgnoringOrder(expected, output)
def test_uninstall_lost_host(self):
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
pa_installer = PrestoadminInstaller(self)
pa_installer.install()
topology = {
"coordinator":
self.cluster.internal_slaves[0],
"workers": [
self.cluster.internal_master, self.cluster.internal_slaves[1],
self.cluster.internal_slaves[2]
]
}
self.upload_topology(topology)
self.installer.install()
start_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(start_output.splitlines())
self.assert_started(process_per_host)
down_node = self.cluster.internal_slaves[0]
self.cluster.stop_host(self.cluster.slaves[0])
expected = self.down_node_connection_error(
self.cluster.internal_slaves[0])
cmd_output = self.run_prestoadmin('server uninstall',
raise_error=False)
self.assertRegexpMatches(cmd_output, expected)
process_per_active_host = []
for host, pid in process_per_host:
if host not in down_node:
process_per_active_host.append((host, pid))
self.assert_stopped(process_per_active_host)
for container in [
self.cluster.internal_master, self.cluster.internal_slaves[1],
self.cluster.internal_slaves[2]
]:
self.assert_uninstalled_dirs_removed(container)
def test_connector_add_wrong_permissions(self):
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
# test add connector without read permissions on file
script = 'chmod 600 /etc/opt/prestoadmin/connectors/tpch.properties;' \
' su app-admin -c "./presto-admin connector add tpch"'
output = self.run_script_from_prestoadmin_dir(script)
with open(
os.path.join(LOCAL_RESOURCES_DIR,
'connector_permissions_warning.txt'), 'r') as f:
expected = f.read() % \
{'master': self.cluster.internal_master,
'slave1': self.cluster.internal_slaves[0],
'slave2': self.cluster.internal_slaves[1],
'slave3': self.cluster.internal_slaves[2]}
self.assertEqualIgnoringOrder(expected, output)
# test add connector directory without read permissions on directory
script = 'chmod 600 /etc/opt/prestoadmin/connectors; ' \
'su app-admin -c "./presto-admin connector add"'
output = self.run_script_from_prestoadmin_dir(script)
permission_error = '\nWarning: [slave3] Permission denied\n\n\n' \
'Warning: [slave2] Permission denied\n\n\n' \
'Warning: [slave1] Permission denied\n\n\n' \
'Warning: [master] Permission denied\n\n'
self.assertEqualIgnoringOrder(output, permission_error)
# test add connector by file without read permissions on directory
script = 'chmod 600 /etc/opt/prestoadmin/connectors; ' \
'su app-admin -c "./presto-admin connector add tpch"'
not_found_error = self.fatal_error(
'Configuration for connector tpch not found')
self.assertRaisesRegexp(OSError, not_found_error,
self.run_script_from_prestoadmin_dir, script)
def test_collect_logs_nonstandard_location(self):
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
version = self.cluster.exec_cmd_on_host(
self.cluster.master,
'rpm -q --qf \"%{VERSION}\\n\" presto-server-rpm')
if '127t' not in version:
print 'test_collect_logs_nonstandard_location only valid for 127t'
return
new_log_location = '/var/presto'
self._add_custom_log_location(new_log_location)
self.run_prestoadmin('server start')
self._collect_logs_and_unzip()
self.assert_path_exists(self.cluster.master,
'/opt/prestoadmin/logs/presto-admin.log')
for host in self.cluster.all_internal_hosts():
self.assert_path_exists(
self.cluster.master,
'/opt/prestoadmin/logs/%s/server.log' % host)
self.assert_path_exists(
self.cluster.master,
'/opt/prestoadmin/logs/%s/launcher.log' % host)
def test_server_starts_no_java_variable(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
self.run_script_from_prestoadmin_dir('rm /etc/presto/env.sh')
# tests that no error is encountered
self.run_prestoadmin('server start')
def setUp(self):
super(TestRpm, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)