def test_catalog_add_remove_coord_worker_using_dash_h(self):
self.setup_cluster_assert_catalogs()
self.run_prestoadmin('catalog remove tpch -H %(master)s,%(slave1)s')
self.run_prestoadmin('server restart')
self.assert_path_removed(self.cluster.master,
os.path.join(get_catalog_directory(),
'tpch.properties'))
self._assert_catalogs_loaded([['system']])
for host in [self.cluster.master, self.cluster.slaves[0]]:
self.assert_path_removed(host,
os.path.join(constants.REMOTE_CATALOG_DIR,
'tpch.properties'))
self.assert_has_default_catalog(self.cluster.slaves[1])
self.assert_has_default_catalog(self.cluster.slaves[2])
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master
)
self.run_prestoadmin('catalog add tpch -H %(master)s,%(slave1)s')
self.run_prestoadmin('server restart')
self.assert_has_default_catalog(self.cluster.master)
self.assert_has_default_catalog(self.cluster.slaves[1])
def test_catalog_add_remove_coord_worker_using_dash_h(self):
self.setup_cluster_assert_catalogs()
self.run_prestoadmin('catalog remove tpch -H %(master)s,%(slave1)s')
self.run_prestoadmin('server restart')
self.assert_path_removed(self.cluster.master,
os.path.join(get_catalog_directory(),
'tpch.properties'))
self._assert_catalogs_loaded([['system']])
for host in [self.cluster.master, self.cluster.slaves[0]]:
self.assert_path_removed(host,
os.path.join(constants.REMOTE_CATALOG_DIR,
'tpch.properties'))
self.assert_has_default_catalog(self.cluster.slaves[1])
self.assert_has_default_catalog(self.cluster.slaves[2])
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master
)
self.run_prestoadmin('catalog add tpch -H %(master)s,%(slave1)s')
self.run_prestoadmin('server restart')
self.assert_has_default_catalog(self.cluster.master)
self.assert_has_default_catalog(self.cluster.slaves[1])
def test_catalog_add_remove(self):
self.setup_cluster_assert_catalogs()
self.run_prestoadmin('catalog remove tpch')
self.assert_path_removed(self.cluster.master, os.path.join(get_catalog_directory(), 'tpch.properties'))
for host in self.cluster.all_hosts():
self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties'))
# test add catalogs from directory with more than one catalog
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master
)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master
)
self.run_prestoadmin('catalog add')
self.run_prestoadmin('server restart')
for host in self.cluster.all_hosts():
filepath = '/etc/presto/catalog/jmx.properties'
self.assert_has_default_catalog(host)
self.assert_config_perms(host, filepath)
self.assert_file_content(host, filepath, 'connector.name=jmx')
self._assert_catalogs_loaded([['system'], ['jmx'], ['tpch']])
def test_catalog_add_remove_non_sudo_user(self):
self.setup_cluster_assert_catalogs()
self.upload_topology(
{"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"}
)
self.run_prestoadmin('catalog remove tpch -p password')
self.assert_path_removed(self.cluster.master,
os.path.join(get_catalog_directory(),
'tpch.properties'))
for host in self.cluster.all_hosts():
self.assert_path_removed(host,
os.path.join(constants.REMOTE_CATALOG_DIR,
'tcph.properties'))
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master
)
self.run_prestoadmin('catalog add -p password')
self.run_prestoadmin('server restart -p password')
for host in self.cluster.all_hosts():
self.assert_has_jmx_catalog(host)
self._assert_catalogs_loaded([['system'], ['jmx']])
def test_catalog_add_remove_non_sudo_user(self):
self.setup_cluster_assert_catalogs()
self.upload_topology({
"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"
})
self.run_prestoadmin('catalog remove tpch -p password')
self.assert_path_removed(
self.cluster.master,
os.path.join(get_catalog_directory(), 'tpch.properties'))
for host in self.cluster.all_hosts():
self.assert_path_removed(
host,
os.path.join(constants.REMOTE_CATALOG_DIR, 'tcph.properties'))
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master)
self.run_prestoadmin('catalog add -p password')
self.run_prestoadmin('server restart -p password')
for host in self.cluster.all_hosts():
self.assert_has_jmx_catalog(host)
self._assert_catalogs_loaded([['system'], ['jmx']])
def test_catalog_add_remove(self):
self.setup_cluster_assert_catalogs()
self.run_prestoadmin('catalog remove tpch')
self.assert_path_removed(
self.cluster.master,
os.path.join(get_catalog_directory(), 'tpch.properties'))
for host in self.cluster.all_hosts():
self.assert_path_removed(
host,
os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties'))
# test add catalogs from directory with more than one catalog
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master)
self.run_prestoadmin('catalog add')
self.run_prestoadmin('server restart')
for host in self.cluster.all_hosts():
filepath = '/etc/presto/catalog/jmx.properties'
self.assert_has_default_catalog(host)
self.assert_config_perms(host, filepath)
self.assert_file_content(host, filepath, 'connector.name=jmx')
self._assert_catalogs_loaded([['system'], ['jmx'], ['tpch']])
def test_catalog_remove(self):
self.setup_cluster(NoHadoopBareImageProvider,
STANDALONE_PRESTO_CLUSTER)
for host in self.cluster.all_hosts():
self.assert_has_default_catalog(host)
missing_catalog_message = """[Errno 1]
Fatal error: [master] Could not remove catalog '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
Fatal error: [slave1] Could not remove catalog '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
Fatal error: [slave2] Could not remove catalog '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
Fatal error: [slave3] Could not remove catalog '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
""" # noqa
success_message = """[master] Catalog removed. Restart the server \
for the change to take effect
[slave1] Catalog removed. Restart the server for the change to take effect
[slave2] Catalog removed. Restart the server for the change to take effect
[slave3] Catalog removed. Restart the server for the change to take effect"""
# test remove catalog does not exist
# expect error
self.assertRaisesMessageIgnoringOrder(
OSError, missing_catalog_message % {'name': 'jmx'},
self.run_prestoadmin, 'catalog remove jmx')
# test remove catalog not in directory, but in presto
self.cluster.exec_cmd_on_host(
self.cluster.master,
'rm %s' % os.path.join(get_catalog_directory(), 'tpch.properties'))
output = self.run_prestoadmin('catalog remove tpch')
self.assertEqualIgnoringOrder(success_message, output)
# test remove catalog in directory but not in presto
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master)
self.assertRaisesMessageIgnoringOrder(
OSError, missing_catalog_message % {'name': 'tpch'},
self.run_prestoadmin, 'catalog remove tpch')
def test_catalog_add_lost_host(self):
installer = StandalonePrestoInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
self.upload_topology()
installer.install()
self.run_prestoadmin('catalog remove tpch')
self.cluster.stop_host(
self.cluster.slaves[0])
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master
)
output = self.run_prestoadmin('catalog add tpch', raise_error=False)
for host in self.cluster.all_internal_hosts():
deploying_message = 'Deploying tpch.properties catalog configurations on: %s'
self.assertTrue(deploying_message % host in output,
'expected %s \n actual %s'
% (deploying_message % host, output))
self.assertRegexpMatches(
output,
self.down_node_connection_error(self.cluster.internal_slaves[0])
)
self.assertEqual(len(output.splitlines()),
len(self.cluster.all_hosts()) +
self.len_down_node_error)
self.run_prestoadmin('server start', raise_error=False)
for host in [self.cluster.master,
self.cluster.slaves[1],
self.cluster.slaves[2]]:
self.assert_has_default_catalog(host)
self._assert_catalogs_loaded([['system'], ['tpch']])
def test_install_with_java_home(self):
installer = StandalonePrestoInstaller(self)
with relocate_jdk_directory(self.cluster, '/usr') as new_java_home:
topology = {
"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"java_home": new_java_home
}
self.upload_topology(topology)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master)
cmd_output = installer.install()
expected = self.format_err_msgs_with_internal_hosts(
installed_all_hosts_output)
actual = cmd_output.splitlines()
self.assertRegexpMatchesLineByLine(actual, expected)
for host in self.cluster.all_hosts():
installer.assert_installed(self, host)
self.assert_has_default_config(host)
self.assert_has_default_catalog(host)
self.assert_has_jmx_catalog(host)
def test_catalog_add_lost_host(self):
installer = StandalonePrestoInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER)
self.upload_topology()
installer.install()
self.run_prestoadmin('catalog remove tpch')
self.cluster.stop_host(self.cluster.slaves[0])
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master)
output = self.run_prestoadmin('catalog add tpch', raise_error=False)
for host in self.cluster.all_internal_hosts():
deploying_message = 'Deploying tpch.properties catalog configurations on: %s'
self.assertTrue(
deploying_message % host in output,
'expected %s \n actual %s' %
(deploying_message % host, output))
self.assertRegexpMatches(
output,
self.down_node_connection_error(self.cluster.internal_slaves[0]))
self.assertEqual(
len(output.splitlines()),
len(self.cluster.all_hosts()) + self.len_down_node_error)
self.run_prestoadmin('server start', raise_error=False)
for host in [
self.cluster.master, self.cluster.slaves[1],
self.cluster.slaves[2]
]:
self.assert_has_default_catalog(host)
self._assert_catalogs_loaded([['system'], ['tpch']])
def test_install_when_catalog_json_exists(self):
installer = StandalonePrestoInstaller(self)
topology = {"coordinator": "master", "workers": ["slave1"]}
self.upload_topology(topology)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master)
cmd_output = installer.install()
expected = [
'Deploying rpm on master...', 'Deploying rpm on slave1...',
'Package deployed successfully on: slave1',
'Package installed successfully on: slave1',
'Package deployed successfully on: master',
'Package installed successfully on: master',
'Deploying configuration on: master',
'Deploying jmx.properties, tpch.properties '
'catalog configurations on: master ',
'Deploying configuration on: slave1',
'Deploying jmx.properties, tpch.properties '
'catalog configurations on: slave1 ',
'Using rpm_specifier as a local path',
'Fetching local presto rpm at path: .*',
'Found existing rpm at: .*'
]
actual = cmd_output.splitlines()
self.assertRegexpMatchesLineByLine(actual, expected)
for container in [self.cluster.master, self.cluster.slaves[0]]:
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
self.assert_has_jmx_catalog(container)
def test_install_interactive(self):
installer = StandalonePrestoInstaller(self)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master)
rpm_name = installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
additional_keywords = {
'user': self.cluster.user,
'rpm_dir': self.cluster.rpm_cache_dir,
'rpm': rpm_name
}
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | '
'./presto-admin server install %(rpm_dir)s/%(rpm)s ',
**additional_keywords)
actual = cmd_output.splitlines()
expected = [
r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Using rpm_specifier as a local path',
r'Package deployed successfully on: ' +
self.cluster.internal_master,
r'Package installed successfully on: ' +
self.cluster.internal_master,
r'Package deployed successfully on: ' +
self.cluster.internal_slaves[0],
r'Package installed successfully on: ' +
self.cluster.internal_slaves[0],
r'Deploying configuration on: ' + self.cluster.internal_master,
r'Deploying jmx.properties, tpch.properties catalog '
r'configurations on: ' + self.cluster.internal_master,
r'Deploying configuration on: ' + self.cluster.internal_slaves[0],
r'Deploying jmx.properties, tpch.properties catalog '
r'configurations on: ' + self.cluster.internal_slaves[0],
r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.',
r'Fetching local presto rpm at path: .*',
r'Found existing rpm at: .*'
]
self.assertRegexpMatchesLineByLine(actual, expected)
for container in [self.cluster.master, self.cluster.slaves[0]]:
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
self.assert_has_jmx_catalog(container)
def test_catalog_add_by_name(self):
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('catalog remove tpch')
# test add catalog by name when it exists
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master
)
self.run_prestoadmin('catalog add tpch')
self.run_prestoadmin('server start')
for host in self.cluster.all_hosts():
self.assert_has_default_catalog(host)
self._assert_catalogs_loaded([['system'], ['tpch']])
def test_catalog_add_by_name(self):
self.setup_cluster(NoHadoopBareImageProvider,
STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('catalog remove tpch')
# test add catalog by name when it exists
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master)
self.run_prestoadmin('catalog add tpch')
self.run_prestoadmin('server start')
for host in self.cluster.all_hosts():
self.assert_has_default_catalog(host)
self._assert_catalogs_loaded([['system'], ['tpch']])
def test_install_when_topology_has_ips(self):
installer = StandalonePrestoInstaller(self)
ips = self.cluster.get_ip_address_dict()
topology = {
"coordinator": ips[self.cluster.internal_master],
"workers": [ips[self.cluster.internal_slaves[0]]]
}
self.upload_topology(topology)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master)
cmd_output = installer.install().splitlines()
expected = [
r'Deploying rpm on %s...' % ips[self.cluster.internal_master],
r'Deploying rpm on %s...' % ips[self.cluster.internal_slaves[0]],
r'Package deployed successfully on: ' +
ips[self.cluster.internal_master],
r'Package installed successfully on: ' +
ips[self.cluster.internal_master],
r'Package deployed successfully on: ' +
ips[self.cluster.internal_slaves[0]],
r'Package installed successfully on: ' +
ips[self.cluster.internal_slaves[0]],
r'Deploying configuration on: ' +
ips[self.cluster.internal_master],
r'Deploying jmx.properties, tpch.properties '
r'catalog configurations on: ' +
ips[self.cluster.internal_master] + r' ',
r'Deploying configuration on: ' +
ips[self.cluster.internal_slaves[0]],
r'Deploying jmx.properties, tpch.properties '
r'catalog configurations on: ' +
ips[self.cluster.internal_slaves[0]] + r' ',
r'Using rpm_specifier as a local path',
r'Fetching local presto rpm at path: .*',
r'Found existing rpm at: .*'
]
cmd_output.sort()
expected.sort()
self.assertRegexpMatchesLineByLine(cmd_output, expected)
self.assert_installed_with_regex_configs(self.cluster.master,
[self.cluster.slaves[0]])
for host in [self.cluster.master, self.cluster.slaves[0]]:
self.assert_has_jmx_catalog(host)
def test_additional_dirs_created(self):
install_dir = '~'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
self.assert_path_exists(self.cluster.master, get_catalog_directory())
self.assert_path_exists(self.cluster.master,
get_coordinator_directory())
self.assert_path_exists(self.cluster.master, get_workers_directory())
def test_additional_dirs_created(self):
install_dir = '~'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
self.assert_path_exists(self.cluster.master, get_catalog_directory())
self.assert_path_exists(self.cluster.master, get_coordinator_directory())
self.assert_path_exists(self.cluster.master, get_workers_directory())
def test_install_interactive(self):
installer = StandalonePrestoInstaller(self)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master)
ips = self.cluster.get_ip_address_dict()
self.upload_topology({
"coordinator":
ips[self.cluster.internal_master],
"workers": [
ips[self.cluster.internal_slaves[0]],
ips[self.cluster.internal_slaves[1]],
ips[self.cluster.internal_slaves[2]]
],
"username":
"******"
})
rpm_name = installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster,
coordinator=ips[self.cluster.internal_master])
additional_keywords = {
'user': "******",
'rpm_dir': self.cluster.rpm_cache_dir,
'rpm': rpm_name
}
self.run_script_from_prestoadmin_dir(
'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | '
'./presto-admin server install %(rpm_dir)s/%(rpm)s -p password',
**additional_keywords)
self.assert_installed_with_regex_configs(self.cluster.master, [
self.cluster.slaves[0], self.cluster.slaves[1],
self.cluster.slaves[2]
])
for container in self.cluster.all_hosts():
installer.assert_installed(self, container)
self.assert_has_default_catalog(container)
self.assert_has_jmx_catalog(container)
def test_catalog_remove(self):
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
for host in self.cluster.all_hosts():
self.assert_has_default_catalog(host)
missing_catalog_message = """[Errno 1]
Fatal error: [master] Could not remove catalog '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
Fatal error: [slave1] Could not remove catalog '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
Fatal error: [slave2] Could not remove catalog '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
Fatal error: [slave3] Could not remove catalog '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
""" # noqa
success_message = """[master] Catalog removed. Restart the server \
for the change to take effect
[slave1] Catalog removed. Restart the server for the change to take effect
[slave2] Catalog removed. Restart the server for the change to take effect
[slave3] Catalog removed. Restart the server for the change to take effect"""
# test remove catalog does not exist
# expect error
self.assertRaisesMessageIgnoringOrder(
OSError,
missing_catalog_message % {'name': 'jmx'},
self.run_prestoadmin,
'catalog remove jmx')
# test remove catalog not in directory, but in presto
self.cluster.exec_cmd_on_host(
self.cluster.master,
'rm %s' % os.path.join(get_catalog_directory(), 'tpch.properties')
)
output = self.run_prestoadmin('catalog remove tpch')
self.assertEqualIgnoringOrder(success_message, output)
# test remove catalog in directory but not in presto
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_catalog_directory(), 'tpch.properties'),
self.cluster.master
)
self.assertRaisesMessageIgnoringOrder(
OSError,
missing_catalog_message % {'name': 'tpch'},
self.run_prestoadmin,
'catalog remove tpch')
def setup_for_catalog_add(self):
connector_script = 'mkdir -p %(catalogs)s\n' \
'echo \'connector.name=tpch\' >> %(catalogs)s/tpch.properties\n' % \
{'catalogs': get_catalog_directory()}
self.run_script_from_prestoadmin_dir(connector_script)
def setup_for_catalog_add(self):
connector_script = 'mkdir -p %(catalogs)s\n' \
'echo \'connector.name=tpch\' >> %(catalogs)s/tpch.properties\n' % \
{'catalogs': get_catalog_directory()}
self.run_script_from_prestoadmin_dir(connector_script)
