def test_catalog_add_remove_coord_worker_using_dash_h(self): self.setup_cluster_assert_catalogs() self.run_prestoadmin('catalog remove tpch -H %(master)s,%(slave1)s') self.run_prestoadmin('server restart') self.assert_path_removed(self.cluster.master, os.path.join(get_catalog_directory(), 'tpch.properties')) self._assert_catalogs_loaded([['system']]) for host in [self.cluster.master, self.cluster.slaves[0]]: self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties')) self.assert_has_default_catalog(self.cluster.slaves[1]) self.assert_has_default_catalog(self.cluster.slaves[2]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) self.run_prestoadmin('catalog add tpch -H %(master)s,%(slave1)s') self.run_prestoadmin('server restart') self.assert_has_default_catalog(self.cluster.master) self.assert_has_default_catalog(self.cluster.slaves[1])
def test_catalog_add_remove(self): self.setup_cluster_assert_catalogs() self.run_prestoadmin('catalog remove tpch') self.assert_path_removed(self.cluster.master, os.path.join(get_catalog_directory(), 'tpch.properties')) for host in self.cluster.all_hosts(): self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties')) # test add catalogs from directory with more than one catalog self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master ) self.run_prestoadmin('catalog add') self.run_prestoadmin('server restart') for host in self.cluster.all_hosts(): filepath = '/etc/presto/catalog/jmx.properties' self.assert_has_default_catalog(host) self.assert_config_perms(host, filepath) self.assert_file_content(host, filepath, 'connector.name=jmx') self._assert_catalogs_loaded([['system'], ['jmx'], ['tpch']])
def test_catalog_add_remove_non_sudo_user(self): self.setup_cluster_assert_catalogs() self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******"} ) self.run_prestoadmin('catalog remove tpch -p password') self.assert_path_removed(self.cluster.master, os.path.join(get_catalog_directory(), 'tpch.properties')) for host in self.cluster.all_hosts(): self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tcph.properties')) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master ) self.run_prestoadmin('catalog add -p password') self.run_prestoadmin('server restart -p password') for host in self.cluster.all_hosts(): self.assert_has_jmx_catalog(host) self._assert_catalogs_loaded([['system'], ['jmx']])
def test_catalog_add_remove_non_sudo_user(self): self.setup_cluster_assert_catalogs() self.upload_topology({ "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******" }) self.run_prestoadmin('catalog remove tpch -p password') self.assert_path_removed( self.cluster.master, os.path.join(get_catalog_directory(), 'tpch.properties')) for host in self.cluster.all_hosts(): self.assert_path_removed( host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tcph.properties')) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) self.run_prestoadmin('catalog add -p password') self.run_prestoadmin('server restart -p password') for host in self.cluster.all_hosts(): self.assert_has_jmx_catalog(host) self._assert_catalogs_loaded([['system'], ['jmx']])
def test_catalog_add_remove(self): self.setup_cluster_assert_catalogs() self.run_prestoadmin('catalog remove tpch') self.assert_path_removed( self.cluster.master, os.path.join(get_catalog_directory(), 'tpch.properties')) for host in self.cluster.all_hosts(): self.assert_path_removed( host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties')) # test add catalogs from directory with more than one catalog self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) self.run_prestoadmin('catalog add') self.run_prestoadmin('server restart') for host in self.cluster.all_hosts(): filepath = '/etc/presto/catalog/jmx.properties' self.assert_has_default_catalog(host) self.assert_config_perms(host, filepath) self.assert_file_content(host, filepath, 'connector.name=jmx') self._assert_catalogs_loaded([['system'], ['jmx'], ['tpch']])
def test_catalog_remove(self): self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER) for host in self.cluster.all_hosts(): self.assert_has_default_catalog(host) missing_catalog_message = """[Errno 1] Fatal error: [master] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave1] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave2] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave3] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. """ # noqa success_message = """[master] Catalog removed. Restart the server \ for the change to take effect [slave1] Catalog removed. Restart the server for the change to take effect [slave2] Catalog removed. Restart the server for the change to take effect [slave3] Catalog removed. Restart the server for the change to take effect""" # test remove catalog does not exist # expect error self.assertRaisesMessageIgnoringOrder( OSError, missing_catalog_message % {'name': 'jmx'}, self.run_prestoadmin, 'catalog remove jmx') # test remove catalog not in directory, but in presto self.cluster.exec_cmd_on_host( self.cluster.master, 'rm %s' % os.path.join(get_catalog_directory(), 'tpch.properties')) output = self.run_prestoadmin('catalog remove tpch') self.assertEqualIgnoringOrder(success_message, output) # test remove catalog in directory but not in presto self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master) self.assertRaisesMessageIgnoringOrder( OSError, missing_catalog_message % {'name': 'tpch'}, self.run_prestoadmin, 'catalog remove tpch')
def test_catalog_add_lost_host(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() installer.install() self.run_prestoadmin('catalog remove tpch') self.cluster.stop_host( self.cluster.slaves[0]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) output = self.run_prestoadmin('catalog add tpch', raise_error=False) for host in self.cluster.all_internal_hosts(): deploying_message = 'Deploying tpch.properties catalog configurations on: %s' self.assertTrue(deploying_message % host in output, 'expected %s \n actual %s' % (deploying_message % host, output)) self.assertRegexpMatches( output, self.down_node_connection_error(self.cluster.internal_slaves[0]) ) self.assertEqual(len(output.splitlines()), len(self.cluster.all_hosts()) + self.len_down_node_error) self.run_prestoadmin('server start', raise_error=False) for host in [self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_has_default_catalog(host) self._assert_catalogs_loaded([['system'], ['tpch']])
def test_install_with_java_home(self): installer = StandalonePrestoInstaller(self) with relocate_jdk_directory(self.cluster, '/usr') as new_java_home: topology = { "coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "java_home": new_java_home } self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) cmd_output = installer.install() expected = self.format_err_msgs_with_internal_hosts( installed_all_hosts_output) actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for host in self.cluster.all_hosts(): installer.assert_installed(self, host) self.assert_has_default_config(host) self.assert_has_default_catalog(host) self.assert_has_jmx_catalog(host)
def test_catalog_add_lost_host(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER) self.upload_topology() installer.install() self.run_prestoadmin('catalog remove tpch') self.cluster.stop_host(self.cluster.slaves[0]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master) output = self.run_prestoadmin('catalog add tpch', raise_error=False) for host in self.cluster.all_internal_hosts(): deploying_message = 'Deploying tpch.properties catalog configurations on: %s' self.assertTrue( deploying_message % host in output, 'expected %s \n actual %s' % (deploying_message % host, output)) self.assertRegexpMatches( output, self.down_node_connection_error(self.cluster.internal_slaves[0])) self.assertEqual( len(output.splitlines()), len(self.cluster.all_hosts()) + self.len_down_node_error) self.run_prestoadmin('server start', raise_error=False) for host in [ self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2] ]: self.assert_has_default_catalog(host) self._assert_catalogs_loaded([['system'], ['tpch']])
def test_install_when_catalog_json_exists(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) cmd_output = installer.install() expected = [ 'Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'catalog configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'catalog configurations on: slave1 ', 'Using rpm_specifier as a local path', 'Fetching local presto rpm at path: .*', 'Found existing rpm at: .*' ] actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container)
def test_install_interactive(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) additional_keywords = { 'user': self.cluster.user, 'rpm_dir': self.cluster.rpm_cache_dir, 'rpm': rpm_name } cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install %(rpm_dir)s/%(rpm)s ', **additional_keywords) actual = cmd_output.splitlines() expected = [ r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Using rpm_specifier as a local path', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties catalog ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties catalog ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*' ] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container)
def test_catalog_add_by_name(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('catalog remove tpch') # test add catalog by name when it exists self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) self.run_prestoadmin('catalog add tpch') self.run_prestoadmin('server start') for host in self.cluster.all_hosts(): self.assert_has_default_catalog(host) self._assert_catalogs_loaded([['system'], ['tpch']])
def test_catalog_add_by_name(self): self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('catalog remove tpch') # test add catalog by name when it exists self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master) self.run_prestoadmin('catalog add tpch') self.run_prestoadmin('server start') for host in self.cluster.all_hosts(): self.assert_has_default_catalog(host) self._assert_catalogs_loaded([['system'], ['tpch']])
def test_install_when_topology_has_ips(self): installer = StandalonePrestoInstaller(self) ips = self.cluster.get_ip_address_dict() topology = { "coordinator": ips[self.cluster.internal_master], "workers": [ips[self.cluster.internal_slaves[0]]] } self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) cmd_output = installer.install().splitlines() expected = [ r'Deploying rpm on %s...' % ips[self.cluster.internal_master], r'Deploying rpm on %s...' % ips[self.cluster.internal_slaves[0]], r'Package deployed successfully on: ' + ips[self.cluster.internal_master], r'Package installed successfully on: ' + ips[self.cluster.internal_master], r'Package deployed successfully on: ' + ips[self.cluster.internal_slaves[0]], r'Package installed successfully on: ' + ips[self.cluster.internal_slaves[0]], r'Deploying configuration on: ' + ips[self.cluster.internal_master], r'Deploying jmx.properties, tpch.properties ' r'catalog configurations on: ' + ips[self.cluster.internal_master] + r' ', r'Deploying configuration on: ' + ips[self.cluster.internal_slaves[0]], r'Deploying jmx.properties, tpch.properties ' r'catalog configurations on: ' + ips[self.cluster.internal_slaves[0]] + r' ', r'Using rpm_specifier as a local path', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*' ] cmd_output.sort() expected.sort() self.assertRegexpMatchesLineByLine(cmd_output, expected) self.assert_installed_with_regex_configs(self.cluster.master, [self.cluster.slaves[0]]) for host in [self.cluster.master, self.cluster.slaves[0]]: self.assert_has_jmx_catalog(host)
def test_additional_dirs_created(self): install_dir = '~' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) self.assert_path_exists(self.cluster.master, get_catalog_directory()) self.assert_path_exists(self.cluster.master, get_coordinator_directory()) self.assert_path_exists(self.cluster.master, get_workers_directory())
def test_install_interactive(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master) ips = self.cluster.get_ip_address_dict() self.upload_topology({ "coordinator": ips[self.cluster.internal_master], "workers": [ ips[self.cluster.internal_slaves[0]], ips[self.cluster.internal_slaves[1]], ips[self.cluster.internal_slaves[2]] ], "username": "******" }) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster, coordinator=ips[self.cluster.internal_master]) additional_keywords = { 'user': "******", 'rpm_dir': self.cluster.rpm_cache_dir, 'rpm': rpm_name } self.run_script_from_prestoadmin_dir( 'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install %(rpm_dir)s/%(rpm)s -p password', **additional_keywords) self.assert_installed_with_regex_configs(self.cluster.master, [ self.cluster.slaves[0], self.cluster.slaves[1], self.cluster.slaves[2] ]) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container)
def test_catalog_remove(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) for host in self.cluster.all_hosts(): self.assert_has_default_catalog(host) missing_catalog_message = """[Errno 1] Fatal error: [master] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave1] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave2] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave3] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. """ # noqa success_message = """[master] Catalog removed. Restart the server \ for the change to take effect [slave1] Catalog removed. Restart the server for the change to take effect [slave2] Catalog removed. Restart the server for the change to take effect [slave3] Catalog removed. Restart the server for the change to take effect""" # test remove catalog does not exist # expect error self.assertRaisesMessageIgnoringOrder( OSError, missing_catalog_message % {'name': 'jmx'}, self.run_prestoadmin, 'catalog remove jmx') # test remove catalog not in directory, but in presto self.cluster.exec_cmd_on_host( self.cluster.master, 'rm %s' % os.path.join(get_catalog_directory(), 'tpch.properties') ) output = self.run_prestoadmin('catalog remove tpch') self.assertEqualIgnoringOrder(success_message, output) # test remove catalog in directory but not in presto self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) self.assertRaisesMessageIgnoringOrder( OSError, missing_catalog_message % {'name': 'tpch'}, self.run_prestoadmin, 'catalog remove tpch')
def setup_for_catalog_add(self): connector_script = 'mkdir -p %(catalogs)s\n' \ 'echo \'connector.name=tpch\' >> %(catalogs)s/tpch.properties\n' % \ {'catalogs': get_catalog_directory()} self.run_script_from_prestoadmin_dir(connector_script)