def test_connector_add_remove_coord_worker_using_dash_h(self): self.setup_cluster_assert_connectors() self.run_prestoadmin('connector remove tpch -H %(master)s,%(slave1)s') self.run_prestoadmin('server restart') self.assert_path_removed(self.cluster.master, os.path.join(get_connectors_directory(), 'tpch.properties')) self._assert_connectors_loaded([['system']]) for host in [self.cluster.master, self.cluster.slaves[0]]: self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties')) self.assert_has_default_connector(self.cluster.slaves[1]) self.assert_has_default_connector(self.cluster.slaves[2]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_connectors_directory(), 'tpch.properties'), self.cluster.master ) self.run_prestoadmin('connector add tpch -H %(master)s,%(slave1)s') self.run_prestoadmin('server restart') self.assert_has_default_connector(self.cluster.master) self.assert_has_default_connector(self.cluster.slaves[1])
def test_connector_add_remove(self): self.setup_cluster_assert_connectors() self.run_prestoadmin('connector remove tpch') self.assert_path_removed(self.cluster.master, os.path.join(get_connectors_directory(), 'tpch.properties')) for host in self.cluster.all_hosts(): self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties')) # test add connectors from directory with more than one connector self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_connectors_directory(), 'tpch.properties'), self.cluster.master ) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_connectors_directory(), 'jmx.properties'), self.cluster.master ) self.run_prestoadmin('connector add') self.run_prestoadmin('server restart') for host in self.cluster.all_hosts(): filepath = '/etc/presto/catalog/jmx.properties' self.assert_has_default_connector(host) self.assert_connector_perms(host, filepath) self.assert_file_content(host, filepath, 'connector.name=jmx') self._assert_connectors_loaded([['system'], ['jmx'], ['tpch']])
def test_connector_add_remove_non_sudo_user(self): self.setup_cluster_assert_connectors() self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "******"} ) self.run_prestoadmin('connector remove tpch -p password') self.assert_path_removed(self.cluster.master, os.path.join(get_connectors_directory(), 'tpch.properties')) for host in self.cluster.all_hosts(): self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tcph.properties')) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_connectors_directory(), 'jmx.properties'), self.cluster.master ) self.run_prestoadmin('connector add -p password') self.run_prestoadmin('server restart -p password') for host in self.cluster.all_hosts(): self.assert_has_jmx_connector(host) self._assert_connectors_loaded([['system'], ['jmx']])
def test_update_config(self, mock_open, mock_fdopen, mock_makedir, mock_path_exists, mock_config, mock_connector): e = ConfigFileNotFoundError( message='problems', config_path='config_path') mock_connector.add.side_effect = e mock_path_exists.side_effect = [False, False] server.update_configs() mock_config.assert_called_with() mock_makedir.assert_called_with(get_connectors_directory()) mock_open.assert_called_with(os.path.join(get_connectors_directory(), 'tpch.properties'), os.O_CREAT | os.O_EXCL | os.O_WRONLY) file_manager = mock_fdopen.return_value.__enter__.return_value file_manager.write.assert_called_with("connector.name=tpch")
def test_connector_add_lost_host(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER) self.upload_topology() installer.install() self.run_prestoadmin('connector remove tpch') self.cluster.stop_host( self.cluster.slaves[0]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_connectors_directory(), 'tpch.properties'), self.cluster.master ) output = self.run_prestoadmin('connector add tpch', raise_error=False) for host in self.cluster.all_internal_hosts(): deploying_message = 'Deploying tpch.properties connector ' \ 'configurations on: %s' self.assertTrue(deploying_message % host in output, 'expected %s \n actual %s' % (deploying_message % host, output)) self.assertRegexpMatches( output, self.down_node_connection_error(self.cluster.internal_slaves[0]) ) self.assertEqual(len(output.splitlines()), len(self.cluster.all_hosts()) + self.len_down_node_error) self.run_prestoadmin('server start', raise_error=False) for host in [self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_has_default_connector(host) self._assert_connectors_loaded([['system'], ['tpch']])
def test_warning_if_connector_dir_empty(self, isdir_mock, listdir_mock): isdir_mock.return_value = True listdir_mock.return_value = [] connector.add() self.assertEqual('\nWarning: Directory %s is empty. No connectors will' ' be deployed\n\n' % get_connectors_directory(), self.test_stderr.getvalue())
def remove(name): """ Remove a connector from the cluster. Parameters: name - Name of the connector to be removed """ _LOGGER.info('[' + env.host + '] Removing connector: ' + name) ret = remove_file(os.path.join(constants.REMOTE_CATALOG_DIR, name + '.properties')) if ret.succeeded: if COULD_NOT_REMOVE in ret: fabric.utils.error(ret) else: print('[%s] Connector removed. Restart the server for the change ' 'to take effect' % env.host) else: fabric.utils.error('Failed to remove connector ' + name + '.\n\t' + ret) local_path = os.path.join(get_connectors_directory(), name + '.properties') try: os.remove(local_path) except OSError as e: if e.errno == errno.ENOENT: pass else: raise
def test_add_all(self, mock_validate, listdir_mock, isdir_mock, deploy_mock): catalogs = ['tpch.properties', 'another.properties'] listdir_mock.return_value = catalogs connector.add() deploy_mock.assert_called_with(catalogs, get_connectors_directory(), constants.REMOTE_CATALOG_DIR, PRESTO_STANDALONE_USER_GROUP)
def test_add_exists(self, isfile_mock, deploy_mock, validate_mock): isfile_mock.return_value = True connector.add('tpch') filenames = ['tpch.properties'] deploy_mock.assert_called_with(filenames, get_connectors_directory(), constants.REMOTE_CATALOG_DIR, PRESTO_STANDALONE_USER_GROUP) validate_mock.assert_called_with(filenames)
def test_remove(self, local_rm_mock, exists_mock, sudo_mock): script = ('if [ -f /etc/presto/catalog/tpch.properties ] ; ' 'then rm /etc/presto/catalog/tpch.properties ; ' 'else echo "Could not remove connector \'tpch\'. ' 'No such file \'/etc/presto/catalog/tpch.properties\'"; fi') exists_mock.return_value = True fabric.api.env.host = 'localhost' connector.remove('tpch') sudo_mock.assert_called_with(script) local_rm_mock.assert_called_with(get_connectors_directory() + '/tpch.properties')
def add(name=None): """ Deploy configuration for a connector onto a cluster. E.g.: 'presto-admin connector add tpch' deploys a configuration file for the tpch connector. The configuration is defined by tpch.properties in /etc/opt/prestoadmin/connectors directory. If no connector name is specified, then configurations for all connectors in the connectors directory will be deployed Parameters: name - Name of the connector to be added """ connectors_dir = get_connectors_directory() if name: filename = name + '.properties' config_path = os.path.join(connectors_dir, filename) if not os.path.isfile(config_path): raise ConfigFileNotFoundError( config_path=config_path, message='Configuration for connector ' + name + ' not found') filenames = [filename] elif not os.path.isdir(connectors_dir): message = ('Cannot add connectors because directory %s does not exist' % connectors_dir) raise ConfigFileNotFoundError(config_path=connectors_dir, message=message) else: try: filenames = os.listdir(connectors_dir) except OSError as e: fabric.utils.error(e.strerror) return if not filenames: fabric.utils.warn( 'Directory %s is empty. No connectors will be deployed' % connectors_dir) return if not validate(filenames): return filenames.sort() _LOGGER.info('Adding connector configurations: ' + str(filenames)) print('Deploying %s connector configurations on: %s ' % (', '.join(filenames), env.host)) deploy_files(filenames, connectors_dir, constants.REMOTE_CATALOG_DIR, PRESTO_STANDALONE_USER_GROUP)
def test_connector_add_by_name(self): self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('connector remove tpch') # test add connector by name when it exists self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_connectors_directory(), 'tpch.properties'), self.cluster.master ) self.run_prestoadmin('connector add tpch') self.run_prestoadmin('server start') for host in self.cluster.all_hosts(): self.assert_has_default_connector(host) self._assert_connectors_loaded([['system'], ['tpch']])
def validate(filenames): for name in filenames: file_path = os.path.join(get_connectors_directory(), name) _LOGGER.info('Validating connector configuration: ' + str(name)) try: with open(file_path) as f: file_content = f.read() if 'connector.name' not in file_content: message = ('Catalog configuration %s does not contain ' 'connector.name' % name) raise ConfigurationError(message) except IOError, e: fabric.utils.error(message='Error validating ' + file_path, exception=e) return False
def test_install_when_topology_has_ips(self): installer = StandalonePrestoInstaller(self) ips = self.cluster.get_ip_address_dict() topology = {"coordinator": ips[self.cluster.internal_master], "workers": [ips[self.cluster.internal_slaves[0]]]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_connectors_directory(), 'jmx.properties'), self.cluster.master ) cmd_output = installer.install().splitlines() expected = [ r'Deploying rpm on %s...' % ips[self.cluster.internal_master], r'Deploying rpm on %s...' % ips[self.cluster.internal_slaves[0]], r'Package deployed successfully on: ' + ips[self.cluster.internal_master], r'Package installed successfully on: ' + ips[self.cluster.internal_master], r'Package deployed successfully on: ' + ips[self.cluster.internal_slaves[0]], r'Package installed successfully on: ' + ips[self.cluster.internal_slaves[0]], r'Deploying configuration on: ' + ips[self.cluster.internal_master], r'Deploying jmx.properties, tpch.properties ' r'connector configurations on: ' + ips[self.cluster.internal_master] + r' ', r'Deploying configuration on: ' + ips[self.cluster.internal_slaves[0]], r'Deploying jmx.properties, tpch.properties ' r'connector configurations on: ' + ips[self.cluster.internal_slaves[0]] + r' ', r'Using rpm_specifier as a local path', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*'] cmd_output.sort() expected.sort() self.assertRegexpMatchesLineByLine(cmd_output, expected) self.assert_installed_with_regex_configs( self.cluster.master, [self.cluster.slaves[0]]) for host in [self.cluster.master, self.cluster.slaves[0]]: self.assert_has_jmx_connector(host)
def test_install_when_connector_json_exists(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_connectors_directory(), 'jmx.properties'), self.cluster.master ) cmd_output = installer.install() expected = ['Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'connector configurations on: slave1 ', 'Using rpm_specifier as a local path', 'Fetching local presto rpm at path: .*', 'Found existing rpm at: .*'] actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container)
def test_install_interactive(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_connectors_directory(), 'jmx.properties'), self.cluster.master ) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) additional_keywords = { 'user': self.cluster.user, 'rpm_dir': self.cluster.rpm_cache_dir, 'rpm': rpm_name } cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install %(rpm_dir)s/%(rpm)s ', **additional_keywords) actual = cmd_output.splitlines() expected = [r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Using rpm_specifier as a local path', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties connector ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*' ] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_connector(container) self.assert_has_jmx_connector(container)
def test_connector_remove(self): self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER) for host in self.cluster.all_hosts(): self.assert_has_default_connector(host) missing_connector_message = """[Errno 1] Fatal error: [master] Could not remove connector '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave1] Could not remove connector '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave2] Could not remove connector '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave3] Could not remove connector '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. """ # noqa success_message = """[master] Connector removed. Restart the server \ for the change to take effect [slave1] Connector removed. Restart the server for the change to take effect [slave2] Connector removed. Restart the server for the change to take effect [slave3] Connector removed. Restart the server for the change to take effect""" # test remove connector does not exist # expect error self.assertRaisesMessageIgnoringOrder( OSError, missing_connector_message % {'name': 'jmx'}, self.run_prestoadmin, 'connector remove jmx') # test remove connector not in directory, but in presto self.cluster.exec_cmd_on_host( self.cluster.master, 'rm /etc/opt/prestoadmin/connectors/tpch.properties' ) output = self.run_prestoadmin('connector remove tpch') self.assertEqualIgnoringOrder(success_message, output) # test remove connector in directory but not in presto self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_connectors_directory(), 'tpch.properties'), self.cluster.master ) self.assertRaisesMessageIgnoringOrder( OSError, missing_connector_message % {'name': 'tpch'}, self.run_prestoadmin, 'connector remove tpch')
def add_tpch_connector(): tpch_connector_config = os.path.join(get_connectors_directory(), 'tpch.properties') util.filesystem.write_to_file_if_not_exists('connector.name=tpch', tpch_connector_config)