def test_connector_add_remove_coord_worker_using_dash_h(self):
self.setup_cluster_assert_connectors()
self.run_prestoadmin('connector remove tpch -H %(master)s,%(slave1)s')
self.run_prestoadmin('server restart')
self.assert_path_removed(self.cluster.master,
os.path.join(get_connectors_directory(),
'tpch.properties'))
self._assert_connectors_loaded([['system']])
for host in [self.cluster.master, self.cluster.slaves[0]]:
self.assert_path_removed(host,
os.path.join(constants.REMOTE_CATALOG_DIR,
'tpch.properties'))
self.assert_has_default_connector(self.cluster.slaves[1])
self.assert_has_default_connector(self.cluster.slaves[2])
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_connectors_directory(), 'tpch.properties'),
self.cluster.master
)
self.run_prestoadmin('connector add tpch -H %(master)s,%(slave1)s')
self.run_prestoadmin('server restart')
self.assert_has_default_connector(self.cluster.master)
self.assert_has_default_connector(self.cluster.slaves[1])
def test_connector_add_remove(self):
self.setup_cluster_assert_connectors()
self.run_prestoadmin('connector remove tpch')
self.assert_path_removed(self.cluster.master, os.path.join(get_connectors_directory(), 'tpch.properties'))
for host in self.cluster.all_hosts():
self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties'))
# test add connectors from directory with more than one connector
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_connectors_directory(), 'tpch.properties'),
self.cluster.master
)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_connectors_directory(), 'jmx.properties'),
self.cluster.master
)
self.run_prestoadmin('connector add')
self.run_prestoadmin('server restart')
for host in self.cluster.all_hosts():
filepath = '/etc/presto/catalog/jmx.properties'
self.assert_has_default_connector(host)
self.assert_connector_perms(host, filepath)
self.assert_file_content(host, filepath, 'connector.name=jmx')
self._assert_connectors_loaded([['system'], ['jmx'], ['tpch']])
def test_connector_add_remove_non_sudo_user(self):
self.setup_cluster_assert_connectors()
self.upload_topology(
{"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"}
)
self.run_prestoadmin('connector remove tpch -p password')
self.assert_path_removed(self.cluster.master,
os.path.join(get_connectors_directory(),
'tpch.properties'))
for host in self.cluster.all_hosts():
self.assert_path_removed(host,
os.path.join(constants.REMOTE_CATALOG_DIR,
'tcph.properties'))
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_connectors_directory(), 'jmx.properties'),
self.cluster.master
)
self.run_prestoadmin('connector add -p password')
self.run_prestoadmin('server restart -p password')
for host in self.cluster.all_hosts():
self.assert_has_jmx_connector(host)
self._assert_connectors_loaded([['system'], ['jmx']])
def test_update_config(self, mock_open, mock_fdopen, mock_makedir,
mock_path_exists, mock_config, mock_connector):
e = ConfigFileNotFoundError(
message='problems', config_path='config_path')
mock_connector.add.side_effect = e
mock_path_exists.side_effect = [False, False]
server.update_configs()
mock_config.assert_called_with()
mock_makedir.assert_called_with(get_connectors_directory())
mock_open.assert_called_with(os.path.join(get_connectors_directory(),
'tpch.properties'),
os.O_CREAT | os.O_EXCL | os.O_WRONLY)
file_manager = mock_fdopen.return_value.__enter__.return_value
file_manager.write.assert_called_with("connector.name=tpch")
def test_connector_add_lost_host(self):
installer = StandalonePrestoInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER)
self.upload_topology()
installer.install()
self.run_prestoadmin('connector remove tpch')
self.cluster.stop_host(
self.cluster.slaves[0])
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_connectors_directory(), 'tpch.properties'),
self.cluster.master
)
output = self.run_prestoadmin('connector add tpch', raise_error=False)
for host in self.cluster.all_internal_hosts():
deploying_message = 'Deploying tpch.properties connector ' \
'configurations on: %s'
self.assertTrue(deploying_message % host in output,
'expected %s \n actual %s'
% (deploying_message % host, output))
self.assertRegexpMatches(
output,
self.down_node_connection_error(self.cluster.internal_slaves[0])
)
self.assertEqual(len(output.splitlines()),
len(self.cluster.all_hosts()) +
self.len_down_node_error)
self.run_prestoadmin('server start', raise_error=False)
for host in [self.cluster.master,
self.cluster.slaves[1],
self.cluster.slaves[2]]:
self.assert_has_default_connector(host)
self._assert_connectors_loaded([['system'], ['tpch']])
def test_warning_if_connector_dir_empty(self, isdir_mock, listdir_mock):
isdir_mock.return_value = True
listdir_mock.return_value = []
connector.add()
self.assertEqual('\nWarning: Directory %s is empty. No connectors will'
' be deployed\n\n' % get_connectors_directory(),
self.test_stderr.getvalue())
def remove(name):
"""
Remove a connector from the cluster.
Parameters:
name - Name of the connector to be removed
"""
_LOGGER.info('[' + env.host + '] Removing connector: ' + name)
ret = remove_file(os.path.join(constants.REMOTE_CATALOG_DIR,
name + '.properties'))
if ret.succeeded:
if COULD_NOT_REMOVE in ret:
fabric.utils.error(ret)
else:
print('[%s] Connector removed. Restart the server for the change '
'to take effect' % env.host)
else:
fabric.utils.error('Failed to remove connector ' + name + '.\n\t' +
ret)
local_path = os.path.join(get_connectors_directory(), name + '.properties')
try:
os.remove(local_path)
except OSError as e:
if e.errno == errno.ENOENT:
pass
else:
raise
def test_add_all(self, mock_validate, listdir_mock, isdir_mock,
deploy_mock):
catalogs = ['tpch.properties', 'another.properties']
listdir_mock.return_value = catalogs
connector.add()
deploy_mock.assert_called_with(catalogs,
get_connectors_directory(),
constants.REMOTE_CATALOG_DIR,
PRESTO_STANDALONE_USER_GROUP)
def test_add_exists(self, isfile_mock, deploy_mock, validate_mock):
isfile_mock.return_value = True
connector.add('tpch')
filenames = ['tpch.properties']
deploy_mock.assert_called_with(filenames,
get_connectors_directory(),
constants.REMOTE_CATALOG_DIR,
PRESTO_STANDALONE_USER_GROUP)
validate_mock.assert_called_with(filenames)
def test_remove(self, local_rm_mock, exists_mock, sudo_mock):
script = ('if [ -f /etc/presto/catalog/tpch.properties ] ; '
'then rm /etc/presto/catalog/tpch.properties ; '
'else echo "Could not remove connector \'tpch\'. '
'No such file \'/etc/presto/catalog/tpch.properties\'"; fi')
exists_mock.return_value = True
fabric.api.env.host = 'localhost'
connector.remove('tpch')
sudo_mock.assert_called_with(script)
local_rm_mock.assert_called_with(get_connectors_directory() +
'/tpch.properties')
def add(name=None):
"""
Deploy configuration for a connector onto a cluster.
E.g.: 'presto-admin connector add tpch'
deploys a configuration file for the tpch connector. The configuration is
defined by tpch.properties in /etc/opt/prestoadmin/connectors directory.
If no connector name is specified, then configurations for all connectors
in the connectors directory will be deployed
Parameters:
name - Name of the connector to be added
"""
connectors_dir = get_connectors_directory()
if name:
filename = name + '.properties'
config_path = os.path.join(connectors_dir, filename)
if not os.path.isfile(config_path):
raise ConfigFileNotFoundError(
config_path=config_path,
message='Configuration for connector ' + name + ' not found')
filenames = [filename]
elif not os.path.isdir(connectors_dir):
message = ('Cannot add connectors because directory %s does not exist'
% connectors_dir)
raise ConfigFileNotFoundError(config_path=connectors_dir,
message=message)
else:
try:
filenames = os.listdir(connectors_dir)
except OSError as e:
fabric.utils.error(e.strerror)
return
if not filenames:
fabric.utils.warn(
'Directory %s is empty. No connectors will be deployed' %
connectors_dir)
return
if not validate(filenames):
return
filenames.sort()
_LOGGER.info('Adding connector configurations: ' + str(filenames))
print('Deploying %s connector configurations on: %s ' %
(', '.join(filenames), env.host))
deploy_files(filenames, connectors_dir,
constants.REMOTE_CATALOG_DIR, PRESTO_STANDALONE_USER_GROUP)
def test_connector_add_by_name(self):
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER)
self.run_prestoadmin('connector remove tpch')
# test add connector by name when it exists
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_connectors_directory(), 'tpch.properties'),
self.cluster.master
)
self.run_prestoadmin('connector add tpch')
self.run_prestoadmin('server start')
for host in self.cluster.all_hosts():
self.assert_has_default_connector(host)
self._assert_connectors_loaded([['system'], ['tpch']])
def validate(filenames):
for name in filenames:
file_path = os.path.join(get_connectors_directory(), name)
_LOGGER.info('Validating connector configuration: ' + str(name))
try:
with open(file_path) as f:
file_content = f.read()
if 'connector.name' not in file_content:
message = ('Catalog configuration %s does not contain '
'connector.name' % name)
raise ConfigurationError(message)
except IOError, e:
fabric.utils.error(message='Error validating ' + file_path,
exception=e)
return False
def test_install_when_topology_has_ips(self):
installer = StandalonePrestoInstaller(self)
ips = self.cluster.get_ip_address_dict()
topology = {"coordinator": ips[self.cluster.internal_master],
"workers": [ips[self.cluster.internal_slaves[0]]]}
self.upload_topology(topology)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_connectors_directory(), 'jmx.properties'),
self.cluster.master
)
cmd_output = installer.install().splitlines()
expected = [
r'Deploying rpm on %s...' % ips[self.cluster.internal_master],
r'Deploying rpm on %s...' % ips[self.cluster.internal_slaves[0]],
r'Package deployed successfully on: ' +
ips[self.cluster.internal_master],
r'Package installed successfully on: ' +
ips[self.cluster.internal_master],
r'Package deployed successfully on: ' +
ips[self.cluster.internal_slaves[0]],
r'Package installed successfully on: ' +
ips[self.cluster.internal_slaves[0]],
r'Deploying configuration on: ' +
ips[self.cluster.internal_master],
r'Deploying jmx.properties, tpch.properties '
r'connector configurations on: ' +
ips[self.cluster.internal_master] + r' ',
r'Deploying configuration on: ' +
ips[self.cluster.internal_slaves[0]],
r'Deploying jmx.properties, tpch.properties '
r'connector configurations on: ' +
ips[self.cluster.internal_slaves[0]] + r' ',
r'Using rpm_specifier as a local path',
r'Fetching local presto rpm at path: .*',
r'Found existing rpm at: .*']
cmd_output.sort()
expected.sort()
self.assertRegexpMatchesLineByLine(cmd_output, expected)
self.assert_installed_with_regex_configs(
self.cluster.master,
[self.cluster.slaves[0]])
for host in [self.cluster.master, self.cluster.slaves[0]]:
self.assert_has_jmx_connector(host)
def test_install_when_connector_json_exists(self):
installer = StandalonePrestoInstaller(self)
topology = {"coordinator": "master",
"workers": ["slave1"]}
self.upload_topology(topology)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_connectors_directory(), 'jmx.properties'),
self.cluster.master
)
cmd_output = installer.install()
expected = ['Deploying rpm on master...',
'Deploying rpm on slave1...',
'Package deployed successfully on: slave1',
'Package installed successfully on: slave1',
'Package deployed successfully on: master',
'Package installed successfully on: master',
'Deploying configuration on: master',
'Deploying jmx.properties, tpch.properties '
'connector configurations on: master ',
'Deploying configuration on: slave1',
'Deploying jmx.properties, tpch.properties '
'connector configurations on: slave1 ',
'Using rpm_specifier as a local path',
'Fetching local presto rpm at path: .*',
'Found existing rpm at: .*']
actual = cmd_output.splitlines()
self.assertRegexpMatchesLineByLine(actual, expected)
for container in [self.cluster.master,
self.cluster.slaves[0]]:
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.assert_has_jmx_connector(container)
def test_install_interactive(self):
installer = StandalonePrestoInstaller(self)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_connectors_directory(), 'jmx.properties'),
self.cluster.master
)
rpm_name = installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
additional_keywords = {
'user': self.cluster.user,
'rpm_dir': self.cluster.rpm_cache_dir,
'rpm': rpm_name
}
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | '
'./presto-admin server install %(rpm_dir)s/%(rpm)s ',
**additional_keywords)
actual = cmd_output.splitlines()
expected = [r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Using rpm_specifier as a local path',
r'Package deployed successfully on: ' +
self.cluster.internal_master,
r'Package installed successfully on: ' +
self.cluster.internal_master,
r'Package deployed successfully on: ' +
self.cluster.internal_slaves[0],
r'Package installed successfully on: ' +
self.cluster.internal_slaves[0],
r'Deploying configuration on: ' +
self.cluster.internal_master,
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' +
self.cluster.internal_master,
r'Deploying configuration on: ' +
self.cluster.internal_slaves[0],
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' +
self.cluster.internal_slaves[0],
r'Deploying rpm on .*\.\.\.',
r'Deploying rpm on .*\.\.\.',
r'Fetching local presto rpm at path: .*',
r'Found existing rpm at: .*'
]
self.assertRegexpMatchesLineByLine(actual, expected)
for container in [self.cluster.master,
self.cluster.slaves[0]]:
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.assert_has_jmx_connector(container)
def test_connector_remove(self):
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PRESTO_CLUSTER)
for host in self.cluster.all_hosts():
self.assert_has_default_connector(host)
missing_connector_message = """[Errno 1]
Fatal error: [master] Could not remove connector '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
Fatal error: [slave1] Could not remove connector '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
Fatal error: [slave2] Could not remove connector '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
Fatal error: [slave3] Could not remove connector '%(name)s'. No such file \
'/etc/presto/catalog/%(name)s.properties'
Aborting.
""" # noqa
success_message = """[master] Connector removed. Restart the server \
for the change to take effect
[slave1] Connector removed. Restart the server for the change to take effect
[slave2] Connector removed. Restart the server for the change to take effect
[slave3] Connector removed. Restart the server for the change to take effect"""
# test remove connector does not exist
# expect error
self.assertRaisesMessageIgnoringOrder(
OSError,
missing_connector_message % {'name': 'jmx'},
self.run_prestoadmin,
'connector remove jmx')
# test remove connector not in directory, but in presto
self.cluster.exec_cmd_on_host(
self.cluster.master,
'rm /etc/opt/prestoadmin/connectors/tpch.properties'
)
output = self.run_prestoadmin('connector remove tpch')
self.assertEqualIgnoringOrder(success_message, output)
# test remove connector in directory but not in presto
self.cluster.write_content_to_host(
'connector.name=tpch',
os.path.join(get_connectors_directory(), 'tpch.properties'),
self.cluster.master
)
self.assertRaisesMessageIgnoringOrder(
OSError,
missing_connector_message % {'name': 'tpch'},
self.run_prestoadmin,
'connector remove tpch')
def add_tpch_connector():
tpch_connector_config = os.path.join(get_connectors_directory(),
'tpch.properties')
util.filesystem.write_to_file_if_not_exists('connector.name=tpch',
tpch_connector_config)
