def write_test_configs(self, cluster, extra_configs=None,
coordinator=None):
if not coordinator:
coordinator = self.cluster.internal_master
config = 'http-server.http.port=8080\n' \
'query.max-memory=50GB\n' \
'query.max-memory-per-node=512MB\n' \
'discovery.uri=http://%s:8080' % coordinator
if extra_configs:
config += '\n' + extra_configs
coordinator_config = '%s\n' \
'coordinator=true\n' \
'node-scheduler.include-coordinator=false\n' \
'discovery-server.enabled=true' % config
workers_config = '%s\ncoordinator=false' % config
cluster.write_content_to_host(
coordinator_config,
os.path.join(get_coordinator_directory(), 'config.properties'),
cluster.master
)
cluster.write_content_to_host(
workers_config,
os.path.join(get_workers_directory(), 'config.properties'),
cluster.master
)
def test_configuration_deploy_show(self):
self.upload_topology()
self.deploy_and_assert_default_config()
node_ids = {}
for host in self.cluster.all_hosts():
node_ids[host] = self._get_node_id(host)
# deploy coordinator configuration only. Has a non-default file
dummy_prop1, dummy_prop2 = self.__write_dummy_config_file()
output = self.run_prestoadmin('configuration deploy coordinator')
deploy_template = 'Deploying configuration on: %s\n'
self.assertEqual(output,
deploy_template % self.cluster.internal_master)
for host in self.cluster.slaves:
self.assert_has_default_config(host)
config_properties_path = os.path.join(
constants.REMOTE_CONF_DIR, 'config.properties')
self.assert_config_perms(self.cluster.master, config_properties_path)
self.assert_file_content(self.cluster.master,
config_properties_path,
dummy_prop1 + '\n' +
dummy_prop2 + '\n' +
self.default_coordinator_test_config_)
# deploy workers configuration only has non-default file
filename = 'node.properties'
path = os.path.join(get_workers_directory(), filename)
self.cluster.write_content_to_host(
'node.environment test', path, self.cluster.master)
path = os.path.join(get_coordinator_directory(), filename)
self.cluster.write_content_to_host(
'node.environment test', path, self.cluster.master)
output = self.run_prestoadmin('configuration deploy workers')
expected = ''
for host in self.cluster.internal_slaves:
expected += deploy_template % host
self.assertEqualIgnoringOrder(output, expected)
for host in self.cluster.slaves:
self.assert_config_perms(host, config_properties_path)
self.assert_file_content(host,
config_properties_path,
dummy_prop1 + '\n' +
dummy_prop2 + '\n' +
self.default_workers_test_config_)
expected = 'node.environment=test\n'
self.assert_node_config(host, expected, node_ids[host])
self.assert_node_config(self.cluster.master,
self.default_node_properties_,
node_ids[self.cluster.master])
def assert_node_config(self, host, expected, expected_node_id=None):
node_properties_path = '/etc/presto/node.properties'
self.assert_config_perms(host, node_properties_path)
node_properties = self.cluster.exec_cmd_on_host(
host, 'cat %s' % (node_properties_path,))
split_properties = node_properties.split('\n', 1)
if expected_node_id:
self.assertEqual(expected_node_id, split_properties[0])
else:
self.assertRegexpMatches(split_properties[0], 'node.id=.*')
actual = split_properties[1]
if host in self.cluster.slaves:
conf_dir = get_workers_directory()
else:
conf_dir = get_coordinator_directory()
self.assertLazyMessage(
lambda: self.file_content_message(actual, expected, os.path.join(conf_dir, 'node.properties')),
self.assertEqual,
actual,
expected)
def assert_file_content(self, host, filepath, expected):
content = self.get_file_content(host, filepath)
split_path = os.path.split(filepath)
pa_file = None
if (split_path[0] == '/etc/presto' and
split_path[1] in ['config.properties',
'log.properties',
'jvm.config']):
if host in self.cluster.slaves:
config_dir = get_workers_directory()
else:
config_dir = get_coordinator_directory()
pa_file = os.path.join(config_dir, split_path[1])
self.assertLazyMessage(
lambda: self.file_content_message(content, expected, pa_file),
self.assertEqual,
content,
expected)
def _get_conf_dir(self):
return get_workers_directory()
def _get_conf_dir(self):
return get_workers_directory()
def test_configuration_show(self):
self.upload_topology()
for host in self.cluster.all_hosts():
self.cluster.exec_cmd_on_host(host, 'rm -rf /etc/presto')
# configuration show no configuration
output = self.run_prestoadmin('configuration show')
with open(os.path.join(LOCAL_RESOURCES_DIR,
'configuration_show_none.txt'), 'r') as f:
expected = f.read()
self.assertEqual(expected, output)
self.run_prestoadmin('configuration deploy')
# configuration show default configuration
output = self.run_prestoadmin('configuration show')
with open(os.path.join(LOCAL_RESOURCES_DIR,
'configuration_show_default.txt'), 'r') as f:
expected = f.read()
self.assertRegexpMatches(output, expected)
# configuration show node
output = self.run_prestoadmin('configuration show node')
with open(os.path.join(LOCAL_RESOURCES_DIR,
'configuration_show_node.txt'), 'r') as f:
expected = f.read()
self.assertRegexpMatches(output, expected)
# configuration show jvm
output = self.run_prestoadmin('configuration show jvm')
with open(os.path.join(LOCAL_RESOURCES_DIR,
'configuration_show_jvm.txt'), 'r') as f:
expected = f.read()
self.assertEqual(output, expected)
# configuration show config
output = self.run_prestoadmin('configuration show config')
with open(os.path.join(LOCAL_RESOURCES_DIR,
'configuration_show_config.txt'), 'r') as f:
expected = f.read()
self.assertEqual(output, expected)
# configuration show log no log.properties
output = self.run_prestoadmin('configuration show log')
with open(os.path.join(LOCAL_RESOURCES_DIR,
'configuration_show_log_none.txt'), 'r') as f:
expected = f.read()
self.assertEqual(output, expected)
# configuration show log has log.properties
log_properties = 'com.facebook.presto=WARN'
filename = 'log.properties'
self.cluster.write_content_to_host(
log_properties,
os.path.join(get_workers_directory(), filename),
self.cluster.master
)
self.cluster.write_content_to_host(
log_properties,
os.path.join(get_coordinator_directory(), filename),
self.cluster.master
)
self.run_prestoadmin('configuration deploy')
output = self.run_prestoadmin('configuration show log')
with open(os.path.join(LOCAL_RESOURCES_DIR,
'configuration_show_log.txt'), 'r') as f:
expected = f.read()
self.assertEqual(output, expected)
