def write_test_configs(self, cluster, extra_configs=None, coordinator=None): if not coordinator: coordinator = self.cluster.internal_master config = 'http-server.http.port=8080\n' \ 'query.max-memory=50GB\n' \ 'query.max-memory-per-node=512MB\n' \ 'discovery.uri=http://%s:8080' % coordinator if extra_configs: config += '\n' + extra_configs coordinator_config = '%s\n' \ 'coordinator=true\n' \ 'node-scheduler.include-coordinator=false\n' \ 'discovery-server.enabled=true' % config workers_config = '%s\ncoordinator=false' % config cluster.write_content_to_host( coordinator_config, os.path.join(get_coordinator_directory(), 'config.properties'), cluster.master ) cluster.write_content_to_host( workers_config, os.path.join(get_workers_directory(), 'config.properties'), cluster.master )
def test_configuration_deploy_show(self): self.upload_topology() self.deploy_and_assert_default_config() node_ids = {} for host in self.cluster.all_hosts(): node_ids[host] = self._get_node_id(host) # deploy coordinator configuration only. Has a non-default file dummy_prop1, dummy_prop2 = self.__write_dummy_config_file() output = self.run_prestoadmin('configuration deploy coordinator') deploy_template = 'Deploying configuration on: %s\n' self.assertEqual(output, deploy_template % self.cluster.internal_master) for host in self.cluster.slaves: self.assert_has_default_config(host) config_properties_path = os.path.join( constants.REMOTE_CONF_DIR, 'config.properties') self.assert_config_perms(self.cluster.master, config_properties_path) self.assert_file_content(self.cluster.master, config_properties_path, dummy_prop1 + '\n' + dummy_prop2 + '\n' + self.default_coordinator_test_config_) # deploy workers configuration only has non-default file filename = 'node.properties' path = os.path.join(get_workers_directory(), filename) self.cluster.write_content_to_host( 'node.environment test', path, self.cluster.master) path = os.path.join(get_coordinator_directory(), filename) self.cluster.write_content_to_host( 'node.environment test', path, self.cluster.master) output = self.run_prestoadmin('configuration deploy workers') expected = '' for host in self.cluster.internal_slaves: expected += deploy_template % host self.assertEqualIgnoringOrder(output, expected) for host in self.cluster.slaves: self.assert_config_perms(host, config_properties_path) self.assert_file_content(host, config_properties_path, dummy_prop1 + '\n' + dummy_prop2 + '\n' + self.default_workers_test_config_) expected = 'node.environment=test\n' self.assert_node_config(host, expected, node_ids[host]) self.assert_node_config(self.cluster.master, self.default_node_properties_, node_ids[self.cluster.master])
def assert_node_config(self, host, expected, expected_node_id=None): node_properties_path = '/etc/presto/node.properties' self.assert_config_perms(host, node_properties_path) node_properties = self.cluster.exec_cmd_on_host( host, 'cat %s' % (node_properties_path,)) split_properties = node_properties.split('\n', 1) if expected_node_id: self.assertEqual(expected_node_id, split_properties[0]) else: self.assertRegexpMatches(split_properties[0], 'node.id=.*') actual = split_properties[1] if host in self.cluster.slaves: conf_dir = get_workers_directory() else: conf_dir = get_coordinator_directory() self.assertLazyMessage( lambda: self.file_content_message(actual, expected, os.path.join(conf_dir, 'node.properties')), self.assertEqual, actual, expected)
def assert_file_content(self, host, filepath, expected): content = self.get_file_content(host, filepath) split_path = os.path.split(filepath) pa_file = None if (split_path[0] == '/etc/presto' and split_path[1] in ['config.properties', 'log.properties', 'jvm.config']): if host in self.cluster.slaves: config_dir = get_workers_directory() else: config_dir = get_coordinator_directory() pa_file = os.path.join(config_dir, split_path[1]) self.assertLazyMessage( lambda: self.file_content_message(content, expected, pa_file), self.assertEqual, content, expected)
def _get_conf_dir(self): return get_workers_directory()
def test_configuration_show(self): self.upload_topology() for host in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host(host, 'rm -rf /etc/presto') # configuration show no configuration output = self.run_prestoadmin('configuration show') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_none.txt'), 'r') as f: expected = f.read() self.assertEqual(expected, output) self.run_prestoadmin('configuration deploy') # configuration show default configuration output = self.run_prestoadmin('configuration show') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_default.txt'), 'r') as f: expected = f.read() self.assertRegexpMatches(output, expected) # configuration show node output = self.run_prestoadmin('configuration show node') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_node.txt'), 'r') as f: expected = f.read() self.assertRegexpMatches(output, expected) # configuration show jvm output = self.run_prestoadmin('configuration show jvm') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_jvm.txt'), 'r') as f: expected = f.read() self.assertEqual(output, expected) # configuration show config output = self.run_prestoadmin('configuration show config') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_config.txt'), 'r') as f: expected = f.read() self.assertEqual(output, expected) # configuration show log no log.properties output = self.run_prestoadmin('configuration show log') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_log_none.txt'), 'r') as f: expected = f.read() self.assertEqual(output, expected) # configuration show log has log.properties log_properties = 'com.facebook.presto=WARN' filename = 'log.properties' self.cluster.write_content_to_host( log_properties, os.path.join(get_workers_directory(), filename), self.cluster.master ) self.cluster.write_content_to_host( log_properties, os.path.join(get_coordinator_directory(), filename), self.cluster.master ) self.run_prestoadmin('configuration deploy') output = self.run_prestoadmin('configuration show log') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_log.txt'), 'r') as f: expected = f.read() self.assertEqual(output, expected)