class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): def __init__(self, base_path, config_dir): ClickHouseCluster.__init__(self, base_path) self.test_config_dir = config_dir def prepare(self, replace_hostnames_with_ips=True): try: main_configs_files = [ "clusters.xml", "zookeeper_session_timeout.xml", "macro.xml", "query_log.xml", "ddl.xml" ] main_configs = [ os.path.join(self.test_config_dir, "config.d", f) for f in main_configs_files ] user_configs = [ os.path.join(self.test_config_dir, "users.d", f) for f in ["restricted_user.xml", "query_log.xml"] ] if self.test_config_dir == "configs_secure": main_configs += [ os.path.join(self.test_config_dir, f) for f in [ "server.crt", "server.key", "dhparam.pem", "config.d/ssl_conf.xml" ] ] for i in xrange(4): self.add_instance('ch{}'.format(i + 1), main_configs=main_configs, user_configs=user_configs, macros={ "layer": 0, "shard": i / 2 + 1, "replica": i % 2 + 1 }, with_zookeeper=True) self.start() # Replace config files for testing ability to set host in DNS and IP formats if replace_hostnames_with_ips: self.replace_domains_to_ip_addresses_in_cluster_config( ['ch1', 'ch3']) # Select sacrifice instance to test CONNECTION_LOSS and server fail on it sacrifice = self.instances['ch4'] self.pm_random_drops = PartitionManager() self.pm_random_drops._add_rule({ 'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, 'action': 'REJECT --reject-with tcp-reset' }) self.pm_random_drops._add_rule({ 'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, 'action': 'REJECT --reject-with tcp-reset' }) # Initialize databases and service tables instance = self.instances['ch1'] self.ddl_check_query( instance, """ CREATE TABLE IF NOT EXISTS all_tables ON CLUSTER 'cluster_no_replicas' (database String, name String, engine String, metadata_modification_time DateTime) ENGINE = Distributed('cluster_no_replicas', 'system', 'tables') """) self.ddl_check_query( instance, "CREATE DATABASE IF NOT EXISTS test ON CLUSTER 'cluster'") except Exception as e: print e raise def sync_replicas(self, table, timeout=5): for instance in self.instances.values(): instance.query("SYSTEM SYNC REPLICA {}".format(table), timeout=timeout) def check_all_hosts_successfully_executed(self, tsv_content, num_hosts=None): if num_hosts is None: num_hosts = len(self.instances) M = TSV.toMat(tsv_content) hosts = [(l[0], l[1]) for l in M] # (host, port) codes = [l[2] for l in M] messages = [l[3] for l in M] assert len(hosts) == num_hosts and len( set(hosts)) == num_hosts, "\n" + tsv_content assert len(set(codes)) == 1, "\n" + tsv_content assert codes[0] == "0", "\n" + tsv_content def ddl_check_query(self, instance, query, num_hosts=None, settings=None): contents = instance.query(query, settings=settings) self.check_all_hosts_successfully_executed(contents, num_hosts) return contents def replace_domains_to_ip_addresses_in_cluster_config( self, instances_to_replace): clusters_config = open( p.join(self.base_dir, '{}/config.d/clusters.xml'.format( self.test_config_dir))).read() for inst_name, inst in self.instances.items(): clusters_config = clusters_config.replace(inst_name, str(inst.ip_address)) for inst_name in instances_to_replace: inst = self.instances[inst_name] self.instances[inst_name].exec_in_container([ 'bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml' ], environment={ "NEW_CONFIG": clusters_config }, privileged=True) # print cluster.instances[inst_name].exec_in_container(['cat', "/etc/clickhouse-server/config.d/clusters.xml"]) @staticmethod def ddl_check_there_are_no_dublicates(instance): query = "SELECT max(c), argMax(q, c) FROM (SELECT lower(query) AS q, count() AS c FROM system.query_log WHERE type=2 AND q LIKE '/* ddl_entry=query-%' GROUP BY query)" rows = instance.query(query) assert len(rows) > 0 and rows[0][ 0] == "1", "dublicates on {} {}, query {}".format( instance.name, instance.ip_address, query) @staticmethod def insert_reliable(instance, query_insert): """ Make retries in case of UNKNOWN_STATUS_OF_INSERT or zkutil::KeeperException errors """ for i in xrange(100): try: instance.query(query_insert) return except Exception as e: last_exception = e s = str(e) if not (s.find('Unknown status, client must retry') >= 0 or s.find('zkutil::KeeperException')): raise e raise last_exception
def test_round_robin(started_cluster): pm = PartitionManager() try: pm._add_rule({ "source": node1.ip_address, "destination": cluster.get_instance_ip("zoo1"), "action": "REJECT --reject-with tcp-reset", }) pm._add_rule({ "source": node2.ip_address, "destination": cluster.get_instance_ip("zoo1"), "action": "REJECT --reject-with tcp-reset", }) pm._add_rule({ "source": node3.ip_address, "destination": cluster.get_instance_ip("zoo1"), "action": "REJECT --reject-with tcp-reset", }) change_balancing("random", "round_robin") print( str( node1.exec_in_container( [ "bash", "-c", "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED", ], privileged=True, user="******", ))) assert ("1" == str( node1.exec_in_container( [ "bash", "-c", "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo2_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l", ], privileged=True, user="******", )).strip()) print( str( node2.exec_in_container( [ "bash", "-c", "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED", ], privileged=True, user="******", ))) assert ("1" == str( node2.exec_in_container( [ "bash", "-c", "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo2_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l", ], privileged=True, user="******", )).strip()) print( str( node3.exec_in_container( [ "bash", "-c", "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED", ], privileged=True, user="******", ))) assert ("1" == str( node3.exec_in_container( [ "bash", "-c", "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo2_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l", ], privileged=True, user="******", )).strip()) finally: pm.heal_all() change_balancing("round_robin", "random", reload=False)