def test_nts_transient_parsing(self): """ Test that we can PARSE a transient replication factor for NTS """ rs = ReplicationStrategy() nts_transient = rs.create('NetworkTopologyStrategy', { 'dc1': '3/1', 'dc2': '5/1' }) self.assertEqual(nts_transient.dc_replication_factors_info['dc1'], ReplicationFactor(3, 1)) self.assertEqual(nts_transient.dc_replication_factors_info['dc2'], ReplicationFactor(5, 1)) self.assertEqual(nts_transient.dc_replication_factors['dc1'], 2) self.assertEqual(nts_transient.dc_replication_factors['dc2'], 4) self.assertIn("'dc1': '3/1', 'dc2': '5/1'", nts_transient.export_for_schema()) nts_str = rs.create('NetworkTopologyStrategy', { 'dc1': '3', 'dc2': '5' }) self.assertNotEqual(nts_transient, nts_str) # make token replica map ring = [MD5Token(0), MD5Token(1), MD5Token(2)] hosts = [ Host('dc1.{}'.format(host), SimpleConvictionPolicy) for host in range(3) ] token_to_host = dict(zip(ring, hosts)) self.assertEqual( nts_transient.make_token_replica_map(token_to_host, ring), nts_str.make_token_replica_map(token_to_host, ring))
def test_getting_replicas(self): tokens = [MD5Token(str(i)) for i in range(0, (2**127 - 1), 2**125)] hosts = [ Host("ip%d" % i, SimpleConvictionPolicy) for i in range(len(tokens)) ] token_to_primary_replica = dict(zip(tokens, hosts)) keyspace = KeyspaceMetadata("ks", True, "SimpleStrategy", {"replication_factor": "1"}) metadata = Mock(spec=Metadata, keyspaces={'ks': keyspace}) token_map = TokenMap(MD5Token, token_to_primary_replica, tokens, metadata) # tokens match node tokens exactly for i, token in enumerate(tokens): expected_host = hosts[(i + 1) % len(hosts)] replicas = token_map.get_replicas("ks", token) self.assertEqual(set(replicas), set([expected_host])) # shift the tokens back by one for token, expected_host in zip(tokens, hosts): replicas = token_map.get_replicas("ks", MD5Token(str(token.value - 1))) self.assertEqual(set(replicas), set([expected_host])) # shift the tokens forward by one for i, token in enumerate(tokens): replicas = token_map.get_replicas("ks", MD5Token(str(token.value + 1))) expected_host = hosts[(i + 1) % len(hosts)] self.assertEqual(set(replicas), set([expected_host]))
def test_nts_replication_parsing(self): """ Test equality between passing numeric and string replication factor for NTS """ rs = ReplicationStrategy() nts_int = rs.create('NetworkTopologyStrategy', {'dc1': 3, 'dc2': 5}) nts_str = rs.create('NetworkTopologyStrategy', { 'dc1': '3', 'dc2': '5' }) self.assertEqual(nts_int.dc_replication_factors['dc1'], 3) self.assertEqual(nts_str.dc_replication_factors['dc1'], 3) self.assertEqual(nts_int.dc_replication_factors_info['dc1'], ReplicationFactor(3)) self.assertEqual(nts_str.dc_replication_factors_info['dc1'], ReplicationFactor(3)) self.assertEqual(nts_int.export_for_schema(), nts_str.export_for_schema()) self.assertEqual(nts_int, nts_str) # make token replica map ring = [MD5Token(0), MD5Token(1), MD5Token(2)] hosts = [ Host('dc1.{}'.format(host), SimpleConvictionPolicy) for host in range(3) ] token_to_host = dict(zip(ring, hosts)) self.assertEqual(nts_int.make_token_replica_map(token_to_host, ring), nts_str.make_token_replica_map(token_to_host, ring))
def test_transient_replication_parsing(self): """ Test that we can PARSE a transient replication factor for SimpleStrategy """ rs = ReplicationStrategy() simple_transient = rs.create('SimpleStrategy', {'replication_factor': '3/1'}) self.assertEqual(simple_transient.replication_factor_info, ReplicationFactor(3, 1)) self.assertEqual(simple_transient.replication_factor, 2) self.assertIn("'replication_factor': '3/1'", simple_transient.export_for_schema()) simple_str = rs.create('SimpleStrategy', {'replication_factor': '2'}) self.assertNotEqual(simple_transient, simple_str) # make token replica map ring = [MD5Token(0), MD5Token(1), MD5Token(2)] hosts = [ Host('dc1.{}'.format(host), SimpleConvictionPolicy) for host in range(3) ] token_to_host = dict(zip(ring, hosts)) self.assertEqual( simple_transient.make_token_replica_map(token_to_host, ring), simple_str.make_token_replica_map(token_to_host, ring))
def test_nts_make_token_replica_map_empty_dc(self): host = Host('1', SimpleConvictionPolicy) host.set_location_info('dc1', 'rack1') token_to_host_owner = {MD5Token(0): host} ring = [MD5Token(0)] nts = NetworkTopologyStrategy({'dc1': 1, 'dc2': 0}) replica_map = nts.make_token_replica_map(token_to_host_owner, ring) self.assertEqual(set(replica_map[MD5Token(0)]), set([host]))
def test_md5_tokens(self): md5_token = MD5Token(cassandra.metadata.MIN_LONG - 1) self.assertEqual(md5_token.hash_fn('123'), 42767516990368493138776584305024125808L) self.assertEqual(md5_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 28528976619278518853815276204542453639L) self.assertEqual(str(md5_token), '<MD5Token: -9223372036854775809L>')
def test_token_values(self): """ Spot check token classes and values """ # spot check murmur3 murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1) self.assertEqual(murmur3_token.hash_fn('123'), -7468325962851647638) self.assertEqual(murmur3_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 7162290910810015547) self.assertEqual(str(murmur3_token), '<Murmur3Token: -9223372036854775809L>') md5_token = MD5Token(cassandra.metadata.MIN_LONG - 1) self.assertEqual(md5_token.hash_fn('123'), 42767516990368493138776584305024125808L) self.assertEqual(md5_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 28528976619278518853815276204542453639L) self.assertEqual(str(md5_token), '<MD5Token: -9223372036854775809L>') bytes_token = BytesToken(str(cassandra.metadata.MIN_LONG - 1)) self.assertEqual(bytes_token.hash_fn('123'), '123') self.assertEqual(bytes_token.hash_fn(123), 123) self.assertEqual(bytes_token.hash_fn(str(cassandra.metadata.MAX_LONG)), str(cassandra.metadata.MAX_LONG)) self.assertEqual(str(bytes_token), "<BytesToken: '-9223372036854775809'>") try: bytes_token = BytesToken(cassandra.metadata.MIN_LONG - 1) self.fail('Tokens for ByteOrderedPartitioner should be only strings') except TypeError: pass
def test_simple_replication_type_parsing(self): """ Test equality between passing numeric and string replication factor for simple strategy """ rs = ReplicationStrategy() simple_int = rs.create('SimpleStrategy', {'replication_factor': 3}) simple_str = rs.create('SimpleStrategy', {'replication_factor': '3'}) self.assertEqual(simple_int.export_for_schema(), simple_str.export_for_schema()) self.assertEqual(simple_int, simple_str) # make token replica map ring = [MD5Token(0), MD5Token(1), MD5Token(2)] hosts = [Host('dc1.{}'.format(host), SimpleConvictionPolicy) for host in range(3)] token_to_host = dict(zip(ring, hosts)) self.assertEqual( simple_int.make_token_replica_map(token_to_host, ring), simple_str.make_token_replica_map(token_to_host, ring) )
def test_getting_replicas(self): tokens = [MD5Token(str(i)) for i in range(0, (2 ** 127 - 1), 2 ** 125)] hosts = [Host("ip%d" % i, SimpleConvictionPolicy) for i in range(len(tokens))] tokens_to_hosts = dict((t, set([h])) for t, h in zip(tokens, hosts)) token_map = TokenMap(MD5Token, tokens_to_hosts, tokens) # tokens match node tokens exactly for token, expected_host in zip(tokens, hosts): replicas = token_map.get_replicas(token) self.assertEqual(replicas, set([expected_host])) # shift the tokens back by one for token, expected_host in zip(tokens[1:], hosts[1:]): replicas = token_map.get_replicas(MD5Token(str(token.value - 1))) self.assertEqual(replicas, set([expected_host])) # shift the tokens forward by one for i, token in enumerate(tokens): replicas = token_map.get_replicas(MD5Token(str(token.value + 1))) expected_host = hosts[(i + 1) % len(hosts)] self.assertEqual(replicas, set([expected_host]))
def test_nts_token_performance(self): """ Tests to ensure that when rf exceeds the number of nodes available, that we dont' needlessly iterate trying to construct tokens for nodes that don't exist. @since 3.7 @jira_ticket PYTHON-379 @expected_result timing with 1500 rf should be same/similar to 3rf if we have 3 nodes @test_category metadata """ token_to_host_owner = {} ring = [] dc1hostnum = 3 current_token = 0 vnodes_per_host = 500 for i in range(dc1hostnum): host = Host('dc1.{0}'.format(i), SimpleConvictionPolicy) host.set_location_info('dc1', "rack1") for vnode_num in range(vnodes_per_host): md5_token = MD5Token(current_token + vnode_num) token_to_host_owner[md5_token] = host ring.append(md5_token) current_token += 1000 nts = NetworkTopologyStrategy({'dc1': 3}) start_time = timeit.default_timer() nts.make_token_replica_map(token_to_host_owner, ring) elapsed_base = timeit.default_timer() - start_time nts = NetworkTopologyStrategy({'dc1': 1500}) start_time = timeit.default_timer() nts.make_token_replica_map(token_to_host_owner, ring) elapsed_bad = timeit.default_timer() - start_time difference = elapsed_bad - elapsed_base self.assertTrue(difference < 1 and difference > -1)
def test_nts_make_token_replica_map(self): token_to_host_owner = {} dc1_1 = Host('dc1.1', SimpleConvictionPolicy) dc1_2 = Host('dc1.2', SimpleConvictionPolicy) dc1_3 = Host('dc1.3', SimpleConvictionPolicy) for host in (dc1_1, dc1_2, dc1_3): host.set_location_info('dc1', 'rack1') token_to_host_owner[MD5Token(0)] = dc1_1 token_to_host_owner[MD5Token(100)] = dc1_2 token_to_host_owner[MD5Token(200)] = dc1_3 dc2_1 = Host('dc2.1', SimpleConvictionPolicy) dc2_2 = Host('dc2.2', SimpleConvictionPolicy) dc2_1.set_location_info('dc2', 'rack1') dc2_2.set_location_info('dc2', 'rack1') token_to_host_owner[MD5Token(1)] = dc2_1 token_to_host_owner[MD5Token(101)] = dc2_2 dc3_1 = Host('dc3.1', SimpleConvictionPolicy) dc3_1.set_location_info('dc3', 'rack3') token_to_host_owner[MD5Token(2)] = dc3_1 ring = [ MD5Token(0), MD5Token(1), MD5Token(2), MD5Token(100), MD5Token(101), MD5Token(200) ] nts = NetworkTopologyStrategy({'dc1': 2, 'dc2': 2, 'dc3': 1}) replica_map = nts.make_token_replica_map(token_to_host_owner, ring) self.assertItemsEqual(replica_map[MD5Token(0)], (dc1_1, dc1_2, dc2_1, dc2_2, dc3_1))
def test_simple_strategy_make_token_replica_map(self): host1 = Host('1', SimpleConvictionPolicy) host2 = Host('2', SimpleConvictionPolicy) host3 = Host('3', SimpleConvictionPolicy) token_to_host_owner = { MD5Token(0): host1, MD5Token(100): host2, MD5Token(200): host3 } ring = [MD5Token(0), MD5Token(100), MD5Token(200)] rf1_replicas = SimpleStrategy({ 'replication_factor': '1' }).make_token_replica_map(token_to_host_owner, ring) self.assertItemsEqual(rf1_replicas[MD5Token(0)], [host1]) self.assertItemsEqual(rf1_replicas[MD5Token(100)], [host2]) self.assertItemsEqual(rf1_replicas[MD5Token(200)], [host3]) rf2_replicas = SimpleStrategy({ 'replication_factor': '2' }).make_token_replica_map(token_to_host_owner, ring) self.assertItemsEqual(rf2_replicas[MD5Token(0)], [host1, host2]) self.assertItemsEqual(rf2_replicas[MD5Token(100)], [host2, host3]) self.assertItemsEqual(rf2_replicas[MD5Token(200)], [host3, host1]) rf3_replicas = SimpleStrategy({ 'replication_factor': '3' }).make_token_replica_map(token_to_host_owner, ring) self.assertItemsEqual(rf3_replicas[MD5Token(0)], [host1, host2, host3]) self.assertItemsEqual(rf3_replicas[MD5Token(100)], [host2, host3, host1]) self.assertItemsEqual(rf3_replicas[MD5Token(200)], [host3, host1, host2])
def test_nts_make_token_replica_map_multi_rack(self): token_to_host_owner = {} # (A) not enough distinct racks, first skipped is used dc1_1 = Host('dc1.1', SimpleConvictionPolicy) dc1_2 = Host('dc1.2', SimpleConvictionPolicy) dc1_3 = Host('dc1.3', SimpleConvictionPolicy) dc1_4 = Host('dc1.4', SimpleConvictionPolicy) dc1_1.set_location_info('dc1', 'rack1') dc1_2.set_location_info('dc1', 'rack1') dc1_3.set_location_info('dc1', 'rack2') dc1_4.set_location_info('dc1', 'rack2') token_to_host_owner[MD5Token(0)] = dc1_1 token_to_host_owner[MD5Token(100)] = dc1_2 token_to_host_owner[MD5Token(200)] = dc1_3 token_to_host_owner[MD5Token(300)] = dc1_4 # (B) distinct racks, but not contiguous dc2_1 = Host('dc2.1', SimpleConvictionPolicy) dc2_2 = Host('dc2.2', SimpleConvictionPolicy) dc2_3 = Host('dc2.3', SimpleConvictionPolicy) dc2_1.set_location_info('dc2', 'rack1') dc2_2.set_location_info('dc2', 'rack1') dc2_3.set_location_info('dc2', 'rack2') token_to_host_owner[MD5Token(1)] = dc2_1 token_to_host_owner[MD5Token(101)] = dc2_2 token_to_host_owner[MD5Token(201)] = dc2_3 ring = [ MD5Token(0), MD5Token(1), MD5Token(100), MD5Token(101), MD5Token(200), MD5Token(201), MD5Token(300) ] nts = NetworkTopologyStrategy({'dc1': 3, 'dc2': 2}) replica_map = nts.make_token_replica_map(token_to_host_owner, ring) token_replicas = replica_map[MD5Token(0)] self.assertItemsEqual(token_replicas, (dc1_1, dc1_2, dc1_3, dc2_1, dc2_3))