def test_nts_transient_parsing(self):
""" Test that we can PARSE a transient replication factor for NTS """
rs = ReplicationStrategy()
nts_transient = rs.create('NetworkTopologyStrategy', {
'dc1': '3/1',
'dc2': '5/1'
})
self.assertEqual(nts_transient.dc_replication_factors_info['dc1'],
ReplicationFactor(3, 1))
self.assertEqual(nts_transient.dc_replication_factors_info['dc2'],
ReplicationFactor(5, 1))
self.assertEqual(nts_transient.dc_replication_factors['dc1'], 2)
self.assertEqual(nts_transient.dc_replication_factors['dc2'], 4)
self.assertIn("'dc1': '3/1', 'dc2': '5/1'",
nts_transient.export_for_schema())
nts_str = rs.create('NetworkTopologyStrategy', {
'dc1': '3',
'dc2': '5'
})
self.assertNotEqual(nts_transient, nts_str)
# make token replica map
ring = [MD5Token(0), MD5Token(1), MD5Token(2)]
hosts = [
Host('dc1.{}'.format(host), SimpleConvictionPolicy)
for host in range(3)
]
token_to_host = dict(zip(ring, hosts))
self.assertEqual(
nts_transient.make_token_replica_map(token_to_host, ring),
nts_str.make_token_replica_map(token_to_host, ring))
def test_getting_replicas(self):
tokens = [MD5Token(str(i)) for i in range(0, (2**127 - 1), 2**125)]
hosts = [
Host("ip%d" % i, SimpleConvictionPolicy)
for i in range(len(tokens))
]
token_to_primary_replica = dict(zip(tokens, hosts))
keyspace = KeyspaceMetadata("ks", True, "SimpleStrategy",
{"replication_factor": "1"})
metadata = Mock(spec=Metadata, keyspaces={'ks': keyspace})
token_map = TokenMap(MD5Token, token_to_primary_replica, tokens,
metadata)
# tokens match node tokens exactly
for i, token in enumerate(tokens):
expected_host = hosts[(i + 1) % len(hosts)]
replicas = token_map.get_replicas("ks", token)
self.assertEqual(set(replicas), set([expected_host]))
# shift the tokens back by one
for token, expected_host in zip(tokens, hosts):
replicas = token_map.get_replicas("ks",
MD5Token(str(token.value - 1)))
self.assertEqual(set(replicas), set([expected_host]))
# shift the tokens forward by one
for i, token in enumerate(tokens):
replicas = token_map.get_replicas("ks",
MD5Token(str(token.value + 1)))
expected_host = hosts[(i + 1) % len(hosts)]
self.assertEqual(set(replicas), set([expected_host]))
def test_nts_replication_parsing(self):
""" Test equality between passing numeric and string replication factor for NTS """
rs = ReplicationStrategy()
nts_int = rs.create('NetworkTopologyStrategy', {'dc1': 3, 'dc2': 5})
nts_str = rs.create('NetworkTopologyStrategy', {
'dc1': '3',
'dc2': '5'
})
self.assertEqual(nts_int.dc_replication_factors['dc1'], 3)
self.assertEqual(nts_str.dc_replication_factors['dc1'], 3)
self.assertEqual(nts_int.dc_replication_factors_info['dc1'],
ReplicationFactor(3))
self.assertEqual(nts_str.dc_replication_factors_info['dc1'],
ReplicationFactor(3))
self.assertEqual(nts_int.export_for_schema(),
nts_str.export_for_schema())
self.assertEqual(nts_int, nts_str)
# make token replica map
ring = [MD5Token(0), MD5Token(1), MD5Token(2)]
hosts = [
Host('dc1.{}'.format(host), SimpleConvictionPolicy)
for host in range(3)
]
token_to_host = dict(zip(ring, hosts))
self.assertEqual(nts_int.make_token_replica_map(token_to_host, ring),
nts_str.make_token_replica_map(token_to_host, ring))
def test_transient_replication_parsing(self):
""" Test that we can PARSE a transient replication factor for SimpleStrategy """
rs = ReplicationStrategy()
simple_transient = rs.create('SimpleStrategy',
{'replication_factor': '3/1'})
self.assertEqual(simple_transient.replication_factor_info,
ReplicationFactor(3, 1))
self.assertEqual(simple_transient.replication_factor, 2)
self.assertIn("'replication_factor': '3/1'",
simple_transient.export_for_schema())
simple_str = rs.create('SimpleStrategy', {'replication_factor': '2'})
self.assertNotEqual(simple_transient, simple_str)
# make token replica map
ring = [MD5Token(0), MD5Token(1), MD5Token(2)]
hosts = [
Host('dc1.{}'.format(host), SimpleConvictionPolicy)
for host in range(3)
]
token_to_host = dict(zip(ring, hosts))
self.assertEqual(
simple_transient.make_token_replica_map(token_to_host, ring),
simple_str.make_token_replica_map(token_to_host, ring))
def test_nts_make_token_replica_map_empty_dc(self):
host = Host('1', SimpleConvictionPolicy)
host.set_location_info('dc1', 'rack1')
token_to_host_owner = {MD5Token(0): host}
ring = [MD5Token(0)]
nts = NetworkTopologyStrategy({'dc1': 1, 'dc2': 0})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
self.assertEqual(set(replica_map[MD5Token(0)]), set([host]))
def test_md5_tokens(self):
md5_token = MD5Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(md5_token.hash_fn('123'),
42767516990368493138776584305024125808L)
self.assertEqual(md5_token.hash_fn(str(cassandra.metadata.MAX_LONG)),
28528976619278518853815276204542453639L)
self.assertEqual(str(md5_token), '<MD5Token: -9223372036854775809L>')
def test_token_values(self):
"""
Spot check token classes and values
"""
# spot check murmur3
murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(murmur3_token.hash_fn('123'), -7468325962851647638)
self.assertEqual(murmur3_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 7162290910810015547)
self.assertEqual(str(murmur3_token), '<Murmur3Token: -9223372036854775809L>')
md5_token = MD5Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(md5_token.hash_fn('123'), 42767516990368493138776584305024125808L)
self.assertEqual(md5_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 28528976619278518853815276204542453639L)
self.assertEqual(str(md5_token), '<MD5Token: -9223372036854775809L>')
bytes_token = BytesToken(str(cassandra.metadata.MIN_LONG - 1))
self.assertEqual(bytes_token.hash_fn('123'), '123')
self.assertEqual(bytes_token.hash_fn(123), 123)
self.assertEqual(bytes_token.hash_fn(str(cassandra.metadata.MAX_LONG)), str(cassandra.metadata.MAX_LONG))
self.assertEqual(str(bytes_token), "<BytesToken: '-9223372036854775809'>")
try:
bytes_token = BytesToken(cassandra.metadata.MIN_LONG - 1)
self.fail('Tokens for ByteOrderedPartitioner should be only strings')
except TypeError:
pass
def test_simple_replication_type_parsing(self):
""" Test equality between passing numeric and string replication factor for simple strategy """
rs = ReplicationStrategy()
simple_int = rs.create('SimpleStrategy', {'replication_factor': 3})
simple_str = rs.create('SimpleStrategy', {'replication_factor': '3'})
self.assertEqual(simple_int.export_for_schema(), simple_str.export_for_schema())
self.assertEqual(simple_int, simple_str)
# make token replica map
ring = [MD5Token(0), MD5Token(1), MD5Token(2)]
hosts = [Host('dc1.{}'.format(host), SimpleConvictionPolicy) for host in range(3)]
token_to_host = dict(zip(ring, hosts))
self.assertEqual(
simple_int.make_token_replica_map(token_to_host, ring),
simple_str.make_token_replica_map(token_to_host, ring)
)
def test_getting_replicas(self):
tokens = [MD5Token(str(i)) for i in range(0, (2 ** 127 - 1), 2 ** 125)]
hosts = [Host("ip%d" % i, SimpleConvictionPolicy) for i in range(len(tokens))]
tokens_to_hosts = dict((t, set([h])) for t, h in zip(tokens, hosts))
token_map = TokenMap(MD5Token, tokens_to_hosts, tokens)
# tokens match node tokens exactly
for token, expected_host in zip(tokens, hosts):
replicas = token_map.get_replicas(token)
self.assertEqual(replicas, set([expected_host]))
# shift the tokens back by one
for token, expected_host in zip(tokens[1:], hosts[1:]):
replicas = token_map.get_replicas(MD5Token(str(token.value - 1)))
self.assertEqual(replicas, set([expected_host]))
# shift the tokens forward by one
for i, token in enumerate(tokens):
replicas = token_map.get_replicas(MD5Token(str(token.value + 1)))
expected_host = hosts[(i + 1) % len(hosts)]
self.assertEqual(replicas, set([expected_host]))
def test_nts_token_performance(self):
"""
Tests to ensure that when rf exceeds the number of nodes available, that we dont'
needlessly iterate trying to construct tokens for nodes that don't exist.
@since 3.7
@jira_ticket PYTHON-379
@expected_result timing with 1500 rf should be same/similar to 3rf if we have 3 nodes
@test_category metadata
"""
token_to_host_owner = {}
ring = []
dc1hostnum = 3
current_token = 0
vnodes_per_host = 500
for i in range(dc1hostnum):
host = Host('dc1.{0}'.format(i), SimpleConvictionPolicy)
host.set_location_info('dc1', "rack1")
for vnode_num in range(vnodes_per_host):
md5_token = MD5Token(current_token + vnode_num)
token_to_host_owner[md5_token] = host
ring.append(md5_token)
current_token += 1000
nts = NetworkTopologyStrategy({'dc1': 3})
start_time = timeit.default_timer()
nts.make_token_replica_map(token_to_host_owner, ring)
elapsed_base = timeit.default_timer() - start_time
nts = NetworkTopologyStrategy({'dc1': 1500})
start_time = timeit.default_timer()
nts.make_token_replica_map(token_to_host_owner, ring)
elapsed_bad = timeit.default_timer() - start_time
difference = elapsed_bad - elapsed_base
self.assertTrue(difference < 1 and difference > -1)
def test_nts_make_token_replica_map(self):
token_to_host_owner = {}
dc1_1 = Host('dc1.1', SimpleConvictionPolicy)
dc1_2 = Host('dc1.2', SimpleConvictionPolicy)
dc1_3 = Host('dc1.3', SimpleConvictionPolicy)
for host in (dc1_1, dc1_2, dc1_3):
host.set_location_info('dc1', 'rack1')
token_to_host_owner[MD5Token(0)] = dc1_1
token_to_host_owner[MD5Token(100)] = dc1_2
token_to_host_owner[MD5Token(200)] = dc1_3
dc2_1 = Host('dc2.1', SimpleConvictionPolicy)
dc2_2 = Host('dc2.2', SimpleConvictionPolicy)
dc2_1.set_location_info('dc2', 'rack1')
dc2_2.set_location_info('dc2', 'rack1')
token_to_host_owner[MD5Token(1)] = dc2_1
token_to_host_owner[MD5Token(101)] = dc2_2
dc3_1 = Host('dc3.1', SimpleConvictionPolicy)
dc3_1.set_location_info('dc3', 'rack3')
token_to_host_owner[MD5Token(2)] = dc3_1
ring = [
MD5Token(0),
MD5Token(1),
MD5Token(2),
MD5Token(100),
MD5Token(101),
MD5Token(200)
]
nts = NetworkTopologyStrategy({'dc1': 2, 'dc2': 2, 'dc3': 1})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
self.assertItemsEqual(replica_map[MD5Token(0)],
(dc1_1, dc1_2, dc2_1, dc2_2, dc3_1))
def test_simple_strategy_make_token_replica_map(self):
host1 = Host('1', SimpleConvictionPolicy)
host2 = Host('2', SimpleConvictionPolicy)
host3 = Host('3', SimpleConvictionPolicy)
token_to_host_owner = {
MD5Token(0): host1,
MD5Token(100): host2,
MD5Token(200): host3
}
ring = [MD5Token(0), MD5Token(100), MD5Token(200)]
rf1_replicas = SimpleStrategy({
'replication_factor': '1'
}).make_token_replica_map(token_to_host_owner, ring)
self.assertItemsEqual(rf1_replicas[MD5Token(0)], [host1])
self.assertItemsEqual(rf1_replicas[MD5Token(100)], [host2])
self.assertItemsEqual(rf1_replicas[MD5Token(200)], [host3])
rf2_replicas = SimpleStrategy({
'replication_factor': '2'
}).make_token_replica_map(token_to_host_owner, ring)
self.assertItemsEqual(rf2_replicas[MD5Token(0)], [host1, host2])
self.assertItemsEqual(rf2_replicas[MD5Token(100)], [host2, host3])
self.assertItemsEqual(rf2_replicas[MD5Token(200)], [host3, host1])
rf3_replicas = SimpleStrategy({
'replication_factor': '3'
}).make_token_replica_map(token_to_host_owner, ring)
self.assertItemsEqual(rf3_replicas[MD5Token(0)], [host1, host2, host3])
self.assertItemsEqual(rf3_replicas[MD5Token(100)],
[host2, host3, host1])
self.assertItemsEqual(rf3_replicas[MD5Token(200)],
[host3, host1, host2])
def test_nts_make_token_replica_map_multi_rack(self):
token_to_host_owner = {}
# (A) not enough distinct racks, first skipped is used
dc1_1 = Host('dc1.1', SimpleConvictionPolicy)
dc1_2 = Host('dc1.2', SimpleConvictionPolicy)
dc1_3 = Host('dc1.3', SimpleConvictionPolicy)
dc1_4 = Host('dc1.4', SimpleConvictionPolicy)
dc1_1.set_location_info('dc1', 'rack1')
dc1_2.set_location_info('dc1', 'rack1')
dc1_3.set_location_info('dc1', 'rack2')
dc1_4.set_location_info('dc1', 'rack2')
token_to_host_owner[MD5Token(0)] = dc1_1
token_to_host_owner[MD5Token(100)] = dc1_2
token_to_host_owner[MD5Token(200)] = dc1_3
token_to_host_owner[MD5Token(300)] = dc1_4
# (B) distinct racks, but not contiguous
dc2_1 = Host('dc2.1', SimpleConvictionPolicy)
dc2_2 = Host('dc2.2', SimpleConvictionPolicy)
dc2_3 = Host('dc2.3', SimpleConvictionPolicy)
dc2_1.set_location_info('dc2', 'rack1')
dc2_2.set_location_info('dc2', 'rack1')
dc2_3.set_location_info('dc2', 'rack2')
token_to_host_owner[MD5Token(1)] = dc2_1
token_to_host_owner[MD5Token(101)] = dc2_2
token_to_host_owner[MD5Token(201)] = dc2_3
ring = [
MD5Token(0),
MD5Token(1),
MD5Token(100),
MD5Token(101),
MD5Token(200),
MD5Token(201),
MD5Token(300)
]
nts = NetworkTopologyStrategy({'dc1': 3, 'dc2': 2})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
token_replicas = replica_map[MD5Token(0)]
self.assertItemsEqual(token_replicas,
(dc1_1, dc1_2, dc1_3, dc2_1, dc2_3))
