def test_nts_make_token_replica_map(self):
token_to_host_owner = {}
dc1_1 = Host('dc1.1', SimpleConvictionPolicy)
dc1_2 = Host('dc1.2', SimpleConvictionPolicy)
dc1_3 = Host('dc1.3', SimpleConvictionPolicy)
for host in (dc1_1, dc1_2, dc1_3):
host.set_location_info('dc1', 'rack1')
token_to_host_owner[MD5Token(0)] = dc1_1
token_to_host_owner[MD5Token(100)] = dc1_2
token_to_host_owner[MD5Token(200)] = dc1_3
dc2_1 = Host('dc2.1', SimpleConvictionPolicy)
dc2_2 = Host('dc2.2', SimpleConvictionPolicy)
dc2_1.set_location_info('dc2', 'rack1')
dc2_2.set_location_info('dc2', 'rack1')
token_to_host_owner[MD5Token(1)] = dc2_1
token_to_host_owner[MD5Token(101)] = dc2_2
dc3_1 = Host('dc3.1', SimpleConvictionPolicy)
dc3_1.set_location_info('dc3', 'rack3')
token_to_host_owner[MD5Token(2)] = dc3_1
ring = [MD5Token(0),
MD5Token(1),
MD5Token(2),
MD5Token(100),
MD5Token(101),
MD5Token(200)]
nts = NetworkTopologyStrategy({'dc1': 2, 'dc2': 2, 'dc3': 1})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
self.assertItemsEqual(replica_map[MD5Token(0)], (dc1_1, dc1_2, dc2_1, dc2_2, dc3_1))
def test_nts_make_token_replica_map(self):
token_to_host_owner = {}
dc1_1 = Host('dc1.1', SimpleConvictionPolicy)
dc1_2 = Host('dc1.2', SimpleConvictionPolicy)
dc1_3 = Host('dc1.3', SimpleConvictionPolicy)
for host in (dc1_1, dc1_2, dc1_3):
host.set_location_info('dc1', 'rack1')
token_to_host_owner[MD5Token(0)] = dc1_1
token_to_host_owner[MD5Token(100)] = dc1_2
token_to_host_owner[MD5Token(200)] = dc1_3
dc2_1 = Host('dc2.1', SimpleConvictionPolicy)
dc2_2 = Host('dc2.2', SimpleConvictionPolicy)
dc2_1.set_location_info('dc2', 'rack1')
dc2_2.set_location_info('dc2', 'rack1')
token_to_host_owner[MD5Token(1)] = dc2_1
token_to_host_owner[MD5Token(101)] = dc2_2
dc3_1 = Host('dc3.1', SimpleConvictionPolicy)
dc3_1.set_location_info('dc3', 'rack3')
token_to_host_owner[MD5Token(2)] = dc3_1
ring = [MD5Token(0),
MD5Token(1),
MD5Token(2),
MD5Token(100),
MD5Token(101),
MD5Token(200)]
nts = NetworkTopologyStrategy({'dc1': 2, 'dc2': 2, 'dc3': 1})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
self.assertItemsEqual(replica_map[MD5Token(0)], (dc1_1, dc1_2, dc2_1, dc2_2, dc3_1))
def test_nts_make_token_replica_map_multi_rack(self):
token_to_host_owner = {}
# (A) not enough distinct racks, first skipped is used
dc1_1 = Host("dc1.1", SimpleConvictionPolicy)
dc1_2 = Host("dc1.2", SimpleConvictionPolicy)
dc1_3 = Host("dc1.3", SimpleConvictionPolicy)
dc1_4 = Host("dc1.4", SimpleConvictionPolicy)
dc1_1.set_location_info("dc1", "rack1")
dc1_2.set_location_info("dc1", "rack1")
dc1_3.set_location_info("dc1", "rack2")
dc1_4.set_location_info("dc1", "rack2")
token_to_host_owner[MD5Token(0)] = dc1_1
token_to_host_owner[MD5Token(100)] = dc1_2
token_to_host_owner[MD5Token(200)] = dc1_3
token_to_host_owner[MD5Token(300)] = dc1_4
# (B) distinct racks, but not contiguous
dc2_1 = Host("dc2.1", SimpleConvictionPolicy)
dc2_2 = Host("dc2.2", SimpleConvictionPolicy)
dc2_3 = Host("dc2.3", SimpleConvictionPolicy)
dc2_1.set_location_info("dc2", "rack1")
dc2_2.set_location_info("dc2", "rack1")
dc2_3.set_location_info("dc2", "rack2")
token_to_host_owner[MD5Token(1)] = dc2_1
token_to_host_owner[MD5Token(101)] = dc2_2
token_to_host_owner[MD5Token(201)] = dc2_3
ring = [MD5Token(0), MD5Token(1), MD5Token(100), MD5Token(101), MD5Token(200), MD5Token(201), MD5Token(300)]
nts = NetworkTopologyStrategy({"dc1": 3, "dc2": 2})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
token_replicas = replica_map[MD5Token(0)]
self.assertItemsEqual(token_replicas, (dc1_1, dc1_2, dc1_3, dc2_1, dc2_3))
def test_nts_make_token_replica_map_empty_dc(self):
host = Host('1', SimpleConvictionPolicy)
host.set_location_info('dc1', 'rack1')
token_to_host_owner = {MD5Token(0): host}
ring = [MD5Token(0)]
nts = NetworkTopologyStrategy({'dc1': 1, 'dc2': 0})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
self.assertEqual(set(replica_map[MD5Token(0)]), set([host]))
def test_nts_make_token_replica_map_empty_dc(self):
host = Host('1', SimpleConvictionPolicy)
host.set_location_info('dc1', 'rack1')
token_to_host_owner = {MD5Token(0): host}
ring = [MD5Token(0)]
nts = NetworkTopologyStrategy({'dc1': 1, 'dc2': 0})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
self.assertEqual(set(replica_map[MD5Token(0)]), set([host]))
def test_replication_strategy(self):
"""
Basic code coverage testing that ensures different ReplicationStrategies
can be initiated using parameters correctly.
"""
rs = ReplicationStrategy()
self.assertEqual(rs.create('OldNetworkTopologyStrategy', None), _UnknownStrategy('OldNetworkTopologyStrategy', None))
fake_options_map = {'options': 'map'}
uks = rs.create('OldNetworkTopologyStrategy', fake_options_map)
self.assertEqual(uks, _UnknownStrategy('OldNetworkTopologyStrategy', fake_options_map))
self.assertEqual(uks.make_token_replica_map({}, []), {})
fake_options_map = {'dc1': '3'}
self.assertIsInstance(rs.create('NetworkTopologyStrategy', fake_options_map), NetworkTopologyStrategy)
self.assertEqual(rs.create('NetworkTopologyStrategy', fake_options_map).dc_replication_factors,
NetworkTopologyStrategy(fake_options_map).dc_replication_factors)
fake_options_map = {'options': 'map'}
self.assertIsNone(rs.create('SimpleStrategy', fake_options_map))
fake_options_map = {'options': 'map'}
self.assertIsInstance(rs.create('LocalStrategy', fake_options_map), LocalStrategy)
fake_options_map = {'options': 'map', 'replication_factor': 3}
self.assertIsInstance(rs.create('SimpleStrategy', fake_options_map), SimpleStrategy)
self.assertEqual(rs.create('SimpleStrategy', fake_options_map).replication_factor,
SimpleStrategy(fake_options_map).replication_factor)
self.assertEqual(rs.create('xxxxxxxx', fake_options_map), _UnknownStrategy('xxxxxxxx', fake_options_map))
self.assertRaises(NotImplementedError, rs.make_token_replica_map, None, None)
self.assertRaises(NotImplementedError, rs.export_for_schema)
def test_nts_make_token_replica_map_multi_rack(self):
token_to_host_owner = {}
# (A) not enough distinct racks, first skipped is used
dc1_1 = Host('dc1.1', SimpleConvictionPolicy)
dc1_2 = Host('dc1.2', SimpleConvictionPolicy)
dc1_3 = Host('dc1.3', SimpleConvictionPolicy)
dc1_4 = Host('dc1.4', SimpleConvictionPolicy)
dc1_1.set_location_info('dc1', 'rack1')
dc1_2.set_location_info('dc1', 'rack1')
dc1_3.set_location_info('dc1', 'rack2')
dc1_4.set_location_info('dc1', 'rack2')
token_to_host_owner[MD5Token(0)] = dc1_1
token_to_host_owner[MD5Token(100)] = dc1_2
token_to_host_owner[MD5Token(200)] = dc1_3
token_to_host_owner[MD5Token(300)] = dc1_4
# (B) distinct racks, but not contiguous
dc2_1 = Host('dc2.1', SimpleConvictionPolicy)
dc2_2 = Host('dc2.2', SimpleConvictionPolicy)
dc2_3 = Host('dc2.3', SimpleConvictionPolicy)
dc2_1.set_location_info('dc2', 'rack1')
dc2_2.set_location_info('dc2', 'rack1')
dc2_3.set_location_info('dc2', 'rack2')
token_to_host_owner[MD5Token(1)] = dc2_1
token_to_host_owner[MD5Token(101)] = dc2_2
token_to_host_owner[MD5Token(201)] = dc2_3
ring = [
MD5Token(0),
MD5Token(1),
MD5Token(100),
MD5Token(101),
MD5Token(200),
MD5Token(201),
MD5Token(300)
]
nts = NetworkTopologyStrategy({'dc1': 3, 'dc2': 2})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
token_replicas = replica_map[MD5Token(0)]
self.assertItemsEqual(token_replicas,
(dc1_1, dc1_2, dc1_3, dc2_1, dc2_3))
def test_nts_token_performance(self):
"""
Tests to ensure that when rf exceeds the number of nodes available, that we dont'
needlessly iterate trying to construct tokens for nodes that don't exist.
@since 3.7
@jira_ticket PYTHON-379
@expected_result timing with 1500 rf should be same/similar to 3rf if we have 3 nodes
@test_category metadata
"""
token_to_host_owner = {}
ring = []
dc1hostnum = 3
current_token = 0
vnodes_per_host = 500
for i in range(dc1hostnum):
host = Host('dc1.{0}'.format(i), SimpleConvictionPolicy)
host.set_location_info('dc1', "rack1")
for vnode_num in range(vnodes_per_host):
md5_token = MD5Token(current_token + vnode_num)
token_to_host_owner[md5_token] = host
ring.append(md5_token)
current_token += 1000
nts = NetworkTopologyStrategy({'dc1': 3})
start_time = timeit.default_timer()
nts.make_token_replica_map(token_to_host_owner, ring)
elapsed_base = timeit.default_timer() - start_time
nts = NetworkTopologyStrategy({'dc1': 1500})
start_time = timeit.default_timer()
nts.make_token_replica_map(token_to_host_owner, ring)
elapsed_bad = timeit.default_timer() - start_time
difference = elapsed_bad - elapsed_base
self.assertTrue(difference < 1 and difference > -1)
def test_nts_token_performance(self):
"""
Tests to ensure that when rf exceeds the number of nodes available, that we dont'
needlessly iterate trying to construct tokens for nodes that don't exist.
@since 3.7
@jira_ticket PYTHON-379
@expected_result timing with 1500 rf should be same/similar to 3rf if we have 3 nodes
@test_category metadata
"""
token_to_host_owner = {}
ring = []
dc1hostnum = 3
current_token = 0
vnodes_per_host = 500
for i in range(dc1hostnum):
host = Host('dc1.{0}'.format(i), SimpleConvictionPolicy)
host.set_location_info('dc1', "rack1")
for vnode_num in range(vnodes_per_host):
md5_token = MD5Token(current_token+vnode_num)
token_to_host_owner[md5_token] = host
ring.append(md5_token)
current_token += 1000
nts = NetworkTopologyStrategy({'dc1': 3})
start_time = timeit.default_timer()
nts.make_token_replica_map(token_to_host_owner, ring)
elapsed_base = timeit.default_timer() - start_time
nts = NetworkTopologyStrategy({'dc1': 1500})
start_time = timeit.default_timer()
nts.make_token_replica_map(token_to_host_owner, ring)
elapsed_bad = timeit.default_timer() - start_time
difference = elapsed_bad - elapsed_base
self.assertTrue(difference < 1 and difference > -1)
def test_nts_export_for_schema(self):
strategy = NetworkTopologyStrategy({'dc1': '1', 'dc2': '2'})
self.assertEqual("{'class': 'NetworkTopologyStrategy', 'dc1': '1', 'dc2': '2'}",
strategy.export_for_schema())
def test_ss_equals(self):
self.assertNotEqual(SimpleStrategy({'replication_factor': '1'}),
NetworkTopologyStrategy({'dc1': 2}))
def test_nts_export_for_schema(self):
strategy = NetworkTopologyStrategy({'dc1': '1', 'dc2': '2'})
self.assertEqual(
"{'class': 'NetworkTopologyStrategy', 'dc1': '1', 'dc2': '2'}",
strategy.export_for_schema())
host1 = Host('1.0.0.0', SimpleConvictionPolicy)
host1.set_location_info('dc1', 'rack1')
host2 = Host('2.0.0.0', SimpleConvictionPolicy)
host2.set_location_info('dc2', 'rack2')
host3 = Host('3.0.0.0', SimpleConvictionPolicy)
host3.set_location_info('dc1', 'rack1')
host4 = Host('4.0.0.0', SimpleConvictionPolicy)
host4.set_location_info('dc2', 'rack2')
token_to_host_owner[Murmur3Token("-9000000000000000000")] = host1
token_to_host_owner[Murmur3Token("-6000000000000000000")] = host2
token_to_host_owner[Murmur3Token("4000000000000000000")] = host3
token_to_host_owner[Murmur3Token("9000000000000000000")] = host4
ring = [
Murmur3Token("-9000000000000000000"),
Murmur3Token("-6000000000000000000"),
Murmur3Token("4000000000000000000"),
Murmur3Token("9000000000000000000")
]
nts = NetworkTopologyStrategy({'dc1': 1, 'dc2': 1})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
for token in replica_map:
print token.value, "[",
for replica in replica_map[token]:
print replica.endpoint._address, " ",
print "]"
def test_ss_equals(self):
self.assertNotEqual(SimpleStrategy(1),
NetworkTopologyStrategy({'dc1': 2}))