def test_tokenmap_dc_rack(self):
host_a = Mock()
host_a.is_up = True
host_a.address = '127.0.0.1'
host_a.datacenter = "dcA"
host_a.rack = "r1"
host_b = Mock()
host_b.is_up = True
host_b.address = '127.0.0.2'
host_b.datacenter = "dcB"
host_b.rack = "r1"
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["127.0.0.1"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-6): host_a,
Murmur3Token(6): host_b
}
s = CqlSession(session, resolve_ip_addresses=False)
token_map = s.tokenmap()
self.assertEqual(
{'127.0.0.1': {'is_up': True, 'tokens': [-6], 'rack': 'r1', 'dc': 'dcA'}},
token_map
)
def _test_get_ranges_murmur3_base(self, opts, expected_ranges):
"""
Set up a mock shell with a simple token map to test the ExportTask get_ranges function.
"""
shell = self.mock_shell()
shell.conn.metadata.partitioner = 'Murmur3Partitioner'
# token range for a cluster of 4 nodes with replication factor 3
shell.get_ring.return_value = {
Murmur3Token(-9223372036854775808):
self.hosts[0:3],
Murmur3Token(-4611686018427387904):
self.hosts[1:4],
Murmur3Token(0): [self.hosts[2], self.hosts[3], self.hosts[0]],
Murmur3Token(4611686018427387904):
[self.hosts[3], self.hosts[0], self.hosts[1]]
}
# merge override options with standard options
overridden_opts = dict(self.opts)
for k, v in opts.items():
overridden_opts[k] = v
export_task = ExportTask(shell, self.ks, self.table, self.columns,
self.fname, overridden_opts,
self.protocol_version, self.config_file)
assert export_task.get_ranges() == expected_ranges
export_task.close()
def test_tokenmap_vnodes(self):
host = Mock()
host.is_up = True
host.address = '127.0.0.1'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["localhost"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-9): host,
Murmur3Token(-6): host,
Murmur3Token(0): host
}
s = CqlSession(session)
token_map = s.tokenmap()
self.assertEqual(True, token_map["localhost"]["is_up"])
self.assertEqual([-9, -6, 0], sorted(token_map["localhost"]["tokens"]))
def tokens_partitions(ksp, table, tokens_ranges):
"""
Method that calculates the new token partitions for a given object
Args:
tokens: current number of tokens of the object
min_tokens_per_worker: defined minimum number of tokens
number_of_workers: defined
Returns:
a partition every time it's called
:type tokens_ranges: list[(long,long)]
"""
from collections import defaultdict
from bisect import bisect_right
from cassandra.metadata import Murmur3Token
splits_per_node = config.splits_per_node
token_range_size = config.token_range_size
target_token_range_size = config.target_token_range_size
tm = config.cluster.metadata.token_map
tmap = tm.tokens_to_hosts_by_ks.get(ksp, None)
tokens_murmur3 = map(lambda a: (Murmur3Token(a[0]), a[1]), tokens_ranges)
if not tmap:
tm.rebuild_keyspace(ksp, build_if_absent=True)
tmap = tm.tokens_to_hosts_by_ks[ksp]
tokens_per_node = defaultdict(list)
for tmumur, t_to in tokens_murmur3:
point = bisect_right(tm.ring, tmumur)
if point == len(tm.ring):
tokens_per_node[tmap[tm.ring[0]][0]].append((tmumur.value, t_to))
else:
tokens_per_node[tmap[tm.ring[point]][0]].append(
(tmumur.value, t_to))
n_nodes = len(tokens_per_node)
step_size = _max_token // (splits_per_node * n_nodes)
if token_range_size:
step_size = token_range_size
elif target_token_range_size:
one = config.session.execute(_size_estimates, [ksp, table]).one()
if one:
(mean_p_size, p_count) = one
estimated_size = mean_p_size * p_count
if estimated_size > 0:
step_size = _max_token // int(
max(estimated_size / target_token_range_size,
splits_per_node * n_nodes))
for tokens_in_node in tokens_per_node.values():
partition = []
for fraction, to in tokens_in_node:
while fraction < to - step_size:
partition.append((fraction, fraction + step_size))
fraction += step_size
partition.append((fraction, to))
group_size = max(len(partition) // splits_per_node, 1)
for i in range(0, len(partition), group_size):
yield partition[i:i + group_size]
def test_token_values(self):
"""
Spot check token classes and values
"""
# spot check murmur3
murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(murmur3_token.hash_fn('123'), -7468325962851647638)
self.assertEqual(murmur3_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 7162290910810015547)
self.assertEqual(str(murmur3_token), '<Murmur3Token: -9223372036854775809L>')
md5_token = MD5Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(md5_token.hash_fn('123'), 42767516990368493138776584305024125808L)
self.assertEqual(md5_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 28528976619278518853815276204542453639L)
self.assertEqual(str(md5_token), '<MD5Token: -9223372036854775809L>')
bytes_token = BytesToken(str(cassandra.metadata.MIN_LONG - 1))
self.assertEqual(bytes_token.hash_fn('123'), '123')
self.assertEqual(bytes_token.hash_fn(123), 123)
self.assertEqual(bytes_token.hash_fn(str(cassandra.metadata.MAX_LONG)), str(cassandra.metadata.MAX_LONG))
self.assertEqual(str(bytes_token), "<BytesToken: '-9223372036854775809'>")
try:
bytes_token = BytesToken(cassandra.metadata.MIN_LONG - 1)
self.fail('Tokens for ByteOrderedPartitioner should be only strings')
except TypeError:
pass
def test_tokenmap_vnodes(self):
host = Mock()
host.is_up = True
host.address = '127.0.0.1'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["localhost"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-9): host,
Murmur3Token(-6): host,
Murmur3Token(0): host
}
s = CqlSession(session)
token_map = s.tokenmap()
self.assertEqual({'localhost': {
'is_up': True,
'tokens': [-9, -6, 0]
}}, token_map)
def test_murmur3_tokens(self):
try:
murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(murmur3_token.hash_fn('123'), -7468325962851647638)
self.assertEqual(murmur3_token.hash_fn(b'\x00\xff\x10\xfa\x99' * 10), 5837342703291459765)
self.assertEqual(murmur3_token.hash_fn(b'\xfe' * 8), -8927430733708461935)
self.assertEqual(murmur3_token.hash_fn(b'\x10' * 8), 1446172840243228796)
self.assertEqual(murmur3_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 7162290910810015547)
self.assertEqual(str(murmur3_token), '<Murmur3Token: -9223372036854775809>')
except NoMurmur3:
raise unittest.SkipTest('The murmur3 extension is not available')
def test_tokenmap_two_dc(self):
host_a = Mock()
host_a.is_up = True
host_a.address = '127.0.0.1'
host_a.datacenter = 'dcA'
host_a.rack = 'r1'
host_b = Mock()
host_b.is_up = False
host_b.address = '127.0.0.2'
host_b.datacenter = 'dcB'
host_b.rack = 'r2'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["127.0.0.1"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-6): host_a,
Murmur3Token(6): host_b
}
s = CqlSession(session, resolve_ip_addresses=False)
token_map = s.tokenmap()
self.assertEqual(
{'127.0.0.1': {'is_up': True, 'tokens': [-6], 'rack': 'r1', 'dc': 'dcA'}},
token_map
)
# A session with different cluster contact points
session2 = Mock()
session2.cluster = Mock()
session2.cluster.contact_points = ["127.0.0.2"]
session2.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-6): host_a,
Murmur3Token(6): host_b
}
s2 = CqlSession(session2, resolve_ip_addresses=False)
token_map2 = s2.tokenmap()
self.assertEqual(
{'127.0.0.2': {'is_up': False, 'tokens': [6], 'rack': 'r2', 'dc': 'dcB'}},
token_map2
)
def test_murmur3_tokens(self):
try:
murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(murmur3_token.hash_fn('123'),
-7468325962851647638)
self.assertEqual(
murmur3_token.hash_fn(str(cassandra.metadata.MAX_LONG)),
7162290910810015547)
self.assertEqual(str(murmur3_token),
'<Murmur3Token: -9223372036854775809L>')
except NoMurmur3:
raise unittest.SkipTest('The murmur3 extension is not available')
def test_tokenmap_two_dc(self):
hostA = Mock()
hostA.is_up = True
hostA.address = '127.0.0.1'
hostA.datacenter = "dcA"
hostB = Mock()
hostB.is_up = True
hostB.address = '127.0.0.2'
hostB.datacenter = "dcB"
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["127.0.0.1"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-6): hostA,
Murmur3Token(6): hostB
}
s = CqlSession(session)
token_map = s.tokenmap()
self.assertEqual({'localhost': {
'is_up': True,
'tokens': [-6]
}}, token_map)
def test_tokenmap_one_token(self):
host = Mock()
host.is_up = True
host.address = '127.0.0.1'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["localhost"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-9): host
}
s = CqlSession(session, resolve_ip_addresses=False)
token_map = s.tokenmap()
self.assertEqual(
{'127.0.0.1': {'is_up': True, 'tokens': [-9]}},
token_map
)
def test_tokenmap_one_token(self):
host = Mock()
host.is_up = True
host.address = '127.0.0.1'
host.rack = 'r1'
host.datacenter = 'dc1'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["localhost"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-9): host
}
s = CqlSession(session, resolve_ip_addresses=self.config.cassandra.resolve_ip_addresses)
token_map = s.tokenmap()
self.assertEqual(
{'127.0.0.1': {'is_up': True, 'tokens': [-9], 'rack': 'r1', 'dc': 'dc1'}}, token_map
)
def test_murmur3_init(self):
murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(str(murmur3_token),
'<Murmur3Token: -9223372036854775809>')
Metadata, TokenMap)
from cassandra.policies import SimpleConvictionPolicy
from cassandra.pool import Host
token_to_host_owner = {}
host1 = Host('1.0.0.0', SimpleConvictionPolicy)
host1.set_location_info('dc1', 'rack1')
host2 = Host('2.0.0.0', SimpleConvictionPolicy)
host2.set_location_info('dc2', 'rack2')
host3 = Host('3.0.0.0', SimpleConvictionPolicy)
host3.set_location_info('dc1', 'rack1')
host4 = Host('4.0.0.0', SimpleConvictionPolicy)
host4.set_location_info('dc2', 'rack2')
token_to_host_owner[Murmur3Token("-9000000000000000000")] = host1
token_to_host_owner[Murmur3Token("-6000000000000000000")] = host2
token_to_host_owner[Murmur3Token("4000000000000000000")] = host3
token_to_host_owner[Murmur3Token("9000000000000000000")] = host4
ring = [
Murmur3Token("-9000000000000000000"),
Murmur3Token("-6000000000000000000"),
Murmur3Token("4000000000000000000"),
Murmur3Token("9000000000000000000")
]
nts = NetworkTopologyStrategy({'dc1': 1, 'dc2': 1})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
for token in replica_map: