def _test_get_ranges_murmur3_base(self, opts, expected_ranges):
"""
Set up a mock shell with a simple token map to test the ExportTask get_ranges function.
"""
shell = self.mock_shell()
shell.conn.metadata.partitioner = 'Murmur3Partitioner'
# token range for a cluster of 4 nodes with replication factor 3
shell.get_ring.return_value = {
Murmur3Token(-9223372036854775808):
self.hosts[0:3],
Murmur3Token(-4611686018427387904):
self.hosts[1:4],
Murmur3Token(0): [self.hosts[2], self.hosts[3], self.hosts[0]],
Murmur3Token(4611686018427387904):
[self.hosts[3], self.hosts[0], self.hosts[1]]
}
# merge override options with standard options
overridden_opts = dict(self.opts)
for k, v in opts.items():
overridden_opts[k] = v
export_task = ExportTask(shell, self.ks, self.table, self.columns,
self.fname, overridden_opts,
self.protocol_version, self.config_file)
assert export_task.get_ranges() == expected_ranges
export_task.close()
def test_parsing_and_calculating_shard_id(self):
"""
Testing the parsing of the options command
and the calculation getting a shard id from a Murmur3 token
"""
class OptionsHolder(object):
options = {
'SCYLLA_SHARD': ['1'],
'SCYLLA_NR_SHARDS': ['12'],
'SCYLLA_PARTITIONER':
['org.apache.cassandra.dht.Murmur3Partitioner'],
'SCYLLA_SHARDING_ALGORITHM': ['biased-token-round-robin'],
'SCYLLA_SHARDING_IGNORE_MSB': ['12']
}
shard_id, shard_info = ShardingInfo.parse_sharding_info(
OptionsHolder())
self.assertEqual(shard_id, 1)
self.assertEqual(
shard_info.shard_id_from_token(Murmur3Token.from_key(b"a")), 4)
self.assertEqual(
shard_info.shard_id_from_token(Murmur3Token.from_key(b"b")), 6)
self.assertEqual(
shard_info.shard_id_from_token(Murmur3Token.from_key(b"c")), 6)
self.assertEqual(
shard_info.shard_id_from_token(Murmur3Token.from_key(b"e")), 4)
self.assertEqual(
shard_info.shard_id_from_token(Murmur3Token.from_key(b"100000")),
2)
def test_tokenmap_dc_rack(self):
host_a = Mock()
host_a.is_up = True
host_a.address = '127.0.0.1'
host_a.datacenter = "dcA"
host_a.rack = "r1"
host_b = Mock()
host_b.is_up = True
host_b.address = '127.0.0.2'
host_b.datacenter = "dcB"
host_b.rack = "r1"
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["127.0.0.1"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-6): host_a,
Murmur3Token(6): host_b
}
s = CqlSession(session, resolve_ip_addresses=False)
token_map = s.tokenmap()
self.assertEqual(
{'127.0.0.1': {'is_up': True, 'tokens': [-6], 'rack': 'r1', 'dc': 'dcA'}},
token_map
)
def test_token_values(self):
"""
Spot check token classes and values
"""
# spot check murmur3
murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(murmur3_token.hash_fn('123'), -7468325962851647638)
self.assertEqual(murmur3_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 7162290910810015547)
self.assertEqual(str(murmur3_token), '<Murmur3Token: -9223372036854775809L>')
md5_token = MD5Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(md5_token.hash_fn('123'), 42767516990368493138776584305024125808L)
self.assertEqual(md5_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 28528976619278518853815276204542453639L)
self.assertEqual(str(md5_token), '<MD5Token: -9223372036854775809L>')
bytes_token = BytesToken(str(cassandra.metadata.MIN_LONG - 1))
self.assertEqual(bytes_token.hash_fn('123'), '123')
self.assertEqual(bytes_token.hash_fn(123), 123)
self.assertEqual(bytes_token.hash_fn(str(cassandra.metadata.MAX_LONG)), str(cassandra.metadata.MAX_LONG))
self.assertEqual(str(bytes_token), "<BytesToken: '-9223372036854775809'>")
try:
bytes_token = BytesToken(cassandra.metadata.MIN_LONG - 1)
self.fail('Tokens for ByteOrderedPartitioner should be only strings')
except TypeError:
pass
def subrange_test(self):
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False,
'num_tokens': 1,
'commitlog_sync_period_in_ms': 500,
'partitioner': 'org.apache.cassandra.dht.Murmur3Partitioner'})
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
for node in cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
# only repair the partition k=0
token = Murmur3Token.from_key(str(bytearray([0, 0, 0, 0])))
# import ipdb; ipdb.set_trace()
# run with force flag
node1.repair(options=['ks', '-st', str(token.value - 1), '-et', str(token.value)])
# verify we have a mix of repaired and unrepaired sstables
self.assertRepairedAndUnrepaired(node1, 'ks')
self.assertRepairedAndUnrepaired(node2, 'ks')
self.assertRepairedAndUnrepaired(node3, 'ks')
def tokens_partitions(ksp, table, tokens_ranges):
"""
Method that calculates the new token partitions for a given object
Args:
tokens: current number of tokens of the object
min_tokens_per_worker: defined minimum number of tokens
number_of_workers: defined
Returns:
a partition every time it's called
:type tokens_ranges: list[(long,long)]
"""
from collections import defaultdict
from bisect import bisect_right
from cassandra.metadata import Murmur3Token
splits_per_node = config.splits_per_node
token_range_size = config.token_range_size
target_token_range_size = config.target_token_range_size
tm = config.cluster.metadata.token_map
tmap = tm.tokens_to_hosts_by_ks.get(ksp, None)
tokens_murmur3 = map(lambda a: (Murmur3Token(a[0]), a[1]), tokens_ranges)
if not tmap:
tm.rebuild_keyspace(ksp, build_if_absent=True)
tmap = tm.tokens_to_hosts_by_ks[ksp]
tokens_per_node = defaultdict(list)
for tmumur, t_to in tokens_murmur3:
point = bisect_right(tm.ring, tmumur)
if point == len(tm.ring):
tokens_per_node[tmap[tm.ring[0]][0]].append((tmumur.value, t_to))
else:
tokens_per_node[tmap[tm.ring[point]][0]].append(
(tmumur.value, t_to))
n_nodes = len(tokens_per_node)
step_size = _max_token // (splits_per_node * n_nodes)
if token_range_size:
step_size = token_range_size
elif target_token_range_size:
one = config.session.execute(_size_estimates, [ksp, table]).one()
if one:
(mean_p_size, p_count) = one
estimated_size = mean_p_size * p_count
if estimated_size > 0:
step_size = _max_token // int(
max(estimated_size / target_token_range_size,
splits_per_node * n_nodes))
for tokens_in_node in tokens_per_node.values():
partition = []
for fraction, to in tokens_in_node:
while fraction < to - step_size:
partition.append((fraction, fraction + step_size))
fraction += step_size
partition.append((fraction, to))
group_size = max(len(partition) // splits_per_node, 1)
for i in range(0, len(partition), group_size):
yield partition[i:i + group_size]
def test_tokenmap_vnodes(self):
host = Mock()
host.is_up = True
host.address = '127.0.0.1'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["localhost"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-9): host,
Murmur3Token(-6): host,
Murmur3Token(0): host
}
s = CqlSession(session)
token_map = s.tokenmap()
self.assertEqual(True, token_map["localhost"]["is_up"])
self.assertEqual([-9, -6, 0], sorted(token_map["localhost"]["tokens"]))
def subrange_test(self):
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False,
'num_tokens': 1,
'commitlog_sync_period_in_ms': 500,
'partitioner': 'org.apache.cassandra.dht.Murmur3Partitioner'})
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
for node in cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
# only repair the partition k=0
token = Murmur3Token.from_key(str(bytearray([0,0,0,0])))
# import ipdb; ipdb.set_trace()
# run with force flag
node1.repair(options=['ks', '-st', str(token.value - 1), '-et', str(token.value)])
# verify we have a mix of repaired and unrepaired sstables
self.assertRepairedAndUnrepaired(node1, 'ks')
self.assertRepairedAndUnrepaired(node2, 'ks')
self.assertRepairedAndUnrepaired(node3, 'ks')
def delete_row(self, pk, ck, session=None, node=None):
session = session or self.exclusive_cql_connection(node or self.node1)
token = Murmur3Token.from_key(pack('>i', pk)).value
assert token < self.tokens[0] or self.tokens[
-1] < token # primary replica should be node1
self.quorum(
session, "DELETE FROM %s.%s WHERE pk = %s AND ck = %s" %
(self.keyspace, self.table, pk, ck))
def insert_row(self, pk, ck, value, session=None, node=None):
session = session or self.exclusive_cql_connection(node or self.node1)
token = Murmur3Token.from_key(pack('>i', pk)).value
assert token < self.tokens[0] or self.tokens[
-1] < token # primary replica should be node1
self.quorum(
session, "INSERT INTO %s.%s (pk, ck, value) VALUES (%s, %s, %s)" %
(self.keyspace, self.table, pk, ck, value))
def test_tokenmap_vnodes(self):
host = Mock()
host.is_up = True
host.address = '127.0.0.1'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["localhost"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-9): host,
Murmur3Token(-6): host,
Murmur3Token(0): host
}
s = CqlSession(session)
token_map = s.tokenmap()
self.assertEqual({'localhost': {
'is_up': True,
'tokens': [-9, -6, 0]
}}, token_map)
def test_murmur3_tokens(self):
try:
murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(murmur3_token.hash_fn('123'), -7468325962851647638)
self.assertEqual(murmur3_token.hash_fn(b'\x00\xff\x10\xfa\x99' * 10), 5837342703291459765)
self.assertEqual(murmur3_token.hash_fn(b'\xfe' * 8), -8927430733708461935)
self.assertEqual(murmur3_token.hash_fn(b'\x10' * 8), 1446172840243228796)
self.assertEqual(murmur3_token.hash_fn(str(cassandra.metadata.MAX_LONG)), 7162290910810015547)
self.assertEqual(str(murmur3_token), '<Murmur3Token: -9223372036854775809>')
except NoMurmur3:
raise unittest.SkipTest('The murmur3 extension is not available')
def test_tokenmap_two_dc(self):
host_a = Mock()
host_a.is_up = True
host_a.address = '127.0.0.1'
host_a.datacenter = 'dcA'
host_a.rack = 'r1'
host_b = Mock()
host_b.is_up = False
host_b.address = '127.0.0.2'
host_b.datacenter = 'dcB'
host_b.rack = 'r2'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["127.0.0.1"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-6): host_a,
Murmur3Token(6): host_b
}
s = CqlSession(session, resolve_ip_addresses=False)
token_map = s.tokenmap()
self.assertEqual(
{'127.0.0.1': {'is_up': True, 'tokens': [-6], 'rack': 'r1', 'dc': 'dcA'}},
token_map
)
# A session with different cluster contact points
session2 = Mock()
session2.cluster = Mock()
session2.cluster.contact_points = ["127.0.0.2"]
session2.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-6): host_a,
Murmur3Token(6): host_b
}
s2 = CqlSession(session2, resolve_ip_addresses=False)
token_map2 = s2.tokenmap()
self.assertEqual(
{'127.0.0.2': {'is_up': False, 'tokens': [6], 'rack': 'r2', 'dc': 'dcB'}},
token_map2
)
def test_murmur3_tokens(self):
try:
murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(murmur3_token.hash_fn('123'),
-7468325962851647638)
self.assertEqual(
murmur3_token.hash_fn(str(cassandra.metadata.MAX_LONG)),
7162290910810015547)
self.assertEqual(str(murmur3_token),
'<Murmur3Token: -9223372036854775809L>')
except NoMurmur3:
raise unittest.SkipTest('The murmur3 extension is not available')
def test_tokenmap_two_dc(self):
hostA = Mock()
hostA.is_up = True
hostA.address = '127.0.0.1'
hostA.datacenter = "dcA"
hostB = Mock()
hostB.is_up = True
hostB.address = '127.0.0.2'
hostB.datacenter = "dcB"
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["127.0.0.1"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-6): hostA,
Murmur3Token(6): hostB
}
s = CqlSession(session)
token_map = s.tokenmap()
self.assertEqual({'localhost': {
'is_up': True,
'tokens': [-6]
}}, token_map)
def test_tokenmap_one_token(self):
host = Mock()
host.is_up = True
host.address = '127.0.0.1'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["localhost"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-9): host
}
s = CqlSession(session, resolve_ip_addresses=False)
token_map = s.tokenmap()
self.assertEqual(
{'127.0.0.1': {'is_up': True, 'tokens': [-9]}},
token_map
)
def test_tokenmap_one_token(self):
host = Mock()
host.is_up = True
host.address = '127.0.0.1'
host.rack = 'r1'
host.datacenter = 'dc1'
session = Mock()
session.cluster = Mock()
session.cluster.contact_points = ["localhost"]
session.cluster.metadata.token_map.token_to_host_owner = {
Murmur3Token(-9): host
}
s = CqlSession(session, resolve_ip_addresses=self.config.cassandra.resolve_ip_addresses)
token_map = s.tokenmap()
self.assertEqual(
{'127.0.0.1': {'is_up': True, 'tokens': [-9], 'rack': 'r1', 'dc': 'dc1'}}, token_map
)
def insert_row(self, pk, ck, value, session=None, node=None):
session = session or self.exclusive_cql_connection(node or self.node1)
token = Murmur3Token.from_key(pack('>i', pk)).value
assert token < self.tokens[0] or self.tokens[-1] < token # primary replica should be node1
self.quorum(session, "INSERT INTO %s.%s (pk, ck, value) VALUES (%s, %s, %s)" % (self.keyspace, self.table, pk, ck, value))
def delete_row(self, pk, ck, session=None, node=None):
session = session or self.exclusive_cql_connection(node or self.node1)
token = Murmur3Token.from_key(pack('>i', pk)).value
assert token < self.tokens[0] or self.tokens[-1] < token # primary replica should be node1
self.quorum(session, "DELETE FROM %s.%s WHERE pk = %s AND ck = %s" % (self.keyspace, self.table, pk, ck))
def test_murmur3_init(self):
murmur3_token = Murmur3Token(cassandra.metadata.MIN_LONG - 1)
self.assertEqual(str(murmur3_token),
'<Murmur3Token: -9223372036854775809>')
Metadata, TokenMap)
from cassandra.policies import SimpleConvictionPolicy
from cassandra.pool import Host
token_to_host_owner = {}
host1 = Host('1.0.0.0', SimpleConvictionPolicy)
host1.set_location_info('dc1', 'rack1')
host2 = Host('2.0.0.0', SimpleConvictionPolicy)
host2.set_location_info('dc2', 'rack2')
host3 = Host('3.0.0.0', SimpleConvictionPolicy)
host3.set_location_info('dc1', 'rack1')
host4 = Host('4.0.0.0', SimpleConvictionPolicy)
host4.set_location_info('dc2', 'rack2')
token_to_host_owner[Murmur3Token("-9000000000000000000")] = host1
token_to_host_owner[Murmur3Token("-6000000000000000000")] = host2
token_to_host_owner[Murmur3Token("4000000000000000000")] = host3
token_to_host_owner[Murmur3Token("9000000000000000000")] = host4
ring = [
Murmur3Token("-9000000000000000000"),
Murmur3Token("-6000000000000000000"),
Murmur3Token("4000000000000000000"),
Murmur3Token("9000000000000000000")
]
nts = NetworkTopologyStrategy({'dc1': 1, 'dc2': 1})
replica_map = nts.make_token_replica_map(token_to_host_owner, ring)
for token in replica_map:
