def test_5_node_positional_itegrity(self):
"""Make a cluster, verify we don't have positional collisions"""
ring = ConsistentHashRing([], hash_type='fnv1a_ch')
for n in range(5):
ring.add_node(("192.168.10.%s" % str(10 + n), "%s" % str(10 + n)))
self.assertEqual(len([n[0] for n in ring.ring]),
len(set([n[0] for n in ring.ring])))
def __init__(self, settings):
replication_factor = settings.REPLICATION_FACTOR
diverse_replicas = settings.DIVERSE_REPLICAS
self.replication_factor = int(replication_factor)
self.diverse_replicas = diverse_replicas
self.instance_ports = {} # { (server, instance) : port }
self.ring = ConsistentHashRing([])
def test_9_node_positional_itegrity(self):
"""Make a cluster, verify we don't have positional collisions"""
ring = ConsistentHashRing([])
for n in range(9):
ring.add_node(("192.168.10.%s" % str(10+n),"%s" % str(10+n)))
self.assertEqual(
len([n[0] for n in ring.ring]),
len(set([n[0] for n in ring.ring])))
def __init__(self, settings):
replication_factor = settings.REPLICATION_FACTOR
diverse_replicas = settings.DIVERSE_REPLICAS
self.replication_factor = int(replication_factor)
self.diverse_replicas = diverse_replicas
self.instance_ports = {} # { (server, instance) : port }
hash_type = settings.ROUTER_HASH_TYPE or 'carbon_ch'
self.ring = ConsistentHashRing([], hash_type=hash_type)
def test_chr_compute_ring_position_fnv1a(self):
hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"),
("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'),
59573)
self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'),
35749)
def test_chr_get_node_fnv1a(self):
hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"),
("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
self.assertEqual(hashring.get_node('hosts.worker1.cpu'),
('127.0.0.1', 'ba603c36342304ed77953f84ac4d357b'))
self.assertEqual(hashring.get_node('hosts.worker2.cpu'),
('127.0.0.3', '866a18b81f2dc4649517a1df13e26f28'))
def test_chr_get_node_fnv1a(self):
hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"),
("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
self.assertEqual(hashring.get_node('hosts.worker1.cpu'),
('127.0.0.1', 'ba603c36342304ed77953f84ac4d357b'))
self.assertEqual(hashring.get_node('hosts.worker2.cpu'),
('127.0.0.3', '866a18b81f2dc4649517a1df13e26f28'))
self.assertEqual(hashring.get_node(
'stats.checkout.cluster.padamski-wro.api.v1.payment-initialize.count'),
('127.0.0.3', '866a18b81f2dc4649517a1df13e26f28'))
class ConsistentHashingRouter(DatapointRouter):
def __init__(self, replication_factor=1):
self.replication_factor = int(replication_factor)
self.instance_ports = {} # { (server, instance) : port }
self.ring = ConsistentHashRing([])
def addDestination(self, destination):
(server, port, instance) = destination
if (server, instance) in self.instance_ports:
raise Exception(
"destination instance (%s, %s) already configured" %
(server, instance))
self.instance_ports[(server, instance)] = port
self.ring.add_node((server, instance))
def removeDestination(self, destination):
(server, port, instance) = destination
if (server, instance) not in self.instance_ports:
raise Exception("destination instance (%s, %s) not configured" %
(server, instance))
del self.instance_ports[(server, instance)]
self.ring.remove_node((server, instance))
def getDestinations(self, metric):
key = self.getKey(metric)
used_servers = set()
for (server, instance) in self.ring.get_nodes(key):
if server in used_servers:
continue
else:
used_servers.add(server)
port = self.instance_ports[(server, instance)]
yield (server, port, instance)
if len(used_servers) >= self.replication_factor:
return
def getKey(self, metric):
return metric
def setKeyFunction(self, func):
self.getKey = func
def setKeyFunctionFromModule(self, keyfunc_spec):
module_path, func_name = keyfunc_spec.rsplit(':', 1)
module_file = open(module_path, 'U')
description = ('.py', 'U', imp.PY_SOURCE)
module = imp.load_module('keyfunc_module', module_file, module_path,
description)
keyfunc = getattr(module, func_name)
self.setKeyFunction(keyfunc)
class ConsistentHashingRouter(DatapointRouter):
def __init__(self, replication_factor=1):
self.replication_factor = int(replication_factor)
self.instance_ports = {} # { (server, instance) : port }
self.ring = ConsistentHashRing([])
def addDestination(self, destination):
(server, port, instance) = destination
if (server, instance) in self.instance_ports:
raise Exception("destination instance (%s, %s) already configured" % (server, instance))
self.instance_ports[ (server, instance) ] = port
self.ring.add_node( (server, instance) )
def removeDestination(self, destination):
(server, port, instance) = destination
if (server, instance) not in self.instance_ports:
raise Exception("destination instance (%s, %s) not configured" % (server, instance))
del self.instance_ports[ (server, instance) ]
self.ring.remove_node( (server, instance) )
def getDestinations(self, metric):
key = self.getKey(metric)
used_servers = set()
for (server, instance) in self.ring.get_nodes(key):
if server in used_servers:
continue
else:
used_servers.add(server)
port = self.instance_ports[ (server, instance) ]
yield (server, port, instance)
if len(used_servers) >= self.replication_factor:
return
def getKey(self, metric):
return metric
def setKeyFunction(self, func):
self.getKey = func
def setKeyFunctionFromModule(self, keyfunc_spec):
module_path, func_name = keyfunc_spec.rsplit(':', 1)
module_file = open(module_path, 'U')
description = ('.py', 'U', imp.PY_SOURCE)
module = imp.load_module('keyfunc_module', module_file, module_path, description)
keyfunc = getattr(module, func_name)
self.setKeyFunction(keyfunc)
def __init__(self, config, cluster='main'):
# Support multiple versions of carbon, the API changed in 0.10.
args = inspect.getargspec(ConsistentHashingRouter.__init__).args
if 'replication_factor' in args:
r = ConsistentHashingRouter(config.replication_factor(cluster))
else:
class Settings(object):
REPLICATION_FACTOR = config.replication_factor(cluster)
DIVERSE_REPLICAS = False
ROUTER_HASH_TYPE = None
r = ConsistentHashingRouter(Settings())
# 'hash_type' was added only in carbon 1.0.2 or master
args = inspect.getargspec(ConsistentHashRing.__init__).args
if 'hash_type' in args:
r.ring = ConsistentHashRing(nodes=[],
hash_type=config.hashing_type(cluster))
self.ring = r
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
class ConsistentHashingRouter(DatapointRouter):
def __init__(self, replication_factor=1):
self.replication_factor = int(replication_factor)
self.instance_ports = {} # { (server, instance) : port }
self.ring = ConsistentHashRing([])
def addDestination(self, destination):
(server, port, instance) = destination
if (server, instance) in self.instance_ports:
raise Exception("destination instance (%s, %s) already configured" % (server, instance))
self.instance_ports[ (server, instance) ] = port
self.ring.add_node( (server, instance) )
def removeDestination(self, destination):
(server, port, instance) = destination
if (server, instance) not in self.instance_ports:
raise Exception("destination instance (%s, %s) not configured" % (server, instance))
del self.instance_ports[ (server, instance) ]
self.ring.remove_node( (server, instance) )
def getDestinations(self, metric):
key = self.getKey(metric)
for count,node in enumerate(self.ring.get_nodes(key)):
if count == self.replication_factor:
return
(server, instance) = node
port = self.instance_ports[ (server, instance) ]
yield (server, port, instance)
#def getKey(self, metric):
# return metric
def getKey(self, metric):
#RBA: modification done to ensure that all the metrics to be aggregated are processed by the same aggregator
return metric.rsplit('.',1)[0]
def setKeyFunction(self, func):
self.getKey = func
def setKeyFunctionFromModule(self, keyfunc_spec):
module_path, func_name = keyfunc_spec.rsplit(':', 1)
module_file = open(module_path, 'U')
description = ('.py', 'U', imp.PY_SOURCE)
module = imp.load_module('keyfunc_module', module_file, module_path, description)
keyfunc = getattr(module, func_name)
self.setKeyFunction(keyfunc)
def __init__(self, settings):
replication_factor = settings.REPLICATION_FACTOR
diverse_replicas = settings.DIVERSE_REPLICAS
self.replication_factor = int(replication_factor)
self.diverse_replicas = diverse_replicas
self.instance_ports = {} # { (server, instance) : port }
self.ring = ConsistentHashRing([])
def test_chr_compute_ring_position_fnv1a(self):
hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"),
("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'),
59573)
self.assertEqual(hashring.compute_ring_position('hosts.worker1.load'),
57163)
self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'),
35749)
self.assertEqual(hashring.compute_ring_position('hosts.worker2.network'),
43584)
self.assertEqual(hashring.compute_ring_position('hosts.worker3.cpu'),
12600)
self.assertEqual(hashring.compute_ring_position('hosts.worker3.irq'),
10052)
def __init__(self, config, cluster='main', aggregation_rules=None):
relay_method = config.relay_method(cluster=cluster)
if relay_method == "consistent-hashing":
# Support multiple versions of carbon, the API changed in 0.10.
args = inspect.getargspec(ConsistentHashingRouter.__init__).args
if 'replication_factor' in args:
r = ConsistentHashingRouter(config.replication_factor(cluster))
else:
class Settings(object):
REPLICATION_FACTOR = config.replication_factor(cluster)
DIVERSE_REPLICAS = False
r = ConsistentHashingRouter(Settings())
# 'hash_type' was added only in carbon 1.0.2 or master
args = inspect.getargspec(ConsistentHashRing.__init__).args
if 'hash_type' in args:
r.ring = ConsistentHashRing(
hash_type=config.hashing_type(cluster))
elif relay_method == "aggregated-consistent-hashing":
if aggregation_rules:
RuleManager.read_from(aggregation_rules)
r = AggregatedConsistentHashingRouter(
RuleManager, config.replication_factor(cluster))
self.ring = r
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def test_11_get_nodes(self):
"""Trigger bisect on identical first key, see: issues/766"""
ring = ConsistentHashRing([], replica_count=1)
ring.add_node(("1", "1"))
n = ring.get_nodes("('1', '1'):0")
self.assertEqual([('1', '1')], list(n))
def __init__(self, replication_factor=1):
self.replication_factor = int(replication_factor)
self.instance_ports = {} # { (server, instance) : port }
self.ring = ConsistentHashRing([])
class ConsistentHashingRouter(DatapointRouter):
plugin_name = 'consistent-hashing'
def __init__(self, settings):
replication_factor = settings.REPLICATION_FACTOR
diverse_replicas = settings.DIVERSE_REPLICAS
self.replication_factor = int(replication_factor)
self.diverse_replicas = diverse_replicas
self.instance_ports = {} # { (server, instance) : port }
hash_type = settings.ROUTER_HASH_TYPE or 'carbon_ch'
self.ring = ConsistentHashRing([], hash_type=hash_type)
def addDestination(self, destination):
(server, port, instance) = destination
if self.hasDestination(destination):
raise Exception("destination instance (%s, %s) already configured" % (server, instance))
self.instance_ports[(server, instance)] = port
self.ring.add_node((server, instance))
def removeDestination(self, destination):
(server, port, instance) = destination
if not self.hasDestination(destination):
raise Exception("destination instance (%s, %s) not configured" % (server, instance))
del self.instance_ports[(server, instance)]
self.ring.remove_node((server, instance))
def hasDestination(self, destination):
(server, _, instance) = destination
return (server, instance) in self.instance_ports
def countDestinations(self):
return len(self.instance_ports)
def getDestinations(self, metric):
key = self.getKey(metric)
if self.diverse_replicas:
used_servers = set()
for (server, instance) in self.ring.get_nodes(key):
if server in used_servers:
continue
else:
used_servers.add(server)
port = self.instance_ports[(server, instance)]
yield (server, port, instance)
if len(used_servers) >= self.replication_factor:
return
else:
for (count, node) in enumerate(self.ring.get_nodes(key)):
if count == self.replication_factor:
return
(server, instance) = node
port = self.instance_ports[(server, instance)]
yield (server, port, instance)
def getKey(self, metric):
return metric
def setKeyFunction(self, func):
self.getKey = func
def setKeyFunctionFromModule(self, keyfunc_spec):
module_path, func_name = keyfunc_spec.rsplit(':', 1)
module_file = open(module_path, 'U')
description = ('.py', 'U', imp.PY_SOURCE)
module = imp.load_module('keyfunc_module', module_file, module_path, description)
keyfunc = getattr(module, func_name)
self.setKeyFunction(keyfunc)
def __init__(self, replication_factor=1):
self.replication_factor = int(replication_factor)
self.instance_ports = {} # { (server, instance) : port }
self.ring = ConsistentHashRing([])
def test_11_get_nodes(self):
"""Trigger bisect on identical first key, see: issues/766"""
ring = ConsistentHashRing([], replica_count=1)
ring.add_node(("1", "1"))
n = ring.get_nodes("('1', '1'):0")
self.assertEqual([('1', '1')], list(n))
