class ConsistentHashingRouter(DatapointRouter): plugin_name = 'consistent-hashing' def __init__(self, settings): replication_factor = settings.REPLICATION_FACTOR diverse_replicas = settings.DIVERSE_REPLICAS self.replication_factor = int(replication_factor) self.diverse_replicas = diverse_replicas self.instance_ports = {} # { (server, instance) : port } self.ring = ConsistentHashRing([]) def addDestination(self, destination): (server, port, instance) = destination if (server, instance) in self.instance_ports: raise Exception("destination instance (%s, %s) already configured" % (server, instance)) self.instance_ports[(server, instance)] = port self.ring.add_node((server, instance)) def removeDestination(self, destination): (server, port, instance) = destination if (server, instance) not in self.instance_ports: raise Exception("destination instance (%s, %s) not configured" % (server, instance)) del self.instance_ports[(server, instance)] self.ring.remove_node((server, instance)) def getDestinations(self, metric): key = self.getKey(metric) if self.diverse_replicas: used_servers = set() for (server, instance) in self.ring.get_nodes(key): if server in used_servers: continue else: used_servers.add(server) port = self.instance_ports[(server, instance)] yield (server, port, instance) if len(used_servers) >= self.replication_factor: return else: for (count, node) in enumerate(self.ring.get_nodes(key)): if count == self.replication_factor: return (server, instance) = node port = self.instance_ports[(server, instance)] yield (server, port, instance) def getKey(self, metric): return metric def setKeyFunction(self, func): self.getKey = func def setKeyFunctionFromModule(self, keyfunc_spec): module_path, func_name = keyfunc_spec.rsplit(':', 1) module_file = open(module_path, 'U') description = ('.py', 'U', imp.PY_SOURCE) module = imp.load_module('keyfunc_module', module_file, module_path, description) keyfunc = getattr(module, func_name) self.setKeyFunction(keyfunc)
def test_5_node_positional_itegrity(self): """Make a cluster, verify we don't have positional collisions""" ring = ConsistentHashRing([], hash_type='fnv1a_ch') for n in range(5): ring.add_node(("192.168.10.%s" % str(10 + n), "%s" % str(10 + n))) self.assertEqual(len([n[0] for n in ring.ring]), len(set([n[0] for n in ring.ring])))
class ConsistentHashingRouter(DatapointRouter): def __init__(self, replication_factor=1, diverse_replicas=False): self.replication_factor = int(replication_factor) self.diverse_replicas = diverse_replicas self.instance_ports = {} # { (server, instance) : port } self.ring = ConsistentHashRing([]) def addDestination(self, destination): (server, port, instance) = destination if (server, instance) in self.instance_ports: raise Exception( "destination instance (%s, %s) already configured" % (server, instance)) self.instance_ports[(server, instance)] = port self.ring.add_node((server, instance)) def removeDestination(self, destination): (server, port, instance) = destination if (server, instance) not in self.instance_ports: raise Exception("destination instance (%s, %s) not configured" % (server, instance)) del self.instance_ports[(server, instance)] self.ring.remove_node((server, instance)) def getDestinations(self, metric): key = self.getKey(metric) if self.diverse_replicas: used_servers = set() for (server, instance) in self.ring.get_nodes(key): if server in used_servers: continue else: used_servers.add(server) port = self.instance_ports[(server, instance)] yield (server, port, instance) if len(used_servers) >= self.replication_factor: return else: for (count, node) in enumerate(self.ring.get_nodes(key)): if count == self.replication_factor: return (server, instance) = node port = self.instance_ports[(server, instance)] yield (server, port, instance) def getKey(self, metric): return metric def setKeyFunction(self, func): self.getKey = func def setKeyFunctionFromModule(self, keyfunc_spec): module_path, func_name = keyfunc_spec.rsplit(':', 1) module_file = open(module_path, 'U') description = ('.py', 'U', imp.PY_SOURCE) module = imp.load_module('keyfunc_module', module_file, module_path, description) keyfunc = getattr(module, func_name) self.setKeyFunction(keyfunc)
def test_9_node_positional_itegrity(self): """Make a cluster, verify we don't have positional collisions""" ring = ConsistentHashRing([]) for n in range(9): ring.add_node(("192.168.10.%s" % str(10+n),"%s" % str(10+n))) self.assertEqual( len([n[0] for n in ring.ring]), len(set([n[0] for n in ring.ring])))
class ConsistentHashingRouter(DatapointRouter): def __init__(self, replication_factor=1): self.replication_factor = int(replication_factor) self.instance_ports = {} # { (server, instance) : port } self.ring = ConsistentHashRing([]) def addDestination(self, destination): (server, port, instance) = destination if (server, instance) in self.instance_ports: raise Exception("destination instance (%s, %s) already configured" % (server, instance)) self.instance_ports[(server, instance)] = port self.ring.add_node((server, instance)) def removeDestination(self, destination): (server, port, instance) = destination if (server, instance) not in self.instance_ports: raise Exception("destination instance (%s, %s) not configured" % (server, instance)) del self.instance_ports[(server, instance)] self.ring.remove_node((server, instance)) def getDestinations(self, metric): key = self.getKey(metric) used_servers = set() for (server, instance) in self.ring.get_nodes(key): if server in used_servers: continue else: used_servers.add(server) port = self.instance_ports[(server, instance)] yield (server, port, instance) if len(used_servers) >= self.replication_factor: return def getKey(self, metric): return metric def setKeyFunction(self, func): self.getKey = func def setKeyFunctionFromModule(self, keyfunc_spec): module_path, func_name = keyfunc_spec.rsplit(":", 1) module_file = open(module_path, "U") description = (".py", "U", imp.PY_SOURCE) module = imp.load_module("keyfunc_module", module_file, module_path, description) keyfunc = getattr(module, func_name) self.setKeyFunction(keyfunc)
class ConsistentHashingRouter(DatapointRouter): def __init__(self, replication_factor=1): self.replication_factor = int(replication_factor) self.instance_ports = {} # { (server, instance) : port } self.ring = ConsistentHashRing([]) def addDestination(self, destination): (server, port, instance) = destination if (server, instance) in self.instance_ports: raise Exception("destination instance (%s, %s) already configured" % (server, instance)) self.instance_ports[ (server, instance) ] = port self.ring.add_node( (server, instance) ) def removeDestination(self, destination): (server, port, instance) = destination if (server, instance) not in self.instance_ports: raise Exception("destination instance (%s, %s) not configured" % (server, instance)) del self.instance_ports[ (server, instance) ] self.ring.remove_node( (server, instance) ) def getDestinations(self, metric): key = self.getKey(metric) for count,node in enumerate(self.ring.get_nodes(key)): if count == self.replication_factor: return (server, instance) = node port = self.instance_ports[ (server, instance) ] yield (server, port, instance) #def getKey(self, metric): # return metric def getKey(self, metric): #RBA: modification done to ensure that all the metrics to be aggregated are processed by the same aggregator return metric.rsplit('.',1)[0] def setKeyFunction(self, func): self.getKey = func def setKeyFunctionFromModule(self, keyfunc_spec): module_path, func_name = keyfunc_spec.rsplit(':', 1) module_file = open(module_path, 'U') description = ('.py', 'U', imp.PY_SOURCE) module = imp.load_module('keyfunc_module', module_file, module_path, description) keyfunc = getattr(module, func_name) self.setKeyFunction(keyfunc)
class ConsistentHashingRouter(DatapointRouter): plugin_name = 'consistent-hashing' def __init__(self, settings): replication_factor = settings.REPLICATION_FACTOR diverse_replicas = settings.DIVERSE_REPLICAS self.replication_factor = int(replication_factor) self.diverse_replicas = diverse_replicas self.instance_ports = {} # { (server, instance) : port } hash_type = settings.ROUTER_HASH_TYPE or 'carbon_ch' self.ring = ConsistentHashRing([], hash_type=hash_type) def addDestination(self, destination): (server, port, instance) = destination if self.hasDestination(destination): raise Exception("destination instance (%s, %s) already configured" % (server, instance)) self.instance_ports[(server, instance)] = port self.ring.add_node((server, instance)) def removeDestination(self, destination): (server, port, instance) = destination if not self.hasDestination(destination): raise Exception("destination instance (%s, %s) not configured" % (server, instance)) del self.instance_ports[(server, instance)] self.ring.remove_node((server, instance)) def hasDestination(self, destination): (server, _, instance) = destination return (server, instance) in self.instance_ports def countDestinations(self): return len(self.instance_ports) def getDestinations(self, metric): key = self.getKey(metric) if self.diverse_replicas: used_servers = set() for (server, instance) in self.ring.get_nodes(key): if server in used_servers: continue else: used_servers.add(server) port = self.instance_ports[(server, instance)] yield (server, port, instance) if len(used_servers) >= self.replication_factor: return else: for (count, node) in enumerate(self.ring.get_nodes(key)): if count == self.replication_factor: return (server, instance) = node port = self.instance_ports[(server, instance)] yield (server, port, instance) def getKey(self, metric): return metric def setKeyFunction(self, func): self.getKey = func def setKeyFunctionFromModule(self, keyfunc_spec): module_path, func_name = keyfunc_spec.rsplit(':', 1) module_file = open(module_path, 'U') description = ('.py', 'U', imp.PY_SOURCE) module = imp.load_module('keyfunc_module', module_file, module_path, description) keyfunc = getattr(module, func_name) self.setKeyFunction(keyfunc)
def test_11_get_nodes(self): """Trigger bisect on identical first key, see: issues/766""" ring = ConsistentHashRing([], replica_count=1) ring.add_node(("1", "1")) n = ring.get_nodes("('1', '1'):0") self.assertEqual([('1', '1')], list(n))
def test_11_get_nodes(self): """Trigger bisect on identical first key, see: issues/766""" ring = ConsistentHashRing([], replica_count=1) ring.add_node(("1", "1")) n = ring.get_nodes("('1', '1'):0") self.assertEqual([('1', '1')], list(n))