def ring():
ring = HashRing(nodes={
'node1': 1,
'node2': 1,
'node3': 1
},
hash_fn='ketama')
return ring
def test_ring_growth_ketama(ring):
add_ring = HashRing(hash_fn='ketama')
for nodename in ring.nodes:
add_ring.add_node(nodename)
assert ring._nodes == add_ring._nodes
assert ring.ring == add_ring.ring
assert ring.distribution == add_ring.distribution
def test_ring_growth_meta(ring_fast):
add_ring = HashRing(compat=False)
for nodename in ring_fast.nodes:
add_ring.add_node(nodename)
assert ring_fast._nodes == add_ring._nodes
assert ring_fast.ring == add_ring.ring
assert ring_fast.distribution == add_ring.distribution
def __init__(self, nodes):
"""
init
:param :
:return:
:rtype:
"""
self.nodes = nodes
self.hr = HashRing(nodes=nodes, hash_fn='ketama')
def __init__(self, load_function):
'''
params:
load_function: on a cache miss, this function will be used to
load a value into the cache, given a key
'''
self.load_function = load_function
self.hr = HashRing(nodes=[])
self.servers = {}
def test_hash_fn():
""""""
def hash_fn(k):
return k + "_hash"
nodes = ["172.31.1.0", "172.31.1.125", "172.31.1.202"]
ring = HashRing(nodes, hash_fn=hash_fn)
assert ring.hashi("coconut") == "coconut_hash"
def test_ketama_ring_shrink_collision():
"""
see issue #6 thanks to @bjhockley
"""
nodes = ["172.31.1.0", "172.31.1.125", "172.31.1.202"]
ring = HashRing(nodes, hash_fn='ketama')
ring.remove_node(nodes[1])
ring.remove_node(nodes[2])
ring.remove_node(nodes[0])
assert ring.ring == {}
def ring():
ring = HashRing(nodes={
'node1': 1,
'node2': 1,
'node3': 1
},
replicas=4,
vnodes=40,
compat=True)
return ring
def ring_fast():
ring = HashRing(nodes={
'node1': 1,
'node2': 1,
'node3': 1
},
replicas=4,
vnodes=40,
compat=False)
return ring
def consistentHashing(trace, nodeNum):
nodes = setUp("consistent", nodeNum)
# create a new consistent hash ring with the nodes
hr = HashRing(nodes, vnodes=200)
servers = np.zeros(len(nodes))
for each in trace:
index = int(hr.get_node(each)) - 1
servers[index] += 1
print "consistentHashing"
print servers
def test_weight_fn():
ring = HashRing(nodes={
'node1': 1,
'node2': 1,
'node3': 1
},
replicas=4,
vnodes=40,
hash_fn='ketama',
weight_fn=weight_fn)
assert ring.distribution['node1'] == 80
assert ring.distribution['node2'] == 160
assert ring.distribution['node3'] == 240
ring.regenerate
assert ring.distribution['node1'] == 80
assert ring.distribution['node2'] == 160
assert ring.distribution['node3'] == 240
with pytest.raises(TypeError):
ring = HashRing(nodes={
'node1': 1,
'node2': 1,
'node3': 1
},
replicas=4,
vnodes=40,
hash_fn='ketama',
weight_fn=12)
with pytest.raises(TypeError):
ring = HashRing(nodes={
'node1': 1,
'node2': 1,
'node3': 1
},
replicas=4,
vnodes=40,
hash_fn='ketama',
weight_fn='coconut')
def get_nodes(self, bucket_name: str) -> str:
hash_ring = HashRing(self._nodes)
member_nodes = []
for _ in range(0,3):
target_node = hash_ring.get(bucket_name)['hostname']
member_nodes.append(target_node)
hash_ring.remove_node(target_node)
return member_nodes
def add_slice(sliceid, hostlist):
slice_host[sliceid] = []
req_list[sliceid] = []
for host in hostlist:
if host not in host_slice:
add_host(host)
host_slice[host].append(sliceid)
slice_host[sliceid].append(host)
host_weight[host] = len(host_slice[host])
#print("hostwe:" + str(host_weight[host]))
#slice_list[sliceid] = HashRing(hostlist, vnodes=40, replicas=0, weight_fn= weight_fn)
slice_list[sliceid] = HashRing(hostlist, replicas=0)
redistribute()
def __init__(self,
servers=('127.0.0.1:11211', ),
username=None,
password=None,
compression=None,
socket_timeout=SOCKET_TIMEOUT,
pickle_protocol=0,
pickler=pickle.Pickler,
unpickler=pickle.Unpickler):
super(DistributedClient,
self).__init__(servers, username, password, compression,
socket_timeout, pickle_protocol, pickler,
unpickler)
self._ring = HashRing(self._servers)
def __init__(self, clientclass, entrypoints, *args, **kwargs):
self.clients = []
nodes = {}
for entrypoint in entrypoints:
client = clientclass(*args, **kwargs)
self.clients.append(client)
nodes[entrypoint] = {'hostname': entrypoint, 'instance': client}
if HashRing is None:
raise Exception('Please install uhashring library.')
self.entrypoints = entrypoints
self.hr = HashRing(nodes=nodes, hash_fn='ketama')
def test_weight_fn():
ring = HashRing(
nodes={"node1": 1, "node2": 1, "node3": 1},
replicas=4,
vnodes=40,
hash_fn="ketama",
weight_fn=weight_fn,
)
assert ring.distribution["node1"] == 80
assert ring.distribution["node2"] == 160
assert ring.distribution["node3"] == 240
ring.regenerate
assert ring.distribution["node1"] == 80
assert ring.distribution["node2"] == 160
assert ring.distribution["node3"] == 240
with pytest.raises(TypeError):
ring = HashRing(
nodes={"node1": 1, "node2": 1, "node3": 1},
replicas=4,
vnodes=40,
hash_fn="ketama",
weight_fn=12,
)
with pytest.raises(TypeError):
ring = HashRing(
nodes={"node1": 1, "node2": 1, "node3": 1},
replicas=4,
vnodes=40,
hash_fn="ketama",
weight_fn="coconut",
)
def __init__(self, args):
super(LbDip, self).__init__(args)
self.nodes_names = [x for x in range(0, self.number_of_servers)]
self.collector = DipCollector(args)
self.hr = HashRing(nodes=self.nodes_names,
vnodes=args.vnodes) #TODO modify vnodes
self.connection_2_bucket = {}
self.past_connection = {
} # dictionay, used to record the bucket that each connection connected to, find connections are moved unnecessarily
self.server_2_buckets_connections = {
x: {}
for x in self.hr.get_nodes()
}
for bucket_id, node in self.hr.get_points():
self.server_2_buckets_connections[node][bucket_id] = []
def test_conf():
ring_1 = HashRing({"node1": 1})
ring_2 = HashRing("node1")
ring_3 = HashRing(["node1"])
ring_4 = HashRing({"node1": {"weight": 1}})
assert ring_1.ring == ring_2.ring == ring_3.ring == ring_4.ring
with pytest.raises(ValueError):
HashRing({"node1": "fail"})
with pytest.raises(ValueError):
HashRing(None)
def test_conf():
ring_1 = HashRing({'node1': 1})
ring_2 = HashRing('node1')
ring_3 = HashRing(['node1'])
ring_4 = HashRing({'node1': {'weight': 1}})
assert ring_1.ring == ring_2.ring == ring_3.ring == ring_4.ring
with pytest.raises(ValueError):
HashRing({'node1': 'fail'})
with pytest.raises(ValueError):
HashRing(None)
def test_ketama_compatibility(ketama_config_file):
if not ketama:
return
ring = HashRing(
nodes={"127.0.0.1:11211": 600, "127.0.0.1:11212": 400},
replicas=4,
vnodes=40,
compat=True,
)
continuum = ketama.Continuum(ketama_config_file)
assert ring.get_points() == continuum.get_points()
numhits = 1000
numvalues = 10000
for i in range(numhits):
key = str(randint(1, numvalues))
assert ring.get_server(key) == continuum.get_server(key)
def initialize(app):
instance_db_path = os.path.join(app.instance_path, app.config['DATABASE'])
# Windows: sqlite:/// vs non-Windows: sqlite:////
db_connection_template = '{0}:///{1}' if os.name == 'nt' else '{0}:////{1}'
app.config['SQLALCHEMY_DATABASE_URI'] = db_connection_template.format(app.config['DB_ENGINE'],instance_db_path)
if app.config['SHARDS'] is not None:
app.config['SQLALCHEMY_BINDS'] = {}
nodes = {}
for key, value in app.config['SHARDS'].iteritems():
shard_db = db_connection_template.format(app.config['DB_ENGINE'] ,instance_db_path + '.' + value )
app.config['SQLALCHEMY_BINDS'][key] = shard_db
nodes[key] = {'instance': key}
app.config['HASHRING'] = HashRing(nodes)
db.init_app(app)
register_command(app)
return db;
def __init__(self):
self._shard1_engine = self.get_engine(5432)
self._shard2_engine = self.get_engine(5433)
self._shard3_engine = self.get_engine(5434)
self._shard1_cur: psycopg2.extensions.cursor = self._shard1_engine.cursor()
self._shard2_cur: psycopg2.extensions.cursor = self._shard2_engine.cursor()
self._shard3_cur: psycopg2.extensions.cursor = self._shard3_engine.cursor()
shard1, shard2, shard3 = "shard1", "shard2", "shard3"
self.ring = HashRing(nodes=[shard1, shard2, shard3])
self.shard_connections = {
shard1: self._shard1_engine,
shard2: self._shard2_engine,
shard3: self._shard3_engine,
}
self.shard_cursors = {
shard1: self._shard1_cur,
shard2: self._shard2_cur,
shard3: self._shard3_cur,
}
def connect_to_redis_servers(self):
print("[ {} ] Connecting to Redis servers...".format(
datetime.datetime.utcnow()))
self.redis_clients = []
self.redis_nodes = dict()
counter = 1
for IP, port in self.redis_endpoints:
print(
"[ {} ] Attempting to connect to Redis server {}/{} at {}:{}".
format(datetime.datetime.utcnow(), counter,
len(self.redis_endpoints), IP, port))
connection = yield aioredis.create_connection(
address=(IP, port),
loop=IOLoop.current().asyncio_loop,
encoding=options.encoding
) #redis.StrictRedis(host=IP, port = port, db = 0)
redis_client = aioredis.Redis(connection)
print("[ {} ] Successfully connected to Redis server {}.".format(
datetime.datetime.utcnow(), counter))
self.redis_clients.append(redis_client)
key_string = "node-" + str(IP) + ":" + str(port)
self.redis_nodes[key_string] = {
"hostname": key_string + ".FQDN",
"nodename": key_string,
"instance": redis_client,
"port": port,
"vnodes": 40
}
counter += 1
self.hash_ring = HashRing(self.redis_nodes)
#for port in self.redis_ports:
# print("[ {} ] Attempting to connect to Redis server {}/{} at {}:{}".format(datetime.datetime.utcnow(), counter, len(self.redis_ports), self.redis_endpoint, port))
# #connection = IOLoop.current().run_sync(lambda: aioredis.create_connection(address = (self.redis_endpoint, port), loop = IOLoop.current().asyncio_loop, encoding = options.encoding))
# connection = yield aioredis.create_connection(address = (self.redis_endpoint, port), loop = IOLoop.current().asyncio_loop, encoding = options.encoding)
# redis_client = aioredis.Redis(connection)
# self.redis_clients.append(redis_client)
# counter = counter + 1
self.num_redis_clients = len(self.redis_clients)
def sync_cluster_config(self, force=False):
"""
Obtain the latest cluster configuration from Elasticsearch and update the hash ring.
This method is called every time there is an operation that requires communication
with Elasticsearch. However, it will only synchronize the cluster configuration if
the time elapsed since the last sync is greater than `SYNC_INTERVAL` or the `force`
parameter is set to `True`.
:param bool force: Force synchronization regardless of the time elapsed since last sync
:raises dmsclient.exceptions.DMSClientException: if there is no cluster configured in Elasticsearch
"""
if not force and (
(datetime.now() - self.last_sync) < self.SYNC_INTERVAL):
return
logger.debug('Syncing cluster configuration')
nodes_ring = {}
try:
clusters = self.clusters.get_all()
for cluster in clusters:
if not cluster.available:
continue
nodes_ring[cluster.cluster_id] = {
'instance': cluster,
'weight': cluster.weight
}
except Exception as e:
raise DMSClientException(
"Could not obtain cluster configuration. %s" % (str(e), ))
logger.debug('Obtained %d clusters from Elasticsearch' %
(len(nodes_ring), ))
if len(nodes_ring) == 0:
raise DMSClientException(
'There are no available cluster configured in Elasticsearch')
self.hashring = HashRing(nodes=nodes_ring)
self.last_sync = datetime.now()
def __init__(self, handler, server, living_list):
super(MessageWorker, self).__init__(target=self.run,
args=(),
daemon=True)
# 具体业务handler对象
self.handler = handler
# 指定服务点
self.server = server
# 接到最新通知的zk存活服务集合
self.living_list = living_list
# 使用中的服务存活状态
self.status_dict = {k: True for k in living_list}
# 应用中的hash环，注意每个worker应用的环并不是统一的，各点可能有自己的更新进度
self.ring = HashRing(living_list)
# 控制hash环状态锁
self.update_lock = Lock()
def __init__(self,address, pwd:str,dbIndex:int,aioLoop,minPoolSize:int=5,maxPoolSize:int=5,encoding=None,*args, **kwargs):
if isinstance(address,list):
self.setAddressList = address
self.setAddress = ""
self.bHashRing = True
self.objHashRing = HashRing([(ad[0] +':'+ str(ad[1])) for ad in address])
else:
self.setAddress = address
self.setAddressList = []
self.bHashRing = False
self.iDb = dbIndex
self.strPwd = pwd
self.objAioLoopObj = aioLoop
self.objAioRedis = None
self.objAioRedisDict = {}
self.iMinPoolSize = minPoolSize
self.iMaxPoolSize = maxPoolSize
self.strEncodeing = encoding
self.dictScriptNameSha = {}
num = 1000000
print('running {} key generation comparison'.format(num))
# ketama C binding
if ketama:
with NamedTemporaryFile(prefix='benchmark_') as ketama_config_file:
ketama_config_file.write("127.0.0.1:11211\t600\n")
ketama_config_file.write("127.0.0.1:11212\t400\n")
ketama_config_file.flush()
kt = ketama.Continuum(ketama_config_file.name)
pt = time()
for i in range(num):
key = 'myval-{}'.format(i)
kt.get_server(key)
print('ketama took {} ms'.format(time() - pt))
# pure python implementation
ring = HashRing(
nodes={'127.0.0.1:11211': 600,
'127.0.0.1:11212': 400},
replicas=4,
vnodes=40,
compat=True)
pt = time()
for i in range(num):
key = 'myval-{}'.format(i)
ring.get_server(key)
print('HashRing took {} ms'.format(time() - pt))
# ketama C binding
if ketama:
with NamedTemporaryFile(prefix="benchmark_") as ketama_config_file:
ketama_config_file.write("127.0.0.1:11211\t600\n")
ketama_config_file.write("127.0.0.1:11212\t400\n")
ketama_config_file.flush()
kt = ketama.Continuum(ketama_config_file.name)
pt = time()
for i in range(num):
key = "myval-{}".format(i)
kt.get_server(key)
print("ketama took {} s".format(time() - pt))
# pure python implementation
ring = HashRing(
nodes={
"127.0.0.1:11211": 600,
"127.0.0.1:11212": 400
},
replicas=4,
vnodes=40,
compat=True,
)
pt = time()
for i in range(num):
key = "myval-{}".format(i)
ring.get_server(key)
print("HashRing took {} s".format(time() - pt))
def __init__(self, nodes = ["node_1"]):
self._nodes = nodes
self._hr = HashRing(nodes=nodes)
self._key_to_node = {}
log.debug(f"Init of Metadata Service is complete. Nodes are: {nodes}")
def test_ketama_hashi():
if not ketama:
return
ring = HashRing()
assert ring.hashi('test') == ketama.hashi('test')
