def _get_collection(self, uid, table):
ring = HashRing([i for i in range(len(DB_SERVERS))])
cli = self._clients[ring.get_node(uid)]
return cli[table]
def _get_repo(self, package):
ring = HashRing(REPOSITORY_SERVERS)
server = ring.get_node(package)
return server
def _reassign_work(self):
log.info('reassign-work')
# Plan
#
# Step 1: calculate desired assignment from current members and
# workload list (e.g., using consistent hashing or any other
# algorithm
# Step 2: collect current assignments from consul
# Step 3: find the delta between the desired and actual assignments:
# these form two lists:
# 1. new assignments to be made
# 2. obsolete assignments to be revoked
# graceful handling may be desirable when moving existing
# assignment from existing member to another member (to make
# sure it is abandoned by old member before new takes charge)
# Step 4: orchestrate the assignment by adding/deleting(/locking)
# entries in consul
#
# We must make sure while we are working on this, we do not re-enter
# into same method!
try:
# Step 1: generate wanted assignment (mapping work to members)
ring = HashRing(self.members)
wanted_assignments = dict() # member_id -> set(work_id)
_ = [
wanted_assignments.setdefault(ring.get_node(work),
set()).add(work)
for work in self.workload
]
for (member, work) in sorted(wanted_assignments.iteritems()):
log.info('assignment', member=member, work_count=len(work))
# Step 2: discover current assignment (from consul)
(_,
results) = yield self.coord.kv_get(self.coord.assignment_prefix,
recurse=True)
matches = [(self.assignment_match(e['Key']), e)
for e in results or []]
current_assignments = dict() # member_id -> set(work_id)
_ = [
current_assignments.setdefault(m.groupdict()['member_id'],
set()).add(
m.groupdict()['work_id'])
for m, e in matches if m is not None
]
# Step 3: handle revoked assignments first on a per member basis
for member_id, current_work in current_assignments.iteritems():
assert isinstance(current_work, set)
wanted_work = wanted_assignments.get(member_id, set())
work_to_revoke = current_work.difference(wanted_work)
# revoking work by simply deleting the assignment entry
# TODO if we want some feedback to see that member abandoned
# work, we could add a consul-based protocol here
for work_id in work_to_revoke:
yield self.coord.kv_delete(self.coord.assignment_prefix +
member_id + '/' + work_id)
# Step 4: assign new work as needed
for member_id, wanted_work in wanted_assignments.iteritems():
assert isinstance(wanted_work, set)
current_work = current_assignments.get(member_id, set())
work_to_assign = wanted_work.difference(current_work)
for work_id in work_to_assign:
yield self.coord.kv_put(
self.coord.assignment_prefix + member_id + '/' +
work_id, '')
except Exception, e:
log.exception('failed-reassignment', e=e)
self._restart_reassignment_soak_timer() # try again in a while
def gen_hash_ring_slow(nodes):
ring = HashRing()
for node, _ in nodes:
ring.add_node(node.name, node.num_replicas)
return ring
def _get_addr(self, package):
ring = HashRing(self._upload_servers)
addr = ring.get_node(package)
return addr
def _get_recorder(self, package):
ring = HashRing(SERVER_RECORDER)
server = ring.get_node(package)
return server
def gen_hash_ring_fast(nodes):
ring = HashRing()
ring.add_nodes([n[0] for n in nodes])
return ring
def _get_user_backend(self, user):
ring = HashRing(BACKEND_SERVERS)
uid = get_uid(user)
server = ring.get_node(uid)
return server
def _get_allocator(self, uid):
ring = HashRing(SERVER_ALLOCATOR)
server = ring.get_node(uid)
return server
#Get the trackers roster from file
try:
trackers_urls = open("data/trackers_roster").read().splitlines()
except:
print "Could not find trackers roster"
print "Please create file data/trackers_roster and add trackers to it."
sys.exit(1)
#And place the valid lines into a list of trackers
trackers = []
for tracker_url in trackers_urls:
if tracker_url != "":
trackers.append(tracker_url)
# Open the cache of nodes location
nodeslocation = shelve.open("data/nodes_location_" + ownuri)
#Distribute the trackers along the ring with 3 replicas for each
# this helps to improve distribution.
ring = HashRing(trackers, 3)
# Initialize the server.
dts = datagramtalk('', ownport, reply_to_message)
# Wait here for user to want to exit.
raw_input("Press enter to stop....")
# We need to stop listening before leaving, otherwise the
# serving thread will stay in memory.
dts.stopListening()
socket.setsockopt(zmq.SUBSCRIBE, '')
# send them seperately
# only need one context
socket2 = context.socket(zmq.PUB)
socket2.bind("tcp://*:5550")
# argument: the ip of all server
# first is ip of this server
addStr = []
for i in range(1, len(sys.argv)):
srv_addr = sys.argv[i]
addStr.append(srv_addr)
# hashring
ring = HashRing(addStr)
##########################################################################################################
# function
# return all zipcode if it has same element
def checksame(zlist):
retlist = []
for temp in zlist:
# how many element have this value
flag = zipList.count(temp)
if flag > 1:
if retlist.count(temp) == 0:
retlist.append(temp)
return retlist
import tornado.web
from hashlib import sha1
import os
import time
from hash_ring import HashRing
memcache_servers = [
'192.168.0.246:11212', '192.168.0.247:11212', '192.168.0.249:11212'
]
weights = {
'192.168.0.246:11212': 2,
'192.168.0.247:11212': 2,
'192.168.0.249:11212': 1
}
ring = HashRing(memcache_servers, weights)
create_session_id = lambda: sha1(
bytes('%s%s' %
(os.urandom(16), time.time()), encoding='utf-8')).hexdigest()
class Gnn(object):
container = {
# '03d025bb5ac29c24c9ca2257634b9b85b2454fb2':{'user':'******','permission':'xx'},
# '03d025bb5ac29c24c9csdfsdfsdfsdf85b2454fb2':{'user':'******','permission':'asdfasdf'}
}
def __init__(self, handler):
"""
# - *- conding: utf-8 -*-
"""
利用Redis介绍一致性哈希
1. pip install hash_ring
2. 根据该网址修改hash_ring文件：https://blog.csdn.net/qq_35225554/article/details/87023110
"""
from hash_ring import HashRing
import redis
# 利用哈希环写入redis数据库
memcache_servers = ['127.0.0.1:6381', '127.0.0.1:6380']
ring = HashRing(memcache_servers)
server = ring.get_node('my_key_11231243').split(":")
r = redis.Redis(host=server[0], port=int(server[1]), db=0)
r.set("my_key_1", "123123123")
print(server)
server = ring.get_node("my_key_2").split(":")
r = redis.Redis(host=server[0], port=int(server[1]), db=0)
r.set('my_key_2', '222222')
print(server)
server = ring.get_node("hello_world_1234").split(":")
r = redis.Redis(host=server[0], port=int(server[1]), db=0)
r.set('my_key_3', '3234234234')
print(server)
# 读取数据库中的数据
server = ring.get_node("my_key_11231243").split(":")
def _get_user_backend(self, user):
ring = HashRing(SERVER_BACKEND)
uid = get_uid(user)
return ring.get_node(uid)
