def ring_slow():
r = HashRing()
nodes = []
for i in range(num_nodes):
r.add_node(b'test-%i' % i, num_replicas=512)
r.add_nodes(nodes)
def test_ring_find_nodes(benchmark, num_nodes):
r = HashRing()
nodes = []
for i in range(num_nodes):
r.add_node(b'test-%i' % i, num_replicas=512)
r.add_nodes(nodes)
def ring_lookup():
return r.find_nodes(b'hello', 3)
assert benchmark(ring_lookup) == r.find_nodes(b'hello', 3)
class DistRedis(MethodMissing):
def __init__(self,hosts):
self.hosts=[]
for h in hosts:
host,port = h.split(':')
self.hosts.append(Redis(host,int(port)))
self.ring = HashRing(self.hosts)
def add_server(self,server):
server,port = server.split(':')
r= Redis(server,port)
self.ring.add_node(r)
def save(self):
for redis in self.ring:
redis.save()
def bgsave(self):
for redis in self.ring:
redis.save(True)
def delete_cloud(self):
for redis in self.ring:
for key in self.ring.keys("*"):
redis.delete(key)
def quit(self):
for redis in self.ring:
redis.quit
def node_for_key(self,key):
if re.match("/\{(.*)\?\}/",key):
l=re.split("/\{(.*)\?\}/",key)
key = l[0]
return self.ring.get_node(key)
def method_missing(self, attr, *args, **kwargs):
redis = self.node_for_key(args[0])
if redis != None:
return redis.__getattribute__(attr)(*args,**kwargs)
'''def node_for_key(self,key):
def test_hash_ring_add():
ring = HashRing()
assert ring.get_num_items() == 0
assert ring.get_node(b'test') is None
assert ring.add_node(b'test', num_replicas=128)
node = ring.get_node(b'test')
assert node.name == b'test'
assert node.num_replicas == 128
assert ring.get_num_items() == 128
assert ring.remove_node(b'test')
assert ring.get_num_items() == 0
assert ring.add_node(b'test', num_replicas=64)
node = ring.get_node(b'test')
assert node.name == b'test'
assert node.num_replicas == 64
assert ring.get_num_items() == 64
def test_hash_ring_basic_flow():
ring = HashRing()
# A ring with no node can't find any nodes.
assert ring.find_node(1) is None
# Adding a node should find the node.
assert ring.add_node(b'test')
assert ring.find_node(1) == b'test'
# Removing it should work.
assert ring.remove_node(b'test')
assert ring.find_node(1) is None
def test_hash_ring_stress_test():
replica_sizes = [2**i for i in range(10)]
ring = HashRing()
config = []
for i in range(20):
name = b'test-%i' % i
num_replicas = replica_sizes[hash(name) % len(replica_sizes)]
config.append((name, num_replicas))
total_items = 0
for name, num_replicas in config:
total_items += num_replicas
assert ring.add_node(name, num_replicas=num_replicas)
assert ring.get_num_items() == total_items
random.shuffle(config)
for name, num_replicas in config:
assert ring.remove_node(name)
total_items -= num_replicas
assert ring.get_num_items() == total_items
assert total_items == 0
class NodeIndexer:
"""
Recebe conexões de supernós e auxilia na estruturação de uma rede overlay
"""
def __init__(self, port):
self.connList = []
self.port = port
# socket vai usar tcp
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# reusar o mesmo socket para outras conexões
self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# classe que vai fazer consistent hashing dos supernós
self.ring = HashRing()
def serve_forever(self):
# configura a escuta no localhost, porta especificada
self.server_socket.bind(('', self.port))
self.server_socket.listen(10)
self.connList.append(self.server_socket)
print "Iniciando o servico na porta " + str(self.port)
try:
while 1:
# pega a lista de sockets que podem ser lidos
read_sockets,write_sockets,error_sockets = select.select(self.connList,[],[])
for sock in read_sockets:
# nova conexão
if sock == self.server_socket:
sockfd, addr = self.server_socket.accept()
self.connList.append(sockfd)
print "Novo superno conectado (%s:%s)" % addr
else:
# mensagem de um supernó
try:
data = sock.recv(BUFFER_SIZE)
if len(data) != 0:
self.handle_data(data, sock)
except(KeyboardInterrupt):
# para que a interrupcao do teclado nao seja tratada no próximo except
raise
except(ValueError):
print "Mensagem fora das especificações recebida de (%s:%s)" % addr
print data
# se saiu do loop, a conexão foi fechada por alguma razão
self.server_socket.close()
except(KeyboardInterrupt):
# interrupção do teclado
print "Fechando servico"
sys.exit(0)
def handle_data(self, data, socket):
opr = data[0]
if opr == 'a':
# append: informa a porta que o supernó está recebendo conexões
port = int(data[1:])
self.handle_add(socket, port)
elif opr == 'r':
# remove: supernó avisa que vai desconectar
port = int(data[1:])
self.handle_remove(socket, port)
elif opr == 'd':
# supernó foi desconectado
self.handle_disconnect(socket)
elif opr == 'q':
# query: requisição de um supernó pelo supernó responsável pelo arquivo descrito por 'data'
if self.ring.is_empty():
response = 'empty'
else:
response = "%s:%s" % self.ring.get_node(data)
socket.send(response)
else:
# mensagem não interpretada pelo serviço de indexação
raise ValueError
def handle_add(self, socket, port):
""" adiciona o supernó no anel hash """
addr = (socket.getpeername()[0], port)
self.ring.add_node(addr)
def handle_remove(self, socket, port):
""" retira o supernó da indexação """
addr = (socket.getpeername()[0], port)
print "Superno desconectando: (%s, %s)" % socket.getpeername()
self.ring.remove_node(addr)
def handle_disconnect(self, socket):
""" remove a conexão para o supernó """
socket.close()
self.connList.remove(socket)
class Scheduler():
def __init__(self,):
'''
init redis with the config. start a inter thread.
hashring with replication
inworker is a thread can push or get worker from it.
'''
self.rs=redis.StrictRedis(host=config.redis_host, port=config.redis_port,db=0)
self.inter=inputer.Inputer(config.input_home)
self.db=Filedb(config.db_file)
self.workers=self.db.get()
self.time=time.time()
#TODO:add the weight for hashring() init
self.hashring=HashRing(nodes=self.workers,replicas=100)
self.nodes={}
def reload_worker(self,):
ctime=time.time()
if ctime-self.time>1:
new_workers=self.db.get()
if new_workers!=self.workers:
news=[var for var in new_workers if var not in self.workers]
for new in news:
self.hashring.add_node(new)
deles=[var for var in self.workers if var not in new_workers]
for dele in deles:
self.hashring.remove_node(dele)
self.workers=new_workers
self.time=ctime
def pushurl(self,url):
#TODO:rs push failed!
node=self.hashring.get_node(url)
self.rs.lpush(node,url)
if node not in self.nodes.keys():
self.nodes[node]=0
self.nodes[node]+=1
return node
def get_nodes(self,):
for node in self.nodes.keys():
print node+":"+str(self.nodes[node])
def run(self,):
self.inter.start()
while self.inter.running:
ifile = self.inter.get_task()
if ifile == None:
time.sleep(3)
continue
if not os.path.exists(ifile):
time.sleep(3)
continue
try:
for line in open(ifile):
self.reload_worker()
if line.strip()!="":
node=self.pushurl(line.strip())
if self.inter.running == 0:
break
self.get_nodes()
except Exception,e:
#TODO:write to log
print str(e)
continue
def gen_hash_ring_slow(nodes):
ring = HashRing()
for node, _ in nodes:
ring.add_node(node.name, node.num_replicas)
return ring
