def update_map(self):
clients = self.clients
if clients:
term, leader = self.term, self._leader_node
for c in clients:
map_ = yield from c.map()
result = json.loads(map_.decode())
print(result)
for p in result['peers']:
self.peers.add(p)
# chose the leader if it's in a higher term
term, leader = max((term, leader),
(result['term'], tuple(result['leader'])))
self.term, self._leader_node = max((self.term, self._leader_node),
(term, leader))
self.update_clients()
if self._leader_node:
self._leader = NodeClient(*self._leader_node,
loop=self.loop,
node=self)
def update_map(self):
clients = self.clients
if clients:
term, leader = self.term, self._leader_node
for c in clients:
map_ = yield from c.map()
result = json.loads(map_.decode())
print(result)
for p in result['peers']:
self.peers.add(p)
# chose the leader if it's in a higher term
term, leader = max((term, leader), (result['term'], tuple(result['leader'])))
self.term, self._leader_node = max((self.term, self._leader_node), (term, leader))
self.update_clients()
if self._leader_node:
self._leader = NodeClient(*self._leader_node, loop=self.loop, node=self)
def __init__(self, host, port, peers=None, loop=None):
self.host = host
self.port = port
self.loop = loop if loop else asyncio.get_event_loop()
self.peers = set(peers) if peers is not None else set()
self.raft_index = 0
self._leader_node = tuple()
self._leader = None
self.heartbeat_timeout = 0.5
self.term = 0
self.current_term = 0
self.data = DirEntry('root', node=self)
self._clients = [
NodeClient(*peer, loop=self.loop, timeout=0.5, node=self)
for peer in self.peers
]
self.pending_logs = {}
class Follower(Node):
implements = set(['get', 'set', 'map', 'replicate'])
def __init__(self, *args, **kwargs):
super(Follower, self).__init__(*args, **kwargs)
self.loop.create_task(self.async_init())
@asyncio.coroutine
def async_init(self):
yield from self.update_map()
if self._leader:
logger.info('joining')
yield from self._leader.join(self.host, self.port)
elif not self.peers:
logger.info('No other peers, I am the leader')
# no peers, promote oneself
self.__class__ = Leader
self._leader = None
self.term = 1
self._leader_node = (self.host, self.port)
@asyncio.coroutine
def broadcast(self, function, attrs, exclude=None, wait_majority=False):
""" Broacast :arg:function to all clients except hosts in :arg:exclude.
If :arg:wait_majority is True, return as soon as the majority of
clients returned without error
If :arg:wait_majority is an integer, return as soon as
:arg:wait_majority clients returned without error
"""
exclude = exclude if exclude is not None else []
coros = [getattr(c, function)(**attrs)
for c in self.clients
if (c.host, c.port) not in exclude
]
if wait_majority:
# let the coro finnish even after we return
coros = [asyncio.shield(c) for c in coros]
if wait_majority is True:
maj = ceil((len(coros) + len(exclude)) / 2.)
else:
maj = wait_majority
success = 0
while True:
w = asyncio.wait(coros,
loop=self.loop, return_when=asyncio.FIRST_COMPLETED)
try:
done, coros = yield from asyncio.wait_for(w,
loop=self.loop, timeout=self.heartbeat_timeout)
except asyncio.TimeoutError:
[c.cancel() for c in coros]
return None, None
success += len([d for d in done if d.exception() is None])
if success >= maj:
return done, coros
if success + len(coros) < maj:
# not any chance to succed
[c.cancel() for c in coros]
return False
else:
done, pending = yield from asyncio.wait(coros, loop=self.loop,
timeout=self.heartbeat_timeout)
[c.cancel() for c in pending]
return done, pending
def update_clients(self):
current_clients = set([(c.remote_host, c.remote_port) for c in self._clients])
new_clients = self.peers.difference(current_clients)
for c in new_clients:
self._clients.append(NodeClient(*c, loop=self.loop, node=self))
@asyncio.coroutine
def update_map(self):
clients = self.clients
if clients:
term, leader = self.term, self._leader_node
for c in clients:
map_ = yield from c.map()
result = json.loads(map_.decode())
print(result)
for p in result['peers']:
self.peers.add(p)
# chose the leader if it's in a higher term
term, leader = max((term, leader), (result['term'], tuple(result['leader'])))
self.term, self._leader_node = max((self.term, self._leader_node), (term, leader))
self.update_clients()
if self._leader_node:
self._leader = NodeClient(*self._leader_node, loop=self.loop, node=self)
@asyncio.coroutine
def get(self, writer, key):
logger.debug('get %s' % key)
data = self.data.get_entry(key, create=False)
writer.write(b'200\n')
writer.write(bytes(json.dumps(dict(
key=key,
value=data.value,
index=data.index)), 'utf-8'))
writer.write(b'\n')
yield from writer.drain()
@asyncio.coroutine
def replicate(self, writer, **kwargs):
index = kwargs['raft_index']
log = self.pending_logs.get(index, None)
if log is None:
action = kwargs.pop('action')
log = action_map[action](self, **kwargs)
self.pending_logs[log.raft_index] = log
else:
log.commit()
writer.write(b'200\n\n')
yield from writer.drain()
@asyncio.coroutine
def join(self, writer, host, port):
result = yield from self._leader.join(host, port)
writer.write(b'200\n%s\n' % result)
yield from writer.drain()
@property
def clients(self):
if self._leader:
return [self._leader] + self._clients
else:
return self._clients
def update_clients(self):
current_clients = set([(c.remote_host, c.remote_port)
for c in self._clients])
new_clients = self.peers.difference(current_clients)
for c in new_clients:
self._clients.append(NodeClient(*c, loop=self.loop, node=self))
class Follower(Node):
implements = set(['get', 'set', 'map', 'replicate'])
def __init__(self, *args, **kwargs):
super(Follower, self).__init__(*args, **kwargs)
self.loop.create_task(self.async_init())
@asyncio.coroutine
def async_init(self):
yield from self.update_map()
if self._leader:
logger.info('joining')
yield from self._leader.join(self.host, self.port)
elif not self.peers:
logger.info('No other peers, I am the leader')
# no peers, promote oneself
self.__class__ = Leader
self._leader = None
self.term = 1
self._leader_node = (self.host, self.port)
@asyncio.coroutine
def broadcast(self, function, attrs, exclude=None, wait_majority=False):
""" Broacast :arg:function to all clients except hosts in :arg:exclude.
If :arg:wait_majority is True, return as soon as the majority of
clients returned without error
If :arg:wait_majority is an integer, return as soon as
:arg:wait_majority clients returned without error
"""
exclude = exclude if exclude is not None else []
coros = [
getattr(c, function)(**attrs) for c in self.clients
if (c.host, c.port) not in exclude
]
if wait_majority:
# let the coro finnish even after we return
coros = [asyncio.shield(c) for c in coros]
if wait_majority is True:
maj = ceil((len(coros) + len(exclude)) / 2.)
else:
maj = wait_majority
success = 0
while True:
w = asyncio.wait(coros,
loop=self.loop,
return_when=asyncio.FIRST_COMPLETED)
try:
done, coros = yield from asyncio.wait_for(
w, loop=self.loop, timeout=self.heartbeat_timeout)
except asyncio.TimeoutError:
[c.cancel() for c in coros]
return None, None
success += len([d for d in done if d.exception() is None])
if success >= maj:
return done, coros
if success + len(coros) < maj:
# not any chance to succed
[c.cancel() for c in coros]
return False
else:
done, pending = yield from asyncio.wait(
coros, loop=self.loop, timeout=self.heartbeat_timeout)
[c.cancel() for c in pending]
return done, pending
def update_clients(self):
current_clients = set([(c.remote_host, c.remote_port)
for c in self._clients])
new_clients = self.peers.difference(current_clients)
for c in new_clients:
self._clients.append(NodeClient(*c, loop=self.loop, node=self))
@asyncio.coroutine
def update_map(self):
clients = self.clients
if clients:
term, leader = self.term, self._leader_node
for c in clients:
map_ = yield from c.map()
result = json.loads(map_.decode())
print(result)
for p in result['peers']:
self.peers.add(p)
# chose the leader if it's in a higher term
term, leader = max((term, leader),
(result['term'], tuple(result['leader'])))
self.term, self._leader_node = max((self.term, self._leader_node),
(term, leader))
self.update_clients()
if self._leader_node:
self._leader = NodeClient(*self._leader_node,
loop=self.loop,
node=self)
@asyncio.coroutine
def get(self, writer, key):
logger.debug('get %s' % key)
data = self.data.get_entry(key, create=False)
writer.write(b'200\n')
writer.write(
bytes(
json.dumps(dict(key=key, value=data.value, index=data.index)),
'utf-8'))
writer.write(b'\n')
yield from writer.drain()
@asyncio.coroutine
def replicate(self, writer, **kwargs):
index = kwargs['raft_index']
log = self.pending_logs.get(index, None)
if log is None:
action = kwargs.pop('action')
log = action_map[action](self, **kwargs)
self.pending_logs[log.raft_index] = log
else:
log.commit()
writer.write(b'200\n\n')
yield from writer.drain()
@asyncio.coroutine
def join(self, writer, host, port):
result = yield from self._leader.join(host, port)
writer.write(b'200\n%s\n' % result)
yield from writer.drain()
@property
def clients(self):
if self._leader:
return [self._leader] + self._clients
else:
return self._clients
