def _add_peer(self, host):
client = gevent.socket.create_connection((host, self.port), source_address=(self.interface, 0))
self.connections[host] = client
for line in util.line_protocol(client):
ack, task_id = line.split(':', 1)
if ack == 'scheduled' and task_id in self.scheduled_acks:
self.scheduled_acks[task_id].put(True)
print "disconnected from peer %s" % host
self._remove_peer(host)
def _run(self):
while True:
try:
self.socket = gevent.socket.create_connection((self.interface, 6002), source_address=(self.interface, 0))
for line in util.line_protocol(self.socket):
event, payload = line.split(':', 1)
self.callback(event, payload)
except IOError:
pass
print "disconnected from dispatcher, retrying..."
def connect(self):
"""
Connects to the currently known leader. It maintains a connection expecting
JSON lists of hosts in the cluster. It should receive a list on connection,
however, if a list of one, this is a redirect to the leader (you hit a node
in the cluster that's not the leader). We also maintain a keepalive. If we
disconnect, it does a leader elect and reconnects.
"""
while True:
logger.info("Connecting to leader %s on port %s" % (self.leader, self.port))
try:
client = util.connect_and_retry((self.leader, self.port),
source_address=(self.interface, 0), max_retries=5)
except IOError:
raise ClusterError("Unable to connect to leader: %s" % self.leader)
logger.info("Connected to leader")
# Use TCP keepalives
keepalive = gevent.spawn_later(5, lambda: client.send('\n'))
try:
for line in util.line_protocol(client, strip=False):
if line == '\n':
# Keepalive ack from leader
keepalive.kill()
keepalive = gevent.spawn_later(5, lambda: client.send('\n'))
else:
new_cluster = json.loads(line)
if len(new_cluster) == 1:
# Cluster of one means you have the wrong leader
self.leader = new_cluster[0]
logger.info("Redirected to %s..." % self.leader)
raise NewLeader()
else:
self.cluster = set(new_cluster)
if self.callback:
self.callback(self.cluster.copy())
self.cluster.remove(self.leader)
candidates = list(self.cluster)
candidates.sort()
self.leader = candidates[0]
logger.info("New leader %s..." % self.leader)
# TODO: if i end up thinking i'm the leader when i'm not
# then i will not rejoin the cluster
raise NewLeader()
except NewLeader:
if self.callback:
self.callback(self.cluster.copy())
if not self.is_leader():
gevent.sleep(1) # TODO: back off loop, not a sleep
else:
break
def _backend_server(self, socket, address):
for line in util.line_protocol(socket):
action, payload = line.split(':', 1)
if action == 'schedule':
task = Task.unserialize(payload)
task.schedule(self.dispatcher)
self.scheduled[task.id] = task
socket.send('scheduled:%s\n' % task.id)
print "scheduled: %s" % task.id
elif action == 'cancel':
task_id = payload
print "canceled: %s" % task_id
self.scheduled.pop(task_id).cancel()
elif action == 'reschedule':
task_id, eta = payload.split(':', 1)
eta = int(eta)
print "rescheduled: %s for %s" % (task_id, eta)
self.scheduled[task_id].reschedule(self.dispatcher, eta)
def _connection_handler(self, socket, address):
"""
If not a leader, a node will simply return a single item list pointing
to the leader. Otherwise, it will add the host of the connected client
to the cluster roster, broadcast to all nodes the new roster, and wait
for keepalives. If no keepalive within timeout or the client drops, it
drops it from the roster and broadcasts to all remaining nodes.
"""
#print 'New connection from %s:%s' % address
if not self.is_leader():
socket.send(json.dumps([self.leader]))
socket.close()
else:
self._update(add={'host': address[0], 'socket': socket})
timeout = gevent.spawn_later(10, lambda: self._shutdown(socket))
for line in util.line_protocol(socket, strip=False):
timeout.kill()
timeout = gevent.spawn_later(10, lambda: self._shutdown(socket))
socket.send('\n')
#print "keepalive from %s:%s" % address
#print "client disconnected"
self._update(remove=address[0])
def _connection_handler(self, socket, address):
print "pair connected"
self.scheduler = socket
for line in util.line_protocol(socket):
self.queue.put(line)
print "pair dropped"