def handle_collect(self, msg):
"""Collect updates from clients"""
kvmsg = KVMsg.from_msg(msg)
self.sequence += 1
kvmsg.sequence = self.sequence
kvmsg.send(self.publisher)
ttl = float(kvmsg.get(b'ttl', 0))
if ttl:
kvmsg[b'ttl'] = b'%f' % (time.time() + ttl)
kvmsg.store(self.kvmap)
logging.info(f"I: publishing update={self.sequence:d}")
def handle_collect(self, msg):
"""Collect updates from clients"""
kvmsg = KVMsg.from_msg(msg)
self.sequence += 1
kvmsg.sequence = self.sequence
kvmsg.send(self.publisher)
ttl = float(kvmsg.get('ttl', 0))
if ttl:
kvmsg['ttl'] = time.time() + ttl
kvmsg.store(self.kvmap)
logging.info("I: publishing update=%d", self.sequence)
def handle_collect(self, msg):
"""Collect updates from clients"""
kvmsg = KVMsg.from_msg(msg)
self.sequence += 1
kvmsg.sequence = self.sequence
kvmsg.send(self.publisher)
ttl = kvmsg.get('ttl')
if ttl is not None:
kvmsg['ttl'] = time.time() + ttl
kvmsg.store(self.kvmap)
logging.info("I: publishing update=%d", self.sequence)
def handle_collect(self, msg):
"""Collect updates from clients"""
kvmsg = KVMsg.from_msg(msg)
logging.info("I: received state update on collector: %s" % kvmsg)
self.sequence += 1
kvmsg.sequence = self.sequence
logging.info("I: publishing update: %s", kvmsg)
kvmsg.send(self.publisher)
ttl = kvmsg.get('ttl')
if ttl is not None:
kvmsg['ttl'] = time.time() + int(ttl)
kvmsg.store(self.kvmap)
def handle_subscriber(self, msg):
"""Collect updates from peer (master)
We're always slave when we get these updates
"""
if self.master:
logging.warn("received subscriber message, but we are master %s",
msg)
return
# Get state snapshot if necessary
if self.kvmap is None:
self.kvmap = {}
snapshot = self.ctx.socket(zmq.DEALER)
snapshot.linger = 0
snapshot.connect("tcp://localhost:%i" % self.peer)
logging.info("I: asking for snapshot from: tcp://localhost:%d",
self.peer)
snapshot.send_multipart([b"ICANHAZ?", b''])
while True:
try:
kvmsg = KVMsg.recv(snapshot)
except KeyboardInterrupt:
# Interrupted
self.bstar.loop.stop()
return
if kvmsg.key == b"KTHXBAI":
self.sequence = kvmsg.sequence
break # Done
kvmsg.store(self.kvmap)
logging.info("I: received snapshot=%d", self.sequence)
# Find and remove update off pending list
kvmsg = KVMsg.from_msg(msg)
# update float ttl -> timestamp
ttl = float(kvmsg.get(b'ttl', 0))
if ttl:
kvmsg[b'ttl'] = b'%f' % (time.time() + ttl)
if kvmsg.key != b"HUGZ":
if not self.was_pending(kvmsg):
# If master update came before client update, flip it
# around, store master update (with sequence) on pending
# list and use to clear client update when it comes later
self.pending.append(kvmsg)
# If update is more recent than our kvmap, apply it
if (kvmsg.sequence > self.sequence):
self.sequence = kvmsg.sequence
kvmsg.store(self.kvmap)
logging.info("I: received update=%d", self.sequence)
def handle_subscriber(self, msg):
"""Collect updates from peer (master)
We're always slave when we get these updates
"""
if self.master:
logging.warn("received subscriber message, but we are master %s", msg)
return
# Get state snapshot if necessary
if self.kvmap is None:
self.kvmap = {}
snapshot = self.ctx.socket(zmq.DEALER)
snapshot.linger = 0
snapshot.connect("tcp://localhost:%i" % self.peer)
logging.info ("I: asking for snapshot from: tcp://localhost:%d",
self.peer)
snapshot.send_multipart(["ICANHAZ?", ''])
while True:
try:
kvmsg = KVMsg.recv(snapshot)
except KeyboardInterrupt:
# Interrupted
self.bstar.loop.stop()
return
if kvmsg.key == "KTHXBAI":
self.sequence = kvmsg.sequence
break # Done
kvmsg.store(self.kvmap)
logging.info ("I: received snapshot=%d", self.sequence)
# Find and remove update off pending list
kvmsg = KVMsg.from_msg(msg)
# update integer ttl -> timestamp
ttl = kvmsg.get('ttl')
if ttl is not None:
kvmsg['ttl'] = time.time() + ttl
if kvmsg.key != "HUGZ":
if not self.was_pending(kvmsg):
# If master update came before client update, flip it
# around, store master update (with sequence) on pending
# list and use to clear client update when it comes later
self.pending.append(kvmsg)
# If update is more recent than our kvmap, apply it
if (kvmsg.sequence > self.sequence):
self.sequence = kvmsg.sequence
kvmsg.store(self.kvmap)
logging.info ("I: received update=%d", self.sequence)
def handle_collect(self, msg):
"""Collect updates from clients
If we're master, we apply these to the kvmap
If we're slave, or unsure, we queue them on our pending list
"""
kvmsg = KVMsg.from_msg(msg)
if self.master:
self.sequence += 1
kvmsg.sequence = self.sequence
kvmsg.send(self.publisher)
ttl = kvmsg.get('ttl')
if ttl is not None:
kvmsg['ttl'] = time.time() + ttl
kvmsg.store(self.kvmap)
logging.info("I: publishing update=%d", self.sequence)
else:
# If we already got message from master, drop it, else
# hold on pending list
if not self.was_pending(kvmsg):
self.pending.append(kvmsg)
def dkv_agent(ctx, pipe, connected_event):
""" Asynchronous agent manages server pool and handles request/reply
dialog when the application asks for it. """
agent = DkvAgent(ctx, pipe, connected_event)
server = None
while True:
poller = zmq.Poller()
poller.register(agent.pipe, zmq.POLLIN)
poll_timer = None
server_socket = None
if agent.state == agent.STATES.INITIAL:
"""In this state we ask the server for a snapshot,
if we have a server to talk to..."""
if agent.servers:
server = agent.servers[agent.cur_server]
logger.debug("Asking for snapshot from %s attempt=%d..", server.address, server.requests)
if (server.requests < 2):
server.snapshot.send_multipart(["ICANHAZ?", agent.subtree])
server.requests += 1
server.expiry = time.time() + SERVER_TTL
agent.state = agent.STATES.SYNCING
server_socket = server.snapshot
elif agent.state == agent.STATES.SYNCING:
"""In this state we read from snapshot and we expect
the server to respond, else we fail over."""
server_socket = server.snapshot
elif agent.state == agent.STATES.ACTIVE:
"""In this state we read from subscriber and we expect
the server to give hugz, else we fail over."""
server_socket = server.subscriber
if server_socket:
"""we have a second socket to poll"""
poller.register(server_socket, zmq.POLLIN)
if server is not None:
poll_timer = 1e3 * max(0, server.expiry - time.time())
try:
# Poll loop
items = dict(poller.poll(poll_timer))
except:
raise # DEBUG
break # Context has been shut down
if agent.pipe in items:
agent.control_message()
elif server_socket in items:
msg = server_socket.recv_multipart()
#logger.debug('server_socket=%s, msg=%s', server_socket, msg)
#logger.debug('msg=%s', msg)
#kvmsg = KVMsg.recv(server_socket)
kvmsg = KVMsg.from_msg(msg)
#pp(kvmsg.__dict__)
server.expiry = time.time() + SERVER_TTL # Anything from server resets its expiry time
if agent.state == agent.STATES.SYNCING:
"""Store in snapshot until we're finished"""
server.requests = 0
#logger.debug('Syncing state msg=%s', msg)
if kvmsg.key == "KTHXBAI":
agent.sequence = kvmsg.sequence
agent.state = agent.STATES.ACTIVE
logger.info("Synced snapshot=%s from %s", agent.sequence, server.address)
logger.info("Connected to %s", server.address)
connected_event.set()
else:
logger.debug("Syncing update=%s from %s", kvmsg.sequence, server.address)
kvmsg.store(agent.kvmap)
elif agent.state == agent.STATES.ACTIVE:
"""Discard out-of-sequence updates, incl. hugz"""
if kvmsg.sequence > agent.sequence:
agent.sequence = kvmsg.sequence
kvmsg.store(agent.kvmap)
action = "update" if kvmsg.body else "delete"
logger.debug("Received %s=%d from %s", action, agent.sequence, server.address)
""" Signal """
if kvmsg.key != 'HUGZ': # Don't send signals if it's just hugz
Dkv.signals.on_sub.send(kvmsg, key=kvmsg.key, value=kvmsg.body, props=kvmsg.properties)
else:
"""Server has died, failover to next"""
if agent.state == agent.STATES.ACTIVE:
level = logging.ERROR
server_state = 'active'
else:
level = logging.WARNING
server_state = 'non-active'
logger.log(level, "Did not receive heartbeat from %s server at %s; failing over.",
server_state, server.address)
agent.cur_server = (agent.cur_server + 1) % len(agent.servers)
agent.state = agent.STATES.INITIAL
