def control_message (self): msg = self.pipe.recv_multipart() command = msg.pop(0) if command == "CONNECT": address = msg.pop(0) port = int(msg.pop(0)) if len(self.servers) < SERVER_MAX: self.servers.append(CloneServer(self.ctx, address, port, self.subtree)) self.publisher.connect("%s:%i" % (address,port+2)) else: logging.error("E: too many servers (max. %i)", SERVER_MAX) elif command == "SET": key,value,sttl = msg ttl = int(sttl) # Send key-value pair on to server kvmsg = KVMsg(0, key=key, body=value) kvmsg.store(self.kvmap) if ttl: kvmsg["ttl"] = ttl kvmsg.send(self.publisher) elif command == "GET": key = msg[0] value = self.kvmap.get(key) self.pipe.send(value.body if value else '')
def handle_snapshot(self, msg): """snapshot requests""" logging.info("I: received state request on snapshot: %s" % msg) if len(msg) != 3 or msg[1] != "ICANHAZ?": print "E: bad request, aborting" dump(msg) self.loop.stop() return identity, request, subtree = msg if subtree: # Send state snapshot to client route = Route(self.snapshot, identity, subtree) # For each entry in kvmap, send kvmsg to client for k, v in self.kvmap.items(): logging.info("I: Sending snapshot: %s" % v) send_single(k, v, route) # Now send END message with sequence number self.snapshot.send(identity, zmq.SNDMORE) kvmsg = KVMsg(self.sequence) kvmsg.key = "KTHXBAI" kvmsg.body = subtree logging.info("I: Sending snapshot end message: %s" % kvmsg) kvmsg.send(self.snapshot)
def handle_subscriber(self, msg): """Collect updates from peer (master) We're always slave when we get these updates """ if self.master: logging.warn("received subscriber message, but we are master %s", msg) return # Get state snapshot if necessary if self.kvmap is None: self.kvmap = {} snapshot = self.ctx.socket(zmq.DEALER) snapshot.linger = 0 snapshot.connect("tcp://localhost:%i" % self.peer) logging.info ("I: asking for snapshot from: tcp://localhost:%d", self.peer) snapshot.send_multipart(["ICANHAZ?", '']) while True: try: kvmsg = KVMsg.recv(snapshot) except KeyboardInterrupt: # Interrupted self.bstar.loop.stop() return if kvmsg.key == "KTHXBAI": self.sequence = kvmsg.sequence break # Done kvmsg.store(self.kvmap) logging.info ("I: received snapshot=%d", self.sequence) # Find and remove update off pending list kvmsg = KVMsg.from_msg(msg) # update integer ttl -> timestamp ttl = kvmsg.get('ttl') if ttl is not None: kvmsg['ttl'] = time.time() + ttl if kvmsg.key != "HUGZ": if not self.was_pending(kvmsg): # If master update came before client update, flip it # around, store master update (with sequence) on pending # list and use to clear client update when it comes later self.pending.append(kvmsg) # If update is more recent than our kvmap, apply it if (kvmsg.sequence > self.sequence): self.sequence = kvmsg.sequence kvmsg.store(self.kvmap) logging.info ("I: received update=%d", self.sequence)
def handle_collect(self, msg): """Collect updates from clients""" kvmsg = KVMsg.from_msg(msg) self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) ttl = float(kvmsg.get('ttl', 0)) if ttl: kvmsg['ttl'] = time.time() + ttl kvmsg.store(self.kvmap) logging.info("I: publishing update=%d", self.sequence)
def handle_collect(self, msg): """Collect updates from clients""" kvmsg = KVMsg.from_msg(msg) logging.info("I: received state update on collector: %s" % kvmsg) self.sequence += 1 kvmsg.sequence = self.sequence logging.info("I: publishing update: %s", kvmsg) kvmsg.send(self.publisher) ttl = kvmsg.get('ttl') if ttl is not None: kvmsg['ttl'] = time.time() + int(ttl) kvmsg.store(self.kvmap)
def handle_snapshot(self, socket, msg): """snapshot requests""" if msg[1] != "ICANHAZ?" or len(msg) != 3: logging.error("E: bad request, aborting") dump(msg) self.bstar.loop.stop() return identity, request = msg[:2] if len(msg) >= 3: subtree = msg[2] # Send state snapshot to client route = Route(socket, identity, subtree) # For each entry in kvmap, send kvmsg to client for k,v in self.kvmap.items(): send_single(k,v,route) # Now send END message with sequence number logging.info("I: Sending state shapshot=%d" % self.sequence) socket.send(identity, zmq.SNDMORE) kvmsg = KVMsg(self.sequence) kvmsg.key = "KTHXBAI" kvmsg.body = subtree kvmsg.send(socket)
def handle_collect(self, msg): """Collect updates from clients If we're master, we apply these to the kvmap If we're slave, or unsure, we queue them on our pending list """ kvmsg = KVMsg.from_msg(msg) if self.master: self.sequence += 1 kvmsg.sequence = self.sequence kvmsg.send(self.publisher) ttl = kvmsg.get('ttl') if ttl is not None: kvmsg['ttl'] = time.time() + ttl kvmsg.store(self.kvmap) logging.info("I: publishing update=%d", self.sequence) else: # If we already got message from master, drop it, else # hold on pending list if not self.was_pending(kvmsg): self.pending.append(kvmsg)
def handle_snapshot(self, socket, msg): """snapshot requests""" if msg[1] != "ICANHAZ?" or len(msg) != 3: logging.error("E: bad request, aborting") dump(msg) self.bstar.loop.stop() return identity, request = msg[:2] if len(msg) >= 3: subtree = msg[2] # Send state snapshot to client route = Route(socket, identity, subtree) # For each entry in kvmap, send kvmsg to client for k, v in self.kvmap.items(): send_single(k, v, route) # Now send END message with sequence number logging.info("I: Sending state shapshot=%d" % self.sequence) socket.send(identity, zmq.SNDMORE) kvmsg = KVMsg(self.sequence) kvmsg.key = "KTHXBAI" kvmsg.body = subtree kvmsg.send(socket)
def clone_agent(ctx, pipe): agent = CloneAgent(ctx, pipe) server = None while True: poller = zmq.Poller() poller.register(agent.pipe, zmq.POLLIN) poll_timer = None server_socket = None if agent.state == STATE_INITIAL: # In this state we ask the server for a snapshot, # if we have a server to talk to... if agent.servers: server = agent.servers[agent.cur_server] logging.info("I: waiting for server at %s:%d...", server.address, server.port) if (server.requests < 2): server.snapshot.send_multipart(["ICANHAZ?", agent.subtree]) server.requests += 1 server.expiry = time.time() + SERVER_TTL agent.state = STATE_SYNCING server_socket = server.snapshot elif agent.state == STATE_SYNCING: # In this state we read from snapshot and we expect # the server to respond, else we fail over. server_socket = server.snapshot elif agent.state == STATE_ACTIVE: # In this state we read from subscriber and we expect # the server to give hugz, else we fail over. server_socket = server.subscriber if server_socket: # we have a second socket to poll: poller.register(server_socket, zmq.POLLIN) if server is not None: poll_timer = 1e3 * max(0, server.expiry - time.time()) # ------------------------------------------------------------ # Poll loop try: items = dict(poller.poll(poll_timer)) except: raise # DEBUG break # Context has been shut down if agent.pipe in items: agent.control_message() elif server_socket in items: kvmsg = KVMsg.recv(server_socket) # Anything from server resets its expiry time server.expiry = time.time() + SERVER_TTL if (agent.state == STATE_SYNCING): # Store in snapshot until we're finished server.requests = 0 if kvmsg.key == "KTHXBAI": agent.sequence = kvmsg.sequence agent.state = STATE_ACTIVE logging.info("I: received from %s:%d snapshot=%d", server.address, server.port, agent.sequence) else: kvmsg.store(agent.kvmap) elif (agent.state == STATE_ACTIVE): # Discard out-of-sequence updates, incl. hugz if (kvmsg.sequence > agent.sequence): agent.sequence = kvmsg.sequence kvmsg.store(agent.kvmap) action = "update" if kvmsg.body else "delete" logging.info("I: received from %s:%d %s=%d", server.address, server.port, action, agent.sequence) else: # Server has died, failover to next logging.info("I: server at %s:%d didn't give hugz", server.address, server.port) agent.cur_server = (agent.cur_server + 1) % len(agent.servers) agent.state = STATE_INITIAL
def send_hugz(self): """Send hugz to anyone listening on the publisher socket""" kvmsg = KVMsg(self.sequence) kvmsg.key = "HUGZ" kvmsg.body = "" kvmsg.send(self.publisher)
def main(): # Prepare our context and subscriber ctx = zmq.Context() snapshot = ctx.socket(zmq.DEALER) snapshot.linger = 0 snapshot.connect("tcp://localhost:5556") subscriber = ctx.socket(zmq.SUB) subscriber.linger = 0 subscriber.setsockopt(zmq.SUBSCRIBE, SUBTREE) subscriber.connect("tcp://localhost:5557") publisher = ctx.socket(zmq.PUB) publisher.linger = 0 publisher.connect("tcp://localhost:5558") random.seed(time.time()) kvmap = {} # Get state snapshot sequence = 0 snapshot.send_multipart(["ICANHAZ?", SUBTREE]) while True: try: kvmsg = KVMsg.recv(snapshot) except: raise return # Interrupted if kvmsg.key == "KTHXBAI": sequence = kvmsg.sequence print "I: Received snapshot=%d" % sequence break # Done kvmsg.store(kvmap) poller = zmq.Poller() poller.register(subscriber, zmq.POLLIN) alarm = time.time()+1. while True: tickless = 1000*max(0, alarm - time.time()) try: items = dict(poller.poll(tickless)) except: break # Interrupted if subscriber in items: kvmsg = KVMsg.recv(subscriber) # Discard out-of-sequence kvmsgs, incl. heartbeats if kvmsg.sequence > sequence: sequence = kvmsg.sequence kvmsg.store(kvmap) action = "update" if kvmsg.body else "delete" print "I: received %s=%d" % (action, sequence) # If we timed-out, generate a random kvmsg if time.time() >= alarm: kvmsg = KVMsg(0) kvmsg.key = SUBTREE + "%d" % random.randint(1,10000) kvmsg.body = "%d" % random.randint(1,1000000) kvmsg['ttl'] = random.randint(0,30) kvmsg.send(publisher) kvmsg.store(kvmap) alarm = time.time() + 1. print " Interrupted\n%d messages in" % sequence
def clone_agent(ctx, pipe): agent = CloneAgent(ctx, pipe) server = None while True: poller = zmq.Poller() poller.register(agent.pipe, zmq.POLLIN) poll_timer = None server_socket = None if agent.state == STATE_INITIAL: # In this state we ask the server for a snapshot, # if we have a server to talk to... if agent.servers: server = agent.servers[agent.cur_server] logging.info ("I: waiting for server at %s:%d...", server.address, server.port) if (server.requests < 2): server.snapshot.send_multipart(["ICANHAZ?", agent.subtree]) server.requests += 1 server.expiry = time.time() + SERVER_TTL agent.state = STATE_SYNCING server_socket = server.snapshot elif agent.state == STATE_SYNCING: # In this state we read from snapshot and we expect # the server to respond, else we fail over. server_socket = server.snapshot elif agent.state == STATE_ACTIVE: # In this state we read from subscriber and we expect # the server to give hugz, else we fail over. server_socket = server.subscriber if server_socket: # we have a second socket to poll: poller.register(server_socket, zmq.POLLIN) if server is not None: poll_timer = 1e3 * max(0,server.expiry - time.time()) # ------------------------------------------------------------ # Poll loop try: items = dict(poller.poll(poll_timer)) except: raise # DEBUG break # Context has been shut down if agent.pipe in items: agent.control_message() elif server_socket in items: kvmsg = KVMsg.recv(server_socket) # Anything from server resets its expiry time server.expiry = time.time() + SERVER_TTL if (agent.state == STATE_SYNCING): # Store in snapshot until we're finished server.requests = 0 if kvmsg.key == "KTHXBAI": agent.sequence = kvmsg.sequence agent.state = STATE_ACTIVE logging.info ("I: received from %s:%d snapshot=%d", server.address, server.port, agent.sequence) else: kvmsg.store(agent.kvmap) elif (agent.state == STATE_ACTIVE): # Discard out-of-sequence updates, incl. hugz if (kvmsg.sequence > agent.sequence): agent.sequence = kvmsg.sequence kvmsg.store(agent.kvmap) action = "update" if kvmsg.body else "delete" logging.info ("I: received from %s:%d %s=%d", server.address, server.port, action, agent.sequence) else: # Server has died, failover to next logging.info ("I: server at %s:%d didn't give hugz", server.address, server.port) agent.cur_server = (agent.cur_server + 1) % len(agent.servers) agent.state = STATE_INITIAL
def main(): # Prepare our context and subscriber ctx = zmq.Context() snapshot = ctx.socket(zmq.DEALER) snapshot.linger = 0 snapshot.connect("tcp://localhost:5556") subscriber = ctx.socket(zmq.SUB) subscriber.linger = 0 subscriber.setsockopt(zmq.SUBSCRIBE, SUBTREE) subscriber.connect("tcp://localhost:5557") publisher = ctx.socket(zmq.PUB) publisher.linger = 0 publisher.connect("tcp://localhost:5558") random.seed(time.time()) kvmap = {} # Get state snapshot sequence = 0 snapshot.send_multipart(["ICANHAZ?", SUBTREE]) while True: try: kvmsg = KVMsg.recv(snapshot) except: raise return # Interrupted if kvmsg.key == "KTHXBAI": sequence = kvmsg.sequence print "I: Received snapshot=%d" % sequence break # Done kvmsg.store(kvmap) poller = zmq.Poller() poller.register(subscriber, zmq.POLLIN) alarm = time.time() + 1. while True: tickless = 1000 * max(0, alarm - time.time()) try: items = dict(poller.poll(tickless)) except: break # Interrupted if subscriber in items: kvmsg = KVMsg.recv(subscriber) # Discard out-of-sequence kvmsgs, incl. heartbeats if kvmsg.sequence > sequence: sequence = kvmsg.sequence kvmsg.store(kvmap) action = "update" if kvmsg.body else "delete" print "I: received %s=%d" % (action, sequence) # If we timed-out, generate a random kvmsg if time.time() >= alarm: kvmsg = KVMsg(0) kvmsg.key = SUBTREE + "%d" % random.randint(1, 10000) kvmsg.body = "%d" % random.randint(1, 1000000) kvmsg['ttl'] = random.randint(0, 30) kvmsg.send(publisher) kvmsg.store(kvmap) alarm = time.time() + 1. print " Interrupted\n%d messages in" % sequence
def main(): # Prepare our context and subscriber ctx = zmq.Context() snapshot = ctx.socket(zmq.DEALER) snapshot.linger = 0 snapshot.connect("tcp://localhost:5556") subscriber = ctx.socket(zmq.SUB) subscriber.linger = 0 subscriber.setsockopt_string(zmq.SUBSCRIBE, "") subscriber.connect("tcp://localhost:5557") publisher = ctx.socket(zmq.PUSH) publisher.linger = 0 publisher.connect("tcp://localhost:5558") random.seed(time.time()) kvmap = {} # Get state snapshot sequence = 0 snapshot.send_string("ICANHAZ?") while True: try: kvmsg = KVMsg.recv(snapshot) except: return # Interrupted if kvmsg.key == b"KTHXBAI": sequence = kvmsg.sequence print(f"S: Received snapshot {sequence} {kvmap}") break # Done kvmsg.store(kvmap) poller = zmq.Poller() poller.register(subscriber, zmq.POLLIN) alarm = time.time() + 1.0 while True: tickless = 1000 * max(0, alarm - time.time()) try: items = dict(poller.poll(tickless)) except: break # Interrupted if subscriber in items: kvmsg = KVMsg.recv(subscriber) # Discard out-of-sequence kvmsgs, incl. heartbeats if kvmsg.sequence > sequence: sequence = kvmsg.sequence kvmsg.store(kvmap) print(f"U: received {sequence} {kvmap}") # If we timed-out, generate a random kvmsg if time.time() >= alarm: kvmsg = KVMsg(0) kvmsg.key = ("%d" % random.randint(1, 10000)).encode() kvmsg.body = ("%d" % random.randint(1, 1000000)).encode() kvmsg.send(publisher) kvmsg.store(kvmap) alarm = time.time() + 1.0 print(f"E: Interrupted\nE: {sequence} messages handled")
def dkv_agent(ctx, pipe, connected_event): """ Asynchronous agent manages server pool and handles request/reply dialog when the application asks for it. """ agent = DkvAgent(ctx, pipe, connected_event) server = None while True: poller = zmq.Poller() poller.register(agent.pipe, zmq.POLLIN) poll_timer = None server_socket = None if agent.state == agent.STATES.INITIAL: """In this state we ask the server for a snapshot, if we have a server to talk to...""" if agent.servers: server = agent.servers[agent.cur_server] logger.debug("Asking for snapshot from %s attempt=%d..", server.address, server.requests) if (server.requests < 2): server.snapshot.send_multipart(["ICANHAZ?", agent.subtree]) server.requests += 1 server.expiry = time.time() + SERVER_TTL agent.state = agent.STATES.SYNCING server_socket = server.snapshot elif agent.state == agent.STATES.SYNCING: """In this state we read from snapshot and we expect the server to respond, else we fail over.""" server_socket = server.snapshot elif agent.state == agent.STATES.ACTIVE: """In this state we read from subscriber and we expect the server to give hugz, else we fail over.""" server_socket = server.subscriber if server_socket: """we have a second socket to poll""" poller.register(server_socket, zmq.POLLIN) if server is not None: poll_timer = 1e3 * max(0, server.expiry - time.time()) try: # Poll loop items = dict(poller.poll(poll_timer)) except: raise # DEBUG break # Context has been shut down if agent.pipe in items: agent.control_message() elif server_socket in items: msg = server_socket.recv_multipart() #logger.debug('server_socket=%s, msg=%s', server_socket, msg) #logger.debug('msg=%s', msg) #kvmsg = KVMsg.recv(server_socket) kvmsg = KVMsg.from_msg(msg) #pp(kvmsg.__dict__) server.expiry = time.time() + SERVER_TTL # Anything from server resets its expiry time if agent.state == agent.STATES.SYNCING: """Store in snapshot until we're finished""" server.requests = 0 #logger.debug('Syncing state msg=%s', msg) if kvmsg.key == "KTHXBAI": agent.sequence = kvmsg.sequence agent.state = agent.STATES.ACTIVE logger.info("Synced snapshot=%s from %s", agent.sequence, server.address) logger.info("Connected to %s", server.address) connected_event.set() else: logger.debug("Syncing update=%s from %s", kvmsg.sequence, server.address) kvmsg.store(agent.kvmap) elif agent.state == agent.STATES.ACTIVE: """Discard out-of-sequence updates, incl. hugz""" if kvmsg.sequence > agent.sequence: agent.sequence = kvmsg.sequence kvmsg.store(agent.kvmap) action = "update" if kvmsg.body else "delete" logger.debug("Received %s=%d from %s", action, agent.sequence, server.address) """ Signal """ if kvmsg.key != 'HUGZ': # Don't send signals if it's just hugz Dkv.signals.on_sub.send(kvmsg, key=kvmsg.key, value=kvmsg.body, props=kvmsg.properties) else: """Server has died, failover to next""" if agent.state == agent.STATES.ACTIVE: level = logging.ERROR server_state = 'active' else: level = logging.WARNING server_state = 'non-active' logger.log(level, "Did not receive heartbeat from %s server at %s; failing over.", server_state, server.address) agent.cur_server = (agent.cur_server + 1) % len(agent.servers) agent.state = agent.STATES.INITIAL
def control_message(self): msg = self.pipe.recv_multipart() command = msg.pop(0) if self.debug: logger.debug('cmd=%s msg=%s', command, msg) pp(msg) if command == "CONNECT": #connect_kwargs = pipeline.load(msg[0]) #self.connect(**connect_kwargs) self.connect(msg[0]) self.pipe.send_multipart(['OK']) elif command == 'CONNECT_DISCOVERY': self.connect_via_discovery() self.pipe.send_multipart(['OK']) elif command == "DISCONNECT": address = msg.pop(0) port = int(msg.pop(0)) self.disconnect(address, port) self.pipe.send_multipart(['OK']) elif command == "SET": #key, value, sttl, serializer = msg key, value, sttl = msg ttl = int(sttl) value = pipeline.load(value) # Send key-value pair on to server kvmsg = KVMsg(0, key=key, body=value) kvmsg.store(self.kvmap) if ttl: kvmsg["ttl"] = ttl #if serializer: # kvmsg["serializer"] = serializer kvmsg.send(self.publisher) self.pipe.send_multipart(['OK']) elif command == "GET": key = msg[0] value = self.kvmap.get(key) if value: body = value.body #serializer = str(value.properties.get('serializer', '')) else: body = '' #serializer = '' body = pipeline.dump(body) #self.pipe.send_multipart([body, serializer]) self.pipe.send_multipart([body]) elif command == "SHOW": key = msg[0].upper() if key == 'SERVERS': self.pipe.send_multipart([ str(','.join([x.__repr__() for x in self.servers])), '']) elif key == 'SERVER': self.pipe.send_multipart([str(self.cur_server), '']) elif key == 'SEQ': self.pipe.send_multipart([str(self.sequence), '']) elif key == 'STATUS': self.pipe.send_multipart([str(self.cur_status), '']) elif key == 'KVMAP': #kvmap_s = pickle.dumps(self.kvmap) #self.pipe.send_multipart([kvmap_s, 'pickle']) kvmap_s = pipeline.dump(self.kvmap) self.pipe.send_multipart([kvmap_s])