def _run_socket(self, sock, name, ep, watched_names=None): if watched_names is None: watched_names = {} validate = self.c.get('validate_signatures', False) # Grab the data off the zeromq internal queue _topic, message = sock.recv_multipart() # zmq hands us byte strings, so let's convert to unicode asap _topic, message = _topic.decode('utf-8'), message.decode('utf-8') # Now, decode the JSON body into a dict. msg = fedmsg.encoding.loads(message) if not validate or fedmsg.crypto.validate(msg, **self.c): # If there is even a slight change of replay, use # check_for_replay if len(self.c.get('replay_endpoints', {})) > 0: for m in check_for_replay( name, watched_names, msg, self.c, self.context): # Revalidate all the replayed messages. if not validate or \ fedmsg.crypto.validate(m, **self.c): return name, ep, m['topic'], m else: raise ValidationError(msg) else: return name, ep, _topic, msg else: raise ValidationError(msg)
def _consume(self, message): # Massage STOMP messages into a more compatible format. if 'topic' not in message['body']: message['body'] = { 'topic': message.get('topic'), 'msg': message['body'], } try: self.validate(message) except RuntimeWarning as e: self.log.warn("Received invalid message {0}".format(e)) return if hasattr(self, "replay_name"): for m in check_for_replay(self.replay_name, self.name_to_seq_id, message, self.hub.config): try: self.validate(m) super(FedmsgConsumer, self)._consume(m) except RuntimeWarning as e: self.log.warn("Received invalid message {}".format(e)) else: super(FedmsgConsumer, self)._consume(message)
def _run_socket(self, sock, name, ep, watched_names=None): if watched_names is None: watched_names = {} validate = self.c.get('validate_signatures', False) _topic, message = sock.recv_multipart() msg = fedmsg.encoding.loads(message) if not validate or fedmsg.crypto.validate(msg, **self.c): # If there is even a slight change of replay, use # check_for_replay if len(self.c.get('replay_endpoints', {})) > 0: for m in check_for_replay( name, watched_names, msg, self.c, self.context): # Revalidate all the replayed messages. if not validate or \ fedmsg.crypto.validate(m, **self.c): return name, ep, m['topic'], m else: raise ValidationError(msg) else: return name, ep, _topic, msg else: raise ValidationError(msg)
def _consume(self, message): try: self.validate(message) except RuntimeWarning as e: self.log.warn("Received invalid message {0}".format(e)) return if hasattr(self, "replay_name"): for m in check_for_replay(self.replay_name, self.name_to_seq_id, message, self.hub.config): try: self.validate(m) super(FedmsgConsumer, self)._consume(m) except RuntimeWarning as e: self.log.warn("Received invalid message {}".format(e)) else: super(FedmsgConsumer, self)._consume(message)
def _consume(self, message): try: self.validate(message) except RuntimeWarning as e: self.log.warn("Received invalid message {}".format(e)) return if hasattr(self, "replay_name"): for m in check_for_replay(self.replay_name, self.name_to_seq_id, message, self.hub.config): try: self.validate(m) self.consume(m) except RuntimeWarning as e: self.log.warn("Received invalid message {}".format(e)) else: self.consume(message)
def tail_messages(self, topic="", passive=False, **kw): """ Tail messages on the bus. Generator that yields tuples of the form: ``(name, endpoint, topic, message)`` """ # TODO -- do the zmq_strict logic dance with "topic" here. # It is buried in moksha.hub, but we need it to work the same way # here. # TODO -- the 'passive' here and the 'active' are ambiguous. They # don't actually mean the same thing. This should be resolved. method = passive and 'bind' or 'connect' failed_hostnames = [] subs = {} watched_names = {} for _name, endpoint_list in self.c['endpoints'].iteritems(): # Listify endpoint_list in case it is a single string endpoint_list = iterate(endpoint_list) for endpoint in endpoint_list: # First, some sanity checking. zeromq will potentially # segfault if we don't do this check. hostname = endpoint.split(':')[1][2:] if hostname in failed_hostnames: continue if hostname != '*': try: socket.gethostbyname_ex(hostname) except: failed_hostnames.append(hostname) self.log.warn("Couldn't resolve %r" % hostname) continue # OK, sanity checks pass. Create the subscriber and connect. subscriber = self.context.socket(zmq.SUB) subscriber.setsockopt(zmq.SUBSCRIBE, topic) set_high_water_mark(subscriber, self.c) set_tcp_keepalive(subscriber, self.c) getattr(subscriber, method)(endpoint) subs[subscriber] = (_name, endpoint) if _name in self.c.get("replay_endpoints", {}): # At first we don't know where the sequence is at. watched_names[_name] = -1 # Register the sockets we just built with a zmq Poller. poller = zmq.Poller() for subscriber in subs: poller.register(subscriber, zmq.POLLIN) # TODO -- what if user wants to pass in validate_signatures in **kw? validate = self.c.get('validate_signatures', False) # Poll that poller. This is much more efficient than it used to be. try: while True: sockets = dict(poller.poll()) for s in sockets: _name, ep = subs[s] _topic, message = s.recv_multipart() msg = fedmsg.encoding.loads(message) if not validate or fedmsg.crypto.validate(msg, **self.c): # If there is even a slight change of replay, use # check_for_replay if len(self.c.get('replay_endpoints', {})) > 0: for m in check_for_replay( _name, watched_names, msg, self.c, self.context): # Revalidate all the replayed messages. if not validate or \ fedmsg.crypto.validate(m, **self.c): yield _name, ep, m['topic'], m else: warnings.warn("!! invalid message " + "received: %r" % msg) else: yield _name, ep, _topic, msg else: # Else.. we are supposed to be validating, but the # message failed validation. # Warn, but don't throw an exception. Keep tailing. warnings.warn("!! invalid message received: %r" % msg) finally: for subscriber in subs: subscriber.close()
def tail_messages(self, topic="", passive=False, **kw): """ Tail messages on the bus. Generator that yields tuples of the form: ``(name, endpoint, topic, message)`` """ # TODO -- do the zmq_strict logic dance with "topic" here. # It is buried in moksha.hub, but we need it to work the same way # here. # TODO -- the 'passive' here and the 'active' are ambiguous. They # don't actually mean the same thing. This should be resolved. method = passive and 'bind' or 'connect' failed_hostnames = [] subs = {} watched_names = {} for _name, endpoint_list in six.iteritems(self.c['endpoints']): # You never want to actually subscribe to this thing, but sometimes # it appears in the endpoints list due to a hack where it gets # added in __init__ above. if _name == 'relay_inbound': continue # Listify endpoint_list in case it is a single string endpoint_list = iterate(endpoint_list) for endpoint in endpoint_list: # First, some sanity checking. zeromq will potentially # segfault if we don't do this check. hostname = endpoint.split(':')[1][2:] if hostname in failed_hostnames: continue if hostname != '*': try: socket.gethostbyname_ex(hostname) except: failed_hostnames.append(hostname) self.log.warn("Couldn't resolve %r" % hostname) continue # OK, sanity checks pass. Create the subscriber and connect. subscriber = self.context.socket(zmq.SUB) subscriber.setsockopt(zmq.SUBSCRIBE, topic) set_high_water_mark(subscriber, self.c) set_tcp_keepalive(subscriber, self.c) set_tcp_reconnect(subscriber, self.c) getattr(subscriber, method)(endpoint) subs[subscriber] = (_name, endpoint) if _name in self.c.get("replay_endpoints", {}): # At first we don't know where the sequence is at. watched_names[_name] = -1 # Register the sockets we just built with a zmq Poller. poller = zmq.Poller() for subscriber in subs: poller.register(subscriber, zmq.POLLIN) # TODO -- what if user wants to pass in validate_signatures in **kw? validate = self.c.get('validate_signatures', False) # Poll that poller. This is much more efficient than it used to be. try: while True: sockets = dict(poller.poll()) for s in sockets: _name, ep = subs[s] _topic, message = s.recv_multipart() msg = fedmsg.encoding.loads(message) if not validate or fedmsg.crypto.validate(msg, **self.c): # If there is even a slight change of replay, use # check_for_replay if len(self.c.get('replay_endpoints', {})) > 0: for m in check_for_replay(_name, watched_names, msg, self.c, self.context): # Revalidate all the replayed messages. if not validate or \ fedmsg.crypto.validate(m, **self.c): yield _name, ep, m['topic'], m else: warnings.warn("!! invalid message " + "received: %r" % msg) else: yield _name, ep, _topic, msg else: # Else.. we are supposed to be validating, but the # message failed validation. # Warn, but don't throw an exception. Keep tailing. warnings.warn("!! invalid message received: %r" % msg) finally: for subscriber in subs: subscriber.close()