Ejemplo n.º 1
0
    def _run_socket(self, sock, name, ep, watched_names=None):
        if watched_names is None:
            watched_names = {}

        validate = self.c.get('validate_signatures', False)

        # Grab the data off the zeromq internal queue
        _topic, message = sock.recv_multipart()

        # zmq hands us byte strings, so let's convert to unicode asap
        _topic, message = _topic.decode('utf-8'), message.decode('utf-8')

        # Now, decode the JSON body into a dict.
        msg = fedmsg.encoding.loads(message)

        if not validate or fedmsg.crypto.validate(msg, **self.c):
            # If there is even a slight change of replay, use
            # check_for_replay
            if len(self.c.get('replay_endpoints', {})) > 0:
                for m in check_for_replay(
                        name, watched_names,
                        msg, self.c, self.context):

                    # Revalidate all the replayed messages.
                    if not validate or \
                            fedmsg.crypto.validate(m, **self.c):
                        return name, ep, m['topic'], m
                    else:
                        raise ValidationError(msg)
            else:
                return name, ep, _topic, msg
        else:
            raise ValidationError(msg)
Ejemplo n.º 2
0
    def _consume(self, message):

        # Massage STOMP messages into a more compatible format.
        if 'topic' not in message['body']:
            message['body'] = {
                'topic': message.get('topic'),
                'msg': message['body'],
            }

        try:
            self.validate(message)
        except RuntimeWarning as e:
            self.log.warn("Received invalid message {0}".format(e))
            return
        if hasattr(self, "replay_name"):
            for m in check_for_replay(self.replay_name, self.name_to_seq_id,
                                      message, self.hub.config):

                try:
                    self.validate(m)
                    super(FedmsgConsumer, self)._consume(m)
                except RuntimeWarning as e:
                    self.log.warn("Received invalid message {}".format(e))
        else:
            super(FedmsgConsumer, self)._consume(message)
Ejemplo n.º 3
0
    def _run_socket(self, sock, name, ep, watched_names=None):
        if watched_names is None:
            watched_names = {}

        validate = self.c.get('validate_signatures', False)

        _topic, message = sock.recv_multipart()
        msg = fedmsg.encoding.loads(message)
        if not validate or fedmsg.crypto.validate(msg, **self.c):
            # If there is even a slight change of replay, use
            # check_for_replay
            if len(self.c.get('replay_endpoints', {})) > 0:
                for m in check_for_replay(
                        name, watched_names,
                        msg, self.c, self.context):

                    # Revalidate all the replayed messages.
                    if not validate or \
                            fedmsg.crypto.validate(m, **self.c):
                        return name, ep, m['topic'], m
                    else:
                        raise ValidationError(msg)
            else:
                return name, ep, _topic, msg
        else:
            raise ValidationError(msg)
Ejemplo n.º 4
0
    def _consume(self, message):
        try:
            self.validate(message)
        except RuntimeWarning as e:
            self.log.warn("Received invalid message {0}".format(e))
            return
        if hasattr(self, "replay_name"):
            for m in check_for_replay(self.replay_name, self.name_to_seq_id, message, self.hub.config):

                try:
                    self.validate(m)
                    super(FedmsgConsumer, self)._consume(m)
                except RuntimeWarning as e:
                    self.log.warn("Received invalid message {}".format(e))
        else:
            super(FedmsgConsumer, self)._consume(message)
Ejemplo n.º 5
0
    def _consume(self, message):
        try:
            self.validate(message)
        except RuntimeWarning as e:
            self.log.warn("Received invalid message {}".format(e))
            return
        if hasattr(self, "replay_name"):
            for m in check_for_replay(self.replay_name, self.name_to_seq_id,
                                      message, self.hub.config):

                try:
                    self.validate(m)
                    self.consume(m)
                except RuntimeWarning as e:
                    self.log.warn("Received invalid message {}".format(e))
        else:
            self.consume(message)
Ejemplo n.º 6
0
    def tail_messages(self, topic="", passive=False, **kw):
        """ Tail messages on the bus.

        Generator that yields tuples of the form:
        ``(name, endpoint, topic, message)``
        """

        # TODO -- do the zmq_strict logic dance with "topic" here.
        # It is buried in moksha.hub, but we need it to work the same way
        # here.

        # TODO -- the 'passive' here and the 'active' are ambiguous.  They
        # don't actually mean the same thing.  This should be resolved.
        method = passive and 'bind' or 'connect'

        failed_hostnames = []
        subs = {}
        watched_names = {}
        for _name, endpoint_list in self.c['endpoints'].iteritems():
            # Listify endpoint_list in case it is a single string
            endpoint_list = iterate(endpoint_list)
            for endpoint in endpoint_list:
                # First, some sanity checking.  zeromq will potentially
                # segfault if we don't do this check.
                hostname = endpoint.split(':')[1][2:]
                if hostname in failed_hostnames:
                    continue

                if hostname != '*':
                    try:
                        socket.gethostbyname_ex(hostname)
                    except:
                        failed_hostnames.append(hostname)
                        self.log.warn("Couldn't resolve %r" % hostname)
                        continue

                # OK, sanity checks pass.  Create the subscriber and connect.
                subscriber = self.context.socket(zmq.SUB)
                subscriber.setsockopt(zmq.SUBSCRIBE, topic)

                set_high_water_mark(subscriber, self.c)
                set_tcp_keepalive(subscriber, self.c)

                getattr(subscriber, method)(endpoint)
                subs[subscriber] = (_name, endpoint)
            if _name in self.c.get("replay_endpoints", {}):
                # At first we don't know where the sequence is at.
                watched_names[_name] = -1

        # Register the sockets we just built with a zmq Poller.
        poller = zmq.Poller()
        for subscriber in subs:
            poller.register(subscriber, zmq.POLLIN)

        # TODO -- what if user wants to pass in validate_signatures in **kw?
        validate = self.c.get('validate_signatures', False)

        # Poll that poller.  This is much more efficient than it used to be.
        try:
            while True:
                sockets = dict(poller.poll())
                for s in sockets:
                    _name, ep = subs[s]
                    _topic, message = s.recv_multipart()
                    msg = fedmsg.encoding.loads(message)
                    if not validate or fedmsg.crypto.validate(msg, **self.c):
                        # If there is even a slight change of replay, use
                        # check_for_replay
                        if len(self.c.get('replay_endpoints', {})) > 0:
                            for m in check_for_replay(
                                    _name, watched_names,
                                    msg, self.c, self.context):

                                # Revalidate all the replayed messages.
                                if not validate or \
                                        fedmsg.crypto.validate(m, **self.c):
                                    yield _name, ep, m['topic'], m
                                else:
                                    warnings.warn("!! invalid message " +
                                                  "received: %r" % msg)
                        else:
                            yield _name, ep, _topic, msg
                    else:
                        # Else.. we are supposed to be validating, but the
                        # message failed validation.

                        # Warn, but don't throw an exception.  Keep tailing.
                        warnings.warn("!! invalid message received: %r" % msg)

        finally:
            for subscriber in subs:
                subscriber.close()
Ejemplo n.º 7
0
    def tail_messages(self, topic="", passive=False, **kw):
        """ Tail messages on the bus.

        Generator that yields tuples of the form:
        ``(name, endpoint, topic, message)``
        """

        # TODO -- do the zmq_strict logic dance with "topic" here.
        # It is buried in moksha.hub, but we need it to work the same way
        # here.

        # TODO -- the 'passive' here and the 'active' are ambiguous.  They
        # don't actually mean the same thing.  This should be resolved.
        method = passive and 'bind' or 'connect'

        failed_hostnames = []
        subs = {}
        watched_names = {}
        for _name, endpoint_list in six.iteritems(self.c['endpoints']):

            # You never want to actually subscribe to this thing, but sometimes
            # it appears in the endpoints list due to a hack where it gets
            # added in __init__ above.
            if _name == 'relay_inbound':
                continue

            # Listify endpoint_list in case it is a single string
            endpoint_list = iterate(endpoint_list)
            for endpoint in endpoint_list:
                # First, some sanity checking.  zeromq will potentially
                # segfault if we don't do this check.
                hostname = endpoint.split(':')[1][2:]
                if hostname in failed_hostnames:
                    continue

                if hostname != '*':
                    try:
                        socket.gethostbyname_ex(hostname)
                    except:
                        failed_hostnames.append(hostname)
                        self.log.warn("Couldn't resolve %r" % hostname)
                        continue

                # OK, sanity checks pass.  Create the subscriber and connect.
                subscriber = self.context.socket(zmq.SUB)
                subscriber.setsockopt(zmq.SUBSCRIBE, topic)

                set_high_water_mark(subscriber, self.c)
                set_tcp_keepalive(subscriber, self.c)
                set_tcp_reconnect(subscriber, self.c)

                getattr(subscriber, method)(endpoint)
                subs[subscriber] = (_name, endpoint)
            if _name in self.c.get("replay_endpoints", {}):
                # At first we don't know where the sequence is at.
                watched_names[_name] = -1

        # Register the sockets we just built with a zmq Poller.
        poller = zmq.Poller()
        for subscriber in subs:
            poller.register(subscriber, zmq.POLLIN)

        # TODO -- what if user wants to pass in validate_signatures in **kw?
        validate = self.c.get('validate_signatures', False)

        # Poll that poller.  This is much more efficient than it used to be.
        try:
            while True:
                sockets = dict(poller.poll())
                for s in sockets:
                    _name, ep = subs[s]
                    _topic, message = s.recv_multipart()
                    msg = fedmsg.encoding.loads(message)
                    if not validate or fedmsg.crypto.validate(msg, **self.c):
                        # If there is even a slight change of replay, use
                        # check_for_replay
                        if len(self.c.get('replay_endpoints', {})) > 0:
                            for m in check_for_replay(_name, watched_names,
                                                      msg, self.c,
                                                      self.context):

                                # Revalidate all the replayed messages.
                                if not validate or \
                                        fedmsg.crypto.validate(m, **self.c):
                                    yield _name, ep, m['topic'], m
                                else:
                                    warnings.warn("!! invalid message " +
                                                  "received: %r" % msg)
                        else:
                            yield _name, ep, _topic, msg
                    else:
                        # Else.. we are supposed to be validating, but the
                        # message failed validation.

                        # Warn, but don't throw an exception.  Keep tailing.
                        warnings.warn("!! invalid message received: %r" % msg)

        finally:
            for subscriber in subs:
                subscriber.close()