def _create_poller(self, topic="", passive=False, **kw): # TODO -- do the zmq_strict logic dance with "topic" here. # It is buried in moksha.hub, but we need it to work the same way # here. # TODO -- the 'passive' here and the 'active' are ambiguous. They # don't actually mean the same thing. This should be resolved. method = passive and 'bind' or 'connect' failed_hostnames = [] subs = {} for _name, endpoint_list in six.iteritems(self.c['endpoints']): # You never want to actually subscribe to this thing, but sometimes # it appears in the endpoints list due to a hack where it gets # added in __init__ above. if _name == 'relay_inbound': continue # Listify endpoint_list in case it is a single string endpoint_list = iterate(endpoint_list) for endpoint in endpoint_list: # First, some sanity checking. zeromq will potentially # segfault if we don't do this check. hostname = endpoint.split(':')[1][2:] if hostname in failed_hostnames: continue if hostname != '*': try: socket.gethostbyname_ex(hostname) except: failed_hostnames.append(hostname) self.log.warn("Couldn't resolve %r" % hostname) continue # OK, sanity checks pass. Create the subscriber and connect. subscriber = self.context.socket(zmq.SUB) subscriber.setsockopt(zmq.SUBSCRIBE, topic.encode('utf-8')) set_high_water_mark(subscriber, self.c) set_tcp_keepalive(subscriber, self.c) set_tcp_reconnect(subscriber, self.c) getattr(subscriber, method)(endpoint) subs[subscriber] = (_name, endpoint) # Register the sockets we just built with a zmq Poller. poller = zmq.Poller() for subscriber in subs: poller.register(subscriber, zmq.POLLIN) return (poller, subs)
def __init__(self, **config): super(FedMsgContext, self).__init__() self.log = logging.getLogger("fedmsg") self.c = config self.hostname = socket.gethostname().split('.', 1)[0] # Prepare our context and publisher self.context = zmq.Context(config['io_threads']) method = ['bind', 'connect'][config['active']] # If no name is provided, use the calling module's __name__ to decide # which publishing endpoint to use. if not config.get("name", None): module_name = guess_calling_module(default="fedmsg") config["name"] = module_name + '.' + self.hostname if any(map(config["name"].startswith, ['fedmsg'])): config["name"] = None # Find my message-signing cert if I need one. if self.c.get('sign_messages', False) and config.get("name"): if not config.get("crypto_backend") == "gpg": if 'cert_prefix' in config: cert_index = "%s.%s" % (config['cert_prefix'], self.hostname) else: cert_index = config['name'] if cert_index == 'relay_inbound': cert_index = "shell.%s" % self.hostname self.c['certname'] = self.c['certnames'][cert_index] else: self.c['gpg_signing_key'] = self.c['gpg_keys'][cert_index] # Do a little special-case mangling. We never want to "listen" to the # relay_inbound address, but in the special case that we want to emit # our messages there, we add it to the :term:`endpoints` dict so that # the code below where we "Actually set up our publisher" can be # simplified. See Issue #37 - http://bit.ly/KN6dEK if config.get('active', False): # If the user has called us with "active=True" then presumably they # have given us a "name" as well. name = config.get("name", "relay_inbound") config['endpoints'][name] = config[name] # Actually set up our publisher if ( not config.get("mute", False) and config.get("name", None) and config.get("endpoints", None) and config['endpoints'].get(config['name']) ): # Construct it. self.publisher = self.context.socket(zmq.PUB) set_high_water_mark(self.publisher, config) set_tcp_keepalive(self.publisher, config) # Set a zmq_linger, thus doing a little bit more to ensure that our # message gets to the fedmsg-relay (*if* we're talking to the relay # which is the case when method == 'connect'). if method == 'connect': self.publisher.setsockopt(zmq.LINGER, config['zmq_linger']) # "Listify" our endpoints. If we're given a list, good. If we're # given a single item, turn it into a list of length 1. config['endpoints'][config['name']] = list(iterate( config['endpoints'][config['name']])) # Try endpoint after endpoint in the list of endpoints. If we # succeed in establishing one, then stop. *That* is our publishing # endpoint. _established = False for endpoint in config['endpoints'][config['name']]: if method == 'bind': endpoint = "tcp://*:{port}".format( port=endpoint.rsplit(':')[-1] ) try: # Call either bind or connect on the new publisher. # This will raise an exception if there's another process # already using the endpoint. getattr(self.publisher, method)(endpoint) # If we can do this successfully, then stop trying. _established = True break except zmq.ZMQError: # If we fail to bind or connect, there's probably another # process already using that endpoint port. Try the next # one. pass # If we make it through the loop without establishing our # connection, then there are not enough endpoints listed in the # config for the number of processes attempting to use fedmsg. if not _established: raise IOError( "Couldn't find an available endpoint " "for name %r" % config.get("name", None)) elif config.get('mute', False): # Our caller doesn't intend to send any messages. Pass silently. pass else: # Something is wrong. warnings.warn( "fedmsg is not configured to send any messages " "for name %r" % config.get("name", None)) # Cleanup. See http://bit.ly/SaGeOr for discussion. weakref.ref(threading.current_thread(), self.destroy) # Sleep just to make sure that the socket gets set up before anyone # tries anything. This is a documented zmq 'feature'. time.sleep(config['post_init_sleep'])
def tail_messages(self, topic="", passive=False, **kw): """ Tail messages on the bus. Generator that yields tuples of the form: ``(name, endpoint, topic, message)`` """ # TODO -- do the zmq_strict logic dance with "topic" here. # It is buried in moksha.hub, but we need it to work the same way # here. # TODO -- the 'passive' here and the 'active' are ambiguous. They # don't actually mean the same thing. This should be resolved. method = passive and 'bind' or 'connect' failed_hostnames = [] subs = {} watched_names = {} for _name, endpoint_list in self.c['endpoints'].iteritems(): # Listify endpoint_list in case it is a single string endpoint_list = iterate(endpoint_list) for endpoint in endpoint_list: # First, some sanity checking. zeromq will potentially # segfault if we don't do this check. hostname = endpoint.split(':')[1][2:] if hostname in failed_hostnames: continue if hostname != '*': try: socket.gethostbyname_ex(hostname) except: failed_hostnames.append(hostname) self.log.warn("Couldn't resolve %r" % hostname) continue # OK, sanity checks pass. Create the subscriber and connect. subscriber = self.context.socket(zmq.SUB) subscriber.setsockopt(zmq.SUBSCRIBE, topic) set_high_water_mark(subscriber, self.c) set_tcp_keepalive(subscriber, self.c) getattr(subscriber, method)(endpoint) subs[subscriber] = (_name, endpoint) if _name in self.c.get("replay_endpoints", {}): # At first we don't know where the sequence is at. watched_names[_name] = -1 # Register the sockets we just built with a zmq Poller. poller = zmq.Poller() for subscriber in subs: poller.register(subscriber, zmq.POLLIN) # TODO -- what if user wants to pass in validate_signatures in **kw? validate = self.c.get('validate_signatures', False) # Poll that poller. This is much more efficient than it used to be. try: while True: sockets = dict(poller.poll()) for s in sockets: _name, ep = subs[s] _topic, message = s.recv_multipart() msg = fedmsg.encoding.loads(message) if not validate or fedmsg.crypto.validate(msg, **self.c): # If there is even a slight change of replay, use # check_for_replay if len(self.c.get('replay_endpoints', {})) > 0: for m in check_for_replay( _name, watched_names, msg, self.c, self.context): # Revalidate all the replayed messages. if not validate or \ fedmsg.crypto.validate(m, **self.c): yield _name, ep, m['topic'], m else: warnings.warn("!! invalid message " + "received: %r" % msg) else: yield _name, ep, _topic, msg else: # Else.. we are supposed to be validating, but the # message failed validation. # Warn, but don't throw an exception. Keep tailing. warnings.warn("!! invalid message received: %r" % msg) finally: for subscriber in subs: subscriber.close()
def __init__(self, **config): super(FedMsgContext, self).__init__() self.log = logging.getLogger("fedmsg") self.c = config self.hostname = socket.gethostname().split('.', 1)[0] # Prepare our context and publisher self.context = zmq.Context(config['io_threads']) method = ['bind', 'connect'][config['active']] # If no name is provided, use the calling module's __name__ to decide # which publishing endpoint to use (unless active=True, in which case # we use "relay_inbound" as set in the subsequent code block). if not config.get("name", None) and not config.get('active', False): module_name = guess_calling_module(default="fedmsg") config["name"] = module_name + '.' + self.hostname if any(map(config["name"].startswith, ['fedmsg'])): config["name"] = None # Do a little special-case mangling. We never want to "listen" to the # relay_inbound address, but in the special case that we want to emit # our messages there, we add it to the :term:`endpoints` dict so that # the code below where we "Actually set up our publisher" can be # simplified. See Issue #37 - https://bit.ly/KN6dEK if config.get('active', False): try: name = config['name'] = config.get("name", "relay_inbound") config['endpoints'][name] = config[name] except KeyError: raise KeyError("Could not find endpoint for fedmsg-relay." " Try installing fedmsg-relay.") # Actually set up our publisher if (not config.get("mute", False) and config.get("name", None) and config.get("endpoints", None) and config['endpoints'].get(config['name'])): # Construct it. self.publisher = self.context.socket(zmq.PUB) set_high_water_mark(self.publisher, config) set_tcp_keepalive(self.publisher, config) # Set a zmq_linger, thus doing a little bit more to ensure that our # message gets to the fedmsg-relay (*if* we're talking to the relay # which is the case when method == 'connect'). if method == 'connect': self.publisher.setsockopt(zmq.LINGER, config['zmq_linger']) # "Listify" our endpoints. If we're given a list, good. If we're # given a single item, turn it into a list of length 1. config['endpoints'][config['name']] = list( iterate(config['endpoints'][config['name']])) # Try endpoint after endpoint in the list of endpoints. If we # succeed in establishing one, then stop. *That* is our publishing # endpoint. _established = False for endpoint in config['endpoints'][config['name']]: self.log.debug("Trying to %s to %s" % (method, endpoint)) if method == 'bind': endpoint = "tcp://*:{port}".format( port=endpoint.rsplit(':')[-1]) try: # Call either bind or connect on the new publisher. # This will raise an exception if there's another process # already using the endpoint. getattr(self.publisher, method)(endpoint) # If we can do this successfully, then stop trying. _established = True break except zmq.ZMQError: # If we fail to bind or connect, there's probably another # process already using that endpoint port. Try the next # one. pass # If we make it through the loop without establishing our # connection, then there are not enough endpoints listed in the # config for the number of processes attempting to use fedmsg. if not _established: raise IOError("Couldn't find an available endpoint " "for name %r" % config.get("name", None)) elif config.get('mute', False): # Our caller doesn't intend to send any messages. Pass silently. pass else: # Something is wrong. warnings.warn("fedmsg is not configured to send any messages " "for name %r" % config.get("name", None)) # Cleanup. See https://bit.ly/SaGeOr for discussion. weakref.ref(threading.current_thread(), self.destroy) # Sleep just to make sure that the socket gets set up before anyone # tries anything. This is a documented zmq 'feature'. time.sleep(config['post_init_sleep'])
def tail_messages(self, topic="", passive=False, **kw): """ Tail messages on the bus. Generator that yields tuples of the form: ``(name, endpoint, topic, message)`` """ # TODO -- do the zmq_strict logic dance with "topic" here. # It is buried in moksha.hub, but we need it to work the same way # here. # TODO -- the 'passive' here and the 'active' are ambiguous. They # don't actually mean the same thing. This should be resolved. method = passive and 'bind' or 'connect' failed_hostnames = [] subs = {} watched_names = {} for _name, endpoint_list in six.iteritems(self.c['endpoints']): # You never want to actually subscribe to this thing, but sometimes # it appears in the endpoints list due to a hack where it gets # added in __init__ above. if _name == 'relay_inbound': continue # Listify endpoint_list in case it is a single string endpoint_list = iterate(endpoint_list) for endpoint in endpoint_list: # First, some sanity checking. zeromq will potentially # segfault if we don't do this check. hostname = endpoint.split(':')[1][2:] if hostname in failed_hostnames: continue if hostname != '*': try: socket.gethostbyname_ex(hostname) except: failed_hostnames.append(hostname) self.log.warn("Couldn't resolve %r" % hostname) continue # OK, sanity checks pass. Create the subscriber and connect. subscriber = self.context.socket(zmq.SUB) subscriber.setsockopt(zmq.SUBSCRIBE, topic) set_high_water_mark(subscriber, self.c) set_tcp_keepalive(subscriber, self.c) set_tcp_reconnect(subscriber, self.c) getattr(subscriber, method)(endpoint) subs[subscriber] = (_name, endpoint) if _name in self.c.get("replay_endpoints", {}): # At first we don't know where the sequence is at. watched_names[_name] = -1 # Register the sockets we just built with a zmq Poller. poller = zmq.Poller() for subscriber in subs: poller.register(subscriber, zmq.POLLIN) # TODO -- what if user wants to pass in validate_signatures in **kw? validate = self.c.get('validate_signatures', False) # Poll that poller. This is much more efficient than it used to be. try: while True: sockets = dict(poller.poll()) for s in sockets: _name, ep = subs[s] _topic, message = s.recv_multipart() msg = fedmsg.encoding.loads(message) if not validate or fedmsg.crypto.validate(msg, **self.c): # If there is even a slight change of replay, use # check_for_replay if len(self.c.get('replay_endpoints', {})) > 0: for m in check_for_replay(_name, watched_names, msg, self.c, self.context): # Revalidate all the replayed messages. if not validate or \ fedmsg.crypto.validate(m, **self.c): yield _name, ep, m['topic'], m else: warnings.warn("!! invalid message " + "received: %r" % msg) else: yield _name, ep, _topic, msg else: # Else.. we are supposed to be validating, but the # message failed validation. # Warn, but don't throw an exception. Keep tailing. warnings.warn("!! invalid message received: %r" % msg) finally: for subscriber in subs: subscriber.close()