def gateway(**kw): """ Rebroadcast messages to a special zmq endpoint. A repeater that rebroadcasts all messages received to a special zmq endpoint. This is used to get messages from inside Fedora Infrastructure out to users. Its communication is uni-directional. It does not relay messages from "outside the bus" back in. The special zmq endpoint is specified by the presence of :term:`fedmsg.consumers.gateway.port` in the config. This service is what makes using ":doc:`consuming`" outside the VPN/firewalled bus environment possible. """ # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict(zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values()), ) kw.update(moksha_options) # Flip the special bit that allows the GatewayConsumer to run kw[GatewayConsumer.config_key] = True from moksha.hub import main main(options=kw, consumers=[GatewayConsumer])
def relay(**kw): """ Relay connections from active loggers to the bus. ``fedmsg-relay`` is a service which binds to two ports, listens for messages on one and emits them on the other. ``fedmsg-logger`` requires that an instance of ``fedmsg-relay`` be running *somewhere* and that it's inbound address be listed in the config as :term:`relay_inbound`. ``fedmsg-relay`` becomes a necessity for integration points that cannot bind consistently to and serve from a port. See :doc:`topology` for the mile-high view. More specifically, ``fedmsg-relay`` is a SUB.bind()->PUB.bind() relay. """ # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_publish_endpoints=",".join(kw['endpoints']["relay_outbound"]), zmq_subscribe_endpoints=kw['relay_inbound'], zmq_subscribe_method="bind", ) kw.update(moksha_options) # Flip the special bit that allows the RelayConsumer to run kw[RelayConsumer.config_key] = True from moksha.hub import main main(options=kw, consumers=[RelayConsumer])
def run(self): # Check if the user wants the websocket server to run if self.config['moksha.livesocket.websocket.port']: self.config['moksha.livesocket.backend'] = 'websocket' # If the user wants to override any consumers installed on the system # and *only* run the ones they want to, they can do that. consumers = None if self.config['explicit_hub_consumers']: locations = self.config['explicit_hub_consumers'].split(',') locations = [load_class(location) for location in locations] # Rephrase the fedmsg-config.py config as moksha *.ini format. # Note that the hub we kick off here cannot send any message. You # should use fedmsg.publish(...) still for that. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values() ), ) self.config.update(moksha_options) from moksha.hub import main main( # Pass in our config dict options=self.config, # Only run the specified consumers if any are so specified. consumers=consumers, # Tell moksha to quiet its logging. framework=False, )
def hub(**kw): """ Run the fedmsg hub. ``fedmsg-hub`` is the all-purpose daemon. This should be run on every host that has services which declare their own consumers. ``fedmsg-hub`` will listen to every endpoint discovered by :mod:`fedmsg.config` and forward messages in-process to the locally-declared consumers. It is a thin wrapper over a moksha-hub. Other commands like ``fedmsg-irc`` are just specialized, restricted versions of ``fedmsg-hub``. ``fedmsg-hub`` is the most general/abstract. ``fedmsg-hub`` also houses the functions to run a websocket server. """ # Check if the user wants the websocket server to run if kw['moksha.livesocket.websocket.port']: kw['moksha.livesocket.backend'] = 'websocket' # Rephrase the fedmsg-config.py config as moksha *.ini format. # Note that the hub we kick off here cannot send any message. You should # use fedmsg.publish(...) still for that. moksha_options = dict(zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values()), ) kw.update(moksha_options) from moksha.hub import main main(options=kw)
def run(self): # Check if the user wants the websocket server to run if self.config['moksha.livesocket.websocket.port']: self.config['moksha.livesocket.backend'] = 'websocket' # If the user wants to override any consumers installed on the system # and *only* run the ones they want to, they can do that. consumers = None if self.config['explicit_hub_consumers']: locations = self.config['explicit_hub_consumers'].split(',') locations = [load_class(location) for location in locations] # Rephrase the fedmsg-config.py config as moksha *.ini format for # zeromq. If we're not using zeromq (say, we're using STOMP), then just # assume that the moksha configuration is specified correctly already # in /etc/fedmsg.d/ if self.config.get('zmq_enabled', True): moksha_options = dict(zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values()), ) self.config.update(moksha_options) self.set_rlimit_nofiles() # Note that the hub we kick off here cannot send any message. You # should use fedmsg.publish(...) still for that. from moksha.hub import main main( # Pass in our config dict options=self.config, # Only run the specified consumers if any are so specified. consumers=consumers, # Tell moksha to quiet its logging. framework=False, )
def hub(**kw): """ Run the fedmsg hub. ``fedmsg-hub`` is the all-purpose daemon. This should be run on every host that has services which declare their own consumers. ``fedmsg-hub`` will listen to every endpoint discovered by :mod:`fedmsg.config` and forward messages in-process to the locally-declared consumers. It is a thin wrapper over a moksha-hub. Other commands like ``fedmsg-irc`` are just specialized, restricted versions of ``fedmsg-hub``. ``fedmsg-hub`` is the most general/abstract. ``fedmsg-hub`` also houses the functions to run a websocket server. """ # Check if the user wants the websocket server to run if kw['moksha.livesocket.websocket.port']: kw['moksha.livesocket.backend'] = 'websocket' # Rephrase the fedmsg-config.py config as moksha *.ini format. # Note that the hub we kick off here cannot send any message. You should # use fedmsg.publish(...) still for that. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values() ), ) kw.update(moksha_options) from moksha.hub import main main(options=kw)
def gateway(**kw): """ Rebroadcast messages to a special zmq endpoint. A repeater that rebroadcasts all messages received to a special zmq endpoint. This is used to get messages from inside Fedora Infrastructure out to users. Its communication is uni-directional. It does not relay messages from "outside the bus" back in. The special zmq endpoint is specified by the presence of :term:`fedmsg.consumers.gateway.port` in the config. This service is what makes using ":doc:`consuming`" outside the VPN/firewalled bus environment possible. """ # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values() ), ) kw.update(moksha_options) # Flip the special bit that allows the GatewayConsumer to run kw[GatewayConsumer.config_key] = True from moksha.hub import main main(options=kw, consumers=[GatewayConsumer])
def relay(**kw): """ Relay connections from active loggers to the bus. ``fedmsg-relay`` is a service which binds to two ports, listens for messages on one and emits them on the other. ``fedmsg-logger`` requires that an instance of ``fedmsg-relay`` be running *somewhere* and that it's inbound address be listed in the config as one of the entries in :term:`relay_inbound`. ``fedmsg-relay`` becomes a necessity for integration points that cannot bind consistently to and serve from a port. See :doc:`topology` for the mile-high view. More specifically, ``fedmsg-relay`` is a SUB.bind()->PUB.bind() relay. """ # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_publish_endpoints=",".join(kw["endpoints"]["relay_outbound"]), zmq_subscribe_endpoints=",".join(list(iterate(kw["relay_inbound"]))), zmq_subscribe_method="bind", ) kw.update(moksha_options) # Flip the special bit that allows the RelayConsumer to run kw[RelayConsumer.config_key] = True from moksha.hub import main main(options=kw, consumers=[RelayConsumer])
def collectd(**kw): """ Print machine-readable information for collectd to monitor the bus. """ # Initialize the processors before CollectdConsumer is instantiated. fedmsg.text.make_processors(**kw) # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_publish_endpoints=",".join(kw['endpoints']["relay_outbound"]), zmq_subscribe_endpoints=",".join(list(iterate(kw['relay_inbound']))), zmq_subscribe_method="bind", ) kw.update(moksha_options) kw[CollectdConsumer.config_key] = True CollectdProducer.frequency = datetime.timedelta( seconds=kw['collectd_interval'] ) # Turn off moksha logging. logging.disable(logging.INFO) from moksha.hub import main main(kw, [CollectdConsumer], [CollectdProducer])
def run(self): # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to # work with moksha's expected configuration. moksha_options = dict(zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values()), ) self.config.update(moksha_options) self.config[IRCBotConsumer.config_key] = True from moksha.hub import main main(options=self.config, consumers=[IRCBotConsumer])
def run(self): # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py # to work with moksha's expected configuration. moksha_options = dict(zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values()), ) self.config.update(moksha_options) # Flip the special bit that allows the GatewayConsumer to run self.config[GatewayConsumer.config_key] = True from moksha.hub import main main(options=self.config, consumers=[GatewayConsumer])
def run(self): # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to # work with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values() ), ) self.config.update(moksha_options) self.config[IRCBotConsumer.config_key] = True from moksha.hub import main main(options=self.config, consumers=[IRCBotConsumer])
def run(self): # Check if the user wants the websocket server to run if self.config['moksha.livesocket.websocket.port']: self.config['moksha.livesocket.backend'] = 'websocket' # Rephrase the fedmsg-config.py config as moksha *.ini format. # Note that the hub we kick off here cannot send any message. You # should use fedmsg.publish(...) still for that. moksha_options = dict(zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values()), ) self.config.update(moksha_options) from moksha.hub import main main(options=self.config)
def notify(**kw): """ Send out desktop notifications over zmq """ moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values()), ) kw.update(moksha_options) kw['fedmsg.consumers.notifyconsumer.enabled'] = True from moksha.hub import main main(options=kw, consumers=[NotifyConsumer])
def run(self): # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_publish_endpoints=",".join(self.config['endpoints']["relay_outbound"]), zmq_subscribe_endpoints=",".join(list(iterate(self.config['relay_inbound']))), zmq_subscribe_method="bind", ) self.config.update(moksha_options) # Flip the special bit that allows the RelayConsumer to run self.config[RelayConsumer.config_key] = True from moksha.hub import main main(options=self.config, consumers=[RelayConsumer])
def run(self): moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values() ), ) self.config.update(moksha_options) self.config['fedmsg.consumers.badges.examplebadge.enabled'] = True from moksha.hub import main main( options=self.config, # If you omit this argument, it will pick up *all* consumers. consumers=[ExampleBadgesConsumer], producers=[], )
def hub(**kw): """ Run the fedmsg hub. """ # Check if the user wants the websocket server to run if kw['moksha.livesocket.websocket.port']: kw['moksha.livesocket.backend'] = 'websocket' # Rephrase the fedmsg-config.py config as moksha *.ini format. # Note that the hub we kick off here cannot send any message. You should # use fedmsg.send_message(...) still for that. moksha_options = dict(zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values()), ) kw.update(moksha_options) from moksha.hub import main main(options=kw)
def run(self): # Do just like in fedmsg.commands.hub and mangle fedmsg.d/ to work # with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=",".join(list(iterate( self.config['relay_inbound'] ))), zmq_subscribe_method="bind", ) self.config.update(moksha_options) # Flip the special bit that allows the RelayConsumer to run self.config[RelayConsumer.config_key] = True from moksha.hub import main for publish_endpoint in self.config['endpoints']['relay_outbound']: self.config['zmq_publish_endpoints'] = publish_endpoint try: return main( # Pass in our config dict options=self.config, # Only run this *one* consumer consumers=[RelayConsumer], # Tell moksha to quiet its logging. framework=False, ) except zmq.ZMQError: self.log.debug("Failed to bind to %r" % publish_endpoint) raise IOError("Failed to bind to any outbound endpoints.")
def run(self): # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py # to work with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values() ), ) self.config.update(moksha_options) # Flip the special bit that allows the GatewayConsumer to run self.config[GatewayConsumer.config_key] = True from moksha.hub import main main(options=self.config, consumers=[GatewayConsumer])
def run(self): # Do just like in fedmsg.commands.hub and mangle fedmsg.d/ to work # with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=",".join( list(iterate(self.config['relay_inbound']))), zmq_subscribe_method="bind", ) self.config.update(moksha_options) # Flip the special bit that allows the RelayConsumer to run self.config[RelayConsumer.config_key] = True from moksha.hub import main for publish_endpoint in self.config['endpoints']['relay_outbound']: self.config['zmq_publish_endpoints'] = publish_endpoint try: return main( # Pass in our config dict options=self.config, # Only run this *one* consumer consumers=[RelayConsumer], # Tell moksha to quiet its logging. framework=False, ) except zmq.ZMQError: self.log.debug("Failed to bind to %r" % publish_endpoint) raise IOError("Failed to bind to any outbound endpoints.")
def ircbot(**kw): """ Relay connections from active loggers to the bus. """ # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values() ), ) kw.update(moksha_options) kw['fedmsg.consumers.ircbot.enabled'] = True from moksha.hub import main main(options=kw)
def ircbot(**kw): """ Relay messages from the bus to any number of IRC channels. This is highly configurable by way of the :term:`irc` config value. """ # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict(zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values()), ) kw.update(moksha_options) kw[IRCBotConsumer.config_key] = True from moksha.hub import main main(options=kw, consumers=[IRCBotConsumer])
def hub(**kw): """ Run the fedmsg hub. """ # Check if the user wants the websocket server to run if 'moksha.livesocket.websocket.port' in kw: kw['moksha.livesocket.backend'] = 'websocket' # Rephrase the fedmsg-config.py config as moksha *.ini format. # Note that the hub we kick off here cannot send any message. You should # use fedmsg.send_message(...) still for that. moksha_options = dict( zmq_subscribe_endpoints=','.join(kw['endpoints'].values()), ) kw.update(moksha_options) from moksha.hub import main main(options=kw)
def run(self): # First, sanity checking. if not self.config.get('tweet_endpoints', None): raise ValueError("Not configured to tweet.") # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to # work with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values() ), ) self.config.update(moksha_options) self.config[TweetBotConsumer.config_key] = True from moksha.hub import main main(options=self.config, consumers=[TweetBotConsumer])
def relay(**kw): """ Relay connections from active loggers to the bus. """ # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_publish_endpoints=",".join(kw['endpoints']["relay_outbound"]), zmq_subscribe_endpoints=kw['relay_inbound'], zmq_subscribe_method="bind", ) kw.update(moksha_options) # Flip the special bit that allows the RelayConsumer to run kw['fedmsg.consumers.relay.enabled'] = True from moksha.hub import main main(options=kw)
def run(self): # Check if the user wants the websocket server to run if self.config['moksha.livesocket.websocket.port']: self.config['moksha.livesocket.backend'] = 'websocket' # Rephrase the fedmsg-config.py config as moksha *.ini format. # Note that the hub we kick off here cannot send any message. You # should use fedmsg.publish(...) still for that. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values() ), ) self.config.update(moksha_options) from moksha.hub import main main(options=self.config)
def run(self): # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to # work with moksha's expected configuration. moksha_options = dict(zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values()), ) self.config.update(moksha_options) self.config[IRCBotConsumer.config_key] = True from moksha.hub import main main( # Pass in our config dict options=self.config, # Only run this *one* consumer consumers=[IRCBotConsumer], # Tell moksah to quiet its logging framework=False, )
def download(**kw): """ Watch the compose topic for rsync.complete messages This is highly configurable by way of the :term:`download` config value. """ # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values() ), ) kw.update(moksha_options) kw[RsyncConsumer.config_key] = True from moksha.hub import main main(options=kw, consumers=[RsyncConsumer])
def ircbot(**kw): """ Relay messages from the bus to any number of IRC channels. This is highly configurable by way of the :term:`irc` config value. """ # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in kw['endpoints'].values() ), ) kw.update(moksha_options) kw[IRCBotConsumer.config_key] = True from moksha.hub import main main(options=kw, consumers=[IRCBotConsumer])
def run(self): # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to # work with moksha's expected configuration. moksha_options = dict( zmq_subscribe_endpoints=",".join(",".join(bunch) for bunch in self.config["endpoints"].values()) ) self.config.update(moksha_options) self.config[IRCBotConsumer.config_key] = True from moksha.hub import main main( # Pass in our config dict options=self.config, # Only run this *one* consumer consumers=[IRCBotConsumer], # Tell moksah to quiet its logging framework=False, )
def run(self): # Initialize the processors before CollectdConsumer is instantiated. fedmsg.meta.make_processors(**self.config) # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py # to work with moksha's expected configuration. moksha_options = dict( mute=True, # Disable some warnings. zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values()), ) self.config.update(moksha_options) self.config[CollectdConsumer.config_key] = True CollectdProducer.frequency = datetime.timedelta( seconds=self.config['collectd_interval']) from moksha.hub import main main(self.config, [CollectdConsumer], [CollectdProducer], framework=False)
def run(self): # Initialize the processors before CollectdConsumer is instantiated. fedmsg.meta.make_processors(**self.config) # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py # to work with moksha's expected configuration. moksha_options = dict( mute=True, # Disable some warnings. zmq_subscribe_endpoints=','.join( ','.join(bunch) for bunch in self.config['endpoints'].values() ), ) self.config.update(moksha_options) self.config[CollectdConsumer.config_key] = True CollectdProducer.frequency = datetime.timedelta( seconds=self.config['collectd_interval'] ) from moksha.hub import main main(self.config, [CollectdConsumer], [CollectdProducer], framework=False)
def collectd(**kw): """ Print machine-readable information for collectd to monitor the bus. """ # Initialize the processors before CollectdConsumer is instantiated. fedmsg.text.make_processors(**kw) # Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work # with moksha's expected configuration. moksha_options = dict( zmq_publish_endpoints=",".join(kw['endpoints']["relay_outbound"]), zmq_subscribe_endpoints=",".join(list(iterate(kw['relay_inbound']))), zmq_subscribe_method="bind", ) kw.update(moksha_options) kw[CollectdConsumer.config_key] = True CollectdProducer.frequency = datetime.timedelta( seconds=kw['collectd_interval']) # Turn off moksha logging. logging.disable(logging.INFO) from moksha.hub import main main(kw, [CollectdConsumer], [CollectdProducer])
self.producers.append(producer_obj) except Exception as e: log.warn("Failed to init %r producer." % producer_class) log.warn(str(e)) @trace def create_topic(self, topic): if AMQPHubExtension and self.amqp_broker: AMQPHubExtension.create_queue(topic) # @@ remove this when we keep track of this in a DB if topic not in self.topics: self.topics[topic] = [] def stop(self): log.debug("Stopping the CentralMokshaHub") MokshaHub.close(self) if self.producers: for producer in self.producers: log.debug("Stopping producer %s" % producer) producer.stop() if self.consumers: for consumer in self.consumers: log.debug("Stopping consumer %s" % consumer) consumer.stop() if __name__ == '__main__': from moksha.hub import main main()
# @@ remove this when we keep track of this in a DB if topic not in self.topics: self.topics[topic] = [] def close(self): log.debug("Stopping the CentralMokshaHub") super(CentralMokshaHub, self).close() if self.producers: while self.producers: producer = self.producers.pop() log.debug("Stopping producer %s" % producer) producer.stop() if self.consumers: while self.consumers: consumer = self.consumers.pop() log.debug("Stopping consumer %s" % consumer) consumer.stop() if hasattr(self, 'websocket_server'): retval = self.websocket_server.stopListening() # For backwards compatibility stop = close if __name__ == '__main__': from moksha.hub import main main()