def gateway(**kw):
""" Rebroadcast messages to a special zmq endpoint.
A repeater that rebroadcasts all messages received to a special zmq
endpoint. This is used to get messages from inside Fedora Infrastructure
out to users. Its communication is uni-directional. It does not relay
messages from "outside the bus" back in.
The special zmq endpoint is specified by the presence of
:term:`fedmsg.consumers.gateway.port` in the config.
This service is what makes using ":doc:`consuming`" outside the
VPN/firewalled bus environment possible.
"""
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in kw['endpoints'].values()), )
kw.update(moksha_options)
# Flip the special bit that allows the GatewayConsumer to run
kw[GatewayConsumer.config_key] = True
from moksha.hub import main
main(options=kw, consumers=[GatewayConsumer])
def relay(**kw):
""" Relay connections from active loggers to the bus.
``fedmsg-relay`` is a service which binds to two ports, listens for
messages on one and emits them on the other. ``fedmsg-logger``
requires that an instance of ``fedmsg-relay`` be running *somewhere*
and that it's inbound address be listed in the config as
:term:`relay_inbound`.
``fedmsg-relay`` becomes a necessity for integration points that cannot
bind consistently to and serve from a port. See :doc:`topology` for the
mile-high view. More specifically, ``fedmsg-relay`` is a
SUB.bind()->PUB.bind() relay.
"""
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_publish_endpoints=",".join(kw['endpoints']["relay_outbound"]),
zmq_subscribe_endpoints=kw['relay_inbound'],
zmq_subscribe_method="bind",
)
kw.update(moksha_options)
# Flip the special bit that allows the RelayConsumer to run
kw[RelayConsumer.config_key] = True
from moksha.hub import main
main(options=kw, consumers=[RelayConsumer])
def run(self):
# Check if the user wants the websocket server to run
if self.config['moksha.livesocket.websocket.port']:
self.config['moksha.livesocket.backend'] = 'websocket'
# If the user wants to override any consumers installed on the system
# and *only* run the ones they want to, they can do that.
consumers = None
if self.config['explicit_hub_consumers']:
locations = self.config['explicit_hub_consumers'].split(',')
locations = [load_class(location) for location in locations]
# Rephrase the fedmsg-config.py config as moksha *.ini format.
# Note that the hub we kick off here cannot send any message. You
# should use fedmsg.publish(...) still for that.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()
),
)
self.config.update(moksha_options)
from moksha.hub import main
main(
# Pass in our config dict
options=self.config,
# Only run the specified consumers if any are so specified.
consumers=consumers,
# Tell moksha to quiet its logging.
framework=False,
)
def hub(**kw):
""" Run the fedmsg hub.
``fedmsg-hub`` is the all-purpose daemon. This should be run on every host
that has services which declare their own consumers. ``fedmsg-hub`` will
listen to every endpoint discovered by :mod:`fedmsg.config` and forward
messages in-process to the locally-declared consumers. It is a thin wrapper
over a moksha-hub.
Other commands like ``fedmsg-irc`` are just specialized, restricted
versions of ``fedmsg-hub``. ``fedmsg-hub`` is the most general/abstract.
``fedmsg-hub`` also houses the functions to run a websocket server.
"""
# Check if the user wants the websocket server to run
if kw['moksha.livesocket.websocket.port']:
kw['moksha.livesocket.backend'] = 'websocket'
# Rephrase the fedmsg-config.py config as moksha *.ini format.
# Note that the hub we kick off here cannot send any message. You should
# use fedmsg.publish(...) still for that.
moksha_options = dict(zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in kw['endpoints'].values()), )
kw.update(moksha_options)
from moksha.hub import main
main(options=kw)
def run(self):
# Check if the user wants the websocket server to run
if self.config['moksha.livesocket.websocket.port']:
self.config['moksha.livesocket.backend'] = 'websocket'
# If the user wants to override any consumers installed on the system
# and *only* run the ones they want to, they can do that.
consumers = None
if self.config['explicit_hub_consumers']:
locations = self.config['explicit_hub_consumers'].split(',')
locations = [load_class(location) for location in locations]
# Rephrase the fedmsg-config.py config as moksha *.ini format for
# zeromq. If we're not using zeromq (say, we're using STOMP), then just
# assume that the moksha configuration is specified correctly already
# in /etc/fedmsg.d/
if self.config.get('zmq_enabled', True):
moksha_options = dict(zmq_subscribe_endpoints=','.join(
','.join(bunch)
for bunch in self.config['endpoints'].values()), )
self.config.update(moksha_options)
self.set_rlimit_nofiles()
# Note that the hub we kick off here cannot send any message. You
# should use fedmsg.publish(...) still for that.
from moksha.hub import main
main(
# Pass in our config dict
options=self.config,
# Only run the specified consumers if any are so specified.
consumers=consumers,
# Tell moksha to quiet its logging.
framework=False,
)
def hub(**kw):
""" Run the fedmsg hub.
``fedmsg-hub`` is the all-purpose daemon. This should be run on every host
that has services which declare their own consumers. ``fedmsg-hub`` will
listen to every endpoint discovered by :mod:`fedmsg.config` and forward
messages in-process to the locally-declared consumers. It is a thin wrapper
over a moksha-hub.
Other commands like ``fedmsg-irc`` are just specialized, restricted
versions of ``fedmsg-hub``. ``fedmsg-hub`` is the most general/abstract.
``fedmsg-hub`` also houses the functions to run a websocket server.
"""
# Check if the user wants the websocket server to run
if kw['moksha.livesocket.websocket.port']:
kw['moksha.livesocket.backend'] = 'websocket'
# Rephrase the fedmsg-config.py config as moksha *.ini format.
# Note that the hub we kick off here cannot send any message. You should
# use fedmsg.publish(...) still for that.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in kw['endpoints'].values()
),
)
kw.update(moksha_options)
from moksha.hub import main
main(options=kw)
def gateway(**kw):
""" Rebroadcast messages to a special zmq endpoint.
A repeater that rebroadcasts all messages received to a special zmq
endpoint. This is used to get messages from inside Fedora Infrastructure
out to users. Its communication is uni-directional. It does not relay
messages from "outside the bus" back in.
The special zmq endpoint is specified by the presence of
:term:`fedmsg.consumers.gateway.port` in the config.
This service is what makes using ":doc:`consuming`" outside the
VPN/firewalled bus environment possible.
"""
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in
kw['endpoints'].values()
),
)
kw.update(moksha_options)
# Flip the special bit that allows the GatewayConsumer to run
kw[GatewayConsumer.config_key] = True
from moksha.hub import main
main(options=kw, consumers=[GatewayConsumer])
def relay(**kw):
""" Relay connections from active loggers to the bus.
``fedmsg-relay`` is a service which binds to two ports, listens for
messages on one and emits them on the other. ``fedmsg-logger``
requires that an instance of ``fedmsg-relay`` be running *somewhere*
and that it's inbound address be listed in the config as one of the entries
in :term:`relay_inbound`.
``fedmsg-relay`` becomes a necessity for integration points that cannot
bind consistently to and serve from a port. See :doc:`topology` for the
mile-high view. More specifically, ``fedmsg-relay`` is a
SUB.bind()->PUB.bind() relay.
"""
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_publish_endpoints=",".join(kw["endpoints"]["relay_outbound"]),
zmq_subscribe_endpoints=",".join(list(iterate(kw["relay_inbound"]))),
zmq_subscribe_method="bind",
)
kw.update(moksha_options)
# Flip the special bit that allows the RelayConsumer to run
kw[RelayConsumer.config_key] = True
from moksha.hub import main
main(options=kw, consumers=[RelayConsumer])
def collectd(**kw):
""" Print machine-readable information for collectd to monitor the bus. """
# Initialize the processors before CollectdConsumer is instantiated.
fedmsg.text.make_processors(**kw)
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_publish_endpoints=",".join(kw['endpoints']["relay_outbound"]),
zmq_subscribe_endpoints=",".join(list(iterate(kw['relay_inbound']))),
zmq_subscribe_method="bind",
)
kw.update(moksha_options)
kw[CollectdConsumer.config_key] = True
CollectdProducer.frequency = datetime.timedelta(
seconds=kw['collectd_interval']
)
# Turn off moksha logging.
logging.disable(logging.INFO)
from moksha.hub import main
main(kw, [CollectdConsumer], [CollectdProducer])
def run(self):
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to
# work with moksha's expected configuration.
moksha_options = dict(zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()), )
self.config.update(moksha_options)
self.config[IRCBotConsumer.config_key] = True
from moksha.hub import main
main(options=self.config, consumers=[IRCBotConsumer])
def run(self):
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py
# to work with moksha's expected configuration.
moksha_options = dict(zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()), )
self.config.update(moksha_options)
# Flip the special bit that allows the GatewayConsumer to run
self.config[GatewayConsumer.config_key] = True
from moksha.hub import main
main(options=self.config, consumers=[GatewayConsumer])
def run(self):
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to
# work with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()
),
)
self.config.update(moksha_options)
self.config[IRCBotConsumer.config_key] = True
from moksha.hub import main
main(options=self.config, consumers=[IRCBotConsumer])
def run(self):
# Check if the user wants the websocket server to run
if self.config['moksha.livesocket.websocket.port']:
self.config['moksha.livesocket.backend'] = 'websocket'
# Rephrase the fedmsg-config.py config as moksha *.ini format.
# Note that the hub we kick off here cannot send any message. You
# should use fedmsg.publish(...) still for that.
moksha_options = dict(zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()), )
self.config.update(moksha_options)
from moksha.hub import main
main(options=self.config)
def notify(**kw):
"""
Send out desktop notifications over zmq
"""
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in kw['endpoints'].values()),
)
kw.update(moksha_options)
kw['fedmsg.consumers.notifyconsumer.enabled'] = True
from moksha.hub import main
main(options=kw, consumers=[NotifyConsumer])
def run(self):
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_publish_endpoints=",".join(self.config['endpoints']["relay_outbound"]),
zmq_subscribe_endpoints=",".join(list(iterate(self.config['relay_inbound']))),
zmq_subscribe_method="bind",
)
self.config.update(moksha_options)
# Flip the special bit that allows the RelayConsumer to run
self.config[RelayConsumer.config_key] = True
from moksha.hub import main
main(options=self.config, consumers=[RelayConsumer])
def run(self):
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()
),
)
self.config.update(moksha_options)
self.config['fedmsg.consumers.badges.examplebadge.enabled'] = True
from moksha.hub import main
main(
options=self.config,
# If you omit this argument, it will pick up *all* consumers.
consumers=[ExampleBadgesConsumer],
producers=[],
)
def hub(**kw):
""" Run the fedmsg hub. """
# Check if the user wants the websocket server to run
if kw['moksha.livesocket.websocket.port']:
kw['moksha.livesocket.backend'] = 'websocket'
# Rephrase the fedmsg-config.py config as moksha *.ini format.
# Note that the hub we kick off here cannot send any message. You should
# use fedmsg.send_message(...) still for that.
moksha_options = dict(zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in kw['endpoints'].values()), )
kw.update(moksha_options)
from moksha.hub import main
main(options=kw)
def run(self):
# Do just like in fedmsg.commands.hub and mangle fedmsg.d/ to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=",".join(list(iterate(
self.config['relay_inbound']
))),
zmq_subscribe_method="bind",
)
self.config.update(moksha_options)
# Flip the special bit that allows the RelayConsumer to run
self.config[RelayConsumer.config_key] = True
from moksha.hub import main
for publish_endpoint in self.config['endpoints']['relay_outbound']:
self.config['zmq_publish_endpoints'] = publish_endpoint
try:
return main(
# Pass in our config dict
options=self.config,
# Only run this *one* consumer
consumers=[RelayConsumer],
# Tell moksha to quiet its logging.
framework=False,
)
except zmq.ZMQError:
self.log.debug("Failed to bind to %r" % publish_endpoint)
raise IOError("Failed to bind to any outbound endpoints.")
def run(self):
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py
# to work with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in
self.config['endpoints'].values()
),
)
self.config.update(moksha_options)
# Flip the special bit that allows the GatewayConsumer to run
self.config[GatewayConsumer.config_key] = True
from moksha.hub import main
main(options=self.config, consumers=[GatewayConsumer])
def run(self):
# Do just like in fedmsg.commands.hub and mangle fedmsg.d/ to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=",".join(
list(iterate(self.config['relay_inbound']))),
zmq_subscribe_method="bind",
)
self.config.update(moksha_options)
# Flip the special bit that allows the RelayConsumer to run
self.config[RelayConsumer.config_key] = True
from moksha.hub import main
for publish_endpoint in self.config['endpoints']['relay_outbound']:
self.config['zmq_publish_endpoints'] = publish_endpoint
try:
return main(
# Pass in our config dict
options=self.config,
# Only run this *one* consumer
consumers=[RelayConsumer],
# Tell moksha to quiet its logging.
framework=False,
)
except zmq.ZMQError:
self.log.debug("Failed to bind to %r" % publish_endpoint)
raise IOError("Failed to bind to any outbound endpoints.")
def ircbot(**kw):
""" Relay connections from active loggers to the bus. """
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in kw['endpoints'].values()
),
)
kw.update(moksha_options)
kw['fedmsg.consumers.ircbot.enabled'] = True
from moksha.hub import main
main(options=kw)
def ircbot(**kw):
""" Relay messages from the bus to any number of IRC channels.
This is highly configurable by way of the :term:`irc` config value.
"""
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in kw['endpoints'].values()), )
kw.update(moksha_options)
kw[IRCBotConsumer.config_key] = True
from moksha.hub import main
main(options=kw, consumers=[IRCBotConsumer])
def hub(**kw):
""" Run the fedmsg hub. """
# Check if the user wants the websocket server to run
if 'moksha.livesocket.websocket.port' in kw:
kw['moksha.livesocket.backend'] = 'websocket'
# Rephrase the fedmsg-config.py config as moksha *.ini format.
# Note that the hub we kick off here cannot send any message. You should
# use fedmsg.send_message(...) still for that.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(kw['endpoints'].values()),
)
kw.update(moksha_options)
from moksha.hub import main
main(options=kw)
def run(self):
# First, sanity checking.
if not self.config.get('tweet_endpoints', None):
raise ValueError("Not configured to tweet.")
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to
# work with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()
),
)
self.config.update(moksha_options)
self.config[TweetBotConsumer.config_key] = True
from moksha.hub import main
main(options=self.config, consumers=[TweetBotConsumer])
def relay(**kw):
""" Relay connections from active loggers to the bus. """
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_publish_endpoints=",".join(kw['endpoints']["relay_outbound"]),
zmq_subscribe_endpoints=kw['relay_inbound'],
zmq_subscribe_method="bind",
)
kw.update(moksha_options)
# Flip the special bit that allows the RelayConsumer to run
kw['fedmsg.consumers.relay.enabled'] = True
from moksha.hub import main
main(options=kw)
def run(self):
# Check if the user wants the websocket server to run
if self.config['moksha.livesocket.websocket.port']:
self.config['moksha.livesocket.backend'] = 'websocket'
# Rephrase the fedmsg-config.py config as moksha *.ini format.
# Note that the hub we kick off here cannot send any message. You
# should use fedmsg.publish(...) still for that.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()
),
)
self.config.update(moksha_options)
from moksha.hub import main
main(options=self.config)
def run(self):
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to
# work with moksha's expected configuration.
moksha_options = dict(zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()), )
self.config.update(moksha_options)
self.config[IRCBotConsumer.config_key] = True
from moksha.hub import main
main(
# Pass in our config dict
options=self.config,
# Only run this *one* consumer
consumers=[IRCBotConsumer],
# Tell moksah to quiet its logging
framework=False,
)
def download(**kw):
""" Watch the compose topic for rsync.complete messages
This is highly configurable by way of the :term:`download` config value.
"""
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in kw['endpoints'].values()
),
)
kw.update(moksha_options)
kw[RsyncConsumer.config_key] = True
from moksha.hub import main
main(options=kw, consumers=[RsyncConsumer])
def ircbot(**kw):
""" Relay messages from the bus to any number of IRC channels.
This is highly configurable by way of the :term:`irc` config value.
"""
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in kw['endpoints'].values()
),
)
kw.update(moksha_options)
kw[IRCBotConsumer.config_key] = True
from moksha.hub import main
main(options=kw, consumers=[IRCBotConsumer])
def run(self):
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to
# work with moksha's expected configuration.
moksha_options = dict(
zmq_subscribe_endpoints=",".join(",".join(bunch) for bunch in self.config["endpoints"].values())
)
self.config.update(moksha_options)
self.config[IRCBotConsumer.config_key] = True
from moksha.hub import main
main(
# Pass in our config dict
options=self.config,
# Only run this *one* consumer
consumers=[IRCBotConsumer],
# Tell moksah to quiet its logging
framework=False,
)
def run(self):
# Initialize the processors before CollectdConsumer is instantiated.
fedmsg.meta.make_processors(**self.config)
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py
# to work with moksha's expected configuration.
moksha_options = dict(
mute=True, # Disable some warnings.
zmq_subscribe_endpoints=','.join(
','.join(bunch)
for bunch in self.config['endpoints'].values()),
)
self.config.update(moksha_options)
self.config[CollectdConsumer.config_key] = True
CollectdProducer.frequency = datetime.timedelta(
seconds=self.config['collectd_interval'])
from moksha.hub import main
main(self.config, [CollectdConsumer], [CollectdProducer],
framework=False)
def run(self):
# Initialize the processors before CollectdConsumer is instantiated.
fedmsg.meta.make_processors(**self.config)
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py
# to work with moksha's expected configuration.
moksha_options = dict(
mute=True, # Disable some warnings.
zmq_subscribe_endpoints=','.join(
','.join(bunch) for bunch in self.config['endpoints'].values()
),
)
self.config.update(moksha_options)
self.config[CollectdConsumer.config_key] = True
CollectdProducer.frequency = datetime.timedelta(
seconds=self.config['collectd_interval']
)
from moksha.hub import main
main(self.config, [CollectdConsumer], [CollectdProducer],
framework=False)
def collectd(**kw):
""" Print machine-readable information for collectd to monitor the bus. """
# Initialize the processors before CollectdConsumer is instantiated.
fedmsg.text.make_processors(**kw)
# Do just like in fedmsg.commands.hub and mangle fedmsg-config.py to work
# with moksha's expected configuration.
moksha_options = dict(
zmq_publish_endpoints=",".join(kw['endpoints']["relay_outbound"]),
zmq_subscribe_endpoints=",".join(list(iterate(kw['relay_inbound']))),
zmq_subscribe_method="bind",
)
kw.update(moksha_options)
kw[CollectdConsumer.config_key] = True
CollectdProducer.frequency = datetime.timedelta(
seconds=kw['collectd_interval'])
# Turn off moksha logging.
logging.disable(logging.INFO)
from moksha.hub import main
main(kw, [CollectdConsumer], [CollectdProducer])
self.producers.append(producer_obj)
except Exception as e:
log.warn("Failed to init %r producer." % producer_class)
log.warn(str(e))
@trace
def create_topic(self, topic):
if AMQPHubExtension and self.amqp_broker:
AMQPHubExtension.create_queue(topic)
# @@ remove this when we keep track of this in a DB
if topic not in self.topics:
self.topics[topic] = []
def stop(self):
log.debug("Stopping the CentralMokshaHub")
MokshaHub.close(self)
if self.producers:
for producer in self.producers:
log.debug("Stopping producer %s" % producer)
producer.stop()
if self.consumers:
for consumer in self.consumers:
log.debug("Stopping consumer %s" % consumer)
consumer.stop()
if __name__ == '__main__':
from moksha.hub import main
main()
# @@ remove this when we keep track of this in a DB
if topic not in self.topics:
self.topics[topic] = []
def close(self):
log.debug("Stopping the CentralMokshaHub")
super(CentralMokshaHub, self).close()
if self.producers:
while self.producers:
producer = self.producers.pop()
log.debug("Stopping producer %s" % producer)
producer.stop()
if self.consumers:
while self.consumers:
consumer = self.consumers.pop()
log.debug("Stopping consumer %s" % consumer)
consumer.stop()
if hasattr(self, 'websocket_server'):
retval = self.websocket_server.stopListening()
# For backwards compatibility
stop = close
if __name__ == '__main__':
from moksha.hub import main
main()
