def _run_socket(self, sock, name, ep, watched_names=None):
if watched_names is None:
watched_names = {}
validate = self.c.get('validate_signatures', False)
# Grab the data off the zeromq internal queue
_topic, message = sock.recv_multipart()
# zmq hands us byte strings, so let's convert to unicode asap
_topic, message = _topic.decode('utf-8'), message.decode('utf-8')
# Now, decode the JSON body into a dict.
msg = fedmsg.encoding.loads(message)
if not validate or fedmsg.crypto.validate(msg, **self.c):
# If there is even a slight change of replay, use
# check_for_replay
if len(self.c.get('replay_endpoints', {})) > 0:
for m in check_for_replay(
name, watched_names,
msg, self.c, self.context):
# Revalidate all the replayed messages.
if not validate or \
fedmsg.crypto.validate(m, **self.c):
return name, ep, m['topic'], m
else:
raise ValidationError(msg)
else:
return name, ep, _topic, msg
else:
raise ValidationError(msg)
def _consume(self, message):
# Massage STOMP messages into a more compatible format.
if 'topic' not in message['body']:
message['body'] = {
'topic': message.get('topic'),
'msg': message['body'],
}
try:
self.validate(message)
except RuntimeWarning as e:
self.log.warn("Received invalid message {0}".format(e))
return
if hasattr(self, "replay_name"):
for m in check_for_replay(self.replay_name, self.name_to_seq_id,
message, self.hub.config):
try:
self.validate(m)
super(FedmsgConsumer, self)._consume(m)
except RuntimeWarning as e:
self.log.warn("Received invalid message {}".format(e))
else:
super(FedmsgConsumer, self)._consume(message)
def _run_socket(self, sock, name, ep, watched_names=None):
if watched_names is None:
watched_names = {}
validate = self.c.get('validate_signatures', False)
_topic, message = sock.recv_multipart()
msg = fedmsg.encoding.loads(message)
if not validate or fedmsg.crypto.validate(msg, **self.c):
# If there is even a slight change of replay, use
# check_for_replay
if len(self.c.get('replay_endpoints', {})) > 0:
for m in check_for_replay(
name, watched_names,
msg, self.c, self.context):
# Revalidate all the replayed messages.
if not validate or \
fedmsg.crypto.validate(m, **self.c):
return name, ep, m['topic'], m
else:
raise ValidationError(msg)
else:
return name, ep, _topic, msg
else:
raise ValidationError(msg)
def _consume(self, message):
try:
self.validate(message)
except RuntimeWarning as e:
self.log.warn("Received invalid message {0}".format(e))
return
if hasattr(self, "replay_name"):
for m in check_for_replay(self.replay_name, self.name_to_seq_id, message, self.hub.config):
try:
self.validate(m)
super(FedmsgConsumer, self)._consume(m)
except RuntimeWarning as e:
self.log.warn("Received invalid message {}".format(e))
else:
super(FedmsgConsumer, self)._consume(message)
def _consume(self, message):
try:
self.validate(message)
except RuntimeWarning as e:
self.log.warn("Received invalid message {}".format(e))
return
if hasattr(self, "replay_name"):
for m in check_for_replay(self.replay_name, self.name_to_seq_id,
message, self.hub.config):
try:
self.validate(m)
self.consume(m)
except RuntimeWarning as e:
self.log.warn("Received invalid message {}".format(e))
else:
self.consume(message)
def tail_messages(self, topic="", passive=False, **kw):
""" Tail messages on the bus.
Generator that yields tuples of the form:
``(name, endpoint, topic, message)``
"""
# TODO -- do the zmq_strict logic dance with "topic" here.
# It is buried in moksha.hub, but we need it to work the same way
# here.
# TODO -- the 'passive' here and the 'active' are ambiguous. They
# don't actually mean the same thing. This should be resolved.
method = passive and 'bind' or 'connect'
failed_hostnames = []
subs = {}
watched_names = {}
for _name, endpoint_list in self.c['endpoints'].iteritems():
# Listify endpoint_list in case it is a single string
endpoint_list = iterate(endpoint_list)
for endpoint in endpoint_list:
# First, some sanity checking. zeromq will potentially
# segfault if we don't do this check.
hostname = endpoint.split(':')[1][2:]
if hostname in failed_hostnames:
continue
if hostname != '*':
try:
socket.gethostbyname_ex(hostname)
except:
failed_hostnames.append(hostname)
self.log.warn("Couldn't resolve %r" % hostname)
continue
# OK, sanity checks pass. Create the subscriber and connect.
subscriber = self.context.socket(zmq.SUB)
subscriber.setsockopt(zmq.SUBSCRIBE, topic)
set_high_water_mark(subscriber, self.c)
set_tcp_keepalive(subscriber, self.c)
getattr(subscriber, method)(endpoint)
subs[subscriber] = (_name, endpoint)
if _name in self.c.get("replay_endpoints", {}):
# At first we don't know where the sequence is at.
watched_names[_name] = -1
# Register the sockets we just built with a zmq Poller.
poller = zmq.Poller()
for subscriber in subs:
poller.register(subscriber, zmq.POLLIN)
# TODO -- what if user wants to pass in validate_signatures in **kw?
validate = self.c.get('validate_signatures', False)
# Poll that poller. This is much more efficient than it used to be.
try:
while True:
sockets = dict(poller.poll())
for s in sockets:
_name, ep = subs[s]
_topic, message = s.recv_multipart()
msg = fedmsg.encoding.loads(message)
if not validate or fedmsg.crypto.validate(msg, **self.c):
# If there is even a slight change of replay, use
# check_for_replay
if len(self.c.get('replay_endpoints', {})) > 0:
for m in check_for_replay(
_name, watched_names,
msg, self.c, self.context):
# Revalidate all the replayed messages.
if not validate or \
fedmsg.crypto.validate(m, **self.c):
yield _name, ep, m['topic'], m
else:
warnings.warn("!! invalid message " +
"received: %r" % msg)
else:
yield _name, ep, _topic, msg
else:
# Else.. we are supposed to be validating, but the
# message failed validation.
# Warn, but don't throw an exception. Keep tailing.
warnings.warn("!! invalid message received: %r" % msg)
finally:
for subscriber in subs:
subscriber.close()
def tail_messages(self, topic="", passive=False, **kw):
""" Tail messages on the bus.
Generator that yields tuples of the form:
``(name, endpoint, topic, message)``
"""
# TODO -- do the zmq_strict logic dance with "topic" here.
# It is buried in moksha.hub, but we need it to work the same way
# here.
# TODO -- the 'passive' here and the 'active' are ambiguous. They
# don't actually mean the same thing. This should be resolved.
method = passive and 'bind' or 'connect'
failed_hostnames = []
subs = {}
watched_names = {}
for _name, endpoint_list in six.iteritems(self.c['endpoints']):
# You never want to actually subscribe to this thing, but sometimes
# it appears in the endpoints list due to a hack where it gets
# added in __init__ above.
if _name == 'relay_inbound':
continue
# Listify endpoint_list in case it is a single string
endpoint_list = iterate(endpoint_list)
for endpoint in endpoint_list:
# First, some sanity checking. zeromq will potentially
# segfault if we don't do this check.
hostname = endpoint.split(':')[1][2:]
if hostname in failed_hostnames:
continue
if hostname != '*':
try:
socket.gethostbyname_ex(hostname)
except:
failed_hostnames.append(hostname)
self.log.warn("Couldn't resolve %r" % hostname)
continue
# OK, sanity checks pass. Create the subscriber and connect.
subscriber = self.context.socket(zmq.SUB)
subscriber.setsockopt(zmq.SUBSCRIBE, topic)
set_high_water_mark(subscriber, self.c)
set_tcp_keepalive(subscriber, self.c)
set_tcp_reconnect(subscriber, self.c)
getattr(subscriber, method)(endpoint)
subs[subscriber] = (_name, endpoint)
if _name in self.c.get("replay_endpoints", {}):
# At first we don't know where the sequence is at.
watched_names[_name] = -1
# Register the sockets we just built with a zmq Poller.
poller = zmq.Poller()
for subscriber in subs:
poller.register(subscriber, zmq.POLLIN)
# TODO -- what if user wants to pass in validate_signatures in **kw?
validate = self.c.get('validate_signatures', False)
# Poll that poller. This is much more efficient than it used to be.
try:
while True:
sockets = dict(poller.poll())
for s in sockets:
_name, ep = subs[s]
_topic, message = s.recv_multipart()
msg = fedmsg.encoding.loads(message)
if not validate or fedmsg.crypto.validate(msg, **self.c):
# If there is even a slight change of replay, use
# check_for_replay
if len(self.c.get('replay_endpoints', {})) > 0:
for m in check_for_replay(_name, watched_names,
msg, self.c,
self.context):
# Revalidate all the replayed messages.
if not validate or \
fedmsg.crypto.validate(m, **self.c):
yield _name, ep, m['topic'], m
else:
warnings.warn("!! invalid message " +
"received: %r" % msg)
else:
yield _name, ep, _topic, msg
else:
# Else.. we are supposed to be validating, but the
# message failed validation.
# Warn, but don't throw an exception. Keep tailing.
warnings.warn("!! invalid message received: %r" % msg)
finally:
for subscriber in subs:
subscriber.close()