def send_membership(self):
member = self.get_random_member()
if member is not None and member is not self.me:
message = GossipMessage()
message.ip = self.me.ip
message.gossip_port = self.me.gossip_port
message.data_port = self.me.data_port
message.milliseconds_since_midnight = time_util.get_milliseconds_since_midnight()
message.public_key = self.me.public_key
message.tags = self.me.tags
for m in self.member_holder.get_active_members():
message.members.append(utils.get_key(m))
if m == self.me:
self.me.sequence = self.me.sequence + 1
message.clock.append(m.sequence)
for key, topic_list in self.listener_registry.get_all_registrations().items():
for topic in topic_list:
if key not in message.listeners:
message.listeners[key] = []
message.listeners[key].append(topic)
member.gossip(message)
return
def decode(data):
buf = BytesIO()
buf.write(data)
buf.seek(0)
unpacker = msgpack.Unpacker(buf)
ip = str(unpacker.unpack()) + '.' + str(unpacker.unpack()) + '.' + str(unpacker.unpack()) + '.' + str(unpacker.unpack())
envelope = Envelope()
envelope.sender_key = utils.get_key(ip=ip, gossip_port=int(unpacker.unpack()), data_port=int(unpacker.unpack()))
envelope.encrypted = unpacker.unpack()
if envelope.encrypted:
envelope.key = unpacker.unpack().decode('utf8')
else:
envelope.key = None
envelope.execute_time = int(unpacker.unpack())
envelope.milliseconds_since_midnight = int(unpacker.unpack())
envelope.topic = unpacker.unpack().decode('utf8')
envelope.partition = unpacker.unpack().decode('utf8')
envelope.type = unpacker.unpack().decode('utf8')
envelope.payload = unpacker.unpack()
spl = envelope.type.split('.')
module = '.'.join(spl[0:-1])
name = spl[-1]
try:
cl = class_for_name(module, name)
envelope.message = generic_codec.decode(envelope.payload, cl)
except ImportError:
logger.warn('Could not locate message type "' + module + '.' + name + '"')
return envelope
def send_membership(self):
member = self.get_random_member()
if member is not None and member is not self.me:
message = GossipMessage()
message.ip = self.me.ip
message.gossip_port = self.me.gossip_port
message.data_port = self.me.data_port
message.milliseconds_since_midnight = time_util.get_milliseconds_since_midnight(
)
message.public_key = self.me.public_key
message.tags = self.me.tags
for m in self.member_holder.get_active_members():
message.members.append(utils.get_key(m))
if m == self.me:
self.me.sequence = self.me.sequence + 1
message.clock.append(m.sequence)
for key, topic_list in self.listener_registry.get_all_registrations(
).items():
for topic in topic_list:
if key not in message.listeners:
message.listeners[key] = []
message.listeners[key].append(topic)
member.gossip(message)
return
def send(self, message, topic, partition=None, offset=None):
envelope = Envelope()
envelope.decoded = False
if offset is not None:
envelope.execute_time = offset
else:
envelope.execute_time = 0
envelope.milliseconds_since_midnight = time_util.get_milliseconds_since_midnight()
envelope.sender_key = utils.get_key(self.me)
envelope.topic = topic
if partition is not None:
envelope.partition = partition
else:
envelope.partition = ''
envelope.type = type(message).__module__ + '.' + message.__class__.__name__
members = self.listener_registry.get_registered_members(topic)
for member in members:
if self.me == member:
envelope.message = message
envelope.decoded = True
else:
envelope.payload = generic_encoder.encode(message)
envelope.decoded = False
member.send(envelope)
def register_member_for_topic(self, topic, member):
key = utils.get_key(member)
if key not in self.map:
self.map[key] = []
if topic not in self.map[key]:
self.map[key].append(topic)
def register_member_for_topic(self, topic, member):
key = utils.get_key(member)
if key not in self.map:
self.map[key] = []
if topic not in self.map[key]:
self.map[key].append(topic)
def handle_gossip_message(self, message):
if self.shutting_down:
return
sender_key = utils.get_key(ip=message.ip, gossip_port=message.gossip_port, data_port=message.data_port)
update_tags = False
for i in range(0, len(message.members)):
key = message.members[i]
m = self.member_holder.get_member(key)
if m is None:
protocol = parameters.get_property(PROTOCOL_PROPERTY, DEFAULT_PROTOCOL)
if protocol == 'udp':
logger.debug("Discovered new UDP member through gossip: " + str(message.getIp()) + ":" + str(message.getGossipPort()) + ":" + str(message.getDataPort()))
m = RemoteMember(use_tcp=False)
else:
logger.debug("Discovered new TCP member through gossip: " + str(message.ip) + ":" + str(message.gossip_port) + ":" + str(message.data_port))
m = RemoteMember(use_tcp=True)
values = key.split(":")
m.ip = values[0]
m.gossip_port = int(values[1])
m.data_port = int(values[2])
if message.public_key:
m.public_key = message.public_key
m.initialize()
self.member_holder.update_member_status(m)
member_clock = message.clock[i]
known_member_clock = m.sequence
if member_clock > known_member_clock:
if key == sender_key:
update_tags = True
m.sequence = member_clock
if key in message.listeners:
topics = message.listeners[key]
else:
topics = []
for key, topic_list in self.listener_registry.get_all_registrations().items():
if key == sender_key:
# member reporting on itself - take its listener list as canon
self.listener_registry.get_all_registrations()[key] = topics
for topic_string in topics:
topic = topic_utils.get_topic(topic_string)
partition = topic_utils.get_partition(topic_string)
if m not in self.listener_registry.get_registered_members(topic):
self.listener_registry.register_member_for_topic(topic, m)
if update_tags:
m = self.member_holder.get_member(sender_key)
m.tags = message.tags
def setup():
global me, key, holder
logging.basicConfig(
format='%(asctime)s %(name)s - %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.DEBUG)
me = MeMember()
key = utils.get_key(me)
holder = MemberHolder()
def setup():
logging.basicConfig(format='%(asctime)s %(name)s - %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG)
envelope.sender_key = utils.get_key(ip='127.0.0.1', gossip_port=8888, data_port=9999)
envelope.execute_time = 1
envelope.milliseconds_since_midnight = time_util.get_milliseconds_since_midnight()
envelope.topic = 'TestTopic'
envelope.partition = 'TestPartition'
envelope.key = None
envelope.encrypted = False
def test_register():
global me, registry
registry.register_member_for_topic('topic2', me)
key = utils.get_key(me)
assert key in registry.map
assert 'topic2' in registry.map[key]
assert len(registry.map[key]) == 1
assert len(registry.get_registered_members('topic2')) == 1
assert me in registry.get_registered_members('topic2')
registry.remove_registration(me, 'topic2')
assert 'topic2' not in registry.map[key]
def decode(data):
buf = BytesIO()
buf.write(data)
buf.seek(0)
unpacker = msgpack.Unpacker(buf)
ip = str(unpacker.unpack()) + '.' + str(unpacker.unpack()) + '.' + str(
unpacker.unpack()) + '.' + str(unpacker.unpack())
envelope = Envelope()
envelope.sender_key = utils.get_key(ip=ip,
gossip_port=int(unpacker.unpack()),
data_port=int(unpacker.unpack()))
envelope.encrypted = unpacker.unpack()
if envelope.encrypted:
envelope.key = unpacker.unpack().decode('utf8')
else:
envelope.key = None
envelope.execute_time = int(unpacker.unpack())
envelope.milliseconds_since_midnight = int(unpacker.unpack())
envelope.topic = unpacker.unpack().decode('utf8')
envelope.partition = unpacker.unpack().decode('utf8')
envelope.type = unpacker.unpack().decode('utf8')
envelope.payload = unpacker.unpack()
spl = envelope.type.split('.')
module = '.'.join(spl[0:-1])
name = spl[-1]
try:
cl = class_for_name(module, name)
envelope.message = generic_codec.decode(envelope.payload, cl)
except ImportError:
logger.warn('Could not locate message type "' + module + '.' + name +
'"')
return envelope
def update_member_status(self, member):
key = utils.get_key(member)
self.lock.acquire()
'''
logger.info('**** Before')
for k, v in self.members.items():
logger.info(k)
logger.info('****')
'''
if key in self.members:
if key in self.active_members and (
member.status == MemberStatus.Failed
or member.status == MemberStatus.Left
or member.status == MemberStatus.Unknown):
self.active_members.pop(key)
self.dead_members[key] = member
elif key in self.dead_members and member.status == MemberStatus.Alive:
self.dead_members.pop(key)
self.active_members[key] = member
else:
logger.info('Adding new member at key ' + str(key))
self.members[key] = member
if MemberStatus.Alive == member.status:
self.active_members[key] = member
else:
self.dead_members[key] = member
'''
logger.info('**** After')
for k, v in self.members.items():
logger.info(k)
logger.info('****')
'''
self.lock.release()
def remove_registration(self, member, topic):
key = utils.get_key(member)
if topic in self.map[key]:
self.map[key].remove(topic)
def __str__(self):
return 'Local member ' + utils.get_key(self)
def setup():
global me, key, holder
logging.basicConfig(format='%(asctime)s %(name)s - %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG)
me = MeMember()
key = utils.get_key(me)
holder = MemberHolder()
def __str__(self):
return 'Local member ' + utils.get_key(self)
def remove_registration(self, member, topic):
key = utils.get_key(member)
if topic in self.map[key]:
self.map[key].remove(topic)