def __init__(self, ip, init_address, port1, port2, init_topic):
self.init_address = init_address
self.port1 = port1
self.init_topic = init_topic
# use local IP address as publisher ID
self.pubID = ip
# Call ZeroMQ API
self.zmqhelper = ZMQHelper()
# 防止线程争夺socket
self.lock = threading.Lock()
# soft_shutoff_check is used to detect soft shutoff
self.soft_shutoff_check = False
# logfile name
self.logfile_name = './Output/' + self.pubID + '-publisher.log'
# port used to receive update IP address msg from broker
self.port2 = port2
self.flag = False
self.flag_lock = threading.Lock()
# socket dictionary
# data formate: {$(topic): $(socket)}
self.sockets = dict()
def __init__(self, ip, init_address, port1, port2, port3, topic,
history_count):
self.init_address = init_address
self.port1 = port1
self.port2 = port2
self.port3 = port3
self.topic = str(topic)
self.history_count = history_count
# Call ZMQ API
self.zmqhelper = ZMQHelper()
# use local ip address as subscriber ID
self.subID = str(ip)
# log file name
self.logfile_name = './Output/' + self.subID + '-subscriber.log'
self.hisIPsocket = None
self.flag = False
self.flag_lock = threading.Lock()
print('\n**************************************\n')
print(ip + ' init with topic ' + topic)
print('\n**************************************\n')
def __init__(self, xsub_port, xpub_port):
self.shutoff_check = False
self.heartbeat_lock = threading.Lock()
self.history_lock = threading.Lock()
# initialize MyBroker class
self.helper = ZMQHelper()
# publisher dictionary
# dictionary format
# $(pubID):{$({$topic:[$history]})}
self.pub_dict = {}
# publisher ownership dictionary
# dictionary format
# $(topic):$({$pubID: $ownership_strength})
self.pub_ownership_dict = {}
# publisher heartbeat dictionary
# This dict is used to record publisher's latest heartbeat timestamp
# $(pubID):$(time)
self.heartbeat_dict = {}
self.xsubsocket, self.xpubsocket = self.helper.prepare_broker(
xsub_port, xpub_port)
print('\n************************************\n')
print('Init MyBroker succeed.')
print('\n************************************\n')
with open(log_file, 'w') as logfile:
logfile.write('Init Broker:\n')
logfile.write('XSUB Port: ' + xsub_port + '\n')
logfile.write('XPUB Port: ' + xpub_port + '\n')
logfile.write('-------------------------------------------\n')
def __init__(self, address, port, topic, history_count):
self.address = address
self.port = port
self.topic = topic
self.history_topic = topic + '-history'
self.history_count = history_count
self.helper = ZMQHelper()
self.myID = str(random.randint(1, 100))
def __init__(self, address, port, init_topic):
self.lock = threading.Lock()
self.shutoff_check = False
self.address = address
self.port = port
self.init_topic = init_topic
self.helper = ZMQHelper()
self.heartbeat_helper = ZMQHelper()
self.myID = str(random.randint(1, 1000))
def __init__(self, zk_server, topic):
self.topic = topic
self.helper = ZMQHelper()
self.myID = str(random.randint(1, 1000))
self.socket = None
zk_connect_addr = zk_server + ':2181'
self.zk = KazooClient(hosts=zk_connect_addr)
self.leader_address = None
self.leader_alive = False
self.init_zk()
def __init__(self, zk_server, topic, history_count):
self.leader_address = None
self.topic = topic
self.history_topic = topic + '-history'
self.history_count = history_count if history_count > 0 else 0
self.helper = ZMQHelper()
self.myID = str(random.randint(1, 100))
self.logfile_name = './Output/' + self.myID + '-subscriber.log'
zk_connect_addr = zk_server + ':2181'
self.zk = KazooClient(zk_connect_addr)
self.isConnected = False
self.socket = None
self.hisIPsocket = None
self.init_zk()
def __init__(self, zk_server, my_address, xsub_port, xpub_port):
'''
:param zk_server: IP address of ZooKeeper Server
:param my_address: IP address of current broker
:param xsub_port: 5556
:param xpub_port: 5557
:sub_history_port: 5558
:sync_with_leader_port:5559
'''
self.zmqhelper = ZMQHelper()
'''
{$(topic): {
$(pubID): {
'publications' : [$(publication)]
'ownership strength': $(ownership_strength)
}
}
}
'''
self.data = {}
self.xsubsocket, self.xpubsocket = self.zmqhelper.prepare_broker(
xsub_port, xpub_port)
self.syncsocket = None
self.historysocket = self.zmqhelper.csrecv('5558')
self.myID = str(random.randint(1, 1000))
self.log_file = './Output/Broker' + self.myID + '.log'
zk_server = zk_server + ':2181'
self.zk = KazooClient(hosts=zk_server)
self.isLeader = False
self.my_address = my_address
self.count = 1
print('\n************************************\n')
print('Init MyBroker % s succeed.' % self.my_address)
print('\n************************************\n')
with open(self.log_file, 'w') as logfile:
logfile.write('Init Broker:\n')
logfile.write('XSUB Port: ' + xsub_port + '\n')
logfile.write('XPUB Port: ' + xpub_port + '\n')
logfile.write('-------------------------------------------\n')
self.init_zk()
class Broker:
def __init__(self, zk_server, my_address, xsub_port, xpub_port):
'''
:param zk_server: IP address of ZooKeeper Server
:param my_address: IP address of current broker
:param xsub_port: 5556
:param xpub_port: 5557
:sub_history_port: 5558
:sync_with_leader_port:5559
'''
self.zmqhelper = ZMQHelper()
'''
{$(topic): {
$(pubID): {
'publications' : [$(publication)]
'ownership strength': $(ownership_strength)
}
}
}
'''
self.data = {}
self.xsubsocket, self.xpubsocket = self.zmqhelper.prepare_broker(
xsub_port, xpub_port)
self.syncsocket = None
self.historysocket = self.zmqhelper.csrecv('5558')
self.myID = str(random.randint(1, 1000))
self.log_file = './Output/Broker' + self.myID + '.log'
zk_server = zk_server + ':2181'
self.zk = KazooClient(hosts=zk_server)
self.isLeader = False
self.my_address = my_address
self.count = 1
print('\n************************************\n')
print('Init MyBroker % s succeed.' % self.my_address)
print('\n************************************\n')
with open(self.log_file, 'w') as logfile:
logfile.write('Init Broker:\n')
logfile.write('XSUB Port: ' + xsub_port + '\n')
logfile.write('XPUB Port: ' + xpub_port + '\n')
logfile.write('-------------------------------------------\n')
self.init_zk()
def init_zk(self):
if self.zk.state != KazooState.CONNECTED:
self.zk.start()
while self.zk.state != KazooState.CONNECTED:
pass
print('Broker %s connected to ZooKeeper server.' % self.myID)
if self.zk.exists('/Brokers') is None:
self.zk.create(path='/Brokers',
value=b'',
ephemeral=False,
makepath=True)
flag = False
while self.zk.exists('/Brokers') is None:
pass
flag = True
if flag:
print('Create Znode Brokers.')
# Create a Znode in ZooKeeper
znode_path = '/Brokers/' + self.myID
self.zk.create(path=znode_path,
value=b'',
ephemeral=True,
makepath=True)
while self.zk.exists(znode_path) is None:
pass
print('Broker %s created a znode in ZooKeeper server.' % self.myID)
# watch publishers znode
pub_watch_path = '/Publishers'
if self.zk.exists(pub_watch_path) is None:
self.zk.create(path=pub_watch_path,
value=b'',
ephemeral=False,
makepath=True)
flag = False
while self.zk.exists(pub_watch_path) is None:
pass
flag = True
if flag:
print('Create Publishers znode.')
@self.zk.ChildrenWatch(path=pub_watch_path)
def watch_publishers(children):
self.publisher_failed(children)
'''
# watch subscriber znode
sub_watch_path = './Subscribers'
@self.zk.ChildrenWatch(client=self.zk, path=sub_watch_path)
def watch_subscribers(children):
self.subscriber_failed(children)
'''
# check if the leader has exists
leader_path = '/Leader'
if self.zk.exists(leader_path):
# If the leader znode already exists, specify this broker as follower
self.isLeader = False
print('Broker %s is follower.' % self.myID)
# followers start watching leader znode for potential election
self.leader_monitor()
# socket for follower to receive msg from leader
leader_address = str(self.zk.get(leader_path)[0])
self.syncsocket = self.zmqhelper.sinkpull(leader_address, '5559')
if self.syncsocket != None:
print('follower: syncsocket ok')
# NOTE: listen sync message from leader and update data storage
self.sync_data()
else:
# If the leader is not existing, create it and receive msg
self.zk.create(leader_path,
value=self.my_address,
ephemeral=True,
makepath=True)
while self.zk.exists(path=leader_path) is None:
pass
print('Broker %s is the first leader.' % self.myID)
self.isLeader = True
# socket for leader to send sync request to followers
self.syncsocket = self.zmqhelper.sourcepush('5559')
if self.syncsocket != None:
print('leader: syncsocket ok')
subscriber_thr = threading.Thread(target=self.receive_hisreq, args=())
threading.Thread.setDaemon(subscriber_thr, True)
subscriber_thr.start()
recv_thr = threading.Thread(target=self.receive_msg, args=())
threading.Thread.setDaemon(recv_thr, True)
recv_thr.start()
def leader_monitor(self):
# Run this method in another thread, because the election.run() method will be blocked until it won
election_path = '/Brokers/'
leader_path = '/Leader'
# watch leader znode
@self.zk.DataWatch(path=leader_path)
def watch_leader(data, state):
if self.zk.exists(path=leader_path) is None:
time.sleep(random.randint(0, 3))
election = self.zk.Election(election_path, self.myID)
election.run(self.win_election)
def publisher_failed(self, children):
'''
:param children: current children list under Publishers Znode
:return:
'''
# delete all entries of the pub which not in the current children list
if self.count != 1:
for key in self.data.keys():
for pubID in self.data[key].keys():
if pubID not in children:
del self.data[key][pubID]
print('delete publisher %s from topic %s\n' %
(pubID, key))
else:
self.count = 0
'''
def subscriber_failed(self, children):
:param children: current children list under Subscribers Znode
:return:
# TODO: Check which subscriber in data storage has failed,
# if you get one, delete the data related to this subscriber
pass
'''
# win the election, start receiving msg from publisher
def win_election(self):
leader_path = '/Leader'
if self.zk.exists(path=leader_path) is None:
self.zk.create(leader_path,
value=self.my_address,
ephemeral=True,
makepath=True)
while self.zk.exists(path=leader_path) is None:
pass
self.isLeader = True
print('Broker %s became new leader' % self.myID)
# self.syncsocket = None
self.syncsocket = self.zmqhelper.sourcepush('5559')
if self.syncsocket != None:
print('Broker %s started receive msg' % self.myID)
# only leader call this method
def receive_msg(self):
'''
Message type:
1. publisher init
2. publication
3. subscriber request
:return:
'''
while self.isLeader is False:
pass
while True:
# Store received data into self data storage
# Send received data to subscribers
# Send received data to followers using PUSH socket
# receive message from publisher
msg = self.xsubsocket.recv_string()
print(msg)
message = msg.split('#')
msg_type = message[0]
# publisher init
if msg_type == 'pub_init':
pubID = message[1]
topic = message[2]
print('\n************************************\n')
print('Init msg: %s init with topic %s' % (pubID, topic))
print('\n************************************\n')
with open(self.log_file, 'a') as logfile:
logfile.write('Init msg: %s init with topic %s\n' %
(pubID, topic))
# update storage
self.update_data('add_pub', pubID, topic, '')
# send msg to followers
#self.syncsocket.send_string('add_pub' + '#' + pubID + '#' + topic + '#')
# publication
elif msg_type == 'publication':
pubID = message[1]
topic = message[2]
publication = message[3]
print('\n************************************\n')
print('Publication: %s published %s with topic %s' %
(pubID, publication, topic))
print('\n************************************\n')
with open(self.log_file, 'a') as logfile:
logfile.write(
'Publication: %s published %s with topic %s\n' %
(pubID, publication, topic))
# update storage
self.update_data('add_pub', pubID, topic, '')
self.update_data('add_publication', pubID, topic, publication)
# send msg to followers
self.syncsocket.send_string('add_publication' + '#' + pubID +
'#' + topic + '#' + publication +
'#')
# check if this pubID has the highest ownership
if self.filter_pub_ownership(pubID, topic) is not None:
# send publication to subscribers using xpubsocket
print('sending to sub')
publication = publication + '--' + str(time.time())
self.zmqhelper.xpub_send_msg(self.xpubsocket, topic,
publication)
def receive_hisreq(self):
while True:
# receive history request msg from subscribers
if self.historysocket is None:
print('historysocket is none')
msg = self.historysocket.recv_string(0, 'utf-8')
print(msg)
message = msg.split('#')
msg_type = message[0]
if msg_type == 'request_history_publication':
with open(self.log_file, 'a') as log:
log.write('\n************************************\n')
log.write(
'Subscriber is requesting history publication.\n')
log.write('\n************************************\n')
topic = str(message[1])
history_count = int(message[2])
i = 0
data = []
# get pubID who has the highest ownership strength
if topic in self.data.keys():
for pub in self.data[topic].keys():
target = self.filter_pub_ownership(pub, topic)
if target is not None:
break
i = len(self.data[topic][target]['publications'])
if i <= history_count:
data.extend(
self.data[topic][target]['publications'][:])
else:
data.extend(self.data[topic][target]
['publications'][-history_count:])
msg = 'history_publication' + '#' + simplejson.dumps(data)
self.historysocket.send_string(msg)
print('\n************************************\n')
print('Send history publications to subscriber...')
print(msg)
print('\n************************************\n')
with open(self.log_file, 'a') as log:
log.write('\n************************************\n')
log.write('Send history publication to subscriber.\n')
log.write('\n************************************\n')
def sync_data(self):
'''
Receive sync msg from leader if this broker is a follower
:return:
'''
# Use a while loop to receive sync msg from leader
print('start sync with leader')
while self.isLeader is False:
try:
msg = self.syncsocket.recv_string()
except Exception as e:
print('Sync data time out')
continue
print('\n************************************\n')
print('received sync msg from leader')
message = msg.split('#')
msg_type = message[0]
if msg_type == 'add_pub':
pubID = message[1]
topic = message[2]
self.update_data(msg_type, pubID, topic, '')
elif msg_type == 'add_publication':
pubID = message[1]
topic = message[2]
publication = message[3]
self.update_data('add_pub', pubID, topic, '')
self.update_data(msg_type, pubID, topic, publication)
print('sync with topic %s pub %s' % (topic, pubID))
print('\n************************************\n')
def update_data(self, update_typ, pubID, topic, publication):
'''
:param update_typ:
1. New publisher registered
2. Received new publication from publisher
:param pubID:
:param topic:
:param publication:
:return:
'''
try:
if update_typ == 'add_pub':
# Assign an ownership strength to the registered publisher
ownership_strength = random.randint(1, 100)
if topic not in self.data.keys():
self.data.update({
topic: {
pubID: {
'publications': [],
'ownership strength': ownership_strength
}
}
})
elif pubID not in self.data[topic].keys():
self.data[topic].update({
pubID: {
'publications': [],
'ownership strength': ownership_strength
}
})
elif update_typ == 'add_publication':
stored_publication = publication + '--' + str(time.time())
self.data[topic][pubID]['publications'].append(
stored_publication)
except KeyError as ex:
print('\n----------------------------------------\n')
print('Error happened while updating publication dictionary...')
print(ex)
print('\n----------------------------------------\n')
# To examine if this pubID has the highest ownership
#
# argument: current publisher & topic
# return publisher ID or None
#
def filter_pub_ownership(self, pubID, topic):
try:
if self.data[topic][pubID]['ownership strength'] == max([
pub['ownership strength']
for pub in self.data[topic].values()
]):
return pubID
else:
return None
except Exception:
return None
class Publisher:
# # # # # # # # # # # # # # # # # # # # # #
# Publisher constructor method:
# Parameters:
# 1. address: initial broker IP address
# 2. port1: used to connect to broker --- socket type: PUB/SUB >> 5556
# 3. port2: used to listen update IP msg from broker --- socket type: Client/Server >> 6002
# 4. init_topic: initial topic of publisher
# # # # # # # # # # # # # # # # # # # # # #
def __init__(self, ip, init_address, port1, port2, init_topic):
self.init_address = init_address
self.port1 = port1
self.init_topic = init_topic
# use local IP address as publisher ID
self.pubID = ip
# Call ZeroMQ API
self.zmqhelper = ZMQHelper()
# 防止线程争夺socket
self.lock = threading.Lock()
# soft_shutoff_check is used to detect soft shutoff
self.soft_shutoff_check = False
# logfile name
self.logfile_name = './Output/' + self.pubID + '-publisher.log'
# port used to receive update IP address msg from broker
self.port2 = port2
self.flag = False
self.flag_lock = threading.Lock()
# socket dictionary
# data formate: {$(topic): $(socket)}
self.sockets = dict()
# # # # # # # # # # # # # # # # # # # # # #
# Register method
# return True: register succeed
# return Fale: register failed
# # # # # # # # # # # # # # # # # # # # # #
def register(self):
update_addr_thr = threading.Thread(target=self.listen_update, args=())
threading.Thread.setDaemon(update_addr_thr, False)
send_pub_thr = threading.Thread(target=self.send_pub,
args=(self.init_topic, ))
threading.Thread.setDaemon(send_pub_thr, False)
if self.connect2broker(self.init_address, self.init_topic):
update_addr_thr.start()
send_pub_thr.start()
# # # # # # # # # # # # # # # # # # # # # #
# Connect publisher to broker
# return True: connection succeed
# return Fale: connection failed
# # # # # # # # # # # # # # # # # # # # # #
def connect2broker(self, address, topic):
with self.lock:
connect_str = 'tcp://' + address + ':' + self.port1
print('Connection info: %s' % connect_str)
self.sockets.update(
{topic: self.zmqhelper.connect_pub2broker(connect_str)})
self.sockets.get(topic).RCVTIMEO = 50000
time.sleep(random.randint(3, 5))
if self.sockets.get(topic) is None:
print('Connection feedback: connected xsub socket failed.')
with open(self.logfile_name, 'a') as log:
log.write('Publisher %s registered to broker failed.\n' %
self.pubID)
self.sockets.pop(topic)
return False
else:
try:
init_str = 'pub_init' + '#' + self.pubID + '#' + topic + '#'
self.zmqhelper.pub_send_msg(self.sockets.get(topic),
init_str)
self.sockets.get(topic).recv_string()
print(
'Connection feedback: connected xsub socket succeed.')
print(
'Connection feedback: %s initialized with initial topic %s succeed.'
% (self.pubID, topic))
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n'
)
log.write('Register info:\n')
log.write('ID: %s\n' % self.pubID)
log.write('Initial topic: %s\n' % topic)
log.write('Connection Info: tcp://' + address + ':' +
self.port1 + '\n')
except Exception as ex:
print(
'\n*************************************************\n'
)
print(ex)
print(
'\n*************************************************\n'
)
return True
# # # # # # # # # # # # # # # # # # # # # #
# Publish message
# Parameters:
# 1. topic
# 2. message
# # # # # # # # # # # # # # # # # # # # # #
def send_pub(self, topic):
file_path = './Input/' + topic + '.txt'
pubs_list = self.get_publications(file_path)
print('PUB ID:', self.pubID)
i = 0
while i < len(pubs_list):
with self.flag_lock:
if self.flag:
try:
with open(self.logfile_name, 'a') as logfile:
logfile.write(
'\n*************************************************\n'
)
logfile.write('Publish Info:\n')
logfile.write('Publish: %s\n' % pubs_list[i])
logfile.write('Time: %s\n' % str(time.time()))
send_str = 'publication' + '#' + self.pubID + '#' + topic + '#' + pubs_list[
i]
with self.lock:
self.zmqhelper.pub_send_msg(
self.sockets.get(topic), send_str)
self.sockets.get(topic).recv_string()
print('Publication: %s\n' % send_str)
i += 1
# send heartbeat msg to broker
send_str = 'pub_heartbeat' + '#' + self.pubID + '#'
with self.lock:
self.zmqhelper.pub_send_msg(
self.sockets.get(topic), send_str)
self.sockets.get(topic).recv_string()
print('Send heartbeat to Broker\n')
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n'
)
log.write('Heartbeat info:\n')
log.write('Time: %s' % str(time.time()))
except Exception as ex:
print(
'\n*************************************************\n'
)
print(ex)
connect_str = 'tcp://' + self.init_address + ':' + self.port1
with self.lock:
self.sockets.get(topic).connect(connect_str)
print(
'\n*************************************************\n'
)
time.sleep(random.randint(3, 8))
# # # # # # # # # # # # # # # # # # # # # #
# Drop topic: notify broker to delete all
# publications releated to given topic
# Parameter: topic
# # # # # # # # # # # # # # # # # # # # # #
def drop_topic(self, topic):
send_str = 'drop_topic' + '#' + self.pubID + '#' + topic + '#'
with self.lock:
self.zmqhelper.pub_send_msg(self.sockets[topic], send_str)
self.sockets.get(topic).recv_string()
# delete correspond socket
with self.lock:
del self.sockets[topic]
print('Drop topic: %s' % topic)
try:
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n')
log.write('Drop topic: %s\n' % topic)
except IOError:
print('Open or write file error.')
# # # # # # # # # # # # # # # # # # # # # #
# Soft shutoff publisher: the publisher won't send
# publications to broker
# # # # # # # # # # # # # # # # # # # # # #
def softshutoff(self):
send_str = 'shutoff' + '#' + self.pubID + '#'
print('Shutoff')
self.soft_shutoff_check = True
# send soft shutoff msg to all brokers
for s in self.sockets.values():
with self.lock:
self.zmqhelper.pub_send_msg(s, send_str)
s.recv_string()
try:
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n')
log.write('Soft shutoff info:\n')
log.write('Time: %s' % str(time.time()))
except IOError:
print('Open or write file error.')
# Listen "Update IP address" msg from brokers
def listen_update(self):
mysocket = self.zmqhelper.csrecv(self.port2)
if mysocket is None:
print('Set up listening update socket for publisher failed')
else:
while True:
msg = mysocket.recv_string()
mysocket.send_string('OK')
msg = msg.split('#')
address = msg[1]
topic = msg[2]
# publisher don't need to update ip
if msg[0] == 'start':
with self.flag_lock:
self.flag = True
print(
'\n---------------------------------------------------------\n'
)
print('Publisher received start sending msg...')
print(
'\n---------------------------------------------------------\n'
)
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n'
)
log.write('Start sending publications.\n')
else:
with self.flag_lock:
self.flag = False
self.init_address = address
time.sleep(2)
print(
'\n---------------------------------------------------------\n'
)
print('Publisher received reconnect to new broker msg...')
print(
'\n---------------------------------------------------------\n'
)
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n'
)
log.write('Reconnect to new broker node.\n')
# connect to new broker using new IP address
self.connect2broker(str(address), str(topic))
# Get publications from file
def get_publications(self, file_path):
try:
with open(file_path, 'r') as file:
pubs = file.readlines()
for i in range(len(pubs)):
pubs[i] = pubs[i][:-1]
return pubs
except IOError:
print('Open or write file error.')
return []
class Subscriber:
# # # # # # # # # # # # # # # # # # # # # #
# Subscriber constructor method
# Parameters:
# 1. init_address: initial broker ip address when sub register to broker
# 2. port1: broker xpub port --- socket type: PUB/SUB >> 5557
# 3. port2: receive update IP address msg from broker --- socket type: Client/Server >> 6001
# 4. port3: get history publication reply & send ip --- socket type: Client/Server >> 6003
# 4. topic: initial subscribe topic
# 5. history_count: history publications count
# # # # # # # # # # # # # # # # # # # # # #
def __init__(self, ip, init_address, port1, port2, port3, topic,
history_count):
self.init_address = init_address
self.port1 = port1
self.port2 = port2
self.port3 = port3
self.topic = str(topic)
self.history_count = history_count
# Call ZMQ API
self.zmqhelper = ZMQHelper()
# use local ip address as subscriber ID
self.subID = str(ip)
# log file name
self.logfile_name = './Output/' + self.subID + '-subscriber.log'
self.hisIPsocket = None
self.flag = False
self.flag_lock = threading.Lock()
print('\n**************************************\n')
print(ip + ' init with topic ' + topic)
print('\n**************************************\n')
# Handler method for subscriber
def handler(self):
# register subscriber
self.register_sub(self.init_address)
# listen update IP message && history publication message from broker
listen_update_thr = threading.Thread(target=self.listen_updateIP,
args=())
threading.Thread.setDaemon(listen_update_thr, True)
listen_update_thr.start()
# listen publication message from broker
listen_pubs_thr = threading.Thread(target=self.receive_msg, args=())
threading.Thread.setDaemon(listen_pubs_thr, True)
listen_pubs_thr.start()
while True:
pass
# # # # # # # # # # # # # # # # # # # # # #
# Register subscriber
# return True: connection succeed
# return False: connection failed
# # # # # # # # # # # # # # # # # # # # # #
def register_sub(self, address):
connect_str = 'tcp://' + address + ':' + self.port1
print('Connection info: %s' % connect_str)
self.socket = self.zmqhelper.connect_sub2broker(connect_str)
self.socket.RCVTIMEO = 30000
time.sleep(random.randint(3, 5))
# connection failed
if self.socket is None:
print('Connection feedback: connected xpub socket failed.')
with open(self.logfile_name, 'w') as logfile:
logfile.write('Subscriber %s registered to broker failed.\n' %
self.subID)
return False
else:
print('Connection feedback: connected xpub socket succeed.')
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n')
log.write('Register info:\n')
log.write('ID: %s\n' % self.subID)
log.write('Topic: %s\n' % self.topic)
log.write('History publications count: %s\n' %
self.history_count)
# send subscriber IP and receive history publications from broker
self.sendIPgetHist(address)
return True
# Note: receive latest publications from brokers
# # # # # # # # # # # # # # # # # # # # # # #
def receive_msg(self):
# self.zmqhelper.subscribe_topic(self.socket, self.topic)
while True:
try:
received_pub = self.zmqhelper.sub_recieve_msg(self.socket)
received_msg = received_pub.split('--')
time_stamp = float(received_msg[1])
received_msg = received_msg[0]
if received_msg.split()[0] == self.topic:
print(
'\n*************************************************\n'
'Receipt Info:\n'
'Publication: %s\n'
'Time Interval: %f\n' %
(received_msg, abs(time.time() - time_stamp)))
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n'
)
log.write('Receipt Info:\n')
log.write('Receive: %s\n' % received_msg)
log.write('Time: %f\n' % abs(time.time() - time_stamp))
log.write(
'\n*************************************************\n'
)
except Exception as ex:
print('\n*************************************************\n')
print(ex)
print('\n*************************************************\n')
# Note: Listen Update IP address msg from broker
def listen_updateIP(self):
mysocket = self.zmqhelper.csrecv(self.port3)
if socket is None:
print('Set up listening update socket for publisher failed')
else:
while True:
msg = mysocket.recv_string()
print('Listening update IP port get new message.... %s ' % msg)
mysocket.send_string('OK')
# TODO: parse msg here
msg = msg.split('#')
new_ip = msg[1]
# update initial IP address
self.init_address = new_ip
with self.flag_lock:
self.flag = True
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n'
)
log.write('Reconnect to new broker...\n')
log.write(
'\n*************************************************\n'
)
self.register_sub(new_ip)
# Note: Send subscriber IP address & history request to broker
# all history publications
def sendIPgetHist(self, address):
# Note: send subscriber IP to broker
ip_msg = 'subscriber_IP' + '#' + self.subID + '#' + self.topic + '#'
self.hisIPsocket = self.zmqhelper.csreq(address, self.port2)
time.sleep(3)
self.hisIPsocket.send_string(ip_msg)
print('Send subscriber IP to broker....')
with open(self.logfile_name, 'a') as log:
log.write('\n*************************************************\n')
log.write('Send subscriber IP to broker...')
log.write('\n*************************************************\n')
recved = str(self.hisIPsocket.recv_string())
print('Received msg from broker: %s' % recved)
if recved == 'OK':
with self.flag_lock:
if self.flag is False:
# TODO: send request history publications msg to broker
msg = 'request_history_publication' + '#' + self.topic + '#' + str(
self.history_count) + '#'
self.hisIPsocket.send_string(msg)
print('Send history request to broker...')
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n'
)
log.write(
'Send history publications request to broker...')
log.write(
'\n*************************************************\n'
)
# Note: receive histroy publications from broker
# we only need to request history publication when subscriber joins the system
# so we only receive msg from broker one time.
histories = self.hisIPsocket.recv_string()
print('Received histories: ' + histories)
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n'
)
log.write(
'Received history publications from broker: ')
log.write(histories)
log.write(
'\n*************************************************\n'
)
# TODO: parse msg
histories = simplejson.loads(histories.split('#')[1])
# TODO: write all history publications into logfile
for history in histories:
history = history.split('--')
print(
'\n*************************************************\n'
)
print('History Publication Receipt Info:\n')
print('Receive: %s\n' % history[0])
print('Time: %f\n' %
abs(time.time() - float(history[1])))
print(
'\n*************************************************\n'
)
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n'
)
log.write('History Publication Receipt Info:\n')
log.write('Receive: %s\n' % history[0])
log.write('Time: %f\n' %
abs(time.time() - float(history[1])))
log.write(
'\n*************************************************\n'
)
# update IP address, then reconnect to new broker
elif recved == 'Wait':
print('Registered topic is unavailable now.........')
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n')
log.write('Registered topic is unavailable now.........')
log.write(
'\n*************************************************\n')
return
else:
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n')
log.write('Reconnect to new broker: %s' %
str(recved.split('#')[1]))
log.write(
'\n*************************************************\n')
self.register_sub(str(recved.split('#')[1]))
class Publisher:
def __init__(self, zk_server, topic):
self.topic = topic
self.helper = ZMQHelper()
self.myID = str(random.randint(1, 1000))
self.socket = None
zk_connect_addr = zk_server + ':2181'
self.zk = KazooClient(hosts=zk_connect_addr)
self.leader_address = None
self.leader_alive = False
self.init_zk()
def init_zk(self):
if self.zk.state != KazooState.CONNECTED:
self.zk.start()
while self.zk.state != KazooState.CONNECTED:
pass
print('Pub %s connected to local ZooKeeper Server.' % self.myID)
# create a Znode for this publisher
znode_path = '/Publishers/' + self.myID
self.zk.create(path=znode_path, value=self.myID, ephemeral=True, makepath=True)
while self.zk.exists(znode_path) is None:
pass
print('Pub %s created Znode in ZooKeeper server.' % self.myID)
leader_path = '/Leader'
# High-level exist watcher to leader znode
@self.zk.DataWatch(path=leader_path)
def watch_leader(data, state):
print('Data in Leader Znode is: %s' % data)
if self.zk.exists(path=leader_path) is None:
self.leader_alive = True
else:
self.leader_address = data
# self.socket = None
print('pub %s try to reconnect with leader' % self.myID)
if self.register_pub():
print('pub %s connected with leader' % self.myID)
self.leader_alive = True
# register publisher, connect with leader
def register_pub(self):
connect_str = 'tcp://' + self.leader_address + ':5556'
print('Connection info: %s' % connect_str)
self.socket = self.helper.connect_pub2broker(connect_str)
time.sleep(0.5)
if self.socket is None:
print('Connection feedback: connected xsub socket failed.')
return False
else:
print('Connection feedback: connected xsub socket succeed.')
init_str = 'pub_init' + '#' + self.myID + '#' + self.topic + '#'
self.helper.pub_send_msg(self.socket, init_str)
print('Connection feedback: %s initialized with initial topic %s succeed.' % (self.myID, self.topic))
return True
# send publication to broker
def send_pub(self, topic, msg):
send_str = 'publication' + '#' + self.myID + '#' + topic + '#' + msg
print('Publication: publishing message %s' % send_str)
self.helper.pub_send_msg(self.socket, send_str)
def main(self, topic, input_file):
with open(input_file, 'r') as f:
for line in f:
while self.leader_alive is False:
time.sleep(0.5)
self.send_pub(topic, line)
time.sleep(random.uniform(0.5, 3.0))
class Publisher:
def __init__(self, address, port, init_topic):
self.lock = threading.Lock()
self.shutoff_check = False
self.address = address
self.port = port
self.init_topic = init_topic
self.helper = ZMQHelper()
self.heartbeat_helper = ZMQHelper()
self.myID = str(random.randint(1, 1000))
def register_handler(self):
if self.register_pub():
heartbeat_thr = threading.Thread(target=self.heartbeat, args=())
threading.Thread.setDaemon(heartbeat_thr, True)
heartbeat_thr.start()
return True
else:
return False
# register publisher, connect with broker
def register_pub(self):
connect_str = 'tcp://' + self.address + ':' + self.port
print('Connection info: %s' % connect_str)
self.socket = self.helper.connect_pub2broker(connect_str)
if self.socket is None:
print('Connecttion feedback: connected xsub socket failed.')
return False
else:
print('Connecttion feedback: connected xsub socket succeed.')
init_str = 'pub_init' + '#' + self.myID + '#' + self.init_topic + '#'
current = time.time()
# repeatedly connect to broker, ensure publisher connect broker succeed
while time.time() - current < 3:
self.helper.pub_send_msg(self.socket, init_str)
print(
'Connecttion feedback: %s initialized with initial topic %s succeed.'
% (self.myID, self.init_topic))
return True
# send publication to broker
def send_pub(self, topic, msg):
send_str = 'publication' + '#' + self.myID + '#' + topic + '#' + msg
print('Publication: publishing message %s' % send_str)
self.helper.pub_send_msg(self.socket, send_str)
# drop a topic
def drop_topic(self, topic):
send_str = 'drop_topic' + '#' + self.myID + '#' + topic + '#'
self.helper.pub_send_msg(self.socket, send_str)
print('Drop topic: %s' % topic)
# send heartbeat
def heartbeat(self):
connect_str = 'tcp://' + self.address + ':' + self.port
self.heartbeat_socket = self.heartbeat_helper.connect_pub2broker(
connect_str)
print('Heartbeat connection info: %s' % connect_str)
if self.heartbeat_socket is None:
print(
'Heartbeat connection feedback: heartbeat connected xsub socket failed.'
)
return False
else:
while True:
with self.lock:
if self.shutoff_check:
break
else:
send_str = 'heartbeat' + '#' + self.myID + '#'
self.heartbeat_helper.pub_send_msg(
self.heartbeat_socket, send_str)
time.sleep(10)
# publisher fails, disconnect with broker
def shutoff(self):
send_str = 'shutoff' + '#' + self.myID + '#'
print('Shutoff')
with self.lock:
self.shutoff_check = True
self.helper.pub_send_msg(self.socket, send_str)
class Broker:
def __init__(self, xsub_port, xpub_port):
self.shutoff_check = False
self.heartbeat_lock = threading.Lock()
self.history_lock = threading.Lock()
# initialize MyBroker class
self.helper = ZMQHelper()
# publisher dictionary
# dictionary format
# $(pubID):{$({$topic:[$history]})}
self.pub_dict = {}
# publisher ownership dictionary
# dictionary format
# $(topic):$({$pubID: $ownership_strength})
self.pub_ownership_dict = {}
# publisher heartbeat dictionary
# This dict is used to record publisher's latest heartbeat timestamp
# $(pubID):$(time)
self.heartbeat_dict = {}
self.xsubsocket, self.xpubsocket = self.helper.prepare_broker(
xsub_port, xpub_port)
print('\n************************************\n')
print('Init MyBroker succeed.')
print('\n************************************\n')
with open(log_file, 'w') as logfile:
logfile.write('Init Broker:\n')
logfile.write('XSUB Port: ' + xsub_port + '\n')
logfile.write('XPUB Port: ' + xpub_port + '\n')
logfile.write('-------------------------------------------\n')
# This handler is used to check if any publisher has dead.
def vice_handler(self):
while True:
with self.heartbeat_lock:
# check if any publisher has failed
for pubID in self.heartbeat_dict.keys():
if time.time() - self.heartbeat_dict[pubID] > 40:
print('\n************************************\n')
print('Publisher dead: %s has dead.' % pubID)
print('\n************************************\n')
with open(log_file, 'a') as logfile:
logfile.write('Publisher dead: %s has dead.\n' %
pubID)
self.update_pub_dict('shutoff', pubID, '', '')
self.update_pub_ownership_dict('shutoff', '', pubID)
del self.heartbeat_dict[pubID]
break
time.sleep(10)
# This helper function is used to send history publications un-interruptedly
def history_helper(self):
try:
while True:
with self.history_lock:
group = self.get_highest_strength_pubs()
for key in list(group.keys())[::-1]:
for pubs in list(self.pub_dict[key][group[key]])[::-1]:
his_topic = group[key] + '-history'
#print('Sending history publications: %s : %s' % (his_topic, pubs))
#with open(log_file, 'a') as logfile:
# logfile.write('Sending history publications: %s : %s\n' % (his_topic, pubs))
self.helper.xpub_send_msg(self.xpubsocket,
his_topic, pubs)
time.sleep(10)
except Exception:
pass
# This method should always be alive to listen message from pubs & subs
# Handler serves for either publisher and subscriber
#
# Message type for publishers:
# 1. Registration msg
# 2. Publication msg
# 3. Drop a topic
# 4. Stop publishing service msg
#
# #########################################
#
# Message type for subscribers:
# 1. request history message
#
def handler(self):
heartbeat_thr = threading.Thread(target=self.vice_handler, args=())
threading.Thread.setDaemon(heartbeat_thr, True)
heartbeat_thr.start()
history_thr = threading.Thread(target=self.history_helper, args=())
threading.Thread.setDaemon(history_thr, True)
history_thr.start()
while True:
# receive message from publisher
msg = self.xsubsocket.recv_string(0, 'utf-8')
message = msg.split('#')
msg_type = message[0]
if msg_type == 'pub_init' and message[1] in self.pub_dict.keys():
continue
print('\n************************************\n')
print('Publication storage info: (pubID, {topic : [history]})')
for x in self.pub_dict.keys():
print('PUB ID: %s' % x)
for y in self.pub_dict[x].keys():
print(' Topic: %s' % y)
for z in self.pub_dict[x][y]:
print(' Publication: %s' % z)
print('\n************************************\n')
print(
'Publisher ownership info: (topic, {pubID : ownership_strength})'
)
for x in self.pub_ownership_dict.keys():
print('Topic: %s' % x)
for y in self.pub_ownership_dict[x].keys():
print('PUB ID: %s ----------> Ownership Strength: %s' %
(y, self.pub_ownership_dict[x][y]))
if msg_type == 'pub_init':
pubID = message[1]
topic = message[2]
print('\n************************************\n')
print('Init msg: %s init with topic %s' % (pubID, topic))
print('\n************************************\n')
with open(log_file, 'a') as logfile:
logfile.write('Init msg: %s init with topic %s\n' %
(pubID, topic))
# publisher registration
self.update_pub_dict('add_pub', pubID, topic, '')
self.update_pub_ownership_dict('add_pub', topic, pubID)
elif msg_type == 'publication':
pubID = message[1]
topic = message[2]
publication = message[3]
print('\n************************************\n')
print('Publication: %s published %s with topic %s' %
(pubID, publication, topic))
print('\n************************************\n')
with open(log_file, 'a') as logfile:
logfile.write(
'Publication: %s published %s with topic %s\n' %
(pubID, publication, topic))
# update storage
self.update_pub_dict('add_publication', pubID, topic,
publication)
# update publisher ownership_strength for a topic
self.update_pub_ownership_dict('add_pub', topic, pubID)
# filter publisher via ownership strength
if self.filter_pub_ownership_dict(pubID, topic) is None:
print(
' Broker filter feedback: %s doesn\'t own highest ownership strength, %s won\'t be forwarded.'
% (pubID, topic))
with open(log_file, 'a') as logfile:
logfile.write(
'Broker filter feedback: %s doesn\'t own highest ownership strength, %s won\'t be forwarded.\n'
% (pubID, topic))
continue
else:
with self.history_lock:
# send publication to subscribers using xpubsocket
publication = publication + '--' + str(time.time())
self.helper.xpub_send_msg(self.xpubsocket, topic,
publication)
elif msg_type == 'drop_topic':
target_pub = message[1]
target_topic = message[2]
print('\n************************************\n')
print('Drop topic: %s droped topic %s' %
(target_pub, target_topic))
print('\n************************************\n')
with open(log_file, 'a') as logfile:
logfile.write('Drop topic: %s droped topic %s\n' %
(target_pub, target_topic))
self.update_pub_dict('drop_topic', target_pub, target_topic,
'')
self.update_pub_ownership_dict('drop_topic', target_topic,
target_pub)
# This shutoff is a soft shutoff, which means publisher would tell broker in advance
elif msg_type == 'shutoff':
with self.heartbeat_lock:
target_pub = message[1]
# update publisher dictionary
self.update_pub_dict('shutoff', target_pub, '', '')
# update publisher ownership dictionary
self.update_pub_ownership_dict('shutoff', '', target_pub)
print('\n************************************\n')
print('Soft shutoff: %s soft shutoff' % target_pub)
print('\n************************************\n')
with open(log_file, 'a') as logfile:
logfile.write('Soft shutoff: %s soft shutoff\n' %
target_pub)
self.shutoff_check = True
elif msg_type == 'heartbeat':
pubID = message[1]
print('\n************************************\n')
with self.heartbeat_lock:
if pubID not in self.heartbeat_dict:
self.heartbeat_dict.update({pubID: time.time()})
else:
self.heartbeat_dict[pubID] = time.time()
print('Heartbeat: %s heartbeat' % pubID)
print('\n************************************\n')
with open(log_file, 'a') as logfile:
logfile.write('Heartbeat: %s heartbeat\n' % pubID)
# update publisher dictionary
#
# arguments: update type, publisher
# update types:
# 1. drop a topic
# 2. add a publisher
# 3. add publication
# 4. shutoff a publisher
#
def update_pub_dict(self, update_typ, pubID, topic, publication):
try:
if update_typ == 'add_pub':
self.pub_dict.update({pubID: {topic: []}})
elif update_typ == 'add_publication':
stored_publication = publication + '--' + str(time.time())
if topic not in self.pub_dict[pubID].keys():
self.pub_dict[pubID].update({topic: [stored_publication]})
else:
self.pub_dict[pubID][topic].append(stored_publication)
elif update_typ == 'drop_topic':
if topic not in self.pub_dict[pubID].keys():
print(
' Broker filter feedback: drop topic %s for publisher %s failed, %s don\'t have this topic.'
% (topic, pubID, pubID))
with open(log_file, 'a') as logfile:
logfile.write(
'Broker filter feedback: drop topic %s for publisher %s failed, %s don\'t have this topic.\n'
% (topic, pubID, pubID))
else:
del self.pub_dict[pubID][topic]
elif update_typ == 'shutoff':
del self.pub_dict[pubID]
except KeyError:
pass
# update publisher ownership strength dictionary
#
# arguments: update type, publisher
# update types:
# 1. add a publisher
# 2. drop a topic
# 3. shutoff a publisher
#
def update_pub_ownership_dict(self, update_type, topic, pubID):
try:
if update_type == 'add_pub':
if topic not in self.pub_ownership_dict.keys():
self.pub_ownership_dict.update(
{topic: {
pubID: random.randint(1, 1000)
}})
else:
if pubID not in self.pub_ownership_dict[topic].keys():
self.pub_ownership_dict[topic].update(
{pubID: random.randint(1, 1000)})
elif update_type == 'drop_topic':
if topic not in self.pub_ownership_dict.keys():
print(
' Broker filter feedback: drop topic %s for publisher %s failed, '
'%s not in publisher ownership dictionary.' %
(topic, pubID, topic))
with open(log_file, 'a') as logfile:
logfile.write(
'Broker filter feedback: drop topic %s for publisher %s failed, '
'%s not in publisher ownership dictionary.\n' %
(topic, pubID, topic))
else:
if pubID not in self.pub_ownership_dict[topic].keys():
print(
' Broker filter feedback: drop topic %s for publisher %s failed, %s don\'t have this topic.'
% (topic, pubID, pubID))
with open(log_file, 'a') as logfile:
logfile.write(
'Broker filter feedback: drop topic %s for publisher %s failed, %s don\'t have this topic.\n'
% (topic, pubID, pubID))
else:
del self.pub_ownership_dict[topic][pubID]
elif update_type == 'shutoff':
for key in self.pub_ownership_dict.keys():
if pubID in self.pub_ownership_dict[key].keys():
del self.pub_ownership_dict[key][pubID]
except KeyError:
pass
# filter publisher ownership strength dictionary
#
# argument: current publisher & topic
# return publisher or None
#
def filter_pub_ownership_dict(self, pubID, topic):
try:
if self.pub_ownership_dict[topic][pubID] == max(
self.pub_ownership_dict[topic].values()):
return pubID
else:
return None
except Exception:
return None
# This function is used to get all publications who have highest ownership strength for all topics
def get_highest_strength_pubs(self):
try:
group = {}
for topic in self.pub_ownership_dict.keys():
max_str = max(self.pub_ownership_dict[topic].values())
for pub in self.pub_ownership_dict[topic].keys():
if self.pub_ownership_dict[topic][pub] == max_str:
group.update({pub: topic})
return group
except Exception:
return {}
class Subscriber:
def __init__(self, zk_server, topic, history_count):
self.leader_address = None
self.topic = topic
self.history_topic = topic + '-history'
self.history_count = history_count if history_count > 0 else 0
self.helper = ZMQHelper()
self.myID = str(random.randint(1, 100))
self.logfile_name = './Output/' + self.myID + '-subscriber.log'
zk_connect_addr = zk_server + ':2181'
self.zk = KazooClient(zk_connect_addr)
self.isConnected = False
self.socket = None
self.hisIPsocket = None
print('\n**************************************\n')
print(' init with topic ' + topic)
print('\n**************************************\n')
self.init_zk()
def init_zk(self):
self.zk.start(timeout=9999999)
while self.zk.state != KazooState.CONNECTED:
pass
print('Sub %s connected to ZooKeeper server.' % self.myID)
# Create a Znode for this subscriber
znode_path = '/Subscribers/' + self.myID
self.zk.create(path=znode_path,
value=b'',
ephemeral=True,
makepath=True)
while self.zk.exists(znode_path) is None:
pass
print('Sub %s created Znode in ZooKeeper server.' % self.myID)
# register this sub with leader once leader created
leader_path = '/Leader'
while self.zk.exists(leader_path) is None:
pass
data, state = self.zk.get(leader_path)
self.leader_address = data.decode("utf-8")
if self.history_count > 0:
self.request_history()
if self.register_sub():
print('Sub %s connected with leader' % self.myID)
self.isConnected = True
# set High-level exist watcher for leader znode
@self.zk.DataWatch(path=leader_path)
def watch_leader(data, state):
if state is None:
self.isConnected = False
print('Sub %s loses connection with old leader' % self.myID)
elif self.isConnected is False:
self.leader_address = data.decode("utf-8")
self.socket = None
if self.register_sub():
print('Sub %s reconnected with new leader' % self.myID)
self.isConnected = True
# only called when a sub join in
def request_history(self):
# Connected to new leader using REQ socket type, using 5558 as port number
self.hisIPsocket = self.helper.csreq(self.leader_address, '5558')
time.sleep(3)
# Send history request message to leader
msg = 'request_history_publication' + '#' + self.topic + '#' + str(
self.history_count) + '#'
self.hisIPsocket.send_string(msg)
print('Send history request to broker...')
with open(self.logfile_name, 'a') as log:
log.write('\n*************************************************\n')
log.write('Send history publications request to broker...')
log.write('\n*************************************************\n')
# receive history publication from leader broker, then store these history publication into log file
histories = self.hisIPsocket.recv_string()
print('Received histories: ' + histories)
with open(self.logfile_name, 'a') as log:
log.write('\n*************************************************\n')
log.write('Received history publications from broker: ')
log.write(histories)
log.write('\n*************************************************\n')
# parse msg
histories = simplejson.loads(histories.split('#')[1])
# write all history publications into logfile
for history in histories:
history = history.split('--')
print('\n*************************************************\n')
print('History Publication Receipt Info:\n')
print('Receive: %s\n' % history[0])
print('Time: %f\n' % abs(time.time() - float(history[1])))
print('\n*************************************************\n')
with open(self.logfile_name, 'a') as log:
log.write(
'\n*************************************************\n')
log.write('History Publication Receipt Info:\n')
log.write('Receive: %s\n' % history[0])
log.write('Time: %f\n' % abs(time.time() - float(history[1])))
log.write(
'\n*************************************************\n')
# register subscriber
def register_sub(self):
connect_str = 'tcp://' + self.leader_address + ':' + '5557'
print('Connection info: %s' % connect_str)
current = time.time()
while time.time() - current < 3:
self.socket = self.helper.connect_sub2broker(connect_str)
if self.socket is None:
print('Connection feedback: connected xpub socket failed.')
self.isConnected = False
return False
else:
print('Connection feedback: connected xpub socket succeed.')
# Add a topic for SUB socket to filter publication
self.add_sub_topic(self.topic)
return True
# receive publications from leader
def receive_publication(self):
while True:
if self.isConnected:
received_pub = self.helper.sub_recieve_msg(self.socket)
message = received_pub.split()
received_msg = ' '.join(message[1:])
received_msg = received_msg.split('--')
time_stamp = float(received_msg[1])
received_msg = received_msg[0]
current = time.time()
print('*************************************************\n'
'Receipt Info:\n'
'Publication: %s\n'
'Time Interval: %f\n' %
(received_msg, abs(current - time_stamp)))
logfile_name = './Output/' + self.myID + '-subscriber.log'
with open(logfile_name, 'a') as log:
log.write(
'*************************************************\n')
log.write('Receipt Info:\n')
log.write('Receive: %s\n' % received_msg)
log.write('Time: %f\n' % abs(current - time_stamp))
# add a subscription topic
def add_sub_topic(self, topic):
print('Add topic for subscriber.')
self.helper.subscribe_topic(self.socket, topic)
class Subscriber:
def __init__(self, address, port, topic, history_count):
self.address = address
self.port = port
self.topic = topic
self.history_topic = topic + '-history'
self.history_count = history_count
self.helper = ZMQHelper()
self.myID = str(random.randint(1, 100))
def prepare(self):
self.register_sub()
self.add_sub_topic(self.history_topic)
self.add_sub_topic(self.topic)
#
# This method should always be alive
# handler is used to receive message from broker
#
# Message Type:
# 1. history publication message
# 2. new publication
#
def handler(self):
current_time = time.time()
new_current_time = time.time()
prev_time = 2e100
# receive publications
count = 0
while True:
received_pub = self.helper.sub_recieve_msg(self.socket)
message = received_pub.split()
received_topic = message[0]
received_msg = ' '.join(message[1:])
received_msg = received_msg.split('--')
time_stamp = float(received_msg[1])
received_msg = received_msg[0]
if count == int(self.history_count) or prev_time <= time_stamp:
self.helper.unsubscribe(self.socket, self.history_topic)
count += 1e10
if time_stamp < current_time and count < int(
self.history_count) and prev_time > time_stamp:
count += 1
print('*************************************************\n'
'Receipt Info:\n'
'History Publication: %s\n'
'Time Interval: %f\n' %
(received_msg, abs(current_time - time_stamp)))
logfile_name = './Output/' + self.myID + '-subscriber.log'
with open(logfile_name, 'a') as log:
log.write(
'*************************************************\n')
log.write('Receipt Info:\n')
log.write('Receive History Publication: %s\n' %
received_msg)
log.write('Time: %f\n' % abs(current_time - time_stamp))
prev_time = time_stamp
if time_stamp >= new_current_time:
new_current_time = time.time()
print('*************************************************\n'
'Receipt Info:\n'
'Publication: %s\n'
'Time Interval: %f\n' %
(received_msg, abs(new_current_time - time_stamp)))
logfile_name = './Output/' + self.myID + '-subscriber.log'
with open(logfile_name, 'a') as log:
log.write(
'*************************************************\n')
log.write('Receipt Info:\n')
log.write('Receive: %s\n' % received_msg)
log.write('Time: %f\n' %
abs(new_current_time - time_stamp))
# register subscriber
def register_sub(self):
connect_str = 'tcp://' + self.address + ':' + self.port
print('Connection info: %s' % connect_str)
current = time.time()
while time.time() - current < 3:
self.socket = self.helper.connect_sub2broker(connect_str)
if self.socket is None:
print('Connection feedback: connected xpub socket failed.')
return False
else:
print('Connection feedback: connected xpub socket succeed.')
return True
# add a subscription topic
def add_sub_topic(self, topic):
self.helper.subscribe_topic(self.socket, topic)
