class ES_zk_interface:
def __init__(self, my_ip, es_interface, es_callback):
self.ip = my_ip
self.es_callback = es_callback
self.es_interface = es_interface
self.zk = KazooClient(hosts=HOST_IP)
self.zk.start()
self.zk.ensure_path("/ass3")
def start_election_process(self):
print "Starting Election process"
#print("Previous Leader: %s" % (self.es_callback.get_leader()))
election = self.zk.Election("/electionpath", self.ip)
election.run(self.leader_function)
def leader_function(self):
print "Won the election: " + str(self.ip)
if self.zk.exists(LEADER_PATH):
self.zk.set(LEADER_PATH, self.ip)
else:
self.zk.create(LEADER_PATH, self.ip)
#print "\nStarting Leader thread.... \n"
self.es_interface.enter_message_loop(self.es_callback, True)
def get_zookeeper_client(self):
return self.zk
def get_leader_path(self):
return LEADER_PATH
def create_zoo(zoo_config):
"""
Create and return a new zoo.
"""
hostname = socket.gethostname()
zoo = KazooClient('hosts=%s' % ','.join(zoo_config))
zoo.start()
return zoo.Election('/deployrepo', hostname)
def run(num, condition):
zk = KazooClient(hosts='127.0.0.1:2181') # Running there
zk.start()
election = zk.Election("/electionpath", num)
# blocks until the election is won, then calls
# my_leader_function()
# WHat is this?
# f****d. It is happening again.
# MY GOD..guess who i am
x = election.run(my_leader_function, num, condition, election)
winner = election.contenders()[0]
print("WINNER:", winner)
print("My x is", x, "and my num is", num)
def action(args, *add):
print(args)
for arc in add:
print("port %s --当前进程%d" % (arc, os.getpid()))
zk = KazooClient(hosts='127.0.0.1:2181')
zk.start()
election = zk.Election("/electionpath")
election.run(leader_func)
zk.stop()
add_and_update('/zookeeper/mykey', b'this is my value!')
read('/zookeeper/mykey')
add_and_update('/zookeeper/mykey', b'this is my setting value!')
def thread_process2(index):
logDS.info("begin %d" % index)
zk = KazooClient(hosts='172.10.3.111:2181', logger=logDS.logger)
zk.start()
root = "/electionpath"
election = zk.Election(root, "identifier:%d" % 0)
# blocks until the election is won, then calls
# my_leader_function()
election.run(my_leader_function)
print(election.contenders())
#logDS.info("thread_process2 88")
#time.sleep(10)
zk.stop()
logDS.info("exit %d" % index)
def main():
name = socket.gethostname()
scheduler = None
# Gracefully shutdown
# Make sure we kill subprocess before abnormal exit
def on_exit(signum, frame):
global scheduler
print scheduler
if scheduler:
scheduler.terminate()
output = scheduler.wait()
print("Scheduler was killed with output: " + str(output))
sys.exit(0)
# Start scheduler as subprocess and wait for it
def start_scheduler():
print(name + " becomes master!")
global scheduler
scheduler = subprocess.Popen(["airflow", "scheduler"],
stderr=subprocess.PIPE)
output = scheduler.communicate()
print("Scheduler was shutdown with output: " + str(output))
# Register to signals
signal.signal(signal.SIGINT, on_exit)
signal.signal(signal.SIGTERM, on_exit)
# Connect to Zookeeper cluster
zk = KazooClient(
hosts='10.0.0.6:2181,10.0.0.11:2181,10.0.0.4:2181,10.0.0.7:2181')
zk.start()
election = zk.Election("/master-scheduler", name)
print(name + " is ready to become master")
# Block until got elected as master
# Then run start_scheduler
election.run(start_scheduler)
zk.stop()
class Broker:
def __init__(self):
self.frontend_socket = None
self.backend_socket = None
self.context = None
self.proxy = None
self.tempPubPort = None
self.zk_path = '/nodes/'
self.zk_type1_leader_path = '/leader1/'
self.zk_type2_leader_path = '/leader2/'
self.replica_path = '/lb_replica/'
self.leader1 = None
self.leader2 = None
self.connection_count = 0
self.zookeeper = KazooClient(hosts='127.0.0.1:2181')
self.zookeeper.start()
self.ip_addr = socket.gethostbyname(socket.gethostname())
def gen_nodes(self):
used_ports = []
for i in range(5):
port1 = str(random.randrange(5000, 9999, 2))
if port1 in used_ports:
port1 = str(random.randrange(5000, 9999, 2))
used_ports.append(port1)
port2 = str(random.randrange(5000, 9999, 3))
if port2 in used_ports:
port2 = str(random.randrange(5000, 9999, 2))
used_ports.append(port2)
node_name = "{}{}".format(self.zk_path, "node" + str(i))
node_info = bytes("{},{},{}".format(port1, port2,
self.ip_addr).encode("utf-8"))
if self.zookeeper.exists(node_name):
pass
else:
self.zookeeper.ensure_path(self.zk_path)
self.zookeeper.create(node_name,
node_info,
None,
ephemeral=True)
self.set_leaders()
def set_leaders(self):
"""Set our Loadbalanced Leaders"""
gen_leader = self.zookeeper.Election(self.zk_path,
"leader").contenders()
if gen_leader:
self.leader1 = gen_leader[-1]
self.leader2 = gen_leader[-2]
else:
_nv = random.randint(0, 4)
self.leader1 = self.zookeeper.get("{}node{}".format(
self.zk_path, _nv))
_nv2 = random.choice([i for i in range(0, 4) if i not in [_nv]])
self.leader2 = self.zookeeper.get("{}node{}".format(
self.zk_path, _nv2))
leader_type1_path = "{}{}".format(self.zk_type1_leader_path,
"leadNode")
leader_type2_path = "{}{}".format(self.zk_type2_leader_path,
"leadNode")
if not self.zookeeper.exists(leader_type1_path):
self.zookeeper.ensure_path(self.zk_type1_leader_path)
self.zookeeper.create(leader_type1_path,
self.leader1[0],
None,
ephemeral=True)
else:
self.zookeeper.ensure_path(self.zk_type1_leader_path)
self.zookeeper.set(leader_type1_path, self.leader1[0])
print("New Type 1 Broker Node added to pool: {}".format(self.leader1))
if not self.zookeeper.exists(leader_type2_path):
self.zookeeper.ensure_path(self.zk_type2_leader_path)
self.zookeeper.create(leader_type2_path,
self.leader2[0],
None,
ephemeral=True)
else:
self.zookeeper.ensure_path(self.zk_type2_leader_path)
self.zookeeper.set(leader_type2_path, self.leader2[0])
print("New Type 2 Broker Node added to pool: {}".format(self.leader2))
def create_loadbalance_replicas(self):
"""Create ephemeral (session bound) replicas of node"""
print("Replicating leader...")
replica_node_type1 = "{}{}".format(self.replica_path, "replicaNodeT1")
replica_node_type2 = "{}{}".format(self.replica_path, "replicaNodeT2")
if self.zookeeper.exists(replica_node_type1):
self.zookeeper.delete(replica_node_type1)
self.zookeeper.ensure_path(self.replica_path)
self.zookeeper.create(replica_node_type1,
self.leader1[0],
None,
ephemeral=True)
if self.zookeeper.exists(replica_node_type2):
self.zookeeper.delete(replica_node_type2)
self.zookeeper.ensure_path(self.replica_path)
self.zookeeper.create(replica_node_type2,
self.leader2[0],
None,
ephemeral=True)
print("... Replication Complete")
def remove_temp_nodes(self):
for i in range(5):
node_name = "{}{}".format(self.zk_path, "node" + str(i))
self.zookeeper.delete(node_name)
def establish_broker(self):
try:
@self.zookeeper.DataWatch(self.zk_type1_leader_path + "leadNode")
def watch_node(data, stat, event):
if event and event.type == "DELETED":
leader_path = "{}{}".format(self.zk_type1_leader_path,
"leadNode")
replica_node_type1 = "{}{}".format(self.replica_path,
"replicaNodeT1")
print("Load balancer event detected")
self.leader1 = self.zookeeper.get(replica_node_type1)
self.zookeeper.ensure_path(self.zk_type1_leader_path)
try:
self.zookeeper.set(leader_path, self.leader1[0])
except:
self.zookeeper.create(leader_path, self.leader1[0])
@self.zookeeper.DataWatch(self.zk_type2_leader_path + "leadNode")
def watch_node(data, stat, event):
if event and event.type == "DELETED":
leader_path = "{}{}".format(self.zk_type2_leader_path,
"leadNode")
replica_node_type2 = "{}{}".format(self.replica_path,
"replicaNodeT2")
print("Load balancer event detected")
self.leader2 = self.zookeeper.get(replica_node_type2)
self.zookeeper.ensure_path(self.zk_type2_leader_path)
try:
self.zookeeper.set(leader_path, self.leader2[0])
except:
self.zookeeper.create(leader_path, self.leader2[0])
except NodeExistsError:
pass
print("Loadbalanced Broker established. RUNNING.")
leader1_connection_addr = self.leader1[0].decode('utf-8').split(',')
leader2_connection_addr = self.leader2[0].decode('utf-8').split(',')
self.context = zmq.Context()
self.frontend_socket = self.context.socket(zmq.XSUB)
self.backend_socket = self.context.socket(zmq.XPUB)
self.frontend_socket.bind(f"tcp://*:{leader1_connection_addr[0]}")
self.backend_socket.bind(f"tcp://*:{leader1_connection_addr[1]}")
self.frontend_socket.bind(f"tcp://*:{leader2_connection_addr[0]}")
self.backend_socket.bind(f"tcp://*:{leader2_connection_addr[1]}")
zmq.proxy(self.frontend_socket, self.backend_socket)
def close_connection(self):
self.frontend_socket.close()
self.backend_socket.close()
class Ingress:
def __init__(self, spout, my_address, zk_address):
# Buffer format:
'''
{
'state': [
{
'state': $state,
'data': $data,
'time': $time,
'status': $status
}
]
}
'''
self.buffer = {}
self.spout = spout
self.zk_address = zk_address
self.id = str(random.randint(1, 1000))
self.my_address = my_address
self.zk = None
self.up_stream_socket = None
self.down_stream_socket = []
self.operators = []
self.parent_path = './' + spout + '/Ingress_operators/'
self.leader_path = self.parent_path + '/leader'
self.znode_path = self.parent_path + '/' + self.my_address
self.operator_path = './' + self.spout + '/operators/'
self.init_zk()
def init_zk(self):
self.zk = KazooClient(self.zk_address)
self.zk.start()
while self.zk.state == KazooState.CONNECTED is False:
pass
print('Connected to ZooKeeper server.')
if self.zk.exists(path=self.parent_path) is False:
self.zk.create(path=self.parent_path,
value=b'',
ephemeral=False,
makepath=True)
while self.zk.exists(path=self.parent_path) is False:
pass
print('Create parent znode path for ingress operator znode.')
self.zk.create(path=self.znode_path, value=b'', ephemeral=True)
while self.zk.exists(self.znode_path) is False:
pass
print('Create zndoe for Ingress operator.')
if self.zk.exists(self.leader_path) is False:
self.zk.create(path=self.leader_path,
value=self.my_address,
ephemeral=True)
while self.zk.exists(self.leader_path) is False:
pass
print('Create leader znode.')
@self.zk.DataWatch(self.leader_path)
def watch_leader(data, stat):
if stat == KazooState.LOST:
election = self.zk.Election(self.parent_path, self.id)
election.run(win_election)
@self.zk.ChildrenWatch(self.zk, self.operator_path)
def watch_operators(children):
for child in children:
if child not in self.operators:
path = self.spout + '/operators/' + child
address = self.operator_path + self.zk.get(path=path)
threading.Thread(target=init_REQ, args=(address, )).start()
def init_REQ(address):
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect('tcp://' + address + ':2341')
self.down_stream_socket.append(socket)
def win_election():
print('Yeah, I won the election.')
self.zk.set(path=self.leader_path, value=self.my_address)
def init_upstream_socket(self):
context = zmq.Context()
self.up_stream_socket = context.socket(zmq.REP)
self.up_stream_socket.bind('tcp://*:2341')
def recv_data(self):
while True:
msg = self.up_stream_socket.recv_string()
self.up_stream_socket.send_sting('OK')
print('Receive msg %s from data source.' % msg)
msg = simplejson.loads(msg)
msg.update({'status': 'recv'})
state = msg['state']
if state in self.buffer.keys():
self.buffer[state].append(msg)
else:
self.buffer.update({state: msg})
def notify_slave(self):
pass
# coding=utf-8
"""
领导选举
当leader被杀死之后从worker中会被选举出来一个新的leader
"""
from kazoo.client import KazooClient
from uuid import uuid4
import time
import sys
my_id = uuid4()
def leader_func():
print "%s is leader" % my_id
while True:
print "%s is working" % my_id
time.sleep(3)
zk = KazooClient(hosts="127.0.0.1:2181")
zk.start()
election = zk.Election("/test/election")
# 会一直阻塞 直到被选举称为leader
election.run(leader_func)
zk.stop()
class ZookeeperLeaderElector:
zk = None
default_node_path = None
zookeeper_connection_url = None
config_file_dir = None
def __init__(self):
# Default zookeeper node path to CRUD
self.default_node_path = "/cluster"
# Locate config file
self._get_config_file_addr()
# Get the subset of properties relevant to zookeeper
self._init_config_props()
# Init zookeeper client instance
self.zk = KazooClient(self.zookeeper_connection_url)
# Locate config file
def _get_config_file_addr(self):
current_dir = path.dirname(path.realpath(__file__))
parent_dir = path.dirname(current_dir)
self.config_file_dir = path.join(path.dirname(parent_dir), 'config',
'zookeeper.config')
# Get the subset of properties relevant to zookeeper
def _init_config_props(self):
server_props = ConfigParser()
server_props.read(self.config_file_dir)
self.zookeeper_connection_url = server_props["connect"]["url"]
def startup(self, server_id, host_ip, broker_server_start):
# Connect to zookeeper server
self.zk.start()
try:
# Ensure a path, create if necessary
self.zk.ensure_path(self.default_node_path)
# Create a node with data
node = "node" + server_id
node_path = self.default_node_path + '/' + node
self.zk.create_async(node_path,
bytes(host_ip, 'utf-8'),
ephemeral=True)
# Elect for leadership
print("[SETUP/ZK] Elect for leadership ...")
election = self.zk.Election(self.default_node_path, node)
election.run(broker_server_start)
# Exit
except KeyboardInterrupt:
self.exit()
# Alwasys stop the zk instance and disconnect
finally:
self.zk.stop()
self.zk.close()
def exit(self):
print("[EXIT] Disconnect from zookeeper server.")
raise KeyboardInterrupt
def ready(self):
if not self.config_file_dir:
sys.exit("[ERR] Doesn't find the config file.")
elif not self.zookeeper_connection_url:
sys.exit("[ERR] Zookeeper server url is EMPTY.")
elif not self.zk:
sys.exit("[ERR] Zookeeper instance instantiation FAILED.")
return True
class ZooAnimal:
def __init__(self):
self.zk = KazooClient(hosts=ZOOKEEPER_LOCATION)
self.zk.start()
# Use util function to get IP address
self.ipaddress = [
ip for ip in list(local_ip4_addr_list())
if ip.startswith(NETWORK_PREFIX)
][0]
# Inheriting children should assign values to fit the scheme
# /role/topic
self.role = None
self.topic = None
#Will only be set by pub and sub
self.broker = None
# Zookeeper
#self.election = None
self.election = self.zk.Election('/broker', self.ipaddress)
self.zk_seq_id = None
self.zk_is_a_master = False
def zookeeper_watcher(self, watch_path):
@self.zk.DataWatch(watch_path)
def zookeeper_election(data, stat, event):
print("Setting election watch.")
print("Watching node -> ", data)
if data is None:
print("Data is none.")
self.election.run(self.zookeeper_register)
#self.election.cancel()
def zookeeper_master(self):
if not self.zk_is_a_master:
print("ZOOANIMAL -> Becoming a master.")
role_topic = "/broker/master"
data = {'ip': self.ipaddress}
data_string = json.dumps(data)
encoded_ip = codecs.encode(data_string, "utf-8")
self.zk.create(role_topic,
ephemeral=True,
makepath=True,
sequence=True,
value=encoded_ip)
self.zk_is_a_master = True
return self.zk_is_a_master
def zookeeper_register(self):
pass
# This is a function stub for the get_broker watch callback
# The child is expected to implement their own logic
# Pub and Sub need to register_sub()
def broker_update(self, data):
print("Broker updated.")
print("Data -> {}".format(data))
pass
def get_broker(self):
for i in range(10):
if self.zk.exists(PATH_TO_MASTER_BROKER):
node_data = self.zk.get(PATH_TO_MASTER_BROKER,
watch=self.broker_update)
broker_data = node_data[0]
master_broker = codecs.decode(broker_data, 'utf-8')
if master_broker != '':
self.broker = master_broker
return self.broker
else:
raise Exception("No master broker.")
time.sleep(0.2)
class ZK_Driver:
#CTOR
def __init__(self, ip_add):
context = zmq.Context()
self.strength = {}
self.pub_history = {}
self.kill = False
self.sub_socket = context.socket(zmq.SUB)
self.pub_socket = context.socket(zmq.PUB)
self.current_topics = []
self.zk_driver = KazooClient(hosts='127.0.0.1:2181')
self.zk_driver.start()
#ROOT DIRECTORY FOR BROKERS
self.home = '/brokers/'
#CREATE ZNODE PATHS FOR BROKERS
self.znode1 = self.home + 'bkr1'
self.znode2 = self.home + 'brk2'
self.znode3 = self.home + 'brk3'
#ENSURE ROOT DIRECTORY IS CREATED
self.zk_driver.ensure_path(self.home)
#CREATE ZNODES WITH PUB + SUB PORT
if not self.zk_driver.exists(self.znode1):
self.zk_driver.create(self.znode1, b'1234:5556')
if not self.zk_driver.exists(self.znode2):
self.zk_driver.create(self.znode2, b'1235:5556')
if not self.zk_driver.exists(self.znode3):
self.zk_driver.create(self.znode3, b'1236:5556')
#HOLD ELECTION TO GET PRESIDENT NODE
self.election = self.zk_driver.Election(self.home, "president")
contenders = self.election.contenders()
self.president = contenders[-1].encode('latin-1') #REPRESENTS THE WINNING PUB/SUB PORT COMBO
ports = self.president.decode('ASCII').split(":")
#FULL BROKER PORT ADDRESSES
self.full_add1 = "tcp://" + str(ip_add) + ":" + ports[0]
self.full_add2 = "tcp://" + str(ip_add) + ":" + ports[1]
#BIND TO ADDRESSES
self.sub_socket.bind(self.full_add1)
self.sub_socket.subscribe("")
self.pub_socket.bind(self.full_add2)
#SET UP WATCH DIRECTORY FOR PRESIDENT
self.president_home = "/president/"
self.pres_znode = "/president/pres"
#CREATE OR UPDATE PRESIDENT ZNODE
if not self.zk_driver.exists(self.pres_znode):
self.zk_driver.ensure_path(self.president_home)
self.zk_driver.create(self.pres_znode, ephemeral=True)
self.zk_driver.set(self.pres_znode, self.president)
# REMOVE PRESIDENT FROM FUTURE ELECTIONS
if ports[0] == "1234":
self.zk_driver.delete(self.znode1)
elif ports[0] == "1235":
self.zk_driver.delete(self.znode2)
elif ports[0] == "1236":
self.zk_driver.delete(self.znode3)
else:
print("No port recognized")
def run(self, stop=None):
@self.zk_driver.DataWatch(self.pres_znode)
def watch_node(data, stat, event):
if event is not None and event.type == "DELETED":
if not self.kill:
# HOLD ELECTION TO GET PRESIDENT NODE
self.election = self.zk_driver.Election(self.home, "president")
contenders = self.election.contenders()
self.president = contenders[-1].encode('latin-1') # REPRESENTS THE WINNING PUB/SUB PORT COMBO
ports = self.president.decode('ASCII').split(":")
# FULL BROKER PORT ADDRESSES
self.full_add1 = "tcp://" + str(ip_add) + ":" + ports[0]
self.full_add2 = "tcp://" + str(ip_add) + ":" + ports[1]
print("Updated Broker to: ", self.full_add1)
# BIND TO ADDRESSES
self.sub_socket.bind(self.full_add1)
self.sub_socket.subscribe("")
# self.pub_socket.bind(self.full_add2)
# UPDATE PRESIDENT ZNODE
if not self.zk_driver.exists(self.pres_znode):
self.zk_driver.ensure_path(self.president_home)
self.zk_driver.create(self.pres_znode, ephemeral=True)
self.zk_driver.set(self.pres_znode, self.president)
# DELETE FROM FUTURE ELECTIONS
if ports[0] == "1234":
self.zk_driver.delete(self.znode1)
elif ports[0] == "1235":
self.zk_driver.delete(self.znode2)
elif ports[0] == "1236":
self.zk_driver.delete(self.znode3)
else:
print("No port recognized")
if not self.kill:
self.kill = True
if stop:
while not stop.is_set():
message = self.sub_socket.recv_string()
topic, info, id = message.split("||")
error_flag = False
if topic == "REGISTER":
error = False
for curr_topic in self.current_topics:
if info.startswith(curr_topic) and info != curr_topic:
print("Topic is too similar to topic of another publisher, choose another")
error = True
if not error:
self.current_topics.append(info)
msgs = info.split("...")
msgs = msgs[0:len(msgs) - 1]
print("Received: %s" % topic + "||" + msgs[len(msgs)-1] + "||Pub ID = " + id)
if info not in self.pub_history.keys():
self.pub_history[info] = []
# maintain ordered list of publishers per topic
if info not in self.strength.keys():
self.strength[info] = [id]
else:
temp = list(self.strength[info])
temp.append(id)
self.strength[info] = temp
self.pub_socket.send_string(message)
else:
### Process strength of publisher for the topic to determine whether to send message to clients
temp = self.strength[topic]
while len(temp) != 0:
first_val = temp[0]
# check if node exists
if self.zk_driver.exists('/' + str(topic) + '/' + str(first_val)):
if str(first_val) == str(id):
# store history
temp = self.pub_history[topic]
temp.append(msgs[len(msgs) - 1])
self.pub_history[topic] = temp
topic, info, id = message.split("||")
self.pub_socket.send_string(
topic + "||" + "...".join(self.pub_history[topic]) + "||" + id)
break
else:
break
# otherwise, delete broken node from out dictionary
else:
temp.remove(first_val)
self.strength[topic] = temp
else:
message = self.sub_socket.recv_string()
#print("Time received: %.20f" % time.time()) # uncomment for measurements purposes
topic, info, id = message.split("||")
error_flag = False
if topic == "REGISTER":
error = False
for curr_topic in self.current_topics:
if info.startswith(curr_topic) and info != curr_topic:
print("Topic is too similar to topic of another publisher, choose another")
error = True
if not error:
self.current_topics.append(info)
print("Addr ", self.full_add1, end=". ")
print("Received: %s" % message)
if info not in self.pub_history.keys():
self.pub_history[info] = []
# maintain ordered list of publishers per topic
if info not in self.strength.keys():
self.strength[info] = [id]
else:
temp = list(self.strength[info])
temp.append(id)
self.strength[info] = temp
self.pub_socket.send_string(message)
else:
if topic in self.current_topics:
msgs = info.split("...")
msgs = msgs[0:len(msgs) - 1]
print("Addr ", self.full_add1, end=". ")
print("Received: %s" % topic + "||" + msgs[len(msgs)-1] + "||Pub ID = " + id)
### Process strength of publisher for the topic to determine whether to send message to clients
temp = self.strength[topic]
while len(temp) != 0:
first_val = temp[0]
# check if node exists
if self.zk_driver.exists('/' + str(topic) + '/' + str(first_val)):
if str(first_val) == str(id):
# store history
temp = self.pub_history[topic]
temp.append(msgs[len(msgs) - 1])
self.pub_history[topic] = temp
topic, info, id = message.split("||")
self.pub_socket.send_string(topic + "||" + "...".join(self.pub_history[topic]) + "||" + id)
#print("Time sent out: %.20f" % time.time()) # uncomment for measurements purposes
break
else:
break
# otherwise, delete broken node from out dictionary
else:
temp.remove(first_val)
self.strength[topic] = temp
else:
print("Please start over with a valid topic")
class ZooKeeper(object):
max_tries = 3
max_delay = 4
delay = 1
def __init__(self, config):
"""
@config: dict like object which contains:
{
"zookeepers": ip:port,ip2:port,
}
"""
self._config = config
self._zk = KazooClient(hosts=config[c.zookeepers])
self._started = False
def start(self):
if self._started:
logger.info("ZookeeperClient has already started.")
return
self._started = True
try:
self._zk.start()
except Exception:
logger.error("Failed to start zookeeper client, error=%s",
traceback.format_exc())
raise
logger.info("ZookeeperClient started...")
def stop(self):
if not self._started:
logger.info("ZookeeperClient has already stopped.")
return
self._started = False
self._zk.stop()
def election(self, election_path, identifier=None, callback=None):
"""
@election_path: zookeeper node path of election
@identifier: identifiy the candidates
@return: block until the caller is the leader, callback will be invoked
"""
election = self._zk.Election(election_path, identifier)
election.run(callback)
def create(self,
path,
value="",
ephemeral=False,
sequence=False,
makepath=False):
retry = KazooRetry(max_tries=self.max_tries,
delay=self.delay,
max_delay=self.max_delay,
ignore_expire=False)
return retry(self._zk.create,
path=path,
value=value,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath)
def ensure_path(self, path):
retry = KazooRetry(max_tries=self.max_tries,
delay=self.delay,
max_delay=self.max_delay,
ignore_expire=False)
return retry(self._zk.ensure_path, path)
def get(self, path, watch=None):
retry = KazooRetry(max_tries=self.max_tries,
delay=self.delay,
max_delay=self.max_delay,
ignore_expire=False)
return retry(self._zk.get, path, watch)
def get_children(self, path, watch=None):
retry = KazooRetry(max_tries=self.max_tries,
delay=self.delay,
max_delay=self.max_delay,
ignore_expire=False)
return retry(self._zk.get_children, path, watch)
def exists(self, path, watch=None):
# retry = KazooRetry(max_tries=self.max_tries, delay=self.delay,
# max_delay=self.max_delay, ignore_expire=False)
# return retry(self._zk.exists, path, watch)
return self._zk.exists(path, watch)
def set(self, path, value):
retry = KazooRetry(max_tries=self.max_tries,
delay=self.delay,
max_delay=self.max_delay,
ignore_expire=False)
return retry(self._zk.set, path, value)
def delete(self, path, recursive=False):
retry = KazooRetry(max_tries=self.max_tries,
delay=self.delay,
max_delay=self.max_delay,
ignore_expire=False)
return retry(self._zk.delete, path, recursive=recursive)
class Ingress:
def __init__(self, spout, my_address, zk_address, db_address, db_user,
db_pwd):
# Connect to MySQL server
self.tb_name = 'IngressOperator'
self.db_name = 'Spout_' + str(spout)
self.columns = ['Time', 'State', 'Data', 'Status']
self.columns_type = ['char(50)', 'char(20)', 'char(100)', 'char(10)']
self.db_connection, self.db_handler = self.init_db(
db_address, db_user, db_pwd)
self.spout = spout
self.zk_address = zk_address
self.id = str(random.randint(1, 1000))
self.my_address = my_address
self.zk = KazooClient(hosts=self.zk_address)
self.up_stream_socket = None
self.down_stream_sockets = []
self.operators = []
self.parent_path = '/Spout--' + str(spout) + '/Ingress_operators'
self.leader_path = '/Spout--' + str(spout) + '/Ingress_leader'
self.znode_path = self.parent_path + '/Ingress--' + self.id
self.operator_path = '/Spout--' + str(self.spout) + '/Operators'
self.isLeader = False
self.lock = threading.Lock()
self.flag = 0
self.init_zk()
def init_db(self, db_address, db_user, db_pwd):
# Connect to Mysql server
db_connection, db_handler = mysqlop.connectMysql(
db_address, db_user, db_pwd)
# Create DB
mysqlop.createDB(db_handler, self.db_name)
# Create Table
mysqlop.createTable(db_handler, self.db_name, self.tb_name,
self.columns, self.columns_type)
# Add primary key for the table
mysqlop.add_primary_key(db_handler, db_connection, self.db_name,
self.tb_name, self.columns[0])
return db_connection, db_handler
def init_zk(self):
self.zk.start()
while (self.zk.state == KazooState.CONNECTED) is False:
pass
print('Connected to ZooKeeper server.')
# Create parent path in zookeeper
if self.zk.exists(path=self.parent_path) is None:
self.zk.create(path=self.parent_path,
value=b'',
ephemeral=False,
makepath=True)
self.zk.create(path=self.znode_path,
value=b'',
ephemeral=False,
makepath=True)
def win_election():
print('Yeah, I won the election.')
# Create leader znode
self.zk.create(path=self.leader_path,
value=self.my_address,
ephemeral=True,
makepath=True)
self.init_upstream_socket()
self.isLeader = True
# Start receiving data from data source
threading.Thread(target=self.recv_sourcedata, args=()).start()
threading.Thread(target=self.distribute_data, args=()).start()
@self.zk.DataWatch(self.leader_path)
def watch_leader(data, stat):
if stat is None:
election = self.zk.Election(self.parent_path, self.id)
election.run(win_election)
def init_REQ(address):
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect('tcp://' + address + ':2341')
# socket.setsockopt(zmq.RCVTIMEO, 30000)
self.down_stream_sockets.append(socket)
@self.zk.ChildrenWatch(self.operator_path)
def watch_operators(children):
for i, op in enumerate(self.operators[:]):
if op not in children:
del self.operators[i]
for child in children:
if child not in self.operators:
self.operators.append(child)
print(child)
path = '/Spout--' + str(self.spout) + '/Operators/' + child
address = self.zk.get(path=path)[0]
print('Address is: ' + address)
init_REQ(address)
while True:
pass
def init_upstream_socket(self):
context = zmq.Context()
self.up_stream_socket = context.socket(zmq.REP)
self.up_stream_socket.bind('tcp://*:2341')
def recv_sourcedata(self):
while True:
msg = self.up_stream_socket.recv_string()
# print('Receive msg %s from data source.' % msg)
msg = simplejson.loads(msg)
temp = []
temp.extend(msg)
temp.append('Recv')
# Store data into DB
self.lock.acquire()
mysqlop.insert_data(self.db_connection, self.db_handler,
self.db_name, self.tb_name, temp)
self.flag += 1
self.lock.release()
self.up_stream_socket.send_string('Ack--OK')
tm.sleep(0.01)
def distribute_data(self):
while True:
if self.flag == 15:
self.flag = 0
# 读取前100/row_count 行数据
self.lock.acquire()
data = mysqlop.query_first_N(self.db_handler, self.db_name,
self.tb_name, 15)
self.lock.release()
print(data)
temp_data = []
for item in data:
temp_data.append({
'Time': item[0],
'State': item[1],
'Data': item[2]
})
data = temp_data
# 开始并行发送
def send_data(socket, my_data):
for __data in my_data:
__data = simplejson.dumps(__data)
socket.send_string(__data)
ack = socket.recv_string()
# Ack msg format: 'ack--' + $time
ack_time = ack.split('--')[1]
print(ack)
# Update DB
self.lock.acquire()
mysqlop.delete_row(self.db_handler, self.db_connection,
self.db_name, self.tb_name, 'Time',
ack_time)
self.lock.release()
socket_count = len(self.down_stream_sockets)
while socket_count == 0:
print('no socket')
each_count = 15 / socket_count
for i in range(socket_count):
if i != socket_count - 1:
threading.Thread(target=send_data,
args=(
self.down_stream_sockets[i],
data[i * each_count:(i + 1) *
each_count],
)).start()
else:
threading.Thread(target=send_data,
args=(
self.down_stream_sockets[i],
data[i * each_count:],
)).start()
class ZookeeperBrokerManager:
zk = None
default_node_path = None
config_file_dir = None
zookeeper_connection_url = None
zookeeper_connection_timeout = None
broker_server_default_port = None
leader_broker_node_path = None
leader_broker_url = None
def __init__(self, role):
print("[SETUP/ZK] Communicate through broker")
# Default zookeeper node path to CRUD
self.default_node_path = "/cluster"
# Get config file
self._get_config_file_addr(role)
# Init publisher props
self._init_config_props()
# Init zookeeper client instance
self.zk = KazooClient(self.zookeeper_connection_url, read_only=True)
# Locate config file
def _get_config_file_addr(self, role):
current_dir = path.dirname(path.realpath(__file__))
parent_dir = path.dirname(path.dirname(current_dir))
filename = role + ".config"
self.config_file_dir = path.join(path.dirname(parent_dir), "config", filename)
# Get the subset of properties relevant to broker server and zookeeper
def _init_config_props(self):
props = ConfigParser()
props.read(self.config_file_dir)
self.broker_server_default_port = props["broker_server"]["port"]
self.zookeeper_connection_url = props["service_discovery"]["connection.url"]
self.zookeeper_connection_timeout = int(props["service_discovery"]["connection.timeout.s"])
# Called when first start
def startup(self, socks_connect, socks_disconnect):
print("[SETUP/ZK] Connect to zookeeper ...")
self.zk.start(timeout=self.zookeeper_connection_timeout)
self.find_leader_broker(socks_connect, socks_disconnect)
# Find the leader broker to connect
def find_leader_broker(self, socks_connect, socks_disconnect):
print("[SETUP/ZK] Find leader broker ...")
# Reconnect if disconnected
if not self.zk.connected:
self.zk.start(timeout=self.zookeeper_connection_timeout)
# Find the leader broker queue
election = self.zk.Election(self.default_node_path)
leader_queue = election.contenders()
# If no leader exists, raise error
if not leader_queue:
sys.exit("[ERR] No active brokers.")
# Identiy the leader broker node
leader_node = leader_queue[0]
self.leader_broker_node_path = self.default_node_path + '/' + leader_node
# Read the leader broker connection url
data, _ = self.zk.get(self.leader_broker_node_path)
leader_host = data.decode("utf-8")
self.leader_broker_url = "tcp://{0}:{1}".format(leader_host, self.broker_server_default_port)
# Connect to leader broker
socks_connect(self.leader_broker_url)
# Watch on the leader node in case it dies
self.watch(socks_connect, socks_disconnect)
# Watch on the leader node, find new leader if the current leader suicides
def watch(self, socks_connect, socks_disconnect):
@self.zk.DataWatch(self.leader_broker_node_path)
def my_func(data, stat, event):
if stat:
print("[SETUP/ZK] Start watching on leader node")
else:
socks_disconnect(self.leader_broker_url)
self.find_leader_broker(socks_connect, socks_disconnect)
# Check if props are ready and error free
def ready(self):
if not self.config_file_dir:
sys.exit("[ERR] Doesn't find the config file.")
elif not self.broker_server_default_port:
sys.exit("[ERR] Broker server default port is not given.")
elif not self.zookeeper_connection_url:
sys.exit("[ERR] Zookeeper server url is EMPTY.")
elif not self.zk:
sys.exit("[ERR] Zookeeper instance instantiation FAILED.")
return True
# Must called on exit
def exit(self):
self.zk.stop()
self.zk.close()
class Broker:
def __init__(self, id, zk_server, ip, xpub, xsub):
self.ID = id
self.IP = ip
self.pubsocket = self.bindp(ip, xpub)
self.subsocket = self.binds(ip, xsub)
self.syncsocket = None
self.pubsynsocket = None
zk_server = zk_server + ':2181'
self.zk = KazooClient(hosts=zk_server)
self.leader_flag = False
#self.ownership = None
# the publisher and the info they store
self.pubdict = {}
self.publisher = {}
self.pubhistory = {}
self.subdict = {}
self.subscriber = {}
self.unable = []
self.init_zk()
print('\n************************************\n')
print('Init MyBroker succeed.')
print('\n************************************\n')
with open(log_file, 'w') as logfile:
logfile.write('Init Broker succeed \n')
def bindp(self,ip, port):
# we use REQ and REP to manage between publisher and broker
context = zmq.Context()
p_socket = context.socket(zmq.REP)
p_socket.setsockopt(zmq.RCVTIMEO, 3000)
p_socket.setsockopt(zmq.SNDTIMEO, 3000)
p_socket.bind('tcp://*:'+ port)
with open(log_file, 'a') as logfile:
logfile.write('5555! \n')
return p_socket
def binds(self, ip, port):
# we use PUB and SUB to manage sub and broker
context = zmq.Context()
s_socket = context.socket(zmq.REP)
s_socket.bind('tcp://*:' + port)
with open(log_file, 'a') as logfile:
logfile.write('5556! \n')
return s_socket
def init_zk(self):
if self.zk.state != KazooState.CONNECTED:
self.zk.start()
while self.zk.state != KazooState.CONNECTED:
pass
# wait until the zookeeper starts
print('Broker %s connected to ZooKeeper server.' % self.ID)
# Create the path /Brokers
if self.zk.exists('/Brokers') is None:
self.zk.create(path='/Brokers', value=b'', ephemeral=False, makepath=True)
while self.zk.exists('/Brokers') is None:
pass
# create my node under the path /Brokers
znode_path = '/Brokers/' + self.ID
self.zk.create(path=znode_path, value=b'', ephemeral=True, makepath=True)
while self.zk.exists(znode_path) is None:
pass
print('Broker %s created a znode in ZooKeeper server.' % self.ID)
def watch_mode(self):
print("In watch mode")
election_path = '/Brokers/'
leader_path = '/Leader'
@self.zk.DataWatch(path=leader_path)
def watch_leader(data, state):
if self.zk.exists(path=leader_path) is None:
# time.sleep(random.randint(0, 3))
election = self.zk.Election(election_path, self.ID)
election.run(self.win_election)
def win_election(self):
print("Win election")
leader_path = '/Leader'
if self.zk.exists(path=leader_path) is None:
try:
self.zk.create(leader_path, value=self.IP.encode('utf-8'), ephemeral=True, makepath=True)
except Exception as ex:
print("shouldn't elect")
while self.zk.exists(path=leader_path) is None:
pass
self.leader_flag = True
#self.start_broker()
#self.syncsocket = None
self.pubsyncsocket = self.sourcepush('5557')
if self.pubsyncsocket != None:
print('Broker %s started sending msg' % self.ID)
def pull_msg(self, address, port):
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.setsockopt(zmq.RCVTIMEO, 30000)
socket.connect('tcp://' + address + ':' + port)
return socket
# Leader push msg to followers
def sourcepush(self, port):
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.bind("tcp://*" + ':' + port)
socket.setsockopt(zmq.SNDTIMEO, 3000)
socket.setsockopt(zmq.RCVTIMEO, 3000)
return socket
def find(self, topic, number1, number2):
number = []
number.append(number1)
number.append(number2)
content ='#'
for t in topic:
i = 0
if t in self.publisher:
#print(t)
owner = self.publisher[t]
#print(owner)
length = len(self.pubhistory[owner][t])
print('Length the sub ask is '+ str(number[i]) +', the length pub has is '+ str(length))
l=0
for n in self.pubhistory:
l = len(self.pubhistory[n][t])+ l
print('We total get: '+ str(l))
if l < number[i]:
content ='Nothing'
break
if length < number[i]:
rest= number[i] - length
s = self.pubdict.get(t)
print(s)
# we turn all the keys into a list and we search the list
for pub in s:
if pub[0] is owner:
pass
else:
print(pub[0])
content = content + t + '#'+ self.pubhistory[pub[0]][t][rest-1] + '#' +str(rest-1) + '#'
#content ='Nothing'
else:
content = content + t + '#'+ self.pubhistory[owner][t][number[i]-1] + '#' + str(number[i]-1) + '#'
i = i+1
else:
content = 'Nothing'
print(content)
return content
def dict_update(self, msg):
if msg[0] == 'init':
pubID = msg[1]
topic = []
topic.append(msg[2])
topic.append(msg[3])
time = float(msg[4])
#print(topic)
self.pubhistory.update({pubID: {}})
for x in topic:
# history[pubID][topic][list]
self.pubhistory[pubID].update( {x:[]})
try:
if self.publisher.get(x) is None:
# pubhistory records what the pub will publisher
self.pubdict.update({x:[]})
# publisher contains the ID of the pub
self.publisher.update({x:pubID})
# pubdict contains the list of pubs who has topic x
self.pubdict[x].append((pubID,time))
# topic: pubID:time
else:
self.pubdict[x].append((pubID,time))
self.pubdict[x]=sorted(self.pubdict[x], key=lambda k: k[1], reverse = False)
#broker.pubhistory.update({pubID: {x:[]}})
# we sort the whole list according to the value, aka the arriving time
# we only edit the pubdict if this pub doesn't show up in the first place
with open(log_file, 'a') as logfile:
logfile.write('PUB-Init msg: %s init with topic %s\n' % (pubID, x))
print('broker.pubdict')
print(self.pubdict)
except KeyError:
pass
elif msg[0] == 'publish':
pubID = msg[1]
topic=msg[2]
publish = msg[3]
# no matter message from which publishers, all of them will restore.
try:
self.pubhistory[pubID][topic].append(publish)
with open(log_file, 'a') as logfile:
logfile.write('Pub No.%s published %s with topic %s\n' % (pubID, publish, topic))
#print(broker.pubhistory)
except KeyError:
pass
elif msg[0] == 'end':
pubID = msg[1]
self.unable.append(pubID)
#list1 = list (broker.publisher.keys()) [list (broker.publisher.values()).index (pubID)]
# we first edit the pubdict, delete all that contain pubID
for topic in self.pubdict:
for x in topic:
if x[0] is pubID:
self.pubdict[topic].remove(x)
# we delete the info we got in the
# we update the publisher
for stopic in self.publisher:
if self.publisher[stopic] is pubID:
#
new = self.pubdict[stopic][0]
self.publisher.update({stopic:new})
# we select the first one to be the new publisher
for pub in self.pubhistory:
if pub.key() is pubID:
self.pubhistory.pop(pub)
def sdict_update(self, msg):
if msg[0] == 'reg':
subID = msg[1]
if self.subdict.get(subID) == None:
self.subdict.update({subID: []})
#sub[topic].append(subID)
elif msg[0] == 'ask':
subID = msg[1]
topic = msg[2:4]
#topic = msg[2]
self.subdict[subID].append(topic)
#sub[topic].append(subID)
elif msg[0] == 'end':
subID = msg[1]
#sub[topic].remove(subID)
dict.pop(subID)
def sync_mode(self):
# we sync data from leader and update its lib
# we only record the info about the dict and sub
while self.leader_flag is False:
print('\n************************************\n')
try:
msg = self.syncsocket.recv_string().split("#")
# we got msg from leader
except Exception as ex:
# print('Time out')
print(ex)
time.sleep(1)
continue
print(msg)
#msg
if msg[0]=='init':
self.dict_update(msg)
elif msg[0]=='publish':
self.dict_update(msg)
else:
self.sdict_update(msg)
#print(broker.pubdict)
print('\n************************************\n')
print('received sync msg from leader')
# update the pub_info
def start(self):
# elect the leader under the path /Leader
leader_path = '/Leader'
'''
if self.zk.exists(leader_path):
# If the leader znode already exists, set the flag=false, specify broker to be the follower
leader_flag = False
self.watch_mode()
# this broker has to watch whether the current broker is down or not
leader_address = str(self.zk.get(leader_path)[0])
leader_address = leader_address[2:]
leader_address = leader_address[:-1]
print(leader_address)
syncsocket = self.pull_msg(leader_address, xleader)
# sync messages from leader and update data storage
self.sync_mode()
else:
# If the leader znode doesn't exist, set the flag=true, specify broker to be the leader
self.zk.create(leader_path, value=self.IP.encode('utf-8'), ephemeral=True, makepath=True)
while self.zk.exists(path=leader_path) is None:
pass
self.leader_flag = True
# if this broker is elected to be the leader, then start broker
# socket for leader to send sync request to followers
#broker.pubsyncsocket = None
self.pubsyncsocket = self.sourcepush(xleader)
if self.pubsyncsocket != None:
print('leader: syncsocket ok')
'''
try:
# If the leader znode doesn't exist, set the flag=true, specify broker to be the leader
self.zk.create(leader_path, value=self.IP.encode('utf-8'), ephemeral=True, makepath=True)
while self.zk.exists(path=leader_path) is None:
pass
self.leader_flag = True
# if this broker is elected to be the leader, then start broker
# socket for leader to send sync request to followers
#broker.pubsyncsocket = None
self.pubsyncsocket = self.sourcepush(xleader)
if self.pubsyncsocket != None:
print('leader: syncsocket ok')
except NodeExistsError:
# If the leader znode already exists, set the flag=false, specify broker to be the follower
self.leader_flag = False
self.watch_mode()
# this broker has to watch whether the current broker is down or not
leader_address = str(self.zk.get(leader_path)[0])[2:-1]
print(leader_address)
self.syncsocket = self.pull_msg(leader_address, xleader)
# sync messages from leader and update data storage
self.sync_mode()
pubsocket = self.pubsocket
print("Bound to port 5555 and waiting for any publisher to contact\n")
subsocket = self.subsocket
print("Bound to port 5556 and waiting for any subscriber to contact\n")
# we first register all the pub and sub then give them ownership strength
while True:
try:
msg1 = pubsocket.recv_string()
print("recved msg: " + msg1)
break
except Exception as ex:
print("fail to recv init msg " + str(ex))
time.sleep(1)
while self.pubsyncsocket != None:
try:
self.pubsyncsocket.send_string(msg1)
#print('-----')
break
except Exception as ex:
print("failed to send sync " + str(ex))
if len(self.zk.get_children('/Brokers/')) <= 1:
self.pubsyncsocket = None
msg11 = msg1.split("#")
print("read val " + str(msg11))
self.dict_update(msg11)
pubsocket.send_string('PUB-Info has been received!')
print("sent confirm msg")
#broker.syncsocket.send_string(msg1.encode('utf-8'))
#pubsocket.send_string("Please send me the publisher \n")
msg2 = subsocket.recv_string()
msg22 = msg2.split("#")
while self.pubsyncsocket != None:
print('*******')
try:
self.pubsyncsocket.send_string(msg2)
break
except Exception as ex:
print("failed to send sync " + str(ex))
continue
self.sdict_update(msg22)
subsocket.send_string('SUB-BROKER has been connected')
def pub_service():
while True:
message = self.pubsocket.recv_string()
msg = message.split("#")
#broker.syncsocket.send_string(msg.encode('utf-8'))
if self.pubsyncsocket is not None:
while True:
try:
self.pubsyncsocket.send_string(message)
break
except Exception as ex:
print("failed to send sync " + str(ex))
continue
#broker.syncsocket.send_string(msg2.encode('utf-8'))
# add the coming info to the dict
self.dict_update(msg22)
# reply to pub that this process is done
self.pubsocket.send_string('Done!!!')
def sub_service():
while True:
message = self.subsocket.recv_string()
msg = message.split("#")
self.sdict_update(msg)
topic = []
topic.append(msg[2])
topic.append(msg[3])
number1 = int(msg[4])
number2 = int(msg[5])
# we get two topics and we deal with it seperately
if msg[0] == 'ask':
# we first find whether the broker has this
pub_msg = self.find(topic, number1, number2)
# msg = topic1 + content1 + number1 + topic1 + content2 + number2
# Broker will only send one topic at one time
print("pub msg: " + pub_msg)
self.subsocket.send_string(pub_msg)
if self.pubsyncsocket is not None:
self.pubsyncsocket.send_string(message)
with open(log_file, 'a') as logfile:
logfile.write('Reply to SUB %s with topic %s and topic %s \n' % (msg[1], msg[2], msg[3]))
elif msg[0] == 'end':
time.sleep(1)
m = 'END!' +'#'
self.subsocket.send_string(m)
elif msg[0] == 'reg':
self.subsocket.send_string('Connected')
pub_thread = threading.Thread(target=pub_service, args=())
sub_thread = threading.Thread(target=sub_service, args=())
pub_thread.start()
sub_thread.start()
pub_thread.join()
sub_thread.join()
class ZK_Driver:
#CTOR
def __init__(self, ip_add):
context = zmq.Context()
self.kill = False
self.sub_socket = context.socket(zmq.SUB)
self.pub_socket = context.socket(zmq.PUB)
self.current_topics = []
self.zk_driver = KazooClient(hosts='127.0.0.1:2181')
self.zk_driver.start()
#ROOT DIRECTORY FOR BROKERS
self.home = '/brokers/'
#CREATE ZNODE PATHS FOR BROKERS
self.znode1 = self.home + 'bkr1'
self.znode2 = self.home + 'brk2'
self.znode3 = self.home + 'brk3'
#ENSURE ROOT DIRECTORY IS CREATED
self.zk_driver.ensure_path(self.home)
#CREATE ZNODES WITH PUB + SUB PORT
if not self.zk_driver.exists(self.znode1):
self.zk_driver.create(self.znode1, b'1234:5556')
if not self.zk_driver.exists(self.znode2):
self.zk_driver.create(self.znode2, b'1235:5556')
if not self.zk_driver.exists(self.znode3):
self.zk_driver.create(self.znode3, b'1236:5556')
#HOLD ELECTION TO GET PRESIDENT NODE
self.election = self.zk_driver.Election(self.home, "president")
contenders = self.election.contenders()
self.president = contenders[-1].encode(
'latin-1') #REPRESENTS THE WINNING PUB/SUB PORT COMBO
ports = self.president.decode('ASCII').split(":")
#FULL BROKER PORT ADDRESSES
self.full_add1 = "tcp://" + str(ip_add) + ":" + ports[0]
self.full_add2 = "tcp://" + str(ip_add) + ":" + ports[1]
#BIND TO ADDRESSES
self.sub_socket.bind(self.full_add1)
self.sub_socket.subscribe("")
self.pub_socket.bind(self.full_add2)
#SET UP WATCH DIRECTORY FOR PRESIDENT
self.president_home = "/president/"
self.pres_znode = "/president/pres"
#CREATE OR UPDATE PRESIDENT ZNODE
if not self.zk_driver.exists(self.pres_znode):
self.zk_driver.ensure_path(self.president_home)
self.zk_driver.create(self.pres_znode, ephemeral=True)
self.zk_driver.set(self.pres_znode, self.president)
# REMOVE PRESIDENT FROM FUTURE ELECTIONS
if ports[0] == "1234":
self.zk_driver.delete(self.znode1)
elif ports[0] == "1235":
self.zk_driver.delete(self.znode2)
elif ports[0] == "1236":
self.zk_driver.delete(self.znode3)
else:
print("No port recognized")
def run(self, stop=None):
@self.zk_driver.DataWatch(self.pres_znode)
def watch_node(data, stat, event):
if event is not None and event.type == "DELETED":
if not self.kill:
# HOLD ELECTION TO GET PRESIDENT NODE
self.election = self.zk_driver.Election(
self.home, "president")
contenders = self.election.contenders()
self.president = contenders[-1].encode(
'latin-1') # REPRESENTS THE WINNING PUB/SUB PORT COMBO
ports = self.president.decode('ASCII').split(":")
# FULL BROKER PORT ADDRESSES
self.full_add1 = "tcp://" + str(ip_add) + ":" + ports[0]
self.full_add2 = "tcp://" + str(ip_add) + ":" + ports[1]
print("Updated Broker to: ", self.full_add1)
# BIND TO ADDRESSES
self.sub_socket.bind(self.full_add1)
self.sub_socket.subscribe("")
# self.pub_socket.bind(self.full_add2)
# UPDATE PRESIDENT ZNODE
if not self.zk_driver.exists(self.pres_znode):
self.zk_driver.ensure_path(self.president_home)
self.zk_driver.create(self.pres_znode, ephemeral=True)
self.zk_driver.set(self.pres_znode, self.president)
# DELETE FROM FUTURE ELECTIONS
if ports[0] == "1234":
self.zk_driver.delete(self.znode1)
elif ports[0] == "1235":
self.zk_driver.delete(self.znode2)
elif ports[0] == "1236":
self.zk_driver.delete(self.znode3)
else:
print("No port recognized")
if not self.kill:
self.kill = True
if stop:
while not stop.is_set():
message = self.sub_socket.recv_string()
topic, info = message.split("||")
error_flag = False
if topic == "REGISTER":
error = False
for curr_topic in self.current_topics:
if info.startswith(curr_topic) and info != curr_topic:
print(
"Topic is too similar to topic of another publisher, choose another"
)
error = True
if not error:
self.current_topics.append(info)
print("Received: %s" % message)
self.pub_socket.send_string(message)
else:
self.pub_socket.send_string(message)
else:
message = self.sub_socket.recv_string()
topic, info = message.split("||")
error_flag = False
if topic == "REGISTER":
error = False
for curr_topic in self.current_topics:
if info.startswith(curr_topic) and info != curr_topic:
print(
"Topic is too similar to topic of another publisher, choose another"
)
error = True
if not error:
self.current_topics.append(info)
print("Addr ", self.full_add1, end=". ")
print("Received: %s" % message)
self.pub_socket.send_string(message)
else:
if topic in self.current_topics:
print("Addr ", self.full_add1, end=". ")
print("Received: %s" % message)
self.pub_socket.send_string(message)
else:
print("Please start over with a valid topic")
class Operator:
def __init__(self, zk_address, my_address, spout, operator_id, db_address, db_user, db_pwd):
'''
:param zk_address: ZooKeeper server address
:param my_address: Address of current operator
:param spout: ID of spout
:param operator_id: Global ID of the operator
'''
self.db_address = db_address
self.db_user = db_user
self.db_pwd = db_pwd
self.db_name = 'Spout_' + str(spout)
self.tb_name = str('Operator_' + str(operator_id))
self.columns = ['ID', 'State', 'Status', 'Sum', 'Mean', 'Max', 'Min']
self.columns_type = ['INT(11)', 'CHAR(20)', 'CHAR(30)', 'DOUBLE(30,4)', 'DOUBLE(30,4)', 'DOUBLE(30,4)','DOUBLE(30,4)']
self.db_connection, self.db_handler = self.init_db()
self.id = 'op' + str(random.randint(1, 1000))
self.my_address = my_address
self.operator_id = operator_id
self.parent_path = '/Spout--' + str(spout) + '/Operator--' + str(operator_id)
self.znode_path = self.parent_path + '/' + str(self.id)
self.leader_path = '/Spout--' + str(spout) + '/Operators/Leader--' + str(operator_id)
self.egress_leader = '/Spout--' + str(spout) + '/Egress_leader'
zk_address = zk_address + ':2181'
self.zk = KazooClient(hosts=zk_address)
self.isLeader = False
self.egress_available = False
self.up_stream_socket = None
self.down_stream_socket = None
self.lock = threading.Lock()
self.flag = 0
self.init_zk()
def init_db(self):
# Connect to Mysql server
db_connection, db_handler = mysqlop.connectMysql(self.db_address, self.db_user, self.db_pwd, db=self.db_name)
# Create Table
mysqlop.createTableAutoInc(db_handler, self.db_name, self.tb_name, self.columns, self.columns_type)
# Add primary key for the table
# mysqlop.add_primary_key(db_handler, db_connection, self.db_name, self.tb_name, self.columns[0])
return db_connection, db_handler
def init_zk(self):
self.zk.start()
while (self.zk.state == KazooState.CONNECTED) is False:
pass
print('Connected to ZK server.')
# Ensure parent path exists
if self.zk.exists(path=self.parent_path) is None:
self.zk.create(path=self.parent_path, value=b'', ephemeral=False, makepath=True)
# Create Znode for this operator in ZK
self.zk.create(path=self.znode_path, value=b'', ephemeral=True, makepath=True)
self.zk.ensure_path(path=self.znode_path)
# Watch the leader of Egress operator
@self.zk.DataWatch(self.egress_leader)
def watch_egress_leader(data, stat):
if stat is not None:
egress_leader_address = str(data)
print(egress_leader_address)
self.down_stream_socket = self.build_egress_socket(egress_leader_address)
print('Connect to egress....')
self.egress_available = True
else:
self.egress_available = False
def win_election():
# create leader node
print('Yeah, I won the election.')
self.zk.create(path=self.leader_path, value=self.my_address, ephemeral=True, makepath=True)
self.up_stream_socket = self.build_socket()
print('Ready to receive msg....')
if self.up_stream_socket is not None:
self.isLeader = True
threading.Thread(target=self.recv_data, args=()).start()
threading.Thread(target=self.distribute_data, args=()).start()
# Watch leader node
@self.zk.DataWatch(self.leader_path)
def watch_leader(data, stat):
if stat is None:
# Old leader failed or no leader was initialized
election = self.zk.Election(self.parent_path)
election.run(win_election)
while True:
pass
def build_socket(self):
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind('tcp://*:2341')
return socket
def build_egress_socket(self, address):
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect('tcp://' + address + ':2341')
return socket
def recv_data(self):
data_set = []
id = 0
i = 0
while True:
i += 1
# print(i)
data = self.up_stream_socket.recv_string()
# print('Recv data :' + data)
data = simplejson.loads(data)
time = data['Time']
self.up_stream_socket.send_string('Ack--' + time)
state = data['State']
data = int(data['Data'])
data_set.append(data)
if len(data_set) == 10:
id += 1
result = self.calculating(data_set)
# 将数据存入数据库
values = [id, state, 'Recv']
values.extend(result)
self.lock.acquire()
mysqlop.insert_data_operator(self.db_connection, self.db_handler, self.db_name, self.tb_name, values)
data_set = []
self.flag += 1
self.lock.release()
tm.sleep(0.01)
def distribute_data(self):
while True:
if self.flag == 5:
self.lock.acquire()
# 读取前20 行数据
data = mysqlop.query_first_N(self.db_handler, self.db_name, self.tb_name, 5)
self.flag = 0
self.lock.release()
temp_data = []
for item in data:
temp_data.append({'ID': item[0], 'State': item[1], 'Sum': item[3], 'Mean': item[4], 'Max': item[5], 'Min': item[6]})
data = temp_data
# 开始发送
for __data in data:
while self.egress_available is False:
pass
print(__data)
__data = simplejson.dumps(__data)
self.down_stream_socket.send_string(__data)
ack = self.down_stream_socket.recv_string()
# Ack msg format: 'ack--' + $ID
ack_id = ack.split('--')[1]
self.lock.acquire()
mysqlop.delete_row(self.db_handler, self.db_connection, self.db_name, self.tb_name, 'ID', ack_id)
self.lock.release()
def calculating(self, data_set):
data_sum = np.sum(data_set)
data_mean = np.mean(data_set)
data_max = np.max(data_set)
data_min = np.min(data_set)
return [data_sum, data_mean, data_max, data_min]
class Broker:
def __init__(self):
# setting up proxy sockets
self.context = zmq.Context()
self.frontend = self.context.socket(zmq.XPUB)
self.backend = self.context.socket(zmq.XSUB)
# Connecting to zookeeper - 2181 from config, ip should/can be changed
self.zk_object = KazooClient(hosts='127.0.0.1:2181')
self.zk_object.start()
# self.path = '/home/gourumv/zoo/data/'
self.path = '/home/'
# creating a znode + path for each broker. We need these for elections and to monitor if it becomes leader
node1 = self.path + "broker1"
if self.zk_object.exists(node1):
pass
else:
# create the file path since zookeeper is file structured.
self.zk_object.ensure_path(self.path)
# create the znode with port info for the pubs + subs to use to find the broker sockets. This is 'data' field used in pub + sub
self.zk_object.create(node1, to_bytes("5555,5556"), makepath=True)
print("Node 1 path created")
# znode2 (same w/ modified port #'s')
node2 = self.path + "broker2"
if self.zk_object.exists(node2):
pass
else:
# make sure the path exists
self.zk_object.ensure_path(self.path)
# create the znode with port info for the pubs + subs to use in addr[]
self.zk_object.create(node2, to_bytes("5557,5558"), makepath=True)
# znode 3 (same as above 2/ modified ports)
node3 = self.path + "broker3"
if self.zk_object.exists(node3):
pass
else:
# make sure the path exists
self.zk_object.ensure_path(self.path)
# create the znode with port info for the pubs + subs to use in addr[]
self.zk_object.create(node3, to_bytes("5559,5560"), makepath=True)
# Select a leader for the first time
self.election = self.zk_object.Election(
self.path, "leader"
) # requirements is the '/home/' (self.path) location in hierarchy, named leader
#self.election = self.zk_object.Election(self.path, "leader") # requirements is the '/home/' (self.path) location in hierarchy, named leader
potential_leaders = self.election.contenders(
) # create a list of broker znodes
self.leader = str(
potential_leaders[-1]
) # always select last one (arbitrary but simple process)
print("Leader ports: " + self.leader)
# use port #'s from the leader to finish connecting the proxy'
addr = self.leader.split(",")
self.frontend.bind("tcp://127.0.0.1:" +
addr[0]) # will want to modify ip as usual
self.backend.bind("tcp://127.0.0.1:" + addr[1])
# set-up znode for the newly minted leader
self.watch_dir = self.path + self.leader
self.leader_path = "/leader/"
self.leader_node = self.leader_path + "node" # saving the path in zookeeper hierarchy to a var
if self.zk_object.exists(self.leader_node):
pass
# if the path doesn't exist -> make it and populate it with a znode
else:
self.zk_object.ensure_path(self.leader_path)
self.zk_object.create(
self.leader_node, ephemeral=True
) # ephemeral so it disappears if the broker dies
# setting
self.zk_object.set(
self.leader_node, to_bytes(self.leader)
) # setting the port info into the leader znode for pubs + subs
def device(self):
# start the proxy device /broker that forwards messages. same as Assignment 1
zmq.device(zmq.FORWARDER, self.frontend, self.backend)
# essentially start - watches the leader node and re-elects if it disappears
def monitor(self):
while True:
# creating the watch
@self.zk_object.DataWatch(self.watch_dir)
def watch_node(data, stat, event):
# url's for unbinding before the information is lost
addr = self.leader.split(",")
front_url = "tcp://127.0.0.1:" + addr[0]
back_url = "tcp://127.0.0.1:" + addr[1]
# re-elect if the znode (and thus by proxy - the broker) dies
if event != None:
if event.type == "DELETED":
# same election code as above
self.election = self.zk_object.Election(
self.path, "leader")
potential_leaders = self.election.contenders()
self.leader = str(potential_leaders[-1])
# set node with new ports info for pubs + subs
self.zk_object.set(self.leader_node, self.leader)
# unbind + re-bind broker ports to new leader's ports
addr = self.leader.split(",")
self.frontend.unbind(front_url)
self.backend.unbind(back_url)
self.frontend.bind("tcp://127.0.0.1:" + addr[0])
self.backend.bind("tcp://127.0.0.1:" + addr[1])
# starts broker
self.election.run(self.device)
class broker_lib:
def __init__(self, id, zk_server, ip, xpub, xsub):
self.ID = id
self.IP = ip
self.pubsocket = self.bindp(ip, xpub)
self.subsocket = self.binds(ip, xsub)
self.syncsocket = None
self.pubsynsocket = None
zk_server = zk_server + ':2181'
self.zk = KazooClient(hosts=zk_server)
self.leader_flag = False
# the publisher and the info they store
self.pubdict = {}
self.publisher = {}
self.subdict = {}
self.subscriber = {}
self.init_zk()
print('\n************************************\n')
print('Init MyBroker succeed.')
print('\n************************************\n')
with open(log_file, 'w') as logfile:
logfile.write('Init Broker succeed \n')
def bindp(self, ip, port):
# we use REQ and REP to manage between publisher and broker
context = zmq.Context()
p_socket = context.socket(zmq.REP)
p_socket.setsockopt(zmq.RCVTIMEO, 3000)
p_socket.setsockopt(zmq.SNDTIMEO, 3000)
p_socket.bind('tcp://*:' + port)
with open(log_file, 'a') as logfile:
logfile.write('5555! \n')
return p_socket
def binds(self, ip, port):
# we use PUB and SUB to manage sub and broker
context = zmq.Context()
s_socket = context.socket(zmq.REP)
s_socket.bind('tcp://*:' + port)
with open(log_file, 'a') as logfile:
logfile.write('5556! \n')
return s_socket
def init_zk(self):
if self.zk.state != KazooState.CONNECTED:
self.zk.start()
while self.zk.state != KazooState.CONNECTED:
pass
# wait until the zookeeper starts
print('Broker %s connected to ZooKeeper server.' % self.ID)
# Create the path /Brokers
if self.zk.exists('/Brokers') is None:
self.zk.create(path='/Brokers',
value=b'',
ephemeral=False,
makepath=True)
while self.zk.exists('/Brokers') is None:
pass
# create my node under the path /Brokers
znode_path = '/Brokers/' + self.ID
self.zk.create(path=znode_path,
value=b'',
ephemeral=True,
makepath=True)
while self.zk.exists(znode_path) is None:
pass
print('Broker %s created a znode in ZooKeeper server.' % self.ID)
def watch_mode(self):
print("In watch mode")
election_path = '/Brokers/'
leader_path = '/Leader'
@self.zk.DataWatch(path=leader_path)
def watch_leader(data, state):
if self.zk.exists(path=leader_path) is None:
# time.sleep(random.randint(0, 3))
election = self.zk.Election(election_path, self.ID)
election.run(self.win_election)
def win_election(self):
print("Win election")
leader_path = '/Leader'
if self.zk.exists(path=leader_path) is None:
try:
self.zk.create(leader_path,
value=self.IP.encode('utf-8'),
ephemeral=True,
makepath=True)
except Exception as ex:
print("shouldn't elect")
while self.zk.exists(path=leader_path) is None:
pass
self.leader_flag = True
#self.start_broker()
#self.syncsocket = None
self.pubsyncsocket = self.sourcepush('5557')
if self.pubsyncsocket != None:
print('Broker %s started sending msg' % self.ID)
def pull_msg(self, address, port):
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.setsockopt(zmq.RCVTIMEO, 30000)
socket.connect('tcp://' + address + ':' + port)
return socket
# Leader push msg to followers
def sourcepush(self, port):
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.bind("tcp://*" + ':' + port)
socket.setsockopt(zmq.SNDTIMEO, 3000)
socket.setsockopt(zmq.RCVTIMEO, 3000)
return socket
@contact: [email protected] @site: @software: PyCharm @file: leader.py @time: 2018/8/9 15:26 """ import time import uuid import logging from kazoo.client import KazooClient logging.basicConfig() my_id = uuid.uuid4() def leader_func(): print "I am the leader {}".format(str(my_id)) while True: print "{} is working! ".format(str(my_id)) time.sleep(3) zk = KazooClient(hosts="10.0.0.130:2181") zk.start() election = zk.Election("/electionpath") election.run(leader_func) zk.stop()
class Egress:
def __init__(self, zk_address, my_address, output_address, spout,
db_address, db_user, db_pwd):
'''
:param zk_address: ZooKeeper server address
:param my_address: Egress operator address
:param output_address: End server address
:param spout: Spout number
:param db_address: Mysql server address
:param db_user: Mysql user
:param db_pwd: Mysql password
'''
self.db_address = db_address
self.db_user = db_user
self.db_pwd = db_pwd
self.db_name = 'Spout_' + str(spout)
self.tb_name = 'EgressOperator'
self.columns = ['ID', 'State', 'Status', 'Sum', 'Mean', 'Max', 'Min']
self.columns_type = [
'INT(11)', 'CHAR(20)', 'CHAR(20)', 'DOUBLE(30,4)', 'DOUBLE(30,4)',
'DOUBLE(30,4)', 'DOUBLE(30,4)'
]
self.db_connection, self.db_handler = self.init_db()
self.output_address = output_address
self.spout = spout
self.zk_address = zk_address
self.id = str(random.randint(1, 1000))
self.my_address = my_address
self.zk = KazooClient(hosts=zk_address)
self.root = 'Spout--' + str(spout)
self.parent_path = '/Spout--' + str(spout) + '/Egress_operators'
self.leader_path = '/Spout--' + str(spout) + '/Egress_leader'
self.znode_path = self.parent_path + '/Egress--' + self.id
self.isLeader = False
self.up_stream_socket = None
self.down_stream_socket = None
self.lock = threading.Lock()
self.flag = 0
self.init_zk()
def init_db(self):
# Connect to Mysql server
db_connection, db_handler = mysqlop.connectMysql(
self.db_address, self.db_user, self.db_pwd)
# Create Table
mysqlop.createTableAutoInc(db_handler, self.db_name, self.tb_name,
self.columns, self.columns_type)
# Add primary key for the table
mysqlop.add_primary_key(db_handler, db_connection, self.db_name,
self.tb_name, self.columns[0])
return db_connection, db_handler
def init_zk(self):
self.zk.start()
while (self.zk.state == KazooState.CONNECTED) is False:
pass
print('Connected to ZK server.')
# Ensure root path exists
if self.zk.exists(path=self.root) is None:
self.zk.create(path=self.root,
value=b'',
ephemeral=False,
makepath=True)
# Ensure parent path exists
if self.zk.exists(path=self.parent_path) is None:
self.zk.create(path=self.parent_path,
value=b'',
ephemeral=False,
makepath=True)
# Create Znode for this operator in ZK
self.zk.create(path=self.znode_path,
value=b'',
ephemeral=True,
makepath=True)
self.zk.ensure_path(path=self.znode_path)
def win_election():
print('Yeah, I won the election.')
self.isLeader = True
if self.zk.exists(path=self.leader_path) is None:
self.zk.create(path=self.leader_path,
value=self.my_address,
ephemeral=True,
makepath=True)
else:
self.zk.set(path=self.leader_path, value=self.my_address)
self.up_stream_socket = self.build_upstream_socket()
self.down_stream_socket = self.build_downstream_socket(
str(self.output_address))
threading.Thread(target=self.recv_data, args=()).start()
threading.Thread(target=self.send_data, args=()).start()
# Watch Leader node
@self.zk.DataWatch(self.leader_path)
def watch_egress_leader(data, state):
if state is None:
print('Suggest election..')
election = self.zk.Election(self.parent_path)
election.run(win_election)
while True:
pass
def build_upstream_socket(self):
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind('tcp://*:2341')
return socket
def build_downstream_socket(self, address):
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect('tcp://' + address + ':2341')
return socket
def recv_data(self):
sum_set = []
mean_set = []
max_set = []
min_set = []
myid = 0
while True:
data = self.up_stream_socket.recv_string()
data = simplejson.loads(data)
# print(data)
id = data['ID']
self.up_stream_socket.send_string('Ack--' + str(id))
state = data['State']
sum_set.append(data['Sum'])
print(sum_set)
mean_set.append(data['Mean'])
max_set.append(data['Max'])
min_set.append(data['Min'])
if len(sum_set) == 3:
myid += 1
data_sum = np.sum(sum_set)
data_max = np.max(max_set)
data_min = np.min(min_set)
data_mean = np.mean(mean_set)
# 将数据存入数据库
values = [id, state, 'Recv']
values.extend([data_sum, data_mean, data_max, data_min])
# print(values)
self.lock.acquire()
mysqlop.insert_data_operator(self.db_connection,
self.db_handler, self.db_name,
self.tb_name, values)
sum_set = []
mean_set = []
max_set = []
min_set = []
self.flag += 1
self.lock.release()
tm.sleep(0.01)
def send_data(self):
while True:
if self.flag == 2:
self.lock.acquire()
self.flag = 0
# 读取前5/row_count 行数据
data = mysqlop.query_first_N(self.db_handler, self.db_name,
self.tb_name, 2)
self.lock.release()
temp_data = []
for item in data:
temp_data.append({
'ID': item[0],
'State': item[1],
'Sum': item[3],
'Mean': item[4],
'Max': item[5],
'Min': item[6]
})
data = temp_data
# 开始发送
for __data in data:
__data = simplejson.dumps(__data)
self.down_stream_socket.send_string(__data)
ack = self.down_stream_socket.recv_string()
# Ack msg format: 'ack--' + $ID
ack_id = ack.split('--')[1]
self.lock.acquire()
mysqlop.delete_row(self.db_handler, self.db_connection,
self.db_name, self.tb_name, 'ID',
ack_id)
self.lock.release()
class ZooAnimal:
def __init__(self):
self.zk = KazooClient(hosts=ZOOKEEPER_LOCATION)
self.zk.start()
# Use util function to get IP address
self.ipaddress = [
ip for ip in list(local_ip4_addr_list())
if ip.startswith(NETWORK_PREFIX)
][0]
# Inheriting children should assign values to fit the scheme
# /approach/role/topic
self.approach = None
self.role = None
self.topic = None
#Will only be set by pub and sub
self.broker = None
# Zookeeper
#self.election = None
self.election = self.zk.Election('/broker/broker', self.ipaddress)
self.zk_seq_id = None
def zookeeper_watcher(self, watch_path):
@self.zk.DataWatch(watch_path)
def zookeeper_election(data, stat, event):
print("Setting election watch.")
print("Watching node -> ", data)
if data is None:
print("Data is none.")
self.election.run(self.zookeeper_register)
#self.election.cancel()
def zookeeper_master(self):
print("Becoming the master.")
role_topic = ZOOKEEPER_PATH_STRING.format(approach=self.approach,
role=self.role,
topic='master')
encoded_ip = codecs.encode(self.ipaddress, "utf-8")
self.zk.create(role_topic,
ephemeral=True,
makepath=True,
value=encoded_ip)
return True
def zookeeper_register(self):
# This will result in a path of /broker/publisher/12345 or whatever
# or /broker/broker/master
role_topic = ZOOKEEPER_PATH_STRING.format(approach=self.approach,
role=self.role,
topic=self.topic)
print("Zooanimal IP-> {}".format(self.ipaddress))
encoded_ip = codecs.encode(self.ipaddress, "utf-8")
if self.role == 'broker':
broker_path = "/{}/broker".format(self.approach)
#for i in range(10):
# if self.zk.exists("/broker/broker/master") == None:
# self.zookeeper_master()
# break
# time.sleep(.44)
if self.zk_seq_id == None:
self.zk.create(role_topic,
ephemeral=True,
sequence=True,
makepath=True,
value=encoded_ip)
brokers = self.zk.get_children(broker_path)
try:
brokers.pop(brokers.index("master"))
except:
pass
brokers = [x for x in brokers if "lock" not in x]
broker_nums = {y: int(y[4:]) for y in brokers}
#sort based on the values
broker_sort = sorted(broker_nums,
key=lambda data: broker_nums[data])
latest_id = broker_sort[-1]
print(latest_id)
self.zk_seq_id = latest_id
for i in range(10):
if self.zk.exists(broker_path + "/master") == None:
self.zookeeper_master()
break
time.sleep(0.2)
if self.zk.exists(broker_path + "/master"):
# Get all the children
path = self.zk.get_children(broker_path)
# Remove the master
path.pop(path.index("master"))
#path.pop(path.index(self.zk_seq_id))
# Process out the locks
path = [x for x in path if "lock" not in x]
#Convert into a dictionary of znode:sequential
#We keep the path name as the key
#Use the sequential number as the value
# e.g. key pool000001 value 000001
path_nums = {y: int(y[4:]) for y in path}
#sort based on the values
path_sort = sorted(path_nums, key=lambda data: path_nums[data])
# Watch the node that is previous to us
# path_sort[0] is the lowest number, [-1] is us, so [-2] is one before usd
#if path_sort.index(self.zk_seq_id) == 0:
# if self.zk.exists("/broker/broker/master") == None:
# self.zookeeper_master()
#else:
previous = path_sort[path_sort.index(self.zk_seq_id) - 1]
#previous = path_sort[-1]
watch_path = broker_path + "/" + previous
self.zookeeper_watcher(watch_path)
elif self.role == 'publisher' or self.role == 'subscriber':
# zk.ensure_path checks if path exists, and if not it creates it
try:
self.zk.create(role_topic, ephemeral=True, makepath=True)
except:
print("Topic already exists.")
# get the string from the path - if it's just created, it will be empty
# if it was created earlier, there should be other ip addresses
other_ips = self.zk.get(role_topic)
# Zookeeper uses byte strings --> b'I'm a byte string'
# We don't like that and need to convert it
other_ips = codecs.decode(other_ips[0], 'utf-8')
# if we just created the path, it will be an empty byte string
# if it's empty, this will be true and we'll add our ip to the end of the other ips
if other_ips != '':
print("Adding to the topics list")
self.zk.set(
role_topic,
codecs.encode(other_ips + ' ' + self.ipaddress, 'utf-8'))
# else the byte string is empty and we can just send our ip_address
else:
self.zk.set(role_topic, codecs.encode(self.ipaddress, 'utf-8'))
# This is a function stub for the get_broker watch callback
# The child is expected to implement their own logic
# Pub and Sub need to register_sub()
def broker_update(self, data):
print("Broker updated.")
print("Data -> {}".format(data))
pass
def get_broker(self):
for i in range(10):
if self.zk.exists(PATH_TO_MASTER[self.approach]):
node_data = self.zk.get(PATH_TO_MASTER[self.approach],
watch=self.broker_update)
broker_data = node_data[0]
master_broker = codecs.decode(broker_data, 'utf-8')
if master_broker != '':
self.broker = master_broker
else:
raise Exception("No master broker.")
time.sleep(0.2)
'''
class Proxy:
"""Implementation of the proxy"""
def __init__(self):
# Use XPUB/XSUB to get multiple contents from different publishers
self.context = zmq.Context()
self.xsubsocket = self.context.socket(zmq.XSUB)
self.xpubsocket = self.context.socket(zmq.XPUB)
self.xpubsocket.setsockopt(zmq.XPUB_VERBOSE, 1)
self.xpubsocket.send_multipart([b'\x01', b'10001'])
# Now we are going to create a poller
self.poller = zmq.Poller()
self.poller.register(self.xsubsocket, zmq.POLLIN)
self.poller.register(self.xpubsocket, zmq.POLLIN)
self.global_url = 0
self.global_port = 0
self.newSub = False
self.topic_info_queue = [
] #the content queue for different topic (zipcode)
self.topicInd = 0
self.zip_list = [
] #the ziplist to keep track with the zipcodes received
self.zk_object = KazooClient(hosts='127.0.0.1:2181')
self.zk_object.start()
self.path = '/home/'
znode1 = self.path + "broker1"
if self.zk_object.exists(znode1):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode1, "5555,5556")
# Print the version of a node and its data
znode2 = self.path + "broker2"
if self.zk_object.exists(znode2):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode2, "5557,5558")
znode3 = self.path + "broker3"
if self.zk_object.exists(znode3):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode3, "5553,5554")
self.election = self.zk_object.Election(self.path, "leader")
leader_list = self.election.contenders()
self.leader = leader_list[-1].encode(
'latin-1') # the leader here is a pair of address
address = self.leader.split(",")
pub_addr = "tcp://*:" + address[0]
sub_addr = "tcp://*:" + address[1]
print("Current elected broker: ", pub_addr + "," + sub_addr)
self.xsubsocket.bind(pub_addr)
self.xpubsocket.bind(sub_addr)
self.watch_dir = self.path + self.leader #use Datawatch
self.leader_path = "/leader/"
self.leader_node = self.leader_path + "node"
if self.zk_object.exists(self.leader_node):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.leader_path)
# Create a node with data
self.zk_object.create(self.leader_node, ephemeral=True)
self.zk_object.set(self.leader_node, self.leader)
self.history_node = '/history/node'
# Now threading1 runs regardless of user input
self.threading1 = threading.Thread(target=self.background_input)
self.threading1.daemon = True
self.threading1.start()
def background_input(self):
print("--- Enter to send history ---")
while True:
addr_input = raw_input()
if addr_input == "":
@self.zk_object.DataWatch(self.history_node)
def watch_node(data, stat, event):
if event == None: #wait for event to be alive and None(stable)
data, stat = self.zk_object.get(self.history_node)
print("Get a new subscriber here")
address = data.split(",")
pub_url = "tcp://" + address[0] + ":" + address[1]
self.global_port = address[1]
self.global_url = pub_url
self.newSub = True
def history_vector(self, h_vec, ind, history, msg):
if len(h_vec[ind]) < history:
h_vec[ind].append(msg)
else:
h_vec[ind].pop(0)
h_vec[ind].append(msg)
return h_vec
def sendToSubscriber(self):
events = dict(self.poller.poll(10000))
# Is there any data from publisher?
if self.xsubsocket in events:
msg = self.xsubsocket.recv_multipart()
content = msg[0]
zipcode, temperature, relhumidity, ownership, history, pub_time = content.split(
" ")
if zipcode not in self.zip_list: # a new topic just come from a new publisher
self.zip_list.append(zipcode)
#for this topic, set initial informations for the ownership and history function
cur_strength = 0
pre_strength = 0
count = 0
history_vec = []
strengh_vec = []
pubInd = 0
pre_msg = []
cur_msg = []
topic_info = [
cur_strength, pre_strength, count, history_vec,
strengh_vec, pubInd, pre_msg, cur_msg
]
self.topic_info_queue.append(topic_info)
#start to collect the msg for the new topic
topic_msg, histry_msg, ownership, strengh_vec = self.scheduleInTopic(
self.topic_info_queue[self.topicInd], msg)
self.topicInd += 1
else:
zipInd = self.zip_list.index(zipcode)
topic_msg, histry_msg, ownership, strengh_vec = self.scheduleInTopic(
self.topic_info_queue[zipInd], msg)
if self.newSub: #Want to send the history message here when only the new subscriber is active
ctx = zmq.Context()
pub = ctx.socket(zmq.PUB)
pub.bind(self.global_url)
if ownership == max(strengh_vec):
curInd = strengh_vec.index(ownership)
time.sleep(1)
for i in range(len(histry_msg)):
pub.send_multipart(histry_msg[i])
# pub.send_multipart(['10001, 0, 0, 0, 0']) # a test message
time.sleep(0.2)
pub.unbind(self.global_url)
pub.close()
ctx.term()
xurl = "tcp://*:" + self.global_port
self.xpubsocket.bind(xurl)
self.newSub = False
print("--- Sent HISTORY ---")
print("--- Enter to send history ---")
else:
self.xpubsocket.send_multipart(
topic_msg) #send the message by xpub
if self.xpubsocket in events: #a subscriber comes here
msg = self.xpubsocket.recv_multipart()
self.xsubsocket.send_multipart(msg)
def scheduleInTopic(self, info, msg):
[
cur_strength, pre_strength, count, history_vec, strengh_vec,
pubInd, pre_msg, cur_msg
] = info
sample_num = 10
content = msg[0]
zipcode, temperature, relhumidity, ownership, history, pub_time = content.split(
" ")
ownership = int(ownership.decode('ascii'))
history = int(history.decode('ascii'))
# creat the history stock for each publisher, should be FIFO
if ownership not in strengh_vec:
strengh_vec.append(ownership)
#create list for this publisher
history_vec.append([])
history_vec = self.history_vector(history_vec, pubInd, history,
msg)
pubInd += 1 # the actual size of the publishers
else:
curInd = strengh_vec.index(ownership)
history_vec = self.history_vector(history_vec, curInd, history,
msg)
#get the highest ownership msg to register the hash ring, using a heartbeat listener
if ownership > cur_strength:
pre_strength = cur_strength
cur_strength = ownership
pre_msg = cur_msg
cur_msg = msg
count = 0
elif ownership == cur_strength:
cur_msg = msg
count = 0
else:
count = count + 1
if count >= sample_num:
cur_strength = pre_strength
cur_msg = pre_msg
count = 0
#update the info vector fro this topic
info[0] = cur_strength
info[1] = pre_strength
info[2] = count
info[3] = history_vec
info[4] = strengh_vec
info[5] = pubInd
info[6] = pre_msg
info[7] = cur_msg
#get the history vector for msg
histInd = strengh_vec.index(cur_strength)
histry_msg = history_vec[histInd]
return cur_msg, histry_msg, ownership, strengh_vec
def schedule(self):
while True:
@self.zk_object.DataWatch(self.watch_dir)
def watch_node(data, stat, event):
if event != None:
print(event.type)
if event.type == "DELETED": #redo a election
self.election = self.zk_object.Election(
self.path, "leader")
leader_list = self.election.contenders()
self.leader = leader_list[-1].encode('latin-1')
self.zk_object.set(self.leader_node, self.leader)
self.xsubsocket.unbind(self.current_pub)
self.xpubsocket.unbind(self.current_sub)
address = self.leader.split(",")
pub_addr = "tcp://*:" + address[0]
sub_addr = "tcp://*:" + address[1]
self.xsubsocket.bind(pub_addr)
self.xpubsocket.bind(sub_addr)
self.election.run(self.sendToSubscriber)
def close(self):
""" This method closes the PyZMQ socket. """
self.xsubsocket.close(0)
self.xpubsocket.close(0)
class Broker:
def __init__(self, zk_server, my_address, xsub_port, xpub_port):
'''
:param zk_server: IP address of ZooKeeper Server
:param my_address: IP address of current broker
:param xsub_port: 5556
:param xpub_port: 5557
:sub_history_port: 5558
:sync_with_leader_port:5559
'''
self.zmqhelper = ZMQHelper()
'''
{$(topic): {
$(pubID): {
'publications' : [$(publication)]
'ownership strength': $(ownership_strength)
}
}
}
'''
self.data = {}
self.xsubsocket, self.xpubsocket = self.zmqhelper.prepare_broker(
xsub_port, xpub_port)
self.syncsocket = None
self.historysocket = self.zmqhelper.csrecv('5558')
self.myID = str(random.randint(1, 1000))
self.log_file = './Output/Broker' + self.myID + '.log'
zk_server = zk_server + ':2181'
self.zk = KazooClient(hosts=zk_server)
self.isLeader = False
self.my_address = my_address
self.count = 1
print('\n************************************\n')
print('Init MyBroker % s succeed.' % self.my_address)
print('\n************************************\n')
with open(self.log_file, 'w') as logfile:
logfile.write('Init Broker:\n')
logfile.write('XSUB Port: ' + xsub_port + '\n')
logfile.write('XPUB Port: ' + xpub_port + '\n')
logfile.write('-------------------------------------------\n')
self.init_zk()
def init_zk(self):
if self.zk.state != KazooState.CONNECTED:
self.zk.start()
while self.zk.state != KazooState.CONNECTED:
pass
print('Broker %s connected to ZooKeeper server.' % self.myID)
if self.zk.exists('/Brokers') is None:
self.zk.create(path='/Brokers',
value=b'',
ephemeral=False,
makepath=True)
flag = False
while self.zk.exists('/Brokers') is None:
pass
flag = True
if flag:
print('Create Znode Brokers.')
# Create a Znode in ZooKeeper
znode_path = '/Brokers/' + self.myID
self.zk.create(path=znode_path,
value=b'',
ephemeral=True,
makepath=True)
while self.zk.exists(znode_path) is None:
pass
print('Broker %s created a znode in ZooKeeper server.' % self.myID)
# watch publishers znode
pub_watch_path = '/Publishers'
if self.zk.exists(pub_watch_path) is None:
self.zk.create(path=pub_watch_path,
value=b'',
ephemeral=False,
makepath=True)
flag = False
while self.zk.exists(pub_watch_path) is None:
pass
flag = True
if flag:
print('Create Publishers znode.')
@self.zk.ChildrenWatch(path=pub_watch_path)
def watch_publishers(children):
self.publisher_failed(children)
'''
# watch subscriber znode
sub_watch_path = './Subscribers'
@self.zk.ChildrenWatch(client=self.zk, path=sub_watch_path)
def watch_subscribers(children):
self.subscriber_failed(children)
'''
# check if the leader has exists
leader_path = '/Leader'
if self.zk.exists(leader_path):
# If the leader znode already exists, specify this broker as follower
self.isLeader = False
print('Broker %s is follower.' % self.myID)
# followers start watching leader znode for potential election
self.leader_monitor()
# socket for follower to receive msg from leader
leader_address = str(self.zk.get(leader_path)[0])
self.syncsocket = self.zmqhelper.sinkpull(leader_address, '5559')
if self.syncsocket != None:
print('follower: syncsocket ok')
# NOTE: listen sync message from leader and update data storage
self.sync_data()
else:
# If the leader is not existing, create it and receive msg
self.zk.create(leader_path,
value=self.my_address,
ephemeral=True,
makepath=True)
while self.zk.exists(path=leader_path) is None:
pass
print('Broker %s is the first leader.' % self.myID)
self.isLeader = True
# socket for leader to send sync request to followers
self.syncsocket = self.zmqhelper.sourcepush('5559')
if self.syncsocket != None:
print('leader: syncsocket ok')
subscriber_thr = threading.Thread(target=self.receive_hisreq, args=())
threading.Thread.setDaemon(subscriber_thr, True)
subscriber_thr.start()
recv_thr = threading.Thread(target=self.receive_msg, args=())
threading.Thread.setDaemon(recv_thr, True)
recv_thr.start()
def leader_monitor(self):
# Run this method in another thread, because the election.run() method will be blocked until it won
election_path = '/Brokers/'
leader_path = '/Leader'
# watch leader znode
@self.zk.DataWatch(path=leader_path)
def watch_leader(data, state):
if self.zk.exists(path=leader_path) is None:
time.sleep(random.randint(0, 3))
election = self.zk.Election(election_path, self.myID)
election.run(self.win_election)
def publisher_failed(self, children):
'''
:param children: current children list under Publishers Znode
:return:
'''
# delete all entries of the pub which not in the current children list
if self.count != 1:
for key in self.data.keys():
for pubID in self.data[key].keys():
if pubID not in children:
del self.data[key][pubID]
print('delete publisher %s from topic %s\n' %
(pubID, key))
else:
self.count = 0
'''
def subscriber_failed(self, children):
:param children: current children list under Subscribers Znode
:return:
# TODO: Check which subscriber in data storage has failed,
# if you get one, delete the data related to this subscriber
pass
'''
# win the election, start receiving msg from publisher
def win_election(self):
leader_path = '/Leader'
if self.zk.exists(path=leader_path) is None:
self.zk.create(leader_path,
value=self.my_address,
ephemeral=True,
makepath=True)
while self.zk.exists(path=leader_path) is None:
pass
self.isLeader = True
print('Broker %s became new leader' % self.myID)
# self.syncsocket = None
self.syncsocket = self.zmqhelper.sourcepush('5559')
if self.syncsocket != None:
print('Broker %s started receive msg' % self.myID)
# only leader call this method
def receive_msg(self):
'''
Message type:
1. publisher init
2. publication
3. subscriber request
:return:
'''
while self.isLeader is False:
pass
while True:
# Store received data into self data storage
# Send received data to subscribers
# Send received data to followers using PUSH socket
# receive message from publisher
msg = self.xsubsocket.recv_string()
print(msg)
message = msg.split('#')
msg_type = message[0]
# publisher init
if msg_type == 'pub_init':
pubID = message[1]
topic = message[2]
print('\n************************************\n')
print('Init msg: %s init with topic %s' % (pubID, topic))
print('\n************************************\n')
with open(self.log_file, 'a') as logfile:
logfile.write('Init msg: %s init with topic %s\n' %
(pubID, topic))
# update storage
self.update_data('add_pub', pubID, topic, '')
# send msg to followers
#self.syncsocket.send_string('add_pub' + '#' + pubID + '#' + topic + '#')
# publication
elif msg_type == 'publication':
pubID = message[1]
topic = message[2]
publication = message[3]
print('\n************************************\n')
print('Publication: %s published %s with topic %s' %
(pubID, publication, topic))
print('\n************************************\n')
with open(self.log_file, 'a') as logfile:
logfile.write(
'Publication: %s published %s with topic %s\n' %
(pubID, publication, topic))
# update storage
self.update_data('add_pub', pubID, topic, '')
self.update_data('add_publication', pubID, topic, publication)
# send msg to followers
self.syncsocket.send_string('add_publication' + '#' + pubID +
'#' + topic + '#' + publication +
'#')
# check if this pubID has the highest ownership
if self.filter_pub_ownership(pubID, topic) is not None:
# send publication to subscribers using xpubsocket
print('sending to sub')
publication = publication + '--' + str(time.time())
self.zmqhelper.xpub_send_msg(self.xpubsocket, topic,
publication)
def receive_hisreq(self):
while True:
# receive history request msg from subscribers
if self.historysocket is None:
print('historysocket is none')
msg = self.historysocket.recv_string(0, 'utf-8')
print(msg)
message = msg.split('#')
msg_type = message[0]
if msg_type == 'request_history_publication':
with open(self.log_file, 'a') as log:
log.write('\n************************************\n')
log.write(
'Subscriber is requesting history publication.\n')
log.write('\n************************************\n')
topic = str(message[1])
history_count = int(message[2])
i = 0
data = []
# get pubID who has the highest ownership strength
if topic in self.data.keys():
for pub in self.data[topic].keys():
target = self.filter_pub_ownership(pub, topic)
if target is not None:
break
i = len(self.data[topic][target]['publications'])
if i <= history_count:
data.extend(
self.data[topic][target]['publications'][:])
else:
data.extend(self.data[topic][target]
['publications'][-history_count:])
msg = 'history_publication' + '#' + simplejson.dumps(data)
self.historysocket.send_string(msg)
print('\n************************************\n')
print('Send history publications to subscriber...')
print(msg)
print('\n************************************\n')
with open(self.log_file, 'a') as log:
log.write('\n************************************\n')
log.write('Send history publication to subscriber.\n')
log.write('\n************************************\n')
def sync_data(self):
'''
Receive sync msg from leader if this broker is a follower
:return:
'''
# Use a while loop to receive sync msg from leader
print('start sync with leader')
while self.isLeader is False:
try:
msg = self.syncsocket.recv_string()
except Exception as e:
print('Sync data time out')
continue
print('\n************************************\n')
print('received sync msg from leader')
message = msg.split('#')
msg_type = message[0]
if msg_type == 'add_pub':
pubID = message[1]
topic = message[2]
self.update_data(msg_type, pubID, topic, '')
elif msg_type == 'add_publication':
pubID = message[1]
topic = message[2]
publication = message[3]
self.update_data('add_pub', pubID, topic, '')
self.update_data(msg_type, pubID, topic, publication)
print('sync with topic %s pub %s' % (topic, pubID))
print('\n************************************\n')
def update_data(self, update_typ, pubID, topic, publication):
'''
:param update_typ:
1. New publisher registered
2. Received new publication from publisher
:param pubID:
:param topic:
:param publication:
:return:
'''
try:
if update_typ == 'add_pub':
# Assign an ownership strength to the registered publisher
ownership_strength = random.randint(1, 100)
if topic not in self.data.keys():
self.data.update({
topic: {
pubID: {
'publications': [],
'ownership strength': ownership_strength
}
}
})
elif pubID not in self.data[topic].keys():
self.data[topic].update({
pubID: {
'publications': [],
'ownership strength': ownership_strength
}
})
elif update_typ == 'add_publication':
stored_publication = publication + '--' + str(time.time())
self.data[topic][pubID]['publications'].append(
stored_publication)
except KeyError as ex:
print('\n----------------------------------------\n')
print('Error happened while updating publication dictionary...')
print(ex)
print('\n----------------------------------------\n')
# To examine if this pubID has the highest ownership
#
# argument: current publisher & topic
# return publisher ID or None
#
def filter_pub_ownership(self, pubID, topic):
try:
if self.data[topic][pubID]['ownership strength'] == max([
pub['ownership strength']
for pub in self.data[topic].values()
]):
return pubID
else:
return None
except Exception:
return None
class ZkThread(threading.Thread):
def __init__(self, crawler_name):
super(ZkThread, self).__init__()
self.crawler_name = crawler_name
self.crawler_path = "/crawler/" + self.crawler_name
self.master_path = self.crawler_path + "/master"
self.node_path = self.crawler_path + "/node"
self.election_path = self.crawler_path + "/election"
self.start_flag = "start".encode('utf-8')
self.master_flag = False
self.crawler_id = str(uuid.uuid4())
self.thread_pool = ThreadPoolExecutor(max_workers=5)
self.zk = KazooClient(hosts=config_util.zk_host)
self.redis_pool = redis.ConnectionPool(host=config_util.redis_host,
port=config_util.redis_port,
db=0)
self.register_node()
self.watch()
def register_node(self):
self.zk.start()
self.zk.ensure_path(self.node_path)
self.zk.create(path=self.node_path + "/" + self.crawler_id,
ephemeral=True)
children = self.zk.get_children(self.node_path)
print("Current children cnt: %d" % len(children))
def load_seed(self):
"""
load seed by master
to be override by user
"""
print("load begin")
while True:
time.sleep(7)
def fetch(self):
"""
fetch data by slave
to be override by user
"""
print("fetch begin")
while True:
time.sleep(7)
def leader_callback(self):
print("[Master] I am the leader {}".format(self.crawler_id))
self.master_flag = True
if not self.zk.exists(self.master_path):
self.zk.create(path=self.master_path,
value=self.start_flag,
ephemeral=True)
print("[Master] {} triggered start flag".format(self.crawler_id))
self.load_seed()
def elect(self):
print("Begin electing")
election = self.zk.Election(self.election_path,
identifier=self.crawler_id)
election.run(self.leader_callback)
def run(self):
self.elect()
def watch(self):
@self.zk.DataWatch(self.master_path)
def watch_crawler_start(data, stat):
if data == self.start_flag:
if not self.master_flag:
print("[Slave] crawling is starting, %s" % data)
self.thread_pool.submit(fn=self.fetch)
@self.zk.ChildrenWatch(self.node_path)
def watch_node(children):
print("watch children, now children cnt: %d" % len(children))
from kazoo.client import KazooClient
from time import sleep
def my_leader_function(name):
print('#'*10, name)
sleep(1)
zk = KazooClient()
zk.start()
election = zk.Election("/electionpath", "my-identifier")
# blocks until the election is won, then calls
# my_leader_function()
print('*'*5, election.contenders())
election.run(my_leader_function, 'test')
print('*'*5, election.contenders())
class Proxy:
"""Implementation of the proxy"""
def __init__(self):
# Use XPUB/XSUB to get multiple contents from different publishers
self.context = zmq.Context()
self.xsubsocket = self.context.socket(zmq.XSUB)
self.xpubsocket = self.context.socket(zmq.XPUB)
self.xpubsocket.setsockopt(zmq.XPUB_VERBOSE, 1)
self.xpubsocket.send_multipart([b'\x01', b'10001'])
# Now we are going to create a poller
self.poller = zmq.Poller()
self.poller.register(self.xsubsocket, zmq.POLLIN)
self.poller.register(self.xpubsocket, zmq.POLLIN)
self.sub_his_dic = {}
self.topic_info_queue = [
] # the content queue for different topic (zipcode)
self.topicInd = 0
self.zip_list = [
] # the ziplist to keep track with the zipcodes received
self.zk_object = KazooClient(hosts='127.0.0.1:2181')
self.zk_object.start()
self.path = '/home/'
znode1 = self.path + "broker1"
if self.zk_object.exists(znode1):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode1, "5555,5556")
# Print the version of a node and its data
znode2 = self.path + "broker2"
if self.zk_object.exists(znode2):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode2, "5557,5558")
znode3 = self.path + "broker3"
if self.zk_object.exists(znode3):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode3, "5553,5554")
self.election = self.zk_object.Election(self.path, "leader")
leader_list = self.election.contenders()
self.leader = leader_list[-1].encode(
'latin-1') # the leader here is a pair of address
address = self.leader.split(",")
pub_addr = "tcp://*:" + address[0]
sub_addr = "tcp://*:" + address[1]
print("Current elected broker: ", pub_addr + "," + sub_addr)
self.xsubsocket.bind(pub_addr)
self.xpubsocket.bind(sub_addr)
self.watch_dir = self.path + self.leader # use Datawatch
self.leader_path = "/leader/"
self.leader_node = self.leader_path + "node"
if self.zk_object.exists(self.leader_node):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.leader_path)
# Create a node with data
self.zk_object.create(self.leader_node, ephemeral=True)
self.zk_object.set(self.leader_node, self.leader)
def history_vector(self, h_vec, ind, history, msg):
if len(h_vec[ind]) < history:
h_vec[ind].append(msg)
else:
h_vec[ind].pop(0)
h_vec[ind].append(msg)
return h_vec
def sendToSubscriber(self):
events = dict(self.poller.poll(10000))
# Is there any data from publisher?
if self.xsubsocket in events:
msg = self.xsubsocket.recv_multipart()
content = msg[0]
zipcode, temperature, relhumidity, ownership, history, pub_time = content.split(
" ")
# if there is a new topic add the topic to the topic list and initialize the history vector fot this topic
# if the topic already in the topic list update the information of the topic if the ownership is larger than the current ownership
# update history vector fot the topic
if zipcode not in self.zip_list:
self.zip_list.append(zipcode)
# for this topic, set initial informations for the ownership and history function
cur_strength = 0
pre_strength = 0
count = 0
history_vec = []
strengh_vec = []
pubInd = 0
pre_msg = []
cur_msg = []
topic_info = [
cur_strength, pre_strength, count, history_vec,
strengh_vec, pubInd, pre_msg, cur_msg
]
self.topic_info_queue.append(topic_info)
# start to collect the msg for the new topic
topic_msg, histry_msg, ownership, strengh_vec = self.scheduleInTopic(
self.topic_info_queue[self.topicInd], msg)
self.topicInd += 1
else:
zipInd = self.zip_list.index(zipcode)
topic_msg, histry_msg, ownership, strengh_vec = self.scheduleInTopic(
self.topic_info_queue[zipInd], msg)
zipInd = self.zip_list.index(zipcode)
msg = self.topic_info_queue[zipInd][3][0]
msg = topic_msg + msg
carry = []
for contex in msg:
carry.append(contex[0])
carry.append('/')
carry = list(carry[2:])
res = ['']
for _ in carry:
res[-1] += _
self.xpubsocket.send_multipart([res[0]
]) # send the message by xpub
if self.xpubsocket in events: # a subscriber comes here
msg1 = self.xpubsocket.recv_multipart()
self.xsubsocket.send_multipart(msg1)
def scheduleInTopic(self, info, msg):
[
cur_strength, pre_strength, count, history_vec, strengh_vec,
pubInd, pre_msg, cur_msg
] = info
sample_num = 10
content = msg[0]
zipcode, temperature, relhumidity, ownership, history, pub_time = content.split(
" ")
ownership = int(ownership.decode('ascii'))
history = int(history.decode('ascii'))
# creat the history stock for each publisher, should be FIFO
# if the ownership is new add the topic information to history vector
# otherwise update the information
if ownership not in strengh_vec:
strengh_vec.append(ownership)
# create list for this publisher
history_vec.append([])
history_vec = self.history_vector(history_vec, pubInd, history,
msg)
pubInd += 1 # the actual size of the publishers
else:
curInd = strengh_vec.index(ownership)
history_vec = self.history_vector(history_vec, curInd, history,
msg)
# get the highest ownership msg to register the hash ring, using a heartbeat listener
if ownership > cur_strength:
pre_strength = cur_strength
cur_strength = ownership
pre_msg = cur_msg
cur_msg = msg
count = 0
elif ownership == cur_strength:
cur_msg = msg
count = 0
else:
count = count + 1
if count >= sample_num:
cur_strength = pre_strength
cur_msg = pre_msg
count = 0
# update the info vector fro this topic to make sure the current information always comes from the publisher with strongest ownership
info[0] = cur_strength
info[1] = pre_strength
info[2] = count
info[3] = history_vec
info[4] = strengh_vec
info[5] = pubInd
info[6] = pre_msg
info[7] = cur_msg
# get the history vector for msg
histInd = strengh_vec.index(cur_strength)
histry_msg = history_vec[histInd]
return cur_msg, histry_msg, ownership, strengh_vec
def schedule(self):
while True:
@self.zk_object.DataWatch(self.watch_dir)
def watch_node(data, stat, event):
if event != None:
print(event.type)
if event.type == "DELETED": # redo a election
self.election = self.zk_object.Election(
self.path, "leader")
leader_list = self.election.contenders()
self.leader = leader_list[-1].encode('latin-1')
self.zk_object.set(self.leader_node, self.leader)
self.xsubsocket.unbind(self.current_pub)
self.xpubsocket.unbind(self.current_sub)
address = self.leader.split(",")
pub_addr = "tcp://*:" + address[0]
sub_addr = "tcp://*:" + address[1]
self.xsubsocket.bind(pub_addr)
self.xpubsocket.bind(sub_addr)
self.election.run(self.sendToSubscriber)
def close(self):
""" This method closes the PyZMQ socket. """
self.xsubsocket.close(0)
self.xpubsocket.close(0)
class ZK(): # Create Interface Class #
'''
This class is Braingenix's interface to zookeeper.
It contains both the Kazoo connection object, as well as some other attributes such as the current mode.
When the class is instantiated, it connects to zk and starts a monitoring thread.
This thread checks if there are any nodes joining or leaving.
It also contains information about the status of the current leader.
If the leader node disconnects, once it's ephemeral zNode is removed, it'll automatically elect another node.
This is to provide failover due to a crash.
It also allows BG to sustain nearly half of all nodes failing without any issues.
'''
def __init__(self, Logger):
'''
Init function for the zookeeper class.
This function sets local pointers to the variable containing the Logger class.
It also defines some variables that will be needed later on.
*You shouldn't have to interact with this class unless you're debugging BrainGenix.*
'''
#########################################################################
# NOTE: The Max zNode Size Is 1MB, So Large Files Will Need To Be Split #
#########################################################################
# Set Local Variables #
self.ZookeeperMode = 'Follower'
self.ZookeeperHaveLeader = False
self.TransactionTime = 0
self.Name = platform.uname().node
self.ConnectedNodes = []
self.ConnectedNodesLastUpdate = []
# Create Local Copy Of Logger Pointer #
self.Logger = Logger
def ConnectToZookeeper(
self, Logger: object,
SystemConfiguration: dict): # Creates Connection With ZK #
'''
This function is used to connect to Zookeeper, hence the name.
*It's automatically called by the main file, so you still shouldn't have to interact with this function.*
The function will also ensure that some essential paths exist for BG-Core.
These paths are as follows: '/BrainGenix/System/Nodes', '/BrainGenix/CLI'
Lastly, It creates a node with it's own hostname to tell the leader that this node has joined.
That will create an event in the log that node {hostname} joined.
'''
# Extract Values From Dictionary #
ZKHost = str(SystemConfiguration.get('ZKHost'))
ZKPort = str(SystemConfiguration.get('ZKPort'))
ZKHost += f':{ZKPort}'
# Save Info About Connection #
self.ZKIP = ZKHost
self.ZKPort = ZKPort
# Connect To Zookeeper #
if Logger != None:
Logger.Log(f'Connecting To Zookeeper Server At Address: {ZKHost}')
self.ZookeeperConnection = KazooClient(hosts=ZKHost)
self.ZookeeperConnection.start()
if Logger != None:
Logger.Log('Established Connection To Zookeeper')
self.ZookeeperConnection.ensure_path('BrainGenix/')
self.ZookeeperConnection.ensure_path('BrainGenix/System')
self.ZookeeperConnection.ensure_path('BrainGenix/System/Nodes')
self.ZookeeperConnection.ensure_path('BrainGenix/CLI')
self.ConnectedNodesLastUpdate = self.ZookeeperConnection.get_children(
'/BrainGenix/System/Nodes')
self.ConnectedNodes = self.ZookeeperConnection.get_children(
'/BrainGenix/System/Nodes')
self.TryCreate(f'/BrainGenix/System/Nodes/{self.Name}/',
zNodeData=b'',
ephemeral=True)
def ConcurrentConnectedNodes(
self): # Return The Number Of Concurrently Connected Nodes #
'''
This function is used to get the number of connected concurrent nodes.
It's used to get the number of connected nodes on the MOTD splash page.
It just returns the number of children znodes under the system/nodes path in ZK.
'''
# Get Number Of Children zNodes #
NodeCount = len(
self.ZookeeperConnection.get_children('/BrainGenix/System/Nodes'))
# Return Count #
return NodeCount
def AutoInitZKLeader(self): # Init ZK Leader #
'''
This function does exactly what the name implies, Initializing the ZK connection.
It's called by the checkthread in the event that the leader is disconnected.
*As with many of the functions in this file, you shouldn't have to interact with it at all.*
'''
self.Logger.Log('Attempting To Locate Zookeeper Leader')
if not self.CheckIfLeaderExists():
self.Logger.Log('Failed To Find ZK Leader, Starting Election')
self.ElectLeader()
self.ZookeeperHaveLeader = True
else:
self.Logger.Log('Leader Located')
self.Logger.Log('This Node Is Running In Follower Mode')
self.ZookeeperHaveLeader = True
def ElectLeader(self): # Elects A Leader From The Pool #
'''
This function is used by the AutoInit function to elect a leader if the current leader is missing.
*It's also not designed to be used by the end user, so you shouldn't have to use it.*
'''
# Create Zookeeper Election Object #
self.Logger.Log('Electing Leader From Zookeeper Ensemble')
UUIDString = str(uuid.uuid1())
ZookeeperElection = self.ZookeeperConnection.Election(
"/BrainGenix/System/Election", UUIDString)
ZookeeperElection.run(self.ElectedLeader)
self.Logger.Log('Election Complete')
self.Logger.Log(f'This Node Is Running In {self.ZookeeperMode} Mode')
def CheckIfLeaderExists(
self): # Checks If A Leader Has Already Been Elected #
'''
This function returns a boolean value indicating if the current leader's ephemeral zNode exists.
*The end user shouldn't have to interact with this function.*
'''
return self.ZookeeperConnection.exists('/BrainGenix/System/Leader')
def ElectedLeader(
self
): # This Function Is Called If We're Elected Leader From The ZK Ensemble #
'''
This function is called when the current node is elected as a leader.
If this function is called and the node is not actually the leader, it can cause some very unstable behavior.
*Please don't use this function unless you're really sure what you're doing.*
'''
# Create Leader Dictionary #
LeaderDictionary = {'Hostname': self.Name, 'IP': self.ZKIP}
LeaderDictionaryString = json.dumps(LeaderDictionary)
# Create LockFile #
if not self.ZookeeperConnection.exists('/BrainGenix/System/Leader'):
self.ZookeeperConnection.create('/BrainGenix/System/Leader',
LeaderDictionaryString.encode(),
ephemeral=True)
self.ZookeeperMode = 'Leader'
else:
self.Logger.Log('Other Node Already Created Lockfile')
def TryCreate(self,
zNodePath: str,
ephemeral: bool = False,
zNodeData: bytes = None):
'''
This function is designed to be used by plugins/modules to create a node even if it might already exist.
It's faster than first checking, and then creating a node.
By default, the nodes it creates are not ephemeral, so if you need ephemeral nodes, pass `ephemeral=False`.
You can also pass data to it as bytes by passing `zNodeData=b'something'`.
'''
if not self.ZookeeperConnection.exists(zNodePath):
self.ZookeeperConnection.create(zNodePath,
ephemeral=ephemeral,
value=zNodeData)
def TryCreateOverwrite(self,
zNodePath: str,
ephemeral: bool = False,
zNodeData: bytes = None):
'''
This function is designed to be used by plugins/modules to create a node even if it might already exist.
It will overwrite the node if it already exists, so please be careful about what you're doing here.
If you're careless, this function can cause undefined behavior.
It's faster than first checking, and then creating a node.
By default, the nodes it creates are not ephemeral, so if you need ephemeral nodes, pass `ephemeral=False`.
You can also pass data to it as bytes by passing `zNodeData=b'something'`.
'''
if not self.ZookeeperConnection.exists(zNodePath):
self.ZookeeperConnection.create(zNodePath,
ephemeral=ephemeral,
value=zNodeData)
else:
self.ZookeeperConnection.set(zNodePath, value=zNodeData)
def LeaderCheckDaemon(
self,
ControlQueue,
RefreshInterval=1): # Constantly Checks If Leader Disconnects #
'''
This function is the actual thread target used by the leader checking process.
*Don't call this function unless you know what you're doing.*
'''
# Enter Loop #
while ControlQueue.empty():
# Catch Errors #
try:
# Check Latency #
StartTime = time.time()
LeaderExists = self.ZookeeperConnection.exists(
'/BrainGenix/System/Leader')
self.TransactionTime = time.time() - StartTime
# Check Leader #
if not LeaderExists:
self.LeaderTimeout()
if not self.ZookeeperConnection.exists(
'/BrainGenix/System/Leader'):
if (self.ZookeeperConnection.get(
'/BrainGenix/System/Leader')[0] !=
self.Name.encode()
and (self.ZookeeperMode == 'Leader')):
self.Logger.Log(
'Node Lock File Overwritten, Degrading To Follower!',
1)
self.ZookeeperMode = 'Follower'
else: # Catch exception if node is destroyed during check
pass
# Check Conn/Disconn Events #
self.GetConnectedNodes()
NewNodes, DelNodes = self.CheckIfNodeChangeEvents()
self.PrintDifferences(NewNodes, DelNodes)
# Delay Until Next Update Interval #
time.sleep(RefreshInterval)
except Exception as E:
if str(E) == 'Connection has been closed':
# Log Connection Destroyed #
self.Logger.Log(
'Zookeeper Connection Destroyed, Shutting Down ZKLeaderCheck Daemon',
8)
sys.exit()
else:
# Log Other Errors #
self.Logger.Log(E, 9)
def GetConnectedNodes(self): # Updates The List Of Connected Nodes In ZK #
'''
This function returns a list of connected nodes' hostnames into a variable called self.ConnectedNodes.
You can call this function or just poll the list as it's updated by the polling thread.
'''
self.ConnectedNodes = self.ZookeeperConnection.get_children(
'/BrainGenix/System/Nodes')
def CheckIfNodeChangeEvents(
self): # Checks If Any Nodes Have Joined Or Left The Cluster #
'''
This function checks if any new nodes have joined or left.
*Don't call this function unless you know what you're doing*
'''
# Check Additions #
AddedNodes = []
for NodeName in self.ConnectedNodes:
if NodeName not in self.ConnectedNodesLastUpdate:
AddedNodes.append(NodeName)
# Check Subtractions #
RemovedNodes = []
for NodeName in self.ConnectedNodesLastUpdate:
if NodeName not in self.ConnectedNodes:
RemovedNodes.append(NodeName)
# Update List Of Nodes Since Last Check #
self.ConnectedNodesLastUpdate = self.ConnectedNodes
return AddedNodes, RemovedNodes
def PrintDifferences(
self, AddedNodes, SubtractedNodes
): # Prints Deltas Between Checks, IE A NODE ADDED OR REMOVED #
'''
This function prints the list of new/disconnected nodes since the last time the CheckIfNodeChangeEvents function was called.
*Don't call this function unless you know what you're doing.*
'''
for NewNode in AddedNodes:
self.Logger.Log(f'Node {NewNode} Connected', 4)
for RemNode in SubtractedNodes:
self.Logger.Log(f'Node {RemNode} Disconnected', 5)
if RemNode == self.Name:
self.Logger.Log('Did Another Instance Time Out?')
self.TryCreateOverwrite(
f'/BrainGenix/System/Nodes/{self.Name}/',
zNodeData=b'',
ephemeral=True)
def LeaderTimeout(self): # Runs if a leader times out #
'''
This function is called when the leader times out. It will elect a new leader and handle transitioning to leader mode if needed.
*Don't call this function unless you know what you're doing.*
'''
self.Logger.Log("Can't find Zookeeper Leader! No Quorum?", 2)
self.ZookeeperHaveLeader = False
self.ZookeeperMode = 'Follower'
self.AutoInitZKLeader()
def Exit(self): # Shutsdown the ZK connection #
self.ZookeeperConnection.stop()
class Proxy:
"""Implementation of the proxy"""
def __init__(self):
# Use XPUB/XSUB to get multiple contents from different publishers
self.context = zmq.Context()
self.xsubsocket = self.context.socket(zmq.XSUB)
self.xpubsocket = self.context.socket(zmq.XPUB)
self.xpubsocket.setsockopt(zmq.XPUB_VERBOSE, 1)
self.xpubsocket.send_multipart([b'\x01', b'10001'])
# Now we are going to create a poller
self.poller = zmq.Poller()
self.poller.register(self.xsubsocket, zmq.POLLIN)
self.poller.register(self.xpubsocket, zmq.POLLIN)
self.global_url = 0
self.global_port = 0
self.newSub = False
self.topic_info_queue = [
] # the content queue for different topic (zipcode)
self.topicInd = 0
self.zip_list = [
] # the ziplist to keep track with the zipcodes received
self.zk_object = KazooClient(hosts='127.0.0.1:2181')
self.zk_object.start()
self.path = '/home/'
znode1 = self.path + "broker1"
if self.zk_object.exists(znode1):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode1, to_bytes("5555,5556"))
# Print the version of a node and its data
znode2 = self.path + "broker2"
if self.zk_object.exists(znode2):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode2, to_bytes("5557,5558"))
znode3 = self.path + "broker3"
if self.zk_object.exists(znode3):
pass
else:
# Ensure a path, create if necessaryto_bytes
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode3, to_bytes("5553,5554"))
'''
elect a leader and put that node under the path of /leader/ and send the port number to the publisher and subscriber
'''
self.election = self.zk_object.Election(self.path, "leader")
leader_list = self.election.contenders()
self.leader = leader_list[-1].encode(
'latin-1') # the leader here is a pair of address
self.leader = str(self.leader)
address = self.leader.split(",")
tem = address[0]
tem = tem.split("'")[1]
address[0] = tem
tem = address[1]
tem = tem.split("'")[0]
address[1] = tem
pub_addr = "tcp://*:" + address[0]
sub_addr = "tcp://*:" + address[1]
print("Current elected broker: ", pub_addr + "," + sub_addr)
self.xsubsocket.bind(pub_addr)
self.xpubsocket.bind(sub_addr)
self.watch_dir = self.path + self.leader # use Datawatch
self.leader_path = "/leader/"
self.leader_node = self.leader_path + "node"
if self.zk_object.exists(self.leader_node):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.leader_path)
# Create a node with data
self.zk_object.create(self.leader_node, ephemeral=True)
self.zk_object.set(self.leader_node, to_bytes(self.leader))
def sendToSubscriber(self):
events = dict(self.poller.poll(10000))
if self.xsubsocket in events:
msg = self.xsubsocket.recv_multipart()
content = msg[0]
content = str(content)
content = content.split("'")[1]
#print("content here is {}".format(content))
zipcode, temperature, relhumidity, pub_time = content.split(
" ")[:4]
cur_msg = [zipcode, temperature, relhumidity, pub_time]
if zipcode not in self.zip_list: # a new topic just come from a new publisher
self.zip_list.append(zipcode)
topic_info = cur_msg
self.topic_info_queue.append(topic_info)
else:
zipInd = self.zip_list.index(zipcode)
self.topic_info_queue[zipInd] = cur_msg
self.xpubsocket.send_string(
"%i %i %i %i " % (int(cur_msg[0]), int(
cur_msg[1]), int(cur_msg[2]), int(cur_msg[3])))
if self.xpubsocket in events: # a subscriber comes here
msg = self.xpubsocket.recv_multipart()
self.xsubsocket.send_multipart(msg)
def schedule(self):
while True:
@self.zk_object.DataWatch(self.watch_dir)
def watch_node(data, stat, event):
if event != None:
print(event.type)
if event.type == "DELETED": # redo a election
self.election = self.zk_object.Election(
self.path, "leader")
leader_list = self.election.contenders()
self.leader = leader_list[-1].encode('latin-1')
self.zk_object.set(self.leader_node, self.leader)
self.xsubsocket.unbind(self.current_pub)
self.xpubsocket.unbind(self.current_sub)
address = self.leader.split(",")
pub_addr = "tcp://*:" + address[0]
sub_addr = "tcp://*:" + address[1]
self.xsubsocket.bind(pub_addr)
self.xpubsocket.bind(sub_addr)
self.election.run(self.sendToSubscriber)
def close(self):
""" This method closes the PyZMQ socket. """
self.xsubsocket.close(0)
self.xpubsocket.close(0)