class GetJob:
def __init__(self):
self.zk = KazooClient(hosts=zk_host)
self.zk.start()
log_handler.info('link to zkserver %s success' % zk_host)
def check(self, path):
try:
result = self.zk.exists(path)
return result
except:
log_handler.error('zk check exits %s error' % path)
def get(self, path):
try:
result = self.zk.get(path)
return result
except:
log_handler.error('zk get path %s error' % path)
def get_children(self, path):
try:
result = self.zk.get_children(path, None)
return result
except:
log_handler.error('zk get children path %s error' % path)
def create(self, path, info, tag):
try:
result = self.zk.create(path, info, ephemeral=tag)
return result
except:
log_handler.error('zk create path %s error' % path)
def delete(self, path):
try:
result = self.zk.delete(path)
return result
except:
log_handler.error('zk delete path %s error' % path)
def init_key(self, server_key):
result = self.check(agent_root)
if not result:
log_handler.info('client key not exits %s result %s' %
(server_key, result))
else:
result = self.get(agent_root)
key_ip = result[0].split('_')[0]
print key_ip, server_ip
if key_ip != server_ip:
log_handler.error(
'client key other ip %s exits %s result %s' %
(key_ip, server_key))
sys.exit(9)
result = self.create(agent_root, '%s_%s' % (server_ip, version), True)
print result
def watch_path(self, path, funt):
self.zk.ChildrenWatch(agent_root, funt)
class Watch():
'''
Watch class
'''
def __init__(self, address, port, contenders):
'''
Constructor
'''
self._contenders = contenders
self._leader = None
self._leader_identifier = None
self._zookeeper = KazooClient(hosts='{}:{}'.format(address, port))
self._zookeeper.start()
self._zookeeper.ChildrenWatch('/{}'.format(contenders), self.watch)
def watch(self, contenders):
'''
Called when tne contenders change
'''
if contenders:
try:
self._leader = min(contenders, key=lambda broker: int(broker[-10:]))
(identifier, _status) = self._zookeeper.get('/{}/{}'.format(self._contenders, self._leader))
self._leader_identifier = identifier.decode('utf-8')
return
except NoNodeError:
pass
self._leader = None
self._leader_identifier = None
def identifier(self):
'''
Get current leader identifier
'''
return self._leader_identifier
class Master(object):
def __init__(self):
self._logger = getLogger('Master')
self._hosts = Global.config.zookeeper.hosts
self._timeout = int(Global.config.zookeeper.timeout)
self._root = Global.config.zookeeper.root
self._master = '{}/master'.format(self._root)
self.is_master = False
self._zk = KazooClient(hosts=self._hosts, timeout=self._timeout)
self._zk.start(timeout=self._timeout)
self.__init_zk()
self.register()
self._zk.ChildrenWatch(self._master, func=self._watch_master)
def __init_zk(self):
nodes = (self._root, self._master)
for node in nodes:
if not self._zk.exists(node):
try:
self._zk.create(node)
except:
self._logger.error('创建节点：{}出错'.format(node))
def _watch_master(self, children):
children.sort()
self._logger.debug('现有children: {}'.format(children))
master = None
if children:
master = children[0]
if master:
if self._path == master:
if self.is_master:
self._logger.info('我本来就是master')
else:
self.is_master = True
self._logger.info('我成为了新的master')
else:
self._logger.info('{}是master, 我是slave'.format(master))
def register(self):
path = self._zk.create('{}/worker'.format(self._master), value=Global.myid, ephemeral=True, sequence=True)
self._path = os.path.basename(path)
self._logger.info('注册一个节点:{}'.format(path))
class DynamicIP(object):
def __init__(self, hosts, watch_node):
self.proxies = dict()
self.watch_node = watch_node
self.zk_client = KazooClient(hosts=hosts)
self.zk_client.start()
self.logger = logging.getLogger(__name__)
def watcher(self, proxy_ids):
current_proxy_ids = set(self.proxies.keys())
newcome_proxy_ids = set(proxy_ids)
# 删除失效的代理
expried_proxy_ids = current_proxy_ids - newcome_proxy_ids
for expried_proxy_id in expried_proxy_ids:
expried_proxy = self.proxies.pop(expried_proxy_id)
# self.logger.info('expried proxy: %s' % expried_proxy)
# 新增代理
new_proxy_ids = newcome_proxy_ids - current_proxy_ids
for new_proxy_id in new_proxy_ids:
try:
ip, stat = self.zk_client.get(self.watch_node + '/' +
new_proxy_id)
data, stat = self.zk_client.get('/adsl_proxy/ip' + '/' + ip)
data = json.loads(data)
self.proxies[new_proxy_id] = data['host']
# self.logger.info('new proxy: %s' % data['host'])
except Exception as e:
pass
def get_proxy(self):
if len(self.proxies):
max_proxy_id = max(self.proxies.keys())
return self.proxies[max_proxy_id]
else:
return None
def run(self):
self.zk_client.ChildrenWatch(self.watch_node, self.watcher)
class AnalyticsDiscovery(gevent.Greenlet):
def _sandesh_connection_info_update(self, status, message):
new_conn_state = getattr(ConnectionStatus, status)
ConnectionState.update(conn_type=ConnectionType.ZOOKEEPER,
name=self._svc_name,
status=new_conn_state,
message=message,
server_addrs=self._zk_server.split(','))
if (self._conn_state and self._conn_state != ConnectionStatus.DOWN
and new_conn_state == ConnectionStatus.DOWN):
msg = 'Connection to Zookeeper down: %s' % (message)
self._logger.error(msg)
if (self._conn_state and self._conn_state != new_conn_state
and new_conn_state == ConnectionStatus.UP):
msg = 'Connection to Zookeeper ESTABLISHED'
self._logger.error(msg)
self._conn_state = new_conn_state
# end _sandesh_connection_info_update
def _zk_listen(self, state):
self._logger.error("Analytics Discovery listen %s" % str(state))
if state == KazooState.CONNECTED:
self._sandesh_connection_info_update(status='UP', message='')
self._logger.error("Analytics Discovery to publish %s" %
str(self._pubinfo))
self._reconnect = True
elif state == KazooState.LOST:
self._logger.error("Analytics Discovery connection LOST")
# Lost the session with ZooKeeper Server
# Best of option we have is to exit the process and restart all
# over again
self._sandesh_connection_info_update(
status='DOWN', message='Connection to Zookeeper lost')
os._exit(2)
elif state == KazooState.SUSPENDED:
self._logger.error("Analytics Discovery connection SUSPENDED")
# Update connection info
self._sandesh_connection_info_update(
status='INIT',
message='Connection to zookeeper lost. Retrying')
def _zk_datawatch(self, watcher, child, data, stat, event="unknown"):
self._logger.error(\
"Analytics Discovery %s ChildData : child %s, data %s, event %s" % \
(watcher, child, data, event))
if data:
data_dict = json.loads(data)
self._wchildren[watcher][child] = OrderedDict(
sorted(data_dict.items()))
else:
if child in self._wchildren[watcher]:
del self._wchildren[watcher][child]
if self._watchers[watcher]:
self._pendingcb.add(watcher)
def _zk_watcher(self, watcher, children):
self._logger.error("Analytics Discovery Children %s" % children)
self._reconnect = True
def __init__(self,
logger,
zkservers,
svc_name,
inst,
watchers={},
zpostfix="",
freq=10):
gevent.Greenlet.__init__(self)
self._svc_name = svc_name
self._inst = inst
self._zk_server = zkservers
# initialize logging and other stuff
if logger is None:
logging.basicConfig()
self._logger = logging
else:
self._logger = logger
self._conn_state = None
self._sandesh_connection_info_update(status='INIT', message='')
self._zkservers = zkservers
self._zk = None
self._pubinfo = None
self._publock = Semaphore()
self._watchers = watchers
self._wchildren = {}
self._pendingcb = set()
self._zpostfix = zpostfix
self._basepath = "/analytics-discovery-" + self._zpostfix
self._reconnect = None
self._freq = freq
def publish(self, pubinfo):
# This function can be called concurrently by the main AlarmDiscovery
# processing loop as well as by clients.
# It is NOT re-entrant
self._publock.acquire()
self._pubinfo = pubinfo
if self._conn_state == ConnectionStatus.UP:
try:
self._logger.error("ensure %s" %
(self._basepath + "/" + self._svc_name))
self._logger.error("zk state %s (%s)" %
(self._zk.state, self._zk.client_state))
self._zk.ensure_path(self._basepath + "/" + self._svc_name)
self._logger.error("check for %s/%s/%s" % \
(self._basepath, self._svc_name, self._inst))
if pubinfo is not None:
if self._zk.exists("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst)):
self._zk.set("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst),
self._pubinfo)
else:
self._zk.create("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst),
self._pubinfo, ephemeral=True)
else:
if self._zk.exists("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst)):
self._logger.error("withdrawing published info!")
self._zk.delete("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst))
except Exception as ex:
template = "Exception {0} in AnalyticsDiscovery publish. Args:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("%s : traceback %s for %s info %s" % \
(messag, traceback.format_exc(), self._svc_name, str(self._pubinfo)))
self._sandesh_connection_info_update(status='DOWN', message='')
self._reconnect = True
else:
self._logger.error("Analytics Discovery cannot publish while down")
self._publock.release()
def _run(self):
while True:
self._logger.error("Analytics Discovery zk start")
self._zk = KazooClient(hosts=self._zkservers)
self._zk.add_listener(self._zk_listen)
try:
self._zk.start()
while self._conn_state != ConnectionStatus.UP:
gevent.sleep(1)
break
except Exception as e:
# Update connection info
self._sandesh_connection_info_update(status='DOWN',
message=str(e))
self._zk.remove_listener(self._zk_listen)
try:
self._zk.stop()
self._zk.close()
except Exception as ex:
template = "Exception {0} in AnalyticsDiscovery zk stop/close. Args:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("%s : traceback %s for %s" % \
(messag, traceback.format_exc(), self._svc_name))
finally:
self._zk = None
gevent.sleep(1)
try:
# Update connection info
self._sandesh_connection_info_update(status='UP', message='')
self._reconnect = False
# Done connecting to ZooKeeper
for wk in self._watchers.keys():
self._zk.ensure_path(self._basepath + "/" + wk)
self._wchildren[wk] = {}
self._zk.ChildrenWatch(self._basepath + "/" + wk,
partial(self._zk_watcher, wk))
# Trigger the initial publish
self._reconnect = True
while True:
try:
if not self._reconnect:
pending_list = list(self._pendingcb)
self._pendingcb = set()
for wk in pending_list:
if self._watchers[wk]:
self._watchers[wk](\
sorted(self._wchildren[wk].values()))
# If a reconnect happens during processing, don't lose it
while self._reconnect:
self._logger.error("Analytics Discovery %s reconnect" \
% self._svc_name)
self._reconnect = False
self._pendingcb = set()
self.publish(self._pubinfo)
for wk in self._watchers.keys():
self._zk.ensure_path(self._basepath + "/" + wk)
children = self._zk.get_children(self._basepath +
"/" + wk)
old_children = set(self._wchildren[wk].keys())
new_children = set(children)
# Remove contents for the children who are gone
# (DO NOT remove the watch)
for elem in old_children - new_children:
del self._wchildren[wk][elem]
# Overwrite existing children, or create new ones
for elem in new_children:
# Create a watch for new children
if elem not in self._wchildren[wk]:
self._zk.DataWatch(self._basepath + "/" + \
wk + "/" + elem,
partial(self._zk_datawatch, wk, elem))
data_str, _ = self._zk.get(\
self._basepath + "/" + wk + "/" + elem)
data_dict = json.loads(data_str)
self._wchildren[wk][elem] = \
OrderedDict(sorted(data_dict.items()))
self._logger.error(\
"Analytics Discovery %s ChildData : child %s, data %s, event %s" % \
(wk, elem, self._wchildren[wk][elem], "GET"))
if self._watchers[wk]:
self._watchers[wk](sorted(
self._wchildren[wk].values()))
gevent.sleep(self._freq)
except gevent.GreenletExit:
self._logger.error("Exiting AnalyticsDiscovery for %s" % \
self._svc_name)
self._zk.remove_listener(self._zk_listen)
gevent.sleep(1)
try:
self._zk.stop()
except:
self._logger.error("Stopping kazooclient failed")
else:
self._logger.error("Stopping kazooclient successful")
try:
self._zk.close()
except:
self._logger.error("Closing kazooclient failed")
else:
self._logger.error("Closing kazooclient successful")
break
except Exception as ex:
template = "Exception {0} in AnalyticsDiscovery reconnect. Args:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("%s : traceback %s for %s info %s" % \
(messag, traceback.format_exc(), self._svc_name, str(self._pubinfo)))
self._reconnect = True
except Exception as ex:
template = "Exception {0} in AnalyticsDiscovery run. Args:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("%s : traceback %s for %s info %s" % \
(messag, traceback.format_exc(), self._svc_name, str(self._pubinfo)))
raise SystemExit
class ClientPool:
def __init__(self, service, server_hosts=None, zk_path=None, zk_hosts=None, logger=None, max_renew_times=3, maxActive=20,
maxIdle=10, get_connection_timeout=30, socket_timeout=30, disable_time=3):
"""
:param service: Thrift的Service名称
:param server_hosts: 服务提供者地址，数组类型，['ip:port','ip:port']
:param zk_path: 服务提供者在zookeeper中的路径
:param zk_hosts: zookeeper的host地址，多个请用逗号隔开
:param max_renew_times: 最大重连次数
:param maxActive: 最大连接数
:param maxIdle: 最大空闲连接数
:param get_connection_timeout:获取连接的超时时间
:param socket_timeout: 读取数据的超时时间
:param disable_time: 连接失效时间
"""
# 负载均衡队列
self.load_balance_queue = deque()
self.service = service
self.lock = threading.RLock()
self.max_renew_times = max_renew_times
self.maxActive = maxActive
self.maxIdle = maxIdle
self.connections = set()
self.pool_size = 0
self.get_connection_timeout = get_connection_timeout
self.no_client_queue = deque()
self.socket_timeout = socket_timeout
self.disable_time = disable_time
self.logger = logging if logger is None else logger
if zk_hosts:
self.kazoo_client = KazooClient(hosts=zk_hosts)
self.kazoo_client.start()
self.zk_path = zk_path
self.zk_hosts = zk_hosts
# 定义Watcher
self.kazoo_client.ChildrenWatch(path=self.zk_path,
func=self.watcher)
# 刷新连接池中的连接对象
self.__refresh_thrift_connections(self.kazoo_client.get_children(self.zk_path))
elif server_hosts:
self.server_hosts = server_hosts
# 复制新的IP地址到负载均衡队列中
self.load_balance_queue.extendleft(self.server_hosts)
else:
raise CTECThriftClientError('没有指定服务器获取方式！')
def get_new_client(self):
"""
轮询在每个ip:port的连接池中获取连接（线程安全）
从当前队列右侧取出ip:port信息，获取client
将连接池对象放回到当前队列的左侧
请求或连接超时时间，默认30秒
:return:
"""
with self.lock:
if self.pool_size < self.maxActive:
try:
ip = self.load_balance_queue.pop()
except IndexError:
raise CTECThriftClientError('没有可用的服务提供者列表！')
if ip:
self.load_balance_queue.appendleft(ip)
# 创建新的thrift client
t_socket = TSocket(ip.split(':')[0], int(ip.split(':')[1]),
socket_timeout=1000 * self.socket_timeout)
proto_factory = TBinaryProtocolFactory()
trans_factory = TBufferedTransportFactory()
transport = trans_factory.get_transport(t_socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
client = TClient(self.service, protocol)
self.pool_size += 1
return client
else:
return None
def close(self):
"""
关闭所有连接池和zk客户端
:return:
"""
if getattr(self, 'kazoo_client', None):
self.kazoo_client.stop()
def watcher(self, children):
"""
zk的watcher方法，负责检测zk的变化，刷新当前双端队列中的连接池
:param children: 子节点，即服务提供方的列表
:return:
"""
self.__refresh_thrift_connections(children)
def __refresh_thrift_connections(self, children):
"""
刷新服务提供者在当前队列中的连接池信息（线程安全），主要用于zk刷新
:param children:
:return:
"""
with self.lock:
# 清空负载均衡队列
self.load_balance_queue.clear()
# 清空连接池
self.connections.clear()
# 复制新的IP地址到负载均衡队列中
self.load_balance_queue.extendleft(children)
def __getattr__(self, name):
"""
函数调用，最大重试次数为max_renew_times
:param name:
:return:
"""
def method(*args, **kwds):
# 从连接池获取连接
client = self.get_client_from_pool()
# 连接池中无连接
if client is None:
# 设置获取连接的超时时间
time_out = Timeout(self.get_connection_timeout)
time_out.start()
try:
async_result = AsyncResult()
self.no_client_queue.appendleft(async_result)
client = async_result.get() # blocking
except:
with self.lock:
if client is None:
self.no_client_queue.remove(async_result)
self.logger.error("Get Connection Timeout!")
finally:
time_out.cancel()
if client is not None:
for i in xrange(self.max_renew_times):
try:
put_back_flag = True
client.last_use_time = time.time()
fun = getattr(client, name, None)
return fun(*args, **kwds)
except socket.timeout:
self.logger.error("Socket Timeout!")
# 关闭连接，不关闭会导致乱序
put_back_flag = False
self.close_one_client(client)
break
except TTransportException, e:
put_back_flag = False
if e.type == TTransportException.END_OF_FILE:
self.logger.warning("Socket Connection Reset Error,%s", e)
with self.lock:
client.close()
self.pool_size -= 1
client = self.get_new_client()
else:
self.logger.error("Socket Error,%s", e)
self.close_one_client(client)
break
except socket.error, e:
put_back_flag = False
if e.errno == socket.errno.ECONNABORTED:
self.logger.warning("Socket Connection aborted Error,%s", e)
with self.lock:
client.close()
self.pool_size -= 1
client = self.get_new_client()
else:
self.logger.error("Socket Error, %s", e)
self.close_one_client(client)
break
except Exception as e:
put_back_flag = False
self.logger.error("Thrift Error, %s", e)
self.close_one_client(client)
break
finally:
class MyDaemon(Daemon):
def __init__(self,
zk_hosts,
pid,
stdin='/dev/null',
stdout='/dev/null',
stderr='/dev/null'):
self._zk_hosts = zk_hosts
self._update_interval = 0
self._zk = None
self._docklist = []
self._image = ''
self._zk_root = ''
super().__init__(pid, stdin=stdin, stdout=stdout, stderr=stderr)
def cleanup(self):
with zk_lock:
self._zk.stop()
# NOT THREAD-SAFE ASYNC CALLBACK (USED BY A GLOBAL)!!!
def update_docklist(self, docks, ev):
with docklist_lock:
removed = set(self._docklist) - set(docks)
added = set(docks) - set(self._docklist)
self._docklist = docks
if removed:
process_removed(removed)
if added:
process_added(added)
def bootstrap(self, zk_root, image, update_interval):
self._image = image
self._zk_root = zk_root
self._update_interval = update_interval
self._zk = KazooClient(hosts=self._zk_hosts)
self._zk.start()
self._zk.ChildrenWatch(self._zk_root, watch_dock_list, send_event=True)
def update_zk_context(self, docks):
with zk_lock:
if not docks:
docks = self._docklist
for dock in docks:
# Set port
data, zns = self._zk.get(self._zk_root + '/' + dock)
# FIXME: Underscore hack, due to a broken zookeeper-shell.sh
t = loads(data.decode('utf-8'))
if not 'dockInfoUpdated' in t and t:
(port, imageId) = getDockInfo(t['virthostip'], self._image,
dock)
# TODO: Currently one broker per dock is assumed.
t['dockInfoUpdated'] = True
t['virtHostPort'] = port
t['imageId'] = imageId
data = dumps(t).encode()
self._zk.set(self._zk_root + '/' + dock, data)
def run(self):
while True:
# with zk_lock:
# with docklist_lock:
self.update_zk_context(self._docklist)
sleep(self._update_interval)
self.cleanup()
zk.delete('/tables/{}'.format(tmp[2]), recursive=True)
if zk.exists('{}/tables/{}'.format(server_path, tmp[2])):
zk.delete('{}/tables/{}'.format(server_path, tmp[2]),
recursive=True)
elif tmp[1] == 'index':
if zk.exists('/indexes/{}'.format(tmp[2])):
zk.delete('/indexes/{}'.format(tmp[2]), recursive=True)
# zk.create('{}/tables/{}/{}'.format(server_path, tmp[2], node_name), bytes(sql, encoding='utf-8'))
if __name__ == '__main__':
# 开始心跳
zk.start()
# party部分，用于检测服务器断线情况
zk.ensure_path('/party')
zk.ChildrenWatch('/party', watch_server_party)
party = zk.Party('/party', server_name)
party.join()
# 构建zookeeper结构
zk.ensure_path('/tables')
zk.ensure_path('/indexes')
zk.ensure_path("{}/tables".format(server_path))
zk.set("{}".format(server_path), b'0')
zk.ensure_path("{}/info".format(server_path))
if not zk.exists("{}/info/recordNum".format(server_path)):
zk.create("{}/info/recordNum".format(server_path), b'0')
if not zk.exists("{}/info/tableNum".format(server_path)):
zk.create("{}/info/tableNum".format(server_path), b'0')
zk.delete("{}/instructions".format(server_path), recursive=True)
zk.ensure_path("{}/instructions".format(server_path))
class AnalyticsDiscovery(gevent.Greenlet):
def _sandesh_connection_info_update(self, status, message):
new_conn_state = getattr(ConnectionStatus, status)
ConnectionState.update(conn_type=ConnectionType.ZOOKEEPER,
name=self._svc_name,
status=new_conn_state,
message=message,
server_addrs=self._zk_server.split(','))
if (self._conn_state and self._conn_state != ConnectionStatus.DOWN
and new_conn_state == ConnectionStatus.DOWN):
msg = 'Connection to Zookeeper down: %s' % (message)
self._logger.error(msg)
if (self._conn_state and self._conn_state != new_conn_state
and new_conn_state == ConnectionStatus.UP):
msg = 'Connection to Zookeeper ESTABLISHED'
self._logger.error(msg)
self._conn_state = new_conn_state
#import pdb; pdb.set_trace()
# end _sandesh_connection_info_update
def _zk_listen(self, state):
self._logger.error("Analytics Discovery listen %s" % str(state))
if state == KazooState.CONNECTED:
if self._conn_state != ConnectionStatus.UP:
self._sandesh_connection_info_update(status='UP', message='')
self._logger.error("Analytics Discovery to publish %s" %
str(self._pubinfo))
self._reconnect = True
else:
self._logger.error("Analytics Discovery already connected")
else:
self._logger.error("Analytics Discovery NOT connected")
if self._conn_state == ConnectionStatus.UP:
self._sandesh_connection_info_update(status='DOWN', message='')
def _zk_datawatch(self, watcher, child, data, stat, event="unknown"):
self._logger.error(\
"Analytics Discovery %s ChildData : child %s, data %s, event %s" % \
(watcher, child, data, event))
if data:
data_dict = json.loads(data)
self._wchildren[watcher][child] = OrderedDict(
sorted(data_dict.items()))
else:
if child in self._wchildren[watcher]:
del self._wchildren[watcher][child]
if self._watchers[watcher]:
self._watchers[watcher](sorted(self._wchildren[watcher].values()))
def _zk_watcher(self, watcher, children):
self._logger.error("Analytics Discovery Children %s" % children)
self._reconnect = True
def __init__(self,
logger,
zkservers,
svc_name,
inst,
watchers={},
zpostfix="",
freq=10):
gevent.Greenlet.__init__(self)
self._svc_name = svc_name
self._inst = inst
self._zk_server = zkservers
# initialize logging and other stuff
if logger is None:
logging.basicConfig()
self._logger = logging
else:
self._logger = logger
self._conn_state = None
self._sandesh_connection_info_update(status='INIT', message='')
self._zk = KazooClient(hosts=zkservers)
self._pubinfo = None
self._watchers = watchers
self._wchildren = {}
self._zpostfix = zpostfix
self._basepath = "/analytics-discovery-" + self._zpostfix
self._reconnect = None
self._freq = freq
def publish(self, pubinfo):
self._pubinfo = pubinfo
#import pdb; pdb.set_trace()
if self._conn_state == ConnectionStatus.UP:
try:
self._logger.error("ensure %s" %
(self._basepath + "/" + self._svc_name))
self._logger.error("zk state %s (%s)" %
(self._zk.state, self._zk.client_state))
self._zk.ensure_path(self._basepath + "/" + self._svc_name)
self._logger.error("check for %s/%s/%s" % \
(self._basepath, self._svc_name, self._inst))
if pubinfo is not None:
if self._zk.exists("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst)):
self._zk.set("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst),
self._pubinfo)
else:
self._zk.create("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst),
self._pubinfo, ephemeral=True)
else:
if self._zk.exists("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst)):
self._logger.error("withdrawing published info!")
self._zk.delete("%s/%s/%s" % \
(self._basepath, self._svc_name, self._inst))
except Exception as ex:
template = "Exception {0} in AnalyticsDiscovery publish. Args:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("%s : traceback %s for %s info %s" % \
(messag, traceback.format_exc(), self._svc_name, str(self._pubinfo)))
self._sandesh_connection_info_update(status='DOWN', message='')
self._reconnect = True
else:
self._logger.error("Analytics Discovery cannot publish while down")
def _run(self):
while True:
try:
self._zk.start()
break
except gevent.event.Timeout as e:
# Update connection info
self._sandesh_connection_info_update(status='DOWN',
message=str(e))
gevent.sleep(1)
# Zookeeper is also throwing exception due to delay in master election
except Exception as e:
# Update connection info
self._sandesh_connection_info_update(status='DOWN',
message=str(e))
gevent.sleep(1)
try:
# Update connection info
self._sandesh_connection_info_update(status='UP', message='')
self._reconnect = False
# Done connecting to ZooKeeper
self._zk.add_listener(self._zk_listen)
for wk in self._watchers.keys():
self._zk.ensure_path(self._basepath + "/" + wk)
self._wchildren[wk] = {}
self._zk.ChildrenWatch(self._basepath + "/" + wk,
partial(self._zk_watcher, wk))
# Trigger the initial publish
self._reconnect = True
while True:
try:
# If a reconnect happens during processing, don't lose it
while self._reconnect:
self._logger.error("Analytics Discovery %s reconnect" \
% self._svc_name)
self._reconnect = False
self.publish(self._pubinfo)
for wk in self._watchers.keys():
self._zk.ensure_path(self._basepath + "/" + wk)
children = self._zk.get_children(self._basepath +
"/" + wk)
old_children = set(self._wchildren[wk].keys())
new_children = set(children)
# Remove contents for the children who are gone
# (DO NOT remove the watch)
for elem in old_children - new_children:
del self._wchildren[wk][elem]
# Overwrite existing children, or create new ones
for elem in new_children:
# Create a watch for new children
if elem not in self._wchildren[wk]:
self._zk.DataWatch(self._basepath + "/" + \
wk + "/" + elem,
partial(self._zk_datawatch, wk, elem))
data_str, _ = self._zk.get(\
self._basepath + "/" + wk + "/" + elem)
data_dict = json.loads(data_str)
self._wchildren[wk][elem] = \
OrderedDict(sorted(data_dict.items()))
self._logger.error(\
"Analytics Discovery %s ChildData : child %s, data %s, event %s" % \
(wk, elem, self._wchildren[wk][elem], "GET"))
if self._watchers[wk]:
self._watchers[wk](sorted(
self._wchildren[wk].values()))
gevent.sleep(self._freq)
except gevent.GreenletExit:
self._logger.error("Exiting AnalyticsDiscovery for %s" % \
self._svc_name)
self._zk.stop()
break
except Exception as ex:
template = "Exception {0} in AnalyticsDiscovery reconnect. Args:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("%s : traceback %s for %s info %s" % \
(messag, traceback.format_exc(), self._svc_name, str(self._pubinfo)))
self._reconnect = True
except Exception as ex:
template = "Exception {0} in AnalyticsDiscovery run. Args:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("%s : traceback %s for %s info %s" % \
(messag, traceback.format_exc(), self._svc_name, str(self._pubinfo)))
raise SystemExit
class Monitor(QObject):
hosts = set() # 主机信息会存放在这里
root_path = "/node/host" # 节点信息存放的根路径
log_path = os.path.join(os.getcwd(), "log.json")
Changed = pyqtSignal(dict) # 数据
Totaled = pyqtSignal(dict)
Lossed = pyqtSignal(set) # 丢失主机
def __init__(self, **kwargs):
super(Monitor, self).__init__(**kwargs)
self.interval = 1000
self.timer = None
self.zk = KazooClient()
self.redis = redis.StrictRedis()
self.__install_listen() # 安装监听
self.zk.start()
self.file = open(self.log_path,'a+')
def __del__(self):
self.zk.stop()
self.file.close()
def __install_listen(self):
""" 集群数量监听: COUNT_OF_HOSTS """
def listen(children): # 监听有没有新节点加入
old_hosts = self.hosts.copy() # 备份,方便比对
self.hosts.clear() # 清空
self.hosts.update({uuid.UUID(_uuid) for _uuid in children}) # 更新主机列表
count_of_hosts = len(self.hosts) # 当前主机的数量
logger.warning(Color.yellow('COUNT_OF_HOSTS is {}'.format(count_of_hosts)))
if len(old_hosts) > len(self.hosts):
self.Lossed.emit(old_hosts - self.hosts) # 警报丢失主机
self.zk.ChildrenWatch(self.root_path, func=listen) # 建立监听
def get_path(self, node_name):
""" 获取 zookeeper 中节点保存的路径 """
return '{path}/{name}'.format(path=self.root_path, name=node_name)
@pyqtSlot()
def get_info(self):
xxx = Counter()
loss_balacing = []
for host in self.hosts:
result, _ = self.zk.get(self.get_path(host.urn))
result = result.decode('utf-8')
if result:
result = json.loads(result)
loss_balacing.append(result.get('localload')) # 负载量 localload 累积起来
xxx.update(result) # Counter 的 update 是累加的
result['node_name'] = host.urn
self.Changed.emit(result)
else:
xxx['node_counts'] = len(self.hosts)
xxx['memoryusage'] /= len(self.hosts) # 内存占用取平均值
xxx['optimize_filter'] /= len(self.hosts) # 去重优化率取平均值
xxx['optimize_queue'] /= len(self.hosts) # 队列优化率取平均值
xxx['loss_balacing'] = 0 if len(loss_balacing) < 2 else statistics.stdev(loss_balacing) # 负载失衡为负载量取方差
self.Totaled.emit(xxx)
self.file.write(json.dumps(xxx)+',') # 存入文件中
self.file.flush()
@pyqtSlot(str, str)
def publishTask(self, spidername, urlseed):
""" 发布任务 """
try:
self.redis.lpush("{}:start_urls".format(spidername), urlseed)
except:
return False
else:
return True
def launch(self):
self.timer = QTimer(self)
self.timer.timeout.connect(self.get_info)
self.timer.start(self.interval) # 每秒钟调用一次
class UsingKazoo(AbstractKoolieZooKeeper):
"""Concrete class to access ZooKeeper using Kazoo."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._kazoo_client: KazooClient = None
@property
def kazoo_client(self):
return self._kazoo_client
def before_start(self):
try:
self._kazoo_client = KazooClient(hosts=self.hosts())
self._kazoo_client.start(timeout=5)
except KazooException as exception:
_logging.warning('Failed to start Kazoo [{}] [{}]'.format(
sys.exc_info()[0], exception))
self._kazoo_client = None
def before_stop(self):
try:
self._kazoo_client.stop()
except KazooException as exception:
_logging.warning('Failed to stop Kazoo [{}] [{}].'.format(
sys.exc_info()[0], exception))
finally:
self._kazoo_client = None
def get_node_value(self, path) -> bytes:
try:
return self._kazoo_client.get(path)[0]
except Exception as exception:
_logging.warning(
'Failed to get value for path [{}] with exception [{}]'.format(
path, exception))
return None
def set_node_value(self, path: str, value=b''):
_logging.debug('ZooKeeper.set_node_value()')
assert path is not None and isinstance(path, str)
assert value is not None and isinstance(value, bytes)
self._kazoo_client.set(path, value, -1)
def get_children(self, path: str) -> typing.List[str]:
return self._kazoo_client.get_children(path)
def watch_children(self, path: str, func: callable):
self._kazoo_client.ChildrenWatch(path, func)
def create_node(self,
path,
value=b'',
acl=None,
ephemeral=False,
sequence=False,
make_path=False):
_logging.debug('create_node()')
self._kazoo_client.create(path, value, acl, ephemeral, sequence,
make_path)
def delete_node(self, path, version=-1, recursive=False):
self._kazoo_client.delete(path, version, recursive)
class ZooKeeperManager():
__zk = None
__hash_range = None
__max_slave_count = None
__node_list = []
def __init__(self, zookeeper_hosts, port_for_client, hash_range,
max_slave_count):
self.__zk = KazooClient(zookeeper_hosts)
self.__zk.start()
self.__zk.ensure_path('/dimint/overlord/host_list')
print('ip : ' + Network.get_ip())
addr = '{0}:{1}'.format(Network.get_ip(), port_for_client)
host_path = '/dimint/overlord/host_list/' + addr
if not self.is_exist(host_path):
self.__zk.create(host_path, b'', ephemeral=True)
self.__zk.ensure_path('/dimint/node/list')
self.__zk.ChildrenWatch('/dimint/node/list', self.check_node_is_dead)
self.__hash_range = hash_range
self.__max_slave_count = max_slave_count
def delete_all_node_role(self):
self.__zk.ensure_path('/dimint/node/role')
master_node_list = self.__zk.get_children('/dimint/node/list')
for master_node in master_node_list:
self.__zk.delete(master_node, recursive=True)
def get_node_list(self):
self.__zk.ensure_path('/dimint/node/list')
self.__node_list = self.__zk.get_children('/dimint/node/list')
return self.__node_list if isinstance(self.__node_list, list) else []
def get_overlord_list(self):
self.__zk.ensure_path('/dimint/overlord/host_list')
overlord_list = self.__zk.get_children('/dimint/overlord/host_list')
return overlord_list if isinstance(overlord_list, list) else []
def get_master_info_list(self):
self.__zk.ensure_path('/dimint/node/role')
master_node_list = self.__zk.get_children('/dimint/node/role')
master_info = {}
for master in master_node_list:
info = json.loads(
self.__zk.get(
'dimint/node/role/{0}'.format(master))[0].decode('utf-8'))
master_info[master] = info
return master_info
def delete_node(self, node, role):
msg = {}
msg['cmd'] = 'die'
if role == 'master':
self.enable_node(node, False)
slaves = self.__zk.get_children(
'/dimint/node/role/{0}'.format(node))
master_info = self.get_master_info_list()
msg['src_node_id'] = node
msg['src_node'] = 'tcp://{0}:{1}'.format(
master_info[node]['ip'], master_info[node]['cmd_receive_port'])
if len(slaves) == 0:
# need to move all keys to another node
if len(master_info) > 1:
msg['cmd'] = 'move_all_key'
node_index = list(master_info.keys()).index(node)
node_index = (node_index + 1) % len(master_info)
next_node = list(master_info.keys())[node_index]
self.enable_node(next_node, False)
msg['key_list'] = master_info[node]['stored_key']
msg['target_node_id'] = next_node
msg['target_node'] = 'tcp://{0}:{1}'.format(
master_info[next_node]['ip'],
master_info[next_node]['transfer_port'])
else:
node_info = self.get_node_info(node)
if node_info is None:
msg['result'] = 'complete'
return msg
msg['src_node_id'] = node
msg['src_node'] = 'tcp://{0}:{1}'.format(
node_info['ip'], node_info['cmd_receive_port'])
self.kill_node([node])
msg['result'] = 'complete'
return msg
def is_exist(self, path):
return self.__zk.exists(path)
def get_identity(self):
while True:
hash_value = Hash.get_node_id()
if not (self.__zk.exists(
'/dimint/node/list/{0}'.format(hash_value))):
return str(hash_value)
def get_node_value(self):
master_info = self.get_master_info_list()
value_list = [m['value'] for m in master_info.values()]
while True:
hash_value = Hash.get_hashed_value(time.time(), self.__hash_range)
if not hash_value in value_list:
return hash_value
def determine_node_role(self, node_id, node_value, msg, enable):
role_path = '/dimint/node/role'
self.__zk.ensure_path(role_path)
masters = self.__zk.get_children(role_path)
msg['enabled'] = True
msg['value'] = node_value
for master in masters:
slaves = self.__zk.get_children(os.path.join(role_path, master))
if len(slaves) < max(1, self.__max_slave_count):
master_info = json.loads(
self.__zk.get(os.path.join(role_path,
master))[0].decode('utf-8'))
master_addr = 'tcp://{0}:{1}'.format(
master_info['ip'], master_info['cmd_receive_port'])
master_push_addr = 'tcp://{0}:{1}'.format(
master_info['ip'], master_info['push_to_slave_port'])
master_receive_addr = 'tcp://{0}:{1}'.format(
master_info['ip'], master_info['receive_slave_port'])
self.__zk.create(os.path.join(role_path, master, node_id),
json.dumps(msg).encode('utf-8'))
return "slave", master_addr, master_push_addr, master_receive_addr
msg['stored_key'] = []
msg['enabled'] = enable
self.__zk.create(os.path.join(role_path, node_id),
json.dumps(msg).encode('utf-8'))
return "master", None, None, None
def select_node(self, key, select_master):
master = str(self.__select_master_node(key))
if master is None:
return [None, None]
master_path = '/dimint/node/role/{0}'.format(master)
slaves = self.__zk.get_children(master_path)
master_info = json.loads(self.__zk.get(master_path)[0].decode('utf-8'))
if not master_info['enabled']:
return [None, None]
if (select_master or len(slaves) < 1):
master_addr = 'tcp://{0}:{1}'.format(
master_info['ip'], master_info['cmd_receive_port'])
return [master, master_addr]
else:
slave = random.choice(slaves)
slave_path = '/dimint/node/role/{0}/{1}'.format(master, slave)
slave_info = json.loads(
self.__zk.get(slave_path)[0].decode('utf-8'))
slave_addr = 'tcp://{0}:{1}'.format(slave_info['ip'],
slave_info['cmd_receive_port'])
return [master, slave_addr]
def __select_master_node(self, key):
master_info_list = self.get_master_info_list()
if (len(master_info_list) == 0):
return None
master_list = sorted(list(master_info_list.items()),
key=lambda tup: int(tup[1]['value']))
hashed_value = Hash.get_hashed_value(key, self.__hash_range)
for master in master_list:
if (hashed_value <= int(master[1]['value'])):
return master[0]
return master_list[0][0]
def __set_master_node_attribute(self, node, attr, value, extra=None):
node_path = '/dimint/node/role/{0}'.format(node)
node_info = json.loads(self.__zk.get(node_path)[0].decode('utf-8'))
if extra is not None:
node_info[attr] = list(set(node_info[attr]) | set([value]))
else:
node_info[attr] = value
self.__zk.set(node_path, json.dumps(node_info).encode('utf-8'))
def add_key_to_node(self, node, key):
self.__set_master_node_attribute(node, 'stored_key', key, 0)
def add_key_list_to_node(self, node, key_list):
node_path = '/dimint/node/role/{0}'.format(node)
node_info = json.loads(self.__zk.get(node_path)[0].decode('utf-8'))
node_info['stored_key'] += key_list
node_info['stored_key'] = list(set(node_info['stored_key']))
self.__zk.set(node_path, json.dumps(node_info).encode('utf-8'))
def remove_key_list_from_node(self, node, key_list):
node_path = '/dimint/node/role/{0}'.format(node)
node_info = json.loads(self.__zk.get(node_path)[0].decode('utf-8'))
node_info['stored_key'] = list(
set(node_info['stored_key']) - set(key_list))
self.__zk.set(node_path, json.dumps(node_info).encode('utf-8'))
def enable_node(self, node, enable=True):
self.__set_master_node_attribute(node, 'enabled', enable)
def change_node_value(self, node, value):
self.__set_master_node_attribute(node, 'value', value)
def get_node_msg(self, node_path):
try:
node = self.__zk.get(node_path)
# node[0] == b''
# b'' == False
if not node[0]:
return {}
return json.loads(node[0].decode('utf-8'))
except NoNodeError as e:
return {}
def set_node_msg(self, node_path, msg):
self.__zk.ensure_path(node_path)
self.__zk.set(node_path, json.dumps(msg).encode('utf-8'))
def check_node_is_dead(self, node_list):
# this file is for exclusive lock(?) if other handler doing this job,
# then delete process should not be executed.
dead_nodes = list(set(self.__node_list) - set(node_list))
if len(dead_nodes) == 1:
self.kill_node(dead_nodes)
self.__node_list = node_list
def kill_node(self, target_nodes):
# this file is for exclusive lock(?) if other handler doing this job
role_path = '/dimint/node/role'
handler_working_file = '/dimint/node/role/dead_node_handler_is_working'
if self.__zk.exists(handler_working_file):
return
self.__zk.create(handler_working_file, b'', ephemeral=True)
for target_node in target_nodes:
node_info = self.get_node_info(target_node)
if node_info is None:
break
if node_info.get('role') == 'slave':
# if slave node is dead, there is nothing to do.
# Just update role information in zookeeper.
self.__zk.delete(
os.path.join(role_path, node_info['master_id'],
target_node))
else:
master_path = os.path.join(role_path, target_node)
try:
nominated_master_info = node_info['slaves'][0]
other_slaves = node_info['slaves'][1:]
write_info = nominated_master_info.copy()
del write_info['node_id']
del write_info['value']
try:
del write_info['role']
except:
pass
master_info = json.loads(
self.__zk.get(master_path)[0].decode('utf-8'))
master_info.update(write_info)
self.__zk.create(
os.path.join(role_path,
nominated_master_info['node_id']),
json.dumps(master_info).encode('utf-8'))
for other_slave in other_slaves:
other_slave_id = other_slave['node_id']
del other_slave['node_id']
try:
del other_slave['role']
except:
pass
self.__zk.create(
os.path.join(role_path,
nominated_master_info['node_id'],
other_slave_id),
json.dumps(other_slave).encode('utf-8'))
self.__zk.delete(master_path, recursive=True)
self.overlord_task_thread.handle_dead_master(
target_node, nominated_master_info, other_slaves)
except IndexError:
self.__zk.delete(master_path)
self.__zk.delete(handler_working_file)
def get_node_info(self, node_id):
role_path = '/dimint/node/role'
for master in self.__zk.get_children(role_path):
master_path = os.path.join(role_path, master)
if master == node_id:
result = {'role': 'master'}
result.update(
json.loads(self.__zk.get(master_path)[0].decode('utf-8')))
result['slaves'] = [
dict(node_id=slave,
**json.loads(
self.__zk.get(os.path.join(
master_path, slave))[0].decode('utf-8')))
for slave in self.__zk.get_children(master_path)
]
return result
for slave in self.__zk.get_children(os.path.join(
role_path, master)):
if slave == node_id:
result = {'role': 'slave', 'master_id': master}
result.update(
json.loads(
self.__zk.get(os.path.join(
role_path, master))[0].decode('utf-8')))
return result
return None
class Proxy(object):
'''
代理类，内部维护所有工作服务器的信息，并监听/server的子节点列表变化
当有新的服务器加入时，其内部维护的工作服务器列表增加一个元素
当有服务器宕机时，将该服务器本身维持的连接分配给其他的服务器
'''
def __init__(self, zk_server_address, server_path):
'''
Constructor
'''
# 服务器父节点路径
self.server_path = server_path
# 登记的正在工作的服务器
self.servers = []
self.zkclient = KazooClient(zk_server_address)
# 添加连接状态监听器
self.zkclient.add_listener(self.connect_listener)
# 添加对服务器父节点的子节点列表的监听
self.zkclient.ChildrenWatch(self.server_path, self.servers_change)
# 连接状态监听器
def connect_listener(self, state):
if state == KeeperState.CONNECTED:
print "代理已经开启..."
elif state == KazooState.LOST:
print "代理停止服务..."
else:
raise Exception("代理未正常开启...")
# 子节点列表的监听器
def servers_change(self, children):
# 更新子节点列表，这个列表是由机器id组成的列表
# 要能够通过机器id映射得到机器实例
self.servers = children
def start(self):
self.zkclient.start()
def stop(self):
self.zkclient.stop()
self.zkclient.close()
# 得到每台服务器的基本信息，包括了负载
def get_server_datas(self):
server_datas = []
# 遍历所有的子节点
for server in self.servers:
# 得到每台工作服务器的负载信息，这个是一直在动态变化的，是否需要控制一下呢？
node_value = self.zkclient.get(self.server_path + "/" + server)
# 这里得到的是序列化的服务器数据
server_data_str = node_value[0]
# 这里将字符串反序列化
server_data = json.loads(server_data_str, object_hook = ServerData.unserialize)
server_datas.append(server_data)
return server_datas
# 均衡策略
def balance_load(self, server_datas):
'''
返回负载最低的服务器
'''
# 根据load排序
server_datas.sort(None, key = lambda server_data: server_data.load)
# 返回
return server_datas[0]
class DeploymentConfig(object):
""" Accesses deployment configuration options. """
# The ZooKeeper node where configuration is stored.
CONFIG_ROOT = '/appscale/config'
def __init__(self, hosts):
""" Creates new DeploymentConfig object.
Args:
hosts: A list of ZooKeeper hosts.
"""
self.logger = logging.getLogger(self.__class__.__name__)
self.update_lock = Lock()
self.state = ConfigStates.LOADING
self.config = {}
self.conn = KazooClient(hosts=hosts, read_only=True)
self.conn.add_listener(self._conn_listener)
self.conn.start()
self.conn.ensure_path(self.CONFIG_ROOT)
self.conn.ChildrenWatch(self.CONFIG_ROOT, func=self._update_config)
def _conn_listener(self, state):
""" Handles changes in ZooKeeper connection state.
Args:
state: A string indicating the new state.
"""
if state == KazooState.LOST:
self.logger.warning('ZK connection lost')
if state == KazooState.SUSPENDED:
self.logger.warning('ZK connection suspended')
else:
self.logger.info('ZK connection established')
def _load_child(self, child):
""" Fetches the data for a configuration node.
Args:
child: A string containing the ZooKeeper node to fetch.
Returns:
A dictionary containing configuration data.
Raises:
InaccessibleConfig if ZooKeeper is not accessible.
"""
node = '/'.join([self.CONFIG_ROOT, child])
try:
data, _ = self.conn.retry(self.conn.get, node)
except (KazooException, ZookeeperError):
raise ConfigInaccessible('ZooKeeper connection not available')
except NoNodeError:
return {}
try:
return json.loads(data)
except ValueError:
self.logger.warning('Invalid deployment config: {}'.format(child))
return {}
def _update_config(self, children):
""" Updates configuration when it changes.
Args:
children: A list of ZooKeeper nodes.
"""
with self.update_lock:
self.state = ConfigStates.LOADING
# Ensure old sections are removed.
self.config = {}
for child in children:
while True:
try:
self.config[child] = self._load_child(child)
break
except ConfigInaccessible as load_error:
self.logger.warning(str(load_error))
time.sleep(SMALL_WAIT)
self.logger.info('Deployment configuration updated')
self.state = ConfigStates.LOADED
def get_config(self, section):
""" Fetches the configuration for a given section.
Args:
section: A string specifying the section to fetch.
Returns:
A dictionary containing configuration data.
Raises:
InaccessibleConfig if ZooKeeper is inaccessible.
"""
# If the connection is established, it should finish loading very soon.
while (self.state == ConfigStates.LOADING and self.conn.state
not in (KazooState.LOST, KazooState.SUSPENDED)):
time.sleep(TINY_WAIT)
if self.state != ConfigStates.LOADED:
raise ConfigInaccessible('ZooKeeper connection not available')
with self.update_lock:
if section not in self.config:
return {}
return self.config[section]
def close(self):
""" Close the ZooKeeper connection. """
self.conn.stop()
print("zk lost") # Register somewhere that the session was lost
elif zk.state == "SUSPENDED":
print("wait for reconnecting")
else:
print("zk fine")
def children_callback(children):
print('****', children)
children = zk.get_children('/zookeeper', children_callback)
my_listener()
zk.create('/goodboy1236456', ephemeral=True, value=b'xxxxxxxxxxxxxxxxxxxx')
#zk.delete('/zookeeper/555555')
xx = zk.ChildrenWatch("/zookeeper/goodboy12364567777")
print(xx.__dict__)
def outprint(event):
print(event)
y = zk.get('/rookie0000000000', watch=outprint)
#zk.add_listener(outprint)
@zk.ChildrenWatch("/")
def watchdemo(children):
print(children)
class ClusterState(object):
uuid = uuid.uuid1() # 生成uuid
hosts = set() # 主机信息会存放在这里
root_path = "/node/host" # 节点信息存放的根路径
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def __init__(self, crawler):
self.crawler = crawler
self.stats = self.crawler.stats
self.zk = KazooClient()
self.__install_listen() # 安装监听
self.__install_signal() # 安装信号
def __install_signal(self):
""" 负责把信号对接起来 """
self.crawler.signals.connect(self.__spider_opened,
signal=signals.spider_opened) # 爬虫开启
self.crawler.signals.connect(self.__spider_closed,
signal=signals.spider_closed) # 爬虫关闭
self.crawler.signals.connect(self.__spider_idle,
signal=signals.spider_idle) # 爬虫闲置
def __install_listen(self):
""" 集群数量监听: COUNT_OF_HOSTS """
def listen(children): # 监听有没有新节点加入
self.hosts.clear() # 清空
self.hosts.update({uuid.UUID(_uuid)
for _uuid in children}) # 更新主机列表
count_of_hosts = len(self.hosts) # 当前主机的数量
self.stats.set_value('COUNT_OF_HOSTS', count_of_hosts)
logger.warning(
Color.yellow('COUNT_OF_HOSTS is {}'.format(count_of_hosts)))
self.zk.ChildrenWatch(self.root_path, func=listen) # 建立监听
@property
def node_name(self):
""" 获取本节点的名称 """
return self.uuid.urn
@property
def node_path(self):
""" 获取 zookeeper 中节点保存的路径 """
return '{path}/{name}'.format(path=self.root_path, name=self.node_name)
def submit_status(self, data):
""" 上传该节点的运作状态 """
bytes_data = json.dumps(data).encode(
'utf-8') # json可对象->strings->bytes
self.zk.set(self.node_path, bytes_data) # 设置结果
def __spider_opened(self, spider):
""" 当爬虫启动: 完成信息注册 """
self.zk.start()
self.zk.ensure_path(self.root_path) # 确保路径存在
if not self.zk.exists(self.node_path): # 不存在则创建
self.zk.create(self.node_path, ephemeral=True,
makepath=True) # 上传节点信息
logger.info(Color.green("has connected to cluster!"))
def __spider_closed(self, spider, reason):
""" 当爬虫关闭: 注销信息注册 """
if self.zk.exists(self.node_path): # 如果存在
self.zk.delete(self.node_path) # 删除节点信息
self.zk.stop()
logger.info(Color.green("has disconnected to cluster!"))
def __spider_idle(self, spider):
""" 当爬虫闲置 """
logger.info(Color.green("self node has idle!"))
class Watch():
'''
Watch class
'''
def __init__(self,
address,
port,
brokers='brokers',
subscribers='subscribers'):
'''
Constructor
'''
self._brokers = brokers
self._subscribers = subscribers
self._threshold = 1
self._hash = None
self._lock = Lock()
self._zookeeper = KazooClient(hosts='{}:{}'.format(address, port))
self._zookeeper.start()
self._zookeeper.ChildrenWatch('/{}'.format(brokers), self.watch)
self._zookeeper.ChildrenWatch('/{}'.format(subscribers), self.watch)
def watch(self, _nodes):
'''
Called when a directory changes
'''
try:
brokers = self._zookeeper.get_children('/{}'.format(self._brokers))
except NoNodeError:
brokers = None
if brokers:
count = (len(
self._zookeeper.get_children('/{}'.format(self._subscribers)))
// self._threshold) + 1
brokers.sort(key=lambda broker: int(broker[-10:]))
brokers = brokers[:count]
with self._lock:
self._hash = Hash(len(brokers))
for broker in brokers:
try:
(identifier, _status) = self._zookeeper.get(
'/{}/{}'.format(self._brokers, broker))
self._hash[broker] = identifier
except NoNodeError:
pass
def all(self):
'''
Get all identifiers
'''
with self._lock:
if self._hash and len(self._hash.values()) != 0:
return list(
map(lambda value: value.decode('utf-8'),
self._hash.values().values()))
return {}
def identifier(self, key):
'''
Get identifier for key
'''
with self._lock:
if self._hash and len(self._hash.values()) != 0:
return self._hash[key].decode('utf-8')
return None
