class ZKClient:
def __init__(self, zk_url, zk_timeout=5):
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
self.logger = logging.getLogger()
try:
self.client = KazooClient(hosts=zk_url)
self.client.start(timeout=zk_timeout)
except handlers.threading.KazooTimeoutError:
self.logger.error("5 seconds connection timeout")
raise
def store_value(self, value, path):
fmt_value = str(value).encode('utf-8')
self.client.ensure_path(path)
self.client.set(path, fmt_value)
self.client.stop()
def retrieve_value(self, path):
self.client.ensure_path(path)
data, stat = self.client.get(path)
self.client.stop()
if not data:
raise ValueError
else:
return int(data.decode('utf-8'))
def register_server(self, path):
self.client.create(path, ephemeral=True, makepath=True)
class ZK:
client = None
def __init__(self, zk_host):
self.client = KazooClient(zk_host)
self.client.start()
def __del__(self):
self.client.stop()
def get_node(self, path):
if not self.client.exists(path):
return None
node = ZKNode(path, self)
return node
def create_node(self, path):
self.client.ensure_path(path)
return self.get_node(path)
def get_transaction(self):
return self.client.transaction()
def get_lock(self, path, id=None):
return self.client.Lock(path + "/lock", id)
def has_lock(self, path):
lock_path = path + "/lock"
if not self.client.exists(lock_path):
return False
if len(self.client.get_children(lock_path)) > 0:
return True
else:
return False
def main(_):
if MODE.STATUS == FLAGS.mode:
request = get_model_status_pb2.GetModelStatusRequest()
request.model_spec.name = 'detection'
request.model_spec.signature_name = 'serving_default'
elif MODE.CONFIG == FLAGS.mode:
request = model_management_pb2.ReloadConfigRequest()
config = request.config.model_config_list.config.add()
config.name = 'detection'
config.base_path = '/models/detection/detection'
config.model_platform = 'tensorflow'
config.model_version_policy.specific.versions.append(5)
config.model_version_policy.specific.versions.append(7)
config2 = request.config.model_config_list.config.add()
config2.name = 'pascal'
config2.base_path = '/models/detection/pascal'
config2.model_platform = 'tensorflow'
elif MODE.ZOOKEEPER == FLAGS.mode:
zk = KazooClient(hosts="10.10.67.225:2181")
zk.start()
zk.ensure_path('/serving/cunan')
zk.set(
'/serving/cunan',
get_config('detection', 5, 224, 'serving_default',
','.join(get_classes('model_data/cci.names')),
"10.12.102.32:8000"))
return
for address in FLAGS.addresses:
channel = grpc.insecure_channel(address)
stub = model_service_pb2_grpc.ModelServiceStub(channel)
if MODE.STATUS == FLAGS.mode:
result = stub.GetModelStatus(request)
elif MODE.CONFIG == FLAGS.mode:
result = stub.HandleReloadConfigRequest(request)
print(result)
class ZKState():
def __init__(self, path, timeout=30):
super(ZKState, self).__init__()
self._zk = KazooClient(hosts=ZK_HOSTS, timeout=timeout)
self._zk.start(timeout=timeout)
self._path = path
self._zk.ensure_path(path)
def processed(self):
return self._zk.exists(self._path + "/complete")
def process_start(self):
if self.processed():
return False
if self._zk.exists(self._path + "/processing"):
return False
try:
self._zk.create(self._path + "/processing", ephemeral=True)
return True
except NodeExistsError: # another process wins
return False
def process_end(self):
self._zk.create(self._path + "/complete")
self._zk.delete(self._path + "/processing")
def process_abort(self):
try:
self._zk.delete(self._path + "/processing")
except NoNodeError:
pass
def close(self):
self._zk.stop()
self._zk.close()
def main_loop():
logging.basicConfig()
zk = KazooClient(hosts=zk_connect_string)
zk.start()
# make sure the root folders for the sendgraph and the schedules exist
zk.ensure_path(metrics_zk_path)
zk.ensure_path(schedule_zk_path)
for topology in zk.get_children(metrics_zk_path):
topology_metrics_zk_path = metrics_zk_path + "/" + topology
print("registering watcher schedule for " + topology_metrics_zk_path)
# register a data watch for each
def watchFunc(data, stat, event):
#print("watch called")
if event is not None and event.type == EventType.CHANGED:
print("new sendgraph data for {0} at {1}".format(topology, byteArrayToInt(data)))
schedule(zk, topology)
return True # returning false will disable the watch
# install data watch
#DataWatch(zk, topology_metrics_zk_path, func=watchFunc)
# if there is some data already, schedule immediately
if len(zk.get_children(topology_metrics_zk_path)):
print("existing sendgraph data for {0}".format(topology))
schedule(zk, topology)
def zookeeper_crud(self, **kwargs):
operation = kwargs['operation']
room = kwargs['room']
room_cn_name = room.comment[:4]
try:
zk = KazooClient(hosts=room.zk_cluster)
zk.start()
if operation == 'retrieve':
path = kwargs['path']
zk.ensure_path(path)
data, stat = zk.get(path)
self.i('%s，获取节点%s数据成功，值为%s' % (room_cn_name, path, data))
return data
elif operation == 'update':
path = kwargs['path']
data = kwargs['data']
zk.ensure_path(path)
zk.set(path, bytes(data))
print ':'.join([str(zk.hosts), 'set', path, data])
self.i('%s，更新节点%s成功，值为%s' % (room_cn_name, path, data))
print ':'.join([str(zk.hosts), 'get', path, zk.get(path)[0]])
return True
except KazooException, e:
self.i('%s，%s操作失败，原因为%s' % (room_cn_name, operation, e.message))
return False
class ES_zk_interface:
def __init__(self, my_ip, es_interface, es_callback):
self.ip = my_ip
self.es_callback = es_callback
self.es_interface = es_interface
self.zk = KazooClient(hosts=HOST_IP)
self.zk.start()
self.zk.ensure_path("/ass3")
def start_election_process(self):
print "Starting Election process"
#print("Previous Leader: %s" % (self.es_callback.get_leader()))
election = self.zk.Election("/electionpath", self.ip)
election.run(self.leader_function)
def leader_function(self):
print "Won the election: " + str(self.ip)
if self.zk.exists(LEADER_PATH):
self.zk.set(LEADER_PATH, self.ip)
else:
self.zk.create(LEADER_PATH, self.ip)
#print "\nStarting Leader thread.... \n"
self.es_interface.enter_message_loop(self.es_callback, True)
def get_zookeeper_client(self):
return self.zk
def get_leader_path(self):
return LEADER_PATH
class ActorAddressBook(object):
def __init__(self, zk_hosts, timeout=60.0):
self.retry = KazooRetry(max_tries=10)
self.zk = KazooClient(hosts=zk_hosts, timeout=timeout)
self.zk.start()
def lookup(self, path):
return self.retry(self._lookup, path)
def _lookup(self, path):
actor_url, stat = self.zk.get(path)
return RemoteActor(actor_url.decode('utf-8'))
def register(self, path, actor_url):
return self.retry(self._register, path, actor_url)
def _register(self, path, actor_url):
self.zk.ensure_path(path)
self.zk.set(path, actor_url.encode('utf-8'))
def delete(self, path):
self.zk.delete(path, recursive=True)
def __del__(self):
self.zk.stop()
class KazooService():
zk = None
rootPath = None
def __init__(self, rootPath):
qs = Zookeeper.objects.filter().first()
if qs is None:
raise LookupError("Unable to find Zookeeper configuration!")
self.zk = KazooClient(hosts=qs.url)
self.rootPath = rootPath
self.zk.start(timeout=qs.timeout)
self.zk.ensure_path(rootPath)
def createOrUpdate(self, pathAffix, nodeValue):
if isinstance(nodeValue, str):
bValue = bytes(nodeValue, 'utf-8')
else:
bValue = bytes(str(nodeValue), 'utf-8')
path = self.rootPath + pathAffix
if self.zk.exists(path) is None:
self.zk.create(path, bValue)
else:
self.zk.set(path, bValue)
def __del__(self):
if self.zk is not None:
self.zk.stop()
class Store(object):
def __init__(self,**kwargs):
self.config = kwargs
self.client = None
def get_client(self):
return self.client
def open(self):
self.client = KazooClient(**self.config)
self.client.add_listener
self.client.start()
def close(self):
self.client.stop()
def read(self,path):
return self.client.get(path)
def write(self,path,value):
base_path = os.path.dirname(path)
self.client.ensure_path(base_path)
self.client.create(path,value)
def overwrite(self,path,value):
self.client.set(path,value)
def exists(self,path):
return self.client.exists(path)
def register_location(host, port):
""" Register service location with ZooKeeper. """
global zk_client
zk_client = KazooClient(hosts=appscale_info.get_zk_locations_string(),
connection_retry=ZK_PERSISTENT_RECONNECTS)
zk_client.start()
server_node = '{}/{}:{}'.format(UA_SERVERS_NODE, host, port)
def create_server_node():
""" Creates a server registration entry in ZooKeeper. """
try:
zk_client.retry(zk_client.create, server_node, ephemeral=True)
except NodeExistsError:
# If the server gets restarted, the old node may exist for a short time.
zk_client.retry(zk_client.delete, server_node)
zk_client.retry(zk_client.create, server_node, ephemeral=True)
logger.info('UAServer registered at {}'.format(server_node))
def zk_state_listener(state):
""" Handles changes to ZooKeeper connection state.
Args:
state: A string specifying the new ZooKeeper connection state.
"""
if state == KazooState.CONNECTED:
threading.Thread(target=create_server_node).start()
zk_client.add_listener(zk_state_listener)
zk_client.ensure_path(UA_SERVERS_NODE)
# Since the client was started before adding the listener, make sure the
# server node gets created.
zk_state_listener(zk_client.state)
class ZookClient():
def __init__(self, hosts):
self.kazooclient = KazooClient(hosts=hosts)
self.kazooclient.start()
def foundApp(self, appname): #Ìí¼Óapp
if self.kazooclient.exists('/' + appname):
pass
else:
self.kazooclient.ensure_path('/' + appname)
def foundNode(self, appname, port, host): #Ìí¼Ó½Úµã
if self.kazooclient.exists('/' + appname + '/' + host + ':' +
str(port)):
pass
else:
self.kazooclient.create('/' + appname + '/' + host + ':' +
str(port))
def close(self):
self.kazooclient.close()
def register(self, appname, port, host):
self.foundApp(appname)
self.foundNode(appname, port, host)
# self.close()
class Register(object):
def __init__(self,
value,
path='/scrapyd-cluster/worker',
hosts='127.0.0.1:2181'):
self.path = path
self.value = value
self.expire = False
self.zk = KazooClient(hosts)
self.zk.start()
self.zk.add_listener(self.listener)
self.zk.ensure_path(os.path.dirname(self.path))
self.path = self.zk.create(self.path,
self.value,
ephemeral=True,
sequence=True)
logger.info('Connected To ZooKeeper(%s) Succeed! Path : %s' %
(hosts, self.path))
def register(self):
if self.expire:
logger.info('Renew Ephemeral Node. Path : %s Value: %s' %
(self.path, self.value))
self.zk.create(self.path, self.value, ephemeral=True)
self.expire = False
def listener(self, state):
if state == KazooState.LOST:
logger.info('ZooKeeper Connection Lost')
self.expire = True
elif state == KazooState.CONNECTED:
self.zk.handler.spawn(self.register)
class ZooWrap():
def __init__(self):
self.zk = KazooClient(hosts="%s,%s" % (
str(cc.conf['zookeeper']['host']),
str(cc.conf['zookeeper']['port'])))
self.zk.start()
self.root = cc.conf['zookeeper']['rootpath']
self.log = logging.getLogger('L.ZOOKEEPER')
self.zk.ensure_path('/%s/sleeping' % (self.root))
self.whoami = cc.conf['whoami']
def get_sleeping(self):
return self.zk.get_children('/%s/sleeping' % (self.root))
def sleep(self):
try:
self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami))
self.log.info('Sleeping correctly')
except NodeExistsError:
self.log.error('Node already sleeping... seems weird')
def wake(self):
try:
self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami))
except NoNodeError:
self.log.error('Node was not sleeping... seems weird')
class KazooWrapper:
"""
Wrapper that ensure that we connect to zookeeper only when need it.
Prevents connection on error request.
"""
log = logging.getLogger('KazooWrapper')
def __init__(self, chroot=None, *args, **kwargs):
self.zk = KazooClient(*args, **kwargs)
self._started = False
self.chroot = chroot
def start(self):
self.log.debug('Connect')
self.zk.start()
if self.chroot:
self.zk.ensure_path(self.chroot)
self.zk.chroot = self.chroot
self._started = True
def stop(self):
if self._started:
self.zk.stop()
def __getattr__(self, name):
if not self._started:
self.start()
return getattr(self.zk, name)
def check_active(port):
zk = KazooClient("localhost:2181")
zk.start()
zk.add_listener(my_listener)
zk.ensure_path("/big_data_assign")
children = zk.get_children("/big_data_assign")
path = "/big_data_assign/"
for each in children:
path_child = path + each
#print(path_child)
c = zk.get(path_child)
c = c[0]
c = c.decode()
c = eval(c, {'OrderedDict': OrderedDict})
#print("here4",c,type(c))
#for key, value in c.items():
key = list(c.items())[0]
key = key[0]
#print(key,type(key))
if (key == port):
active = True
break
else:
active = False
#print(active)
return active
class ZKStore:
def __init__(self, hosts):
self.zk = KazooClient(hosts=hosts)
self.zk.add_listener(listener)
self.zk.start()
def isConnected(self):
if __state__ == 1:
return True
return False
def write(self, path, node, value):
self.zk.ensure_path(path)
if self.zk.exists(path+"/"+node):
self.zk.set(path+"/"+node, value)
else:
self.zk.create(path + "/" + node, value)
def read(self, path):
if self.zk.exists(path):
data, stat = self.zk.get(path)
return data
return None
class ZooWrap():
def __init__(self):
self.zk = KazooClient(hosts="%s,%s" %
(str(cc.conf['zookeeper']['host']),
str(cc.conf['zookeeper']['port'])))
self.zk.start()
self.root = cc.conf['zookeeper']['rootpath']
self.log = logging.getLogger('L.ZOOKEEPER')
self.zk.ensure_path('/%s/sleeping' % (self.root))
self.whoami = cc.conf['whoami']
def get_sleeping(self):
return self.zk.get_children('/%s/sleeping' % (self.root))
def sleep(self):
try:
self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami))
self.log.info('Sleeping correctly')
except NodeExistsError:
self.log.error('Node already sleeping... seems weird')
def wake(self):
try:
self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami))
except NoNodeError:
self.log.error('Node was not sleeping... seems weird')
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-p",
"--project",
help="project name kafka or schema-registry",
default="kafka")
parser.add_argument(
"-z",
"--zookeeper",
help="zookeeper url with port",
default='zookeeper.wgdp.io:2181',
)
args = parser.parse_args()
conf_dict = collect_info(args.project)
print(conf_dict)
region = define_region_name(conf_dict["hostname"])
if region == 'edcenter':
if args.project == 'kafka':
conf_dict['url'] = 'kafka.wgdp.io'
else:
conf_dict['url'] = 'schemas-registry.wgdp.io'
else:
conf_dict['url'] = '{}-kafka.wgdp.io'.format(region)
zk = KazooClient(hosts=args.zookeeper)
zk.start()
zk.ensure_path("/app/{}/{}/{}".format(args.project, region,
conf_dict["hostname"]))
zk.set(
"/app/{}/{}/{}".format(args.project, region, conf_dict["hostname"]),
json.dumps(conf_dict),
)
def _start(self, err_msg, spawn_monit=False):
if self._is_destroyed:
return
self._client = None
# Increase the session timeout from 10 to 25 seconds.
try:
host_list = self.zk_hosts
client = KazooClient(
hosts=",".join(host_list),
timeout=self._get_session_timeout(),
max_retries=3,
handler=SequentialGeventHandler())
# Increase the start timeout to 20 seconds from 15 seconds.
# Guard this with explicit gevent timeout to protect us from
# some corner cases where starting client failed to respect
# start timeout passed in below.
with gevent.Timeout(seconds=self._get_start_timeout() + 5):
client.start(timeout=self._get_start_timeout())
client.ensure_path("/")
self._last_success_health_check_ts = time.time()
log.info("Successfully started kazoo client.")
self._client = client
except (Exception, gevent.Timeout):
self._sc.increment("errors.zk.client.start.failure",
tags={'host': hostname},
sample_rate=1)
log.exception(err_msg)
finally:
if spawn_monit:
self._monit_greenlet = gevent.spawn(self._monit)
gevent.sleep(0)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--hosts", required=False, help="zk hosts, example:127.0.0.1:2181")
parser.add_argument("--module", required=False, help="module name")
parser.add_argument("--module_dependence", required=False, help="dependence module names, separated by comma")
parser.add_argument("--redis_names", required=False, help="redis names")
parser.add_argument("--redis_data", required=False, help="redis data")
parser.add_argument("--paths", required=False, help="zk absolute path")
args = parser.parse_args()
module = args.module
module_dependence = args.module_dependence
zk = KazooClient(hosts=args.hosts)
zk.start()
if module is not None:
create_node(zk, module, module + "-test", module_dependence)
if module_dependence is not None:
for s in module_dependence.split(","):
create_node(zk, s, s + "-test")
if args.redis_names is not None and args.redis_data is not None:
for s in args.redis_names.split(","):
create_redis(zk, s, args.redis_data)
if args.paths is not None:
for s in args.paths.split(","):
zk.ensure_path(s)
def test_zookeeper(zookeeper_server):
test_value = str(uuid.uuid4()).encode("utf-8")
zk_client1 = KazooClient(
hosts="127.0.0.1:{}".format(zookeeper_server.port))
zk_client2 = KazooClient(
hosts="127.0.0.1:{}".format(zookeeper_server.port))
zk_client1.start()
zk_client2.start()
zk_client1.ensure_path("/my")
assert zk_client1.exists("/you") is None
assert zk_client2.exists("/you") is None
assert zk_client1.exists("/my") is not None
assert zk_client2.exists("/my") is not None
zk_client1.create("/my/testplan", test_value)
data, _ = zk_client1.get("/my/testplan")
assert data == test_value
data, _ = zk_client2.get("/my/testplan")
assert data == test_value
zk_client2.set("/my/testplan", str(uuid.uuid4()).encode("utf-8"))
data, _ = zk_client1.get("/my/testplan")
assert data != test_value
data, _ = zk_client2.get("/my/testplan")
assert data != test_value
def test_subscriber_connect(self):
test_zk = KazooClient(self.zk_sub.zookeeper_connection_url)
try:
test_zk.start(5)
except Exception:
self.assertRaises("Couldn't connect to zookeeper server")
# Ensure a path, create if necessary
test_zk.ensure_path(self.zk_sub.default_node_path)
fake_host_ip_lst = ["0.0.0.0", "0.0.0.1", "0.0.0.2", "0.0.0.3"]
try:
for host_ip in fake_host_ip_lst:
# Create a node with data
node = "node" + str(uuid.uuid4())
path = self.default_node_path + '/' + node
test_zk.create(path, bytes(host_ip, "utf-8"))
self.zk_sub.subscriber_connect("subscriber")
finally:
nodes_to_delete = test_zk.get_children(self.default_node_path)
for node in nodes_to_delete:
path = self.default_node_path + '/' + node
test_zk.delete(path)
children = test_zk.get_children(self.default_node_path)
self.assertEqual(children, [])
def zookeeper_watcher(self, q, room):
service = q.get()
try:
zk = KazooClient(hosts=room.zk_cluster)
zk.start()
count = 0
path = '/FlagsCenter/%s/gray_%s/gray_%s.result' % (self.full_app_name, service, service)
zk.ensure_path(path)
@zk.DataWatch(path)
def watch_node(data, stat):
if count == 0:
self.service_dict[room] = self.service_dict.get(room, dict())
self.service_dict[room][stat.pzxid] = self.service_dict[room].get(stat.pzxid, dict())
self.service_dict[room][stat.pzxid]['name'] = service
self.service_dict[room][stat.pzxid]['data'] = False
else:
self.service_dict[room][stat.pzxid]['data'] = data
while count < (ZOOKEEPER['WATCHER_TIMEOUT'] + ZOOKEEPER['WATCHER_DELAY']) * ZOOKEEPER['WATCHER_TIMES']:
if all([self.service_dict[room][pzxid]['data'] for pzxid in self.service_dict[room]]):
q.task_done()
return
count += 1
time.sleep(1.0 / ZOOKEEPER['WATCHER_TIMES'])
q.task_done()
return
except KazooException, e:
self.service_dict[room] = e.message
self.i(u'%s，watcher操作失败，原因为%s' % (room.comment[:4], self.service_dict))
q.task_done()
return
def zk_init(server_address, zk_node, zk_address = ''):
#初始化zk节点
zk_cli = KazooClient(hosts = zk_address)
try:
zk_cli.start()
except:
logging.error('zk Init error, can not connect %s' %(str(zk_address)))
return -1
try:
zk_cli.get(zk_node)
except:
logging.warn('can not find zk path %s, creat it' %(str(zk_node)))
zk_cli.ensure_path(zk_node)
if zk_node[-1] != '/':
zk_node += '/'
try:
zk_cli.create(zk_node + server_address, '1', ephemeral=True)
except:
if zk_cli.get(zk_node + server_address):
return 0
else:
logging.error('create zk_node error, can not create node %s' %(str(zk_node) + str(zk_address)))
return -1
return 0
class DistributedLock:
def __init__(self, hosts, logger=None):
self.client = KazooClient(hosts=hosts, logger=logger)
def ensure_path(self, path):
self.client.start()
self.client.ensure_path(path)
self.client.stop()
def try_lock(self, path):
try:
self.client.start()
self.client.create(path, ephemeral=True)
return True
except:
logging.exception("try lock exception. path: {}".format(path))
return False
def try_unlock(self, path):
try:
self.client.delete(path)
self.client.stop()
return True
except:
logging.exception("try unlock exception. path: {}".format(path))
return False
class ZooWatch:
def __init__(self):
print("Zoo watch init")
logging.basicConfig()
self.zk = KazooClient(hosts='zoo:2181')
self.zk.start()
self.temp = []
self.master_db_name = "mongomaster"
def start(self):
print("[*] Starting zoo watch", file=sys.stdout)
self.zk.ensure_path("/worker")
@self.zk.ChildrenWatch("/worker")
def callback_worker(workers):
print("[*] Changes detected", file=sys.stdout)
print(workers, self.temp)
if len(workers) < len(self.temp):
node = listdiff(self.temp, workers)
print("[-] Node deleted: " + node, file=sys.stdout)
print("[*] Current workers: " + str(workers), file=sys.stdout)
if "slv_node" in node:
killed_containers = client.containers.list(
all=True, filters={"exited": "137"})
print("[*] killed killed ++++++++", killed_containers)
#slave_cnt = client.containers.get(node)
i = node[-1]
#name ="slv"+i
#slave_cnt = client.containers.get(name)
global prune_cont
#if(name != prune_cont):
#print("[&&] slave cont",slave_cnt)
#slave_db_cnt = client.containers.get("mongodb" + i)
if prune_cont not in killed_containers and len(
killed_containers) > 0:
#slave_cnt.remove()
#slave_db_cnt.remove()
#random_name = "".join(random.choices(string.ascii_lowercase + string.digits, k=7))
bring_up_new_worker_container(
slave_name="slv" + "newwwwwwwwwwwwwwwww",
db_name="mongodb" + "newwwwwwwwwwwwwwwwwwww")
else:
print("[*] Scaling down - removing " + node)
#client.containers.prune()
#print("[*] Or newly elected master is deleting its old node")
else:
print("[-] Master failed", file=sys.stdout)
elif len(workers) > len(self.temp):
print("[+] Node added: " + listdiff(self.temp, workers),
file=sys.stdout)
print("[*] Current workers: " + str(workers), file=sys.stdout)
else:
pass
self.temp = workers
while True:
pass
def main():
logging.basicConfig()
zk = KazooClient(hosts=zookeeper_hostname)
zk.start()
zk.ensure_path("/worker")
node_name = "/worker/" + os.environ["NODE_NAME"]
if not zk.exists(node_name):
msg = "Creating node: " + node_name
print(msg, file=sys.stdout)
db_name = os.environ["DB_HOSTNAME"]
zk.create(node_name, db_name.encode(), ephemeral=True)
if os.environ["WORKER_TYPE"] == "master":
rpc_server = RpcServer(queue_name='writeQ', func=writedb, is_master=True)
rpc_server.start()
else:
try:
if node_name != "/worker/slave1":
master_db = zk.get("/worker/master")[0].decode()
print("[*] Cloning database from master db: " + master_db, file=sys.stdout)
subprocess.call(
"mongodump --host " + master_db + " --port 27017 --db rideshare && mongorestore --host " + os.environ[
'DB_HOSTNAME'] + " --port 27017",
stdout=sys.stdout,
stderr=sys.stdout,
shell=True
)
except Exception as e:
print(e, file=sys.stdout)
old_name = os.environ["NODE_NAME"]
p1 = multiprocessing.Process(target=slave_rpc_server)
p2 = multiprocessing.Process(target=become_master, args=(p1, old_name,))
p1.start()
p2.start()
def list_worker():
logging.basicConfig()
zk = KazooClient(hosts='zoo:2181')
zk.start()
zk.ensure_path("/workers")
children = zk.get_children("/workers")
return Response(json.dumps(children),status=200,mimetype='application/json')
class ZkHelper(object):
def __init__(self, address='', port=''):
assert address and port
self.zk_address = address
self.zk_port = port
self.retry = KazooRetry(max_delay=10000, max_tries=None)
self.zk = KazooClient(hosts='%s:%s' % (self.zk_address, self.zk_port),
connection_retry=self.retry,
timeout=20)
self.zk.add_listener(self._listener)
self.zk.start()
logging.info("instance zk client start (%s:%s)" %
(self.zk_address, self.zk_port))
@staticmethod
def _listener(state):
if state == KazooState.LOST:
logging.info(
"zk connect lost, stop this connection and then start new one!"
)
elif state == KazooState.SUSPENDED:
logging.info(
"zk connect suspended, stop this connection and then start new one!"
)
def write(self, path, data):
self.zk.ensure_path(path)
self.retry(self.zk.set, path, data)
logging.info("write data:%s to path:%s" % (data, path))
def ensure_path(self, path):
self.zk.ensure_path(path)
def get_lock(self, path):
return self.zk.Lock(path, threading.currentThread())
def read(self, path):
if self.zk.exists(path):
data = self.retry(self.zk.get, path)
logging.info("read data:%s from path:%s" % (data, path))
return data[0]
logging.info("path:%s not exist" % path)
def get_children_list(self, path):
if self.zk.exists(path):
data = self.retry(self.zk.get_children, path)
logging.info("get children:%s from path:%s" % (data, path))
return data
logging.info("path:%s not exist" % path)
def exists(self, path):
return self.zk.exists(path)
def get_lock(self, path):
lock = self.retry(self.zk.Lock, path, threading.current_thread())
return lock
def close(self):
self.zk.close()
def setup(self, env, result):
"""
Setup method that will be executed before all testcases. It is
used to ensure the path that the testcases require.
"""
zk = KazooClient(hosts="127.0.0.1:{}".format(env.zk.port))
zk.start()
zk.ensure_path("/testplan")
def createZkNode(zk_host, zk_port, host_group, host, serverset_member):
zk_connection_string = zk_host + ":" + zk_port
zk = KazooClient(hosts=zk_connection_string)
zk.start()
zk.ensure_path("inventory")
zk_node = "inventory/" + host_group + "/" + host
zk.ensure_path(zk_node)
zk.set(zk_node, str(serverset_member).encode())
class ZkHelper:
def __init__(self,
zk_addresses,
zk_timeout_sec,
zk_root,
redis_address,
redis_id=None):
""" create zookeeper client, keep connection and auto re-register when session expired
:type zk_addresses: str
:type zk_timeout_sec: int
:type zk_root: str
:type redis_address: str
"""
self.__zk_addresses = zk_addresses # "zookeeper.n.miliao.com:2181"
self.__zk_timeout = zk_timeout_sec
self.__redis_address = bytes(redis_address) # "192.168.1.1:6379"
self.__zk_registered_redis_id = redis_id
self.__zk_redis_path = None
self.__zk_root = zk_root # "/xmss/cluster/{clusterid}/filecache/partition_{id}"
self.__zk_redis_pool = "%s/%s" % (self.__zk_root, "pool")
self.__zk_lock_path = "%s/%s" % (self.__zk_root, "lock")
# init zookeeper client
zk_retry = KazooRetry(max_tries=3, delay=1.0,
ignore_expire=False) # delay in sec
self.__zk_client = KazooClient(hosts=self.__zk_addresses,
logger=KAZOO_LOGGER,
timeout=self.__zk_timeout,
connection_retry=zk_retry)
self.__zk_client.add_listener(self.listener)
self.__zk_lock = lock.Lock(self.__zk_client,
self.__zk_lock_path,
identifier=self.__redis_address)
# auto re-register worker
self.__lost_zk_connection = False
self.__re_register_event = self.__zk_client.handler.event_object()
self.__re_register_thread = None
self.__supervisor = None
self.__stopped = False
def list_pool(self):
try:
if not self.__zk_client.exists(self.__zk_redis_pool):
LOGGER.warn("zk path[%s] doesn`t exists. create it.",
self.__zk_redis_pool)
self.__zk_client.ensure_path(self.__zk_redis_pool)
children_str = self.__zk_client.get_children(
path=self.__zk_redis_pool)
children = [int(i) for i in children_str]
LOGGER.info("current pool:[%s]", children)
return children
except (ConnectionLoss, ZookeeperError), e:
LOGGER.error("zk exception when list pool.")
raise ZkHelperException(e)
def print_children():
zk = KazooClient("localhost:2181")
zk.start()
zk.add_listener(my_listener)
zk.ensure_path("/big_data_assign")
#print("All children")
children = zk.get_children("/big_data_assign")
print("children: ", children)
return children
def register(self, zk, version='1.0.0', revision='1.0.0'):
client = KazooClient(zk)
client.start()
for service, methods in self._services.items():
logging.info(
f'register service "{service}", methods "{methods}" to zookeeper "{zk}"'
)
url = f'dubbo://{self._host}:{self._port}/{service}?anyhost=true&application={self._app}&dubbo={self._dubbo_version}&interface={service}&methods={",".join(methods)}&pid={next(_pid_gen)}&revision={revision}&side=provider×tamp={get_timestamp()}&version={version}'
client.ensure_path(f'/dubbo/{service}/providers/{quote_plus(url)}')
class dzk:
def __init__(self):
self.BasePath = "/my/"
self.zk = KazooClient(hosts='x.24.79.51:2181,x.24.79.53:2181',retry_max_delay=2000)
self.zk.start()
self.zk.add_listener(self.listener)
def listener(state):
if state == KazooState.LOST:
self.zk.start()
elif state == KazooState.SUSPENDED:
print "*******listener saw KazooState.LOST"
else:
print "*******listener saw KazooState.CONNECT"
def getIpHost(self):
self.myname = socket.getfqdn(socket.gethostname())
myip = socket.gethostbyname(self.myname)
return myip
def register(self):
ip = self.getIpHost()
if ip:
NODE = self.BasePath + ip
print "register:",NODE
else:
print "[ERROR:] %s does not exist " %(NODE)
sys.exit(2)
if not self.zk.exists(NODE):
self.zk.ensure_path(NODE)
def getData(self):
ip = self.getIpHost()
if ip:
NODE = self.BasePath + ip
else:
print "[ERROR:] %s does not exist " %(NODE)
if self.zk.exists(NODE):
data, stat = self.zk.get(NODE)
print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
def monitor(self):
pass
def heartbeat(self):
pass
def role(self):
pass
def command(self):
pass
def build(self, **connection_kwargs):
assert self.zk_hosts
assert self.zk_proxy_dir
retry = {
"max_delay": self.zk_max_delay,
"max_tries": self.zk_max_tries
}
zk_client = KazooClient(hosts=self.zk_hosts, connection_retry=retry)
zk_client.start()
zk_client.ensure_path(self.zk_proxy_dir)
return CodisConnectionPool(zk_client, self.zk_proxy_dir, True, **connection_kwargs)
def main():
'''
A manual configuration file pusher for the crawlers. This will update
Zookeeper with the contents of the file specified in the args.
'''
import argparse
from kazoo.client import KazooClient
parser = argparse.ArgumentParser(
description="Crawler config file pusher to Zookeeper")
parser.add_argument('-f', '--file', action='store', required=True,
help="The yaml file to use")
parser.add_argument('-i', '--id', action='store', default="all",
help="The crawler id to use in zookeeper")
parser.add_argument('-p', '--path', action='store',
default="/scrapy-cluster/crawler/",
help="The zookeeper path to use")
parser.add_argument('-w', '--wipe', action='store_const', const=True,
help="Remove the current config")
parser.add_argument('-z', '--zoo-keeper', action='store', required=True,
help="The Zookeeper connection <host>:<port>")
args = vars(parser.parse_args())
filename = args['file']
id = args['id']
wipe = args['wipe']
zoo = args['zoo_keeper']
path = args['path']
zk = KazooClient(hosts=zoo)
zk.start()
# ensure path exists
zk.ensure_path(path)
bytes = open(filename, 'rb').read()
if zk.exists(path):
# push the conf file
if not zk.exists(path + id) and not wipe:
print "creaing conf node"
zk.create(path + id, bytes)
elif not wipe:
print "updating conf file"
zk.set(path + id, bytes)
if wipe:
zk.set(path + id, None)
zk.stop()
def main():
zk = KazooClient(hosts="127.0.0.1:2181", timeout=2.0)
zk.add_listener(my_listener)
zk.start()
if zk.exists("/ELECTION") == None:
zk.ensure_path("/ELECTION")
c = 1
node_pathes = []
while c < 10:
c += 1
node_path = zk.create("/ELECTION/guid-n_", b"a value", ephemeral=True, sequence=True)
node_pathes.append(node_path)
my_path = random.choice(node_pathes)
my_path = my_path.replace("/ELECTION/", "")
# print "print my_path", my_path
children = zk.get_children("/ELECTION/", watch=election_child_watcher)
get_next_path = False
prev_path = None
for child_path in sorted(children):
if child_path == my_path:
break
prev_path = child_path
# I'm the leader
if prev_path == None:
print "OK I'm leader don't have to watch"
return
# fires twice, once on creation ignore
@zk.DataWatch("/ELECTION/" + prev_path)
def watch_node(data, stat):
# only watch for first change
if stat.version == 1:
print ("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
print "setting watch on " + prev_path
print "my", my_path
zk.set("/ELECTION/" + prev_path, b"some data")
print "boom. watch triggered?"
# time.sleep(10)
print "bye"
zk.stop()
def ensure_kafka_zk_path(retries=3):
while retries >= 0:
# Connect to the ZooKeeper nodes. Use a pretty large timeout in case they were
# just started. We should wait for them for a little while.
zk = KazooClient(hosts=ZOOKEEPER_NODE_LIST, timeout=30000)
try:
zk.start()
zk.ensure_path(KAFKA_ZOOKEEPER_BASE)
return True
except:
retries -= 1
finally:
zk.stop()
return False
def put_topology(topology, config=config):
zk = KazooClient(**config)
zk.start()
for service, links in topology.iteritems():
if zk.exists("/sdn/services/{name}".format(name=service)):
zk.set("/sdn/services/{name}".format(name=service), json.dumps(links))
else:
zk.ensure_path("/sdn/services")
zk.create("/sdn/services/{name}".format(name=service), json.dumps(links))
ret_check = zk.get_children("/sdn/services")
zk.stop()
return ret_check
def test_zookeeper_fault_tolerance():
"""
Kill zookeeper1 and see if other zookeeper instances are in quorum
"""
zookeper_deployer = runtime.get_deployer("zookeeper")
kazoo_connection_url = str(runtime.get_active_config('zookeeper_host') + ':2181')
zkclient = KazooClient(hosts=kazoo_connection_url)
zkclient.start()
zkclient.ensure_path("/my/zookeeper_errorinjection")
# kill the Zookeeper1 instance
print "killing zoookeeper instance1"
zookeper_deployer.kill("zookeeper1")
time.sleep(20)
zkclient.stop()
def test_multiclient_rebalance(self, *args):
"""Test rebalancing with many connected clients
This test is primarily good at ferreting out concurrency bugs
and therefore doesn't test one specific thing, since such
bugs are fundamentally hard to trap.
To test, it simulates 10 consumer connecting over time, rebalancing,
and eventually disconnecting one by one. In order to accomplish
this, it creates a separate kazoo client for each consumer. This
is required because otherwise there is too much thread contention
over the single kazoo client. Some callbacks won't happen until
much too late because there aren't enough threads to go around.
"""
n_partitions = 10
n_consumers = 10
t = Topic(self.c, 'testtopic')
# with self.client, new clients don't see the brokers -- unsure why
from kazoo.client import KazooClient
zk_hosts = ','.join(
['%s:%s' % (h,p) for h,p in self.client.hosts.hosts])
zkclient = KazooClient(hosts=zk_hosts)
zkclient.start()
self._register_fake_brokers(n_partitions, client=zkclient)
zkclient.ensure_path('/brokers/topics')
# bring up consumers
consumers = []
for i in xrange(n_consumers):
newclient = KazooClient(hosts=zk_hosts)
newclient.start()
cluster = Cluster(newclient)
topic = Topic(cluster, 'testtopic')
consumers.append((newclient, topic.subscribe('group1')))
time.sleep(1)
time.sleep(5) # let things settle
# bring down consumers
for client,consumer in consumers:
consumer.stop_partitions()
client.stop()
time.sleep(2) # a little more time so we don't kill during rebalance
newclient.stop()
class NodeMonitor:
STATIC_NODE_ID = 0
global t
def __init__(self):
self.zk = None
self.SERVER_IP_AND_PORT = "localhost:2181"
self.NODE_ID = str(NodeMonitor.STATIC_NODE_ID)
NodeMonitor.STATIC_NODE_ID += 1
def start_zk(self):
self.zk = KazooClient(hosts=self.SERVER_IP_AND_PORT)
self.zk.add_listener(self._connection_listener)
self.zk.start();
self.zk.ensure_path("/monitorData/"+ self.NODE_ID)
def start_update_info(self):
t = threading.Timer(0.0, self._update_info)
t.start()
def _update_info_once(self):
cmi = CollectMachineInfo()
async_obj = self.zk.set_async("/monitorData/"+ self.NODE_ID, (cmi.collectInfo()).encode(encoding="utf-8"))
async_obj.rawlink(self._update_info_callback)
def _connection_listener(self, state):
if state == KazooState.LOST:
print "connection lost, going to connect again"
self.start_zk();
elif state == KazooState.SUSPENDED:
print "suspended"
else:
print "connected ok"
def _update_info_callback(self, async_obj):
try:
print "update success"
except (ConnectionLossException, NoAuthException):
print "exception!"
def _update_info(self):
print "begin to update"
self._update_info_once()
t = threading.Timer(5.0, self._update_info)
t.start()
class ZookeeperSession(BaseClient):
conext_manager = ZookeeperResponseContextManager
loose_policy = {}
strict_policy = {}
def __init__(self,server_list='127.0.0.1:2181',*args,**kwargs):
super(ZookeeperSession,self).__init__(*args,**kwargs)
self.session_policy = "loose_policy"
self._zookeeper_client = None
self.server_list = server_list
def set_session_policy(self,session_policy="loose"):
'''prototype not currenlty used.
'''
self.session_policy = session_policy+"_policy"
def connect(self,*args,**kwargs):
'''See http://kazoo.readthedocs.org/en/latest/api/client.html
for details regarding available options. Any provided client
start() parameters provided will override defaults.
'''
defaults = {
"hosts" : self.server_list,
"handler" : SequentialGeventHandler()
}
defaults.update(getattr(self,self.session_policy))
defaults.update(kwargs)
self._state = KazooState.LOST
self._zookeeper_client = KazooClient(**defaults)
self._zookeeper_client.add_listener(self._state_tracker)
watchable = self._zookeeper_client.start_async()
watchable.wait(30)
if not self._zookeeper_client.connected:
err = "Could not connect to Zookeeper server(s) %(server_list)s" % defaults
raise ResponseError(err)
@require_state(KazooState.CONNECTED)
@record_stats
def ensure_path(self,path,watcher=None):
self._zookeeper_client.ensure_path(path,watcher)
def _state_tracker(self,state):
self._state = state
def __del__(self):
if isinstance(self._zookeeper_client, KazooClient):
self._zookeeper_client.stop()
def test_zookeeper_process_tracking():
"""
Tests if process register node correctly with zookeeper and zookeeper deletes it when process terminates
"""
#Wait for zookeeper to start so that kazoo client can connect correctly
time.sleep(5)
#"connecting to esnure /my/zookeeper_test"
kazoo_connection_url = str(runtime.get_active_config('zookeeper_host') + ':2181')
zkclient = KazooClient(hosts=kazoo_connection_url)
zkclient.start()
zkclient.ensure_path("/my/zookeeper_test")
#spawn a python multiprocess which creates an ephermeral node
#once the process ends the node will be deleted.
p = Process(target=zookeeper_ephemeral_node, args=("process1",))
p.start()
zkclient.stop()
class ZkServiceRegister:
def __init__(self, zk_address, zk_timeout):
self.__zkClient = KazooClient(hosts=zk_address, timeout=zk_timeout, read_only=False)
self.__zkListener = ZkServiceRegisterListener(self.__zkClient)
self.__zkClient.add_listener(self.__zkListener)
self.__zkClient.start()
def register(self, path, host, port, weight=DEFAULT_HOST_WEIGHT):
try:
if not self.__zkClient.exists(path):
self.__zkClient.ensure_path(path)
except Exception, e:
print e.message
reg_path = path + '/' + host + ':' + str(port) + ':' + str(weight)
if self.__zkClient.exists(reg_path):
self.__zkClient.delete(reg_path)
self.__zkClient.create(reg_path, value='', ephemeral=True)
def zkset():
file = "/tmp/redis_config.properties"
zk = KazooClient(hosts='172.29.22.15:2181')
zk.start()
f=open(file,'rb')
m=f.read()
#Ensure a path , Create if necessary.
zk.ensure_path("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes")
#Create a node with data.
try:
zk.create("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)
except kazoo.exceptions.NodeExistsError:
zk.set("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)
zk.stop()
class MasterElection(object):
def __init__(self):
self.zk = KazooClient(hosts=state.ARGS.zookeeper)
self.master_notified = False
self.my_node = None
self.zk.start() # Stop never called
self.zk.ensure_path(ELECTION_PATH)
def start_election(self, master_callback):
"""
Start the master election.
If this node is the master, the callback will be called once.
:params master_callback: Called if this node is the master
"""
self.callback = master_callback
self.my_node = self.zk.create(ELECTION_PATH + '/n_',
ephemeral=True, sequence=True)
self.zk.get_children(ELECTION_PATH, watch=self._election_update)
self._election_update()
def _election_update(self, data=None):
def worker():
try:
self.master_notified = True
self.callback()
except Exception as e:
self.master_notified = False
log.info("Failed to activate master, redoing election: %r", e)
self.zk.delete(self.my_node)
self.my_node = self.zk.create(ELECTION_PATH + '/n_',
ephemeral=True, sequence=True)
self._election_update()
if not self.master_notified and \
sorted(self.zk.get_children(ELECTION_PATH))[0] == \
os.path.basename(self.my_node):
t = threading.Thread(target=worker)
t.daemon = True
t.start()
class MicroService(object):
"""docstring for MicroService"""
def __init__(self, hosts, app):
from kazoo.client import KazooClient
self.zk = KazooClient(hosts=hosts)
self.app = "/" + app
def zkStart(self):
self.zk.start()
def zkStop(self):
self.zk.stop()
def registerService(self, service, url):
if self.zk.exists(self.app + "/" + service):
self.zk.delete(self.app + "/" + service, recursive=True)
self.zk.delete(self.app, recursive=True)
self.zk.ensure_path(self.app + "/" + service)
self.zk.set(self.app + "/" + service, bytes(url, encoding="utf-8"))
self.__watchServiceChanged(service)
return 'Service registed'
def unregisterService(self, service):
self.zk.delete(self.app + "/" + service, recursive=True)
return "Service deleted"
def findService(self, service):
data, stat = self.zk.get(self.app + "/" + service)
return data.decode("utf-8")
def __watchServiceChanged(self, service):
@self.zk.DataWatch(self.app + "/" + service)
def changed(data, stat):
print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
class ZooDict(object):
def __init__(self, path_root=None):
self.path_root = path_root or default_path_root
def __enter__(self):
self.zk = KazooClient(hosts=zk_hosts)
self.zk.start(timeout=5)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.zk.stop()
def start(self):
return self.__enter__()
def stop(self):
return self.__exit__()
def get(self, key):
try:
r = self.zk.get(self.path_root + '/' + key)
if r and r[0]:
return json.loads(r[0])
except NoNodeError as no_node:
return None
def set(self, key, data_dict):
self.zk.ensure_path(self.path_root + '/' + key)
self.zk.set(self.path_root + '/' + key, json.dumps(data_dict))
def delete(self, key):
self.zk.delete(self.path_root + '/' + key)
def items(self):
return [ (child, self.get(child)) for child in self.zk.get_children(self.path_root) ]
def doSync(ip, src, dst, deleteFile):
zk = KazooClient(hosts=ip)
zk.start()
if not os.path.exists(src):
print src + ' not exists'
return
if not os.path.isdir(src):
print 'copy file'
copyFile(zk, src, dst)
zk.stop()
return
for f in os.listdir(src):
if f.startswith('.svn'):
continue
fullPath = os.path.join(src, f)
zk_path = os.path.join(dst, f)
if os.path.isdir(fullPath):
zk.ensure_path(zk_path)
syncChild(zk, fullPath, zk_path, deleteFile)
continue
zk.ensure_path(dst)
copyFile(zk, fullPath, zk_path)
zk.stop()
class ZookClient(object):
def __init__(self, api_client=None):
logging.basicConfig()
# Create a client and start it
self.zk = KazooClient()
self.zk.start()
def create_accounts_path(self, name, **kwargs):
path = "/dso/" + name
self.zk.ensure_path(path)
self.zk.set(path, b"id: 7b4235ca-00fb-4dca-ad3e-8b6e3662631a\ngroupname: hr\ndescription: 人力资源")
def create_accountinfo_path(self, account_id, **kwargs):
self.zk.ensure_path("/app/someservice")
def create_path(self, path, **kwargs):
self.zk.ensure_path(path)
def get_data(self, path):
return self.zk.get(path)
def test_tran(self):
self.zk.delete("/app22")
self.zk.create("/app22", b"" + '{"12": "12"}')
tran = self.zk.transaction()
tran.delete("/app22")
tran.create("/app22", b"" + '{"22": "22"}')
tran.commit()
print "commit"
def stop(self):
# In the end, stop it
self.zk.stop()
class ZookeeperRegistry(Registry):
_app_config = ApplicationConfig('default_app')
_connect_state = 'UNCONNECT'
def __init__(self, zk_hosts, application_config=None):
Registry.__init__(self)
if application_config:
self._app_config = application_config
self.__zk = KazooClient(hosts=zk_hosts)
self.__zk.add_listener(self.__state_listener)
self.__zk.start()
def __state_listener(self, state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
self._connect_state = state
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
# print 'disconnect from zookeeper'
self._connect_state = state
else:
# Handle being connected/reconnected to Zookeeper
# print 'connected'
self._connect_state = state
def __unquote(self, origin_nodes):
return (urllib.unquote(child_node).decode('utf8') for child_node in origin_nodes if child_node)
def _do_event(self, event):
# event.path 是类似/dubbo/com.ofpay.demo.api.UserProvider/providers 这样的
# 如果要删除，必须先把/dubbo/和最后的/providers去掉
# 将zookeeper中查询到的服务节点列表加入到一个dict中
# zookeeper中保持的节点url类似如下
logger.info("receive event is {0}, event state is {1}".format(event, event.state))
provide_name = event.path[7:event.path.rfind('/')]
if event.state in ['CONNECTED', 'DELETED']:
children = self.__zk.get_children(event.path, watch=self.event_listener)
self._compare_swap_nodes(provide_name, self.__unquote(children))
configurators_nodes = self._get_provider_configuration(provide_name)
self._set_provider_configuration(provide_name, configurators_nodes)
print self._service_providers
def _do_config_event(self, event):
"""
zk的目录路径为 /dubbo/com.qianmi.pc.api.es.item.EsGoodsQueryProvider/configurators
:param event:
:return:
"""
logger.info("receive config event is {0}, event state is {1}".format(event, event.state))
provide_name = event.path[7:event.path.rfind('/')]
configurators_nodes = self._get_provider_configuration(provide_name)
self._set_provider_configuration(provide_name, configurators_nodes)
print self._service_providers
def register(self, interface, **kwargs):
ip = self.__zk._connection._socket.getsockname()[0]
params = {
'interface': interface,
'application': self._app_config.name,
'application.version': self._app_config.version,
'category': 'consumer',
'dubbo': 'dubbo-client-py-1.0.0',
'environment': self._app_config.environment,
'method': '',
'owner': self._app_config.owner,
'side': 'consumer',
'pid': os.getpid(),
'version': '1.0'
}
url = 'consumer://{0}/{1}?{2}'.format(ip, interface, urllib.urlencode(params))
# print urllib.quote(url, safe='')
consumer_path = '{0}/{1}/{2}'.format('dubbo', interface, 'consumers')
self.__zk.ensure_path(consumer_path)
self.__zk.create(consumer_path + '/' + urllib.quote(url, safe=''), ephemeral=True)
def subscribe(self, interface, **kwargs):
"""
监听注册中心的服务上下线
:param interface: 类似com.ofpay.demo.api.UserProvider这样的服务名
:return: 无返回
"""
version = kwargs.get('version', '')
group = kwargs.get('group', '')
providers_children = self.__zk.get_children('{0}/{1}/{2}'.format('dubbo', interface, 'providers'),
watch=self.event_listener)
logger.debug("watch node is {0}".format(providers_children))
self.__zk.get_children('{0}/{1}/{2}'.format('dubbo', interface, 'configurators'),
watch=self.configuration_listener)
# 全部重新添加
self._compare_swap_nodes(interface, self.__unquote(providers_children))
configurators_nodes = self._get_provider_configuration(interface)
self._set_provider_configuration(interface, configurators_nodes)
def _get_provider_configuration(self, interface):
"""
获取dubbo自定义配置数据，从"/dubbo/{interface}/configurators" 路径下获取配置
:param interface:
:return:
"""
try:
configurators_nodes = self.__zk.get_children('{0}/{1}/{2}'.format('dubbo', interface, 'configurators'),
watch=self.configuration_listener)
logger.debug("configurators node is {0}".format(configurators_nodes))
return self.__unquote(configurators_nodes)
except Exception as e:
logger.warn("get provider %s configuration error %s", interface, str(e))
class SimpleSwitch13(app_manager.RyuApp):
OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]
def __init__(self, *args, **kwargs):
super(SimpleSwitch13, self).__init__(*args, **kwargs)
self.mac_to_port = {}
self.zkConf = {'root':'/multicontroller', 'topo':'/topology',
'swstat':'/swstat', 'counter': '/counter'}
self.zk = KazooClient('127.0.0.1:2181')
self.zk.start()
self.ip = '202.201.3.51'
self.sws = {}
self.gid = random.randint(0, 10000)
self.dps = {}
self.links = []
self.interval = 5
self.role = OFPCR_ROLE_EQUAL
self.topoThread = hub.spawn(self._topoThread)
self.linkThread = hub.spawn(self._linkDiscover)
self.clearLinkThread = hub.spawn(self._cleanLinks)
self.clearLinkThread = hub.spawn(self._cleanSwitches)
def _cleanSwitches(self):
while True:
self.sws = {k:self.sws[k] for k in self.sws if self.sws[k]}
hub.sleep(self.interval)
def _topoThread(self):
linkNode = self.zkConf['root'] + self.zkConf['topo'] + self.ip
if self.zk.exists(linkNode):
self.zk.set(linkNode, json.dumps(self.links))
else:
self.zk.create(linkNode, json.dumps(self.links))
hub.sleep(self.interval)
def _linkDiscover(self):
while True:
for dpid in self.dps:
self.sendSlldp(dpid)
hub.sleep(self.interval)
def sendSlldp(self, dpid):
dp = self.dps.get(dpid)
if dp is None:
return
actions = [dp.ofproto_parser.OFPActionOutput(dp.ofproto.OFPP_FLOOD)]
pkt = packet.Packet()
pkt.add_protocol(ethernet.ethernet(ethertype=ETH_TYPE_SLLDP,
dst=SLLDP_MAC_DST, src=SLLDP_MAC_SRC))
pkt.add_protocol(slldp(dp.id))
pkt.serialize()
slldpPacket = pkt.data
out = dp.ofproto_parser.OFPPacketOut(
datapath=dp, in_port=dp.ofproto.OFPP_CONTROLLER,
buffer_id=dp.ofproto.OFP_NO_BUFFER, actions=actions,
data=slldpPacket)
dp.send_msg(out)
def getLinks(self):
topoNode = self.zkConf['root'] + self.zkConf['topo']
ips = self.zk.get_children(topoNode)
res = []
for ip in ips:
links = self.zk.get(topoNode + ip)[0]
for link in links:
res.append(link)
return res
@set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
def _packet_in_handler(self, ev):
msg = ev.msg
pkt = packet.Packet(msg.data)
eth = pkt.get_protocols(ethernet.ethernet)[0]
dst = eth.dst
# SLLDP packet
if dst == SLLDP_MAC_DST:
self.handleSlldp(ev)
return
# process packet_in message in subclass
self.packet_in_process(ev)
def handleSlldp(self, ev):
msg = ev.msg
datapath = msg.datapath
dpid = datapath.id
inPort = msg.match['in_port']
pkt = packet.Packet(msg.data)
slldpBuff = pkt.get_protocols(ethernet.ethernet)[2]
dpidSrc, _ = slldp.parser(slldpBuff)
self.links.append({'srcdpid': dpidSrc,
'dst': {'dpid': dpid, 'port': inPort},
'time': time.time()})
def _cleanLinks(self):
while True:
now = time.time()
self.links = [l for l in self.links if now - l['time'] < self.interval]
hub.sleep(self.interval)
@abc.abstractmethod
def packet_in_process(self, ev):
pass
@set_ev_cls(event.EventSwitchEnter)
def switch_enter(self, ev):
dpid = ev.datapath.id
self.sws[dpid] = True
self.dps[dpid] = ev.datapath
dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid)
self.zk.ensure_path(dpNode)
if self.election(dpid):
self.role = OFPCR_ROLE_MASTER
else:
self.role = OFPCR_ROLE_SLAVE
self.countUp(dpid)
self.roleRequest(dpid, self.role)
mflag = dpNode + '/' + 'master'
DataWatch(self.zk, mflag, self.masterWatcher)
def masterWatcher(self, data, stat, ev):
if ev and ev.type == 'DELETED':
_, _, dpid, _ = ev.path.split('/')
dpid = str_to_dpid(dpid)
if self.sws.get(dpid):
if self.election(dpid):
self.role = OFPCR_ROLE_MASTER
self.roleRequest(dpid, self.role)
return self.sws.get(dpid)
def election(self, dpid):
dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid)
mflag = dpNode + '/' + 'master'
while not self.zk.exists(mflag):
mlock = self.zk.Lock(dpNode + '/' + 'mlock', self.ip)
with mlock:
if not self.zk.exists(mflag):
self.zk.create(mflag, self.ip, ephemeral=True)
if self.zk.exists(mflag):
if self.zk.get(mflag) == self.ip:
return True
else:
return False
else:
time.sleep(random.randint(0, 100)/500.0)
return False
def roleRequest(self, dp, role):
msg = dp.ofproto_parser.OFPRoleRequest(dp, role, self.gid)
dp.send_msg(msg)
def getCount(self, dpid):
dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid)
countNode = dpNode + self.zkConf['counter']
counters = self.zk.get_children(countNode)
return len(counters)
def countUp(self, dpid):
countNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid) + self.zkConf['counter']
self.zk.ensure_path(countNode)
self.zk.create(countNode+uuid4().hex, 'alive', ephemeral=True)
@set_ev_cls(event.EventSwitchLeave)
def switch_levave(self, ev):
dpid = ev.datapath
count = self.getCount(dpid)
self.sws[dpid] = False
if count == 0:
dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid)
self.zk.delete(dpNode, recursive=True)
class RPCServer(asyncore.dispatcher):
zk_root = "/demo"
zk_rpc = zk_root + "/rpc"
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.host = host
self.port = port
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(1)
self.child_pids = []
if self.prefork(10): # 产生子进程
self.register_zk() # 注册服务
self.register_parent_signal() # 父进程善后处理
else:
self.register_child_signal() # 子进程善后处理
def prefork(self, n):
for i in range(n):
pid = os.fork()
if pid < 0: # fork error
raise
if pid > 0: # parent process
self.child_pids.append(pid)
continue
if pid == 0:
return False # child process
return True
def register_zk(self):
self.zk = KazooClient(hosts='127.0.0.1:2181')
self.zk.start()
self.zk.ensure_path(self.zk_root) # 创建根节点
value = json.dumps({"host": self.host, "port": self.port})
# 创建服务子节点
self.zk.create(self.zk_rpc, value, ephemeral=True, sequence=True)
def exit_parent(self, sig, frame):
self.zk.stop() # 关闭zk客户端
self.close() # 关闭serversocket
asyncore.close_all() # 关闭所有clientsocket
pids = []
for pid in self.child_pids:
print 'before kill'
try:
os.kill(pid, signal.SIGINT) # 关闭子进程
pids.append(pid)
except OSError, ex:
if ex.args[0] == errno.ECHILD: # 目标子进程已经提前挂了
continue
raise ex
print 'after kill', pid
for pid in pids:
while True:
try:
os.waitpid(pid, 0) # 收割目标子进程
break
except OSError, ex:
if ex.args[0] == errno.ECHILD: # 子进程已经割过了
break
if ex.args[0] != errno.EINTR:
raise ex # 被其它信号打断了，要重试
print 'wait over', pid
class Store:
"""
Uses zoo keeper as a key vaue store. This store is as thread safe as the underlying client.
"""
def __init__(self, host_list):
hosts = ",".join(host_list)
print("zk, connecting to: '%s'" % hosts)
self.zk = KazooClient(hosts)
def __enter__(self):
self.zk.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.zk.stop()
def get(self, uid, ns, key):
"""
Get the value associated with the key.
:param uid: The user id to use.
:param ns: The namespace to use.
:param key: The key to look up.
:return: A tuple of (version, value) stored.
"""
path = "/jwl/sync/%s/%s/%s" % (uid, ns, key)
print("zk, fetching: '%s'" % path)
try:
r = self.zk.get(path)
print("zk, value at: '%s'='%s'" % (path, r))
return r[1].version, r[0]
except NoNodeError:
print("zk, no value at: '%s'" % path)
return None, None
def put(self, uid, ns, expected_version, key, value):
"""
If the version of the value in the system is what we expect, then update the key with
the value given. Otherwise return a tuple of (False, cur_serial, cur_value).
:param uid: The user id to use.
:param ns: The namespace to use.
:param expected_version: The version of the record we expect.
:param key: The key to update.
:param value: The value to associate with the key.
:return: A tuple of (True, new_version, new_value) if it works, (False, cur_version, cur_value) on failure.
"""
path = "/jwl/sync/%s/%s/%s" % (uid, ns, key)
self.zk.ensure_path(path)
if expected_version is None:
self.zk.set(path, value)
return True, 1, value
try:
print("zk, set '%s'='%s'" % (path, value))
self.zk.set(path, value, version=expected_version)
return True, expected_version + 1, value
except BadVersionError:
print("zk, set failed '%s'='%s'" % (path, value))
r = self.zk.get(path)
return False, r[1].version, r[0]