class ZookeeperClient:
def __init__(self, zk_host):
self.zk = KazooClient(hosts=zk_host)
def start(self):
logging.info("Starting zookeeper client")
self.zk.start()
def exists(self, node_path):
logging.info("Verifying zk node: %s", node_path)
return self.zk.exists(node_path)
def get_children(self, node_path):
logging.info("Getting children for node: {}".format(node_path))
return self.zk.get_children(node_path)
def get_as_json(self, node_path):
logging.info("Getting zk node: %s", node_path)
(data, stats) = self.zk.get(node_path)
return json.loads(data)
def set_as_json(self, node_path, object):
logging.info("Setting value to zk node: %s", node_path)
self.zk.set(node_path, json.dumps(object, indent=2))
def main():
if len(sys.argv) < 5:
print(help_msg, "\n")
print(sys.argv[0], 'zookeeper_server broker_uri_1 broker_uri_2 broker_uri_3')
print('Example:', sys.argv[0], 'localhost:2181 socket://localhost:10001/broker1 '
'socket://localhost:10002/broker2 socket://localhost:10003/broker3')
exit()
zk_server = sys.argv[1]
broker_uris = sys.argv[2:5]
shift_cmd = shift_cmd_template.format(*broker_uris)
print('Deploying', shift_cmd)
zk_client = KazooClient(zk_server, timeout=10 * 60)
print('Connecting to Zookeeper at', zk_server)
zk_client.start()
for uri in broker_uris:
broker_status[uri] = False
bid = get_id(uri)
# make sure broker is free
data, stats = zk_client.get(ZK_BROKER_OPS_STATUS_STR.format(bid))
op_status = OpStatus(data.decode('utf-8').upper())
if op_status not in [OpStatus.Null, OpStatus.Finished]:
raise RuntimeError('Cannot start {}, {} is in {} state'.format(shift_cmd, bid, op_status.name))
# update broker's ops status
zk_client.set(ZK_BROKER_OPS_STATUS_STR.format(bid), OpStatus.Null.value.encode('utf-8'))
# write the cmd to the broker's ops
zk_client.set(ZK_BROKER_OPS_STR.format(bid), shift_cmd.encode('utf-8'))
# set watches for this broker's op status
DataWatch(zk_client, ZK_BROKER_OPS_STATUS_STR.format(bid), func=get_broker_op_data_watcher(uri))
print('Waiting for brokers ...')
while not all_done():
time.sleep(1)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-p",
"--project",
help="project name kafka or schema-registry",
default="kafka")
parser.add_argument(
"-z",
"--zookeeper",
help="zookeeper url with port",
default='zookeeper.wgdp.io:2181',
)
args = parser.parse_args()
conf_dict = collect_info(args.project)
print(conf_dict)
region = define_region_name(conf_dict["hostname"])
if region == 'edcenter':
if args.project == 'kafka':
conf_dict['url'] = 'kafka.wgdp.io'
else:
conf_dict['url'] = 'schemas-registry.wgdp.io'
else:
conf_dict['url'] = '{}-kafka.wgdp.io'.format(region)
zk = KazooClient(hosts=args.zookeeper)
zk.start()
zk.ensure_path("/app/{}/{}/{}".format(args.project, region,
conf_dict["hostname"]))
zk.set(
"/app/{}/{}/{}".format(args.project, region, conf_dict["hostname"]),
json.dumps(conf_dict),
)
class ES_zk_interface:
def __init__(self, my_ip, es_interface, es_callback):
self.ip = my_ip
self.es_callback = es_callback
self.es_interface = es_interface
self.zk = KazooClient(hosts=HOST_IP)
self.zk.start()
self.zk.ensure_path("/ass3")
def start_election_process(self):
print "Starting Election process"
#print("Previous Leader: %s" % (self.es_callback.get_leader()))
election = self.zk.Election("/electionpath", self.ip)
election.run(self.leader_function)
def leader_function(self):
print "Won the election: " + str(self.ip)
if self.zk.exists(LEADER_PATH):
self.zk.set(LEADER_PATH, self.ip)
else:
self.zk.create(LEADER_PATH, self.ip)
#print "\nStarting Leader thread.... \n"
self.es_interface.enter_message_loop(self.es_callback, True)
def get_zookeeper_client(self):
return self.zk
def get_leader_path(self):
return LEADER_PATH
def zookeeper_crud(self, **kwargs):
operation = kwargs['operation']
room = kwargs['room']
room_cn_name = room.comment[:4]
try:
zk = KazooClient(hosts=room.zk_cluster)
zk.start()
if operation == 'retrieve':
path = kwargs['path']
zk.ensure_path(path)
data, stat = zk.get(path)
self.i('%s，获取节点%s数据成功，值为%s' % (room_cn_name, path, data))
return data
elif operation == 'update':
path = kwargs['path']
data = kwargs['data']
zk.ensure_path(path)
zk.set(path, bytes(data))
print ':'.join([str(zk.hosts), 'set', path, data])
self.i('%s，更新节点%s成功，值为%s' % (room_cn_name, path, data))
print ':'.join([str(zk.hosts), 'get', path, zk.get(path)[0]])
return True
except KazooException, e:
self.i('%s，%s操作失败，原因为%s' % (room_cn_name, operation, e.message))
return False
def main(_):
if MODE.STATUS == FLAGS.mode:
request = get_model_status_pb2.GetModelStatusRequest()
request.model_spec.name = 'detection'
request.model_spec.signature_name = 'serving_default'
elif MODE.CONFIG == FLAGS.mode:
request = model_management_pb2.ReloadConfigRequest()
config = request.config.model_config_list.config.add()
config.name = 'detection'
config.base_path = '/models/detection/detection'
config.model_platform = 'tensorflow'
config.model_version_policy.specific.versions.append(5)
config.model_version_policy.specific.versions.append(7)
config2 = request.config.model_config_list.config.add()
config2.name = 'pascal'
config2.base_path = '/models/detection/pascal'
config2.model_platform = 'tensorflow'
elif MODE.ZOOKEEPER == FLAGS.mode:
zk = KazooClient(hosts="10.10.67.225:2181")
zk.start()
zk.ensure_path('/serving/cunan')
zk.set(
'/serving/cunan',
get_config('detection', 5, 224, 'serving_default',
','.join(get_classes('model_data/cci.names')),
"10.12.102.32:8000"))
return
for address in FLAGS.addresses:
channel = grpc.insecure_channel(address)
stub = model_service_pb2_grpc.ModelServiceStub(channel)
if MODE.STATUS == FLAGS.mode:
result = stub.GetModelStatus(request)
elif MODE.CONFIG == FLAGS.mode:
result = stub.HandleReloadConfigRequest(request)
print(result)
class ZookeeperSession(object):
def __init__(self, locations, name_prefix, root_prefix='/frontera'):
self._zk = KazooClient(hosts=locations)
self._zk.add_listener(self.zookeeper_listener)
self._zk.start()
self.root_prefix = root_prefix
self.znode_path = self._zk.create("%s/%s" %
(self.root_prefix, name_prefix),
ephemeral=True,
sequence=True,
makepath=True)
def zookeeper_listener(self, state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
pass
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
pass
else:
# Handle being connected/reconnected to Zookeeper
pass
def set(self, value):
self._zk.set(self.znode_path, value)
def get_workers(self, prefix='', exclude_prefix=''):
for znode_name in self._zk.get_children(self.root_prefix):
if prefix and not znode_name.startswith(prefix):
continue
if exclude_prefix and znode_name.startswith(exclude_prefix):
continue
location, _ = self._zk.get(self.root_prefix + "/" + znode_name)
yield location
def execute(self, request):
zk = KazooClient(hosts=request['zk_url'])
try:
zk.start()
path_ = request['node_path']
value_ = request['node_value'].encode('utf-8')
is_update_timestamp_ = request.get('is_update_timestamp', False)
is_delete_node_ = request.get('is_delete_node', None)
if zk.exists(path_):
if is_delete_node_:
zk.delete(path_, recursive=True)
return self.resp_dict, ''
if is_update_timestamp_:
zk.set(path_ + '/TIMESTAMP', ZkTestPoint.get_timestamp_bytes())
if value_:
zk.set(path_, value_)
else:
data_, stat_ = zk.get(path_)
if data_:
self.resp_dict['result_value'] = data_.decode()
elif value_:
# 新加节点时也需要添加 '/TIMESTAMP' 节点
zk.create(path_, value_, makepath=True)
zk.create(path_ + '/TIMESTAMP', ZkTestPoint.get_timestamp_bytes(), makepath=True)
return self.resp_dict, ''
except Exception as e:
logger.exception('zkTestPoint execute error, cause: %s', str(e))
raise Exception('zkTestPoint execute error, cause: ' + str(e))
finally:
zk.stop()
class ZKMData(object):
def __init__(self):
super(ZKMData, self).__init__()
self._zk = KazooClient(hosts=ZK_HOSTS)
self._zk.start()
def set(self, path, value):
value = json.dumps(value).encode('utf-8')
if self._zk.exists(path):
try:
self._zk.set(path, value)
except NoNodeError:
pass
try:
self._zk.create(path, value, makepath=True)
except NodeExistsError:
pass
def get(self, path):
try:
value, stat = self._zk.get(path)
if not value: return {}
return json.loads(value.decode('utf-8'))
except Exception as e:
return {}
def close(self):
self._zk.stop()
self._zk.close()
class ZKStore:
def __init__(self, hosts):
self.zk = KazooClient(hosts=hosts)
self.zk.add_listener(listener)
self.zk.start()
def isConnected(self):
if __state__ == 1:
return True
return False
def write(self, path, node, value):
self.zk.ensure_path(path)
if self.zk.exists(path+"/"+node):
self.zk.set(path+"/"+node, value)
else:
self.zk.create(path + "/" + node, value)
def read(self, path):
if self.zk.exists(path):
data, stat = self.zk.get(path)
return data
return None
def _register_cluster_membership(zk: KazooClient, zk_path: str,
ip: str) -> List[str]:
"""
Add `ip` to the list of cluster members registered in ZooKeeper.
The ZK lock must be held around the call to this function.
Args:
zk:
The client to use to communicate with ZooKeeper.
zk_path:
The path of the ZNode to use for node registration.
ip:
The ip to add to the list of cluster member IPs in ZooKeeper.
"""
log.info("Registering cluster membership for `{}`".format(ip))
# Get the latest list of cluster members.
nodes = _get_registered_nodes(zk=zk, zk_path=zk_path)
if ip in nodes:
# We're already registered with ZK.
log.info(
"Cluster member `{}` already registered in ZooKeeper. Skipping.".
format(ip))
return nodes
log.info("Adding `{}` to list of nodes `{}`".format(ip, nodes))
nodes.append(ip)
zk.set(zk_path, json.dumps({"nodes": nodes}).encode("ascii"))
zk.sync(zk_path)
log.info("Successfully registered cluster membership for `{}`".format(ip))
return nodes
def do_zookeeper_update(addr, path, value): print(path) zk = KazooClient(addr) zk.start() zk.set(path, bytes(value, 'utf-8')) do_zookeeper_read(addr, path)
def add_and_update(key, value):
print(key, value, '#' * 20)
zk = KazooClient(hosts='127.0.0.1:2181')
zk.start()
# children = zk.get_children('/')
# 创建节点：makepath 设置为 True ，父节点不存在则创建，其他参数不填均为默认
# zk.create('/zookeeper/goodboy')
# 操作完后，别忘了关闭zk连接
# zk.stop()
print('#' * 20)
data, stat = zk.get(key)
print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
if zk.exists(key):
zk.set(key, value)
print('@' * 10, 'setting')
else:
zk.create(key, value, makepath=True)
# zk.create(key)
# print(value, type(value),'#'*10)
# if value != b"":
# zk.set(key, value)
zk.stop()
class ZKData(object):
def __init__(self):
super(ZKData,self).__init__()
options={"max_tries":-1,"max_delay":5,"ignore_expire":True}
self._zk=KazooClient(hosts=ZK_HOSTS,connection_retry=options)
try:
self._zk.start(timeout=3600)
except:
print(traceback.format_exc(), flush=True)
def set(self, path, value):
value=json.dumps(value).encode('utf-8')
try:
self._zk.create(path, value, makepath=True)
except NodeExistsError:
self._zk.set(path,value)
def get(self, path):
try:
value, stat= self._zk.get(path)
if not value: return {}
return json.loads(value.decode('utf-8'))
except Exception as e:
return {}
def close(self):
self._zk.stop()
self._zk.close()
class Zookeeper(KeyManager):
def __init__(self, hosts):
self._hosts = hosts
self.zk = KazooClient(hosts=hosts)
self.zk.start()
def get(self, key):
result = self.zk.get(key)[0]
if result == "":
result = []
children = self.zk.get_children(key)
for i in children:
result.append({'name': i, 'value': self.zk.get(os.path.join(key, i))[0]})
return result
else:
return self.zk.get(key)[0]
def set(self, key, data):
try:
self.zk.set(key, data.encode('utf-8'))
except:
self.zk.create(key, data.encode('utf-8'))
def mkdir(self, key):
self.set(key, "")
def close(self):
self.zk.stop()
self.zk.close()
@property
def hosts(self):
return self._hosts
def zk_flag_set(zk: KazooClient, name: str, value: str):
path = "{}/{}".format(ZK_PREFIX, name)
data = value.encode('utf-8')
try:
zk.set(path, data)
except NoNodeError:
zk.create(path, data, makepath=True)
def _register_cluster_membership(zk: KazooClient, zk_path: str, ip: str) -> List[str]:
"""
Add `ip` to the list of cluster members registered in ZooKeeper.
The ZK lock must be held around the call to this function.
Args:
zk:
The client to use to communicate with ZooKeeper.
zk_path:
The path of the ZNode to use for node registration.
ip:
The ip to add to the list of cluster member IPs in ZooKeeper.
"""
log.info("Registering cluster membership for `{}`".format(ip))
# Get the latest list of cluster members.
nodes = _get_registered_nodes(zk=zk, zk_path=zk_path)
if ip in nodes:
# We're already registered with ZK.
log.info("Cluster member `{}` already registered in ZooKeeper. Skipping.".format(ip))
return nodes
log.info("Adding `{}` to list of nodes `{}`".format(ip, nodes))
nodes.append(ip)
zk.set(zk_path, json.dumps({"nodes": nodes}).encode("ascii"))
zk.sync(zk_path)
log.info("Successfully registered cluster membership for `{}`".format(ip))
return nodes
class Store(object):
def __init__(self,**kwargs):
self.config = kwargs
self.client = None
def get_client(self):
return self.client
def open(self):
self.client = KazooClient(**self.config)
self.client.add_listener
self.client.start()
def close(self):
self.client.stop()
def read(self,path):
return self.client.get(path)
def write(self,path,value):
base_path = os.path.dirname(path)
self.client.ensure_path(base_path)
self.client.create(path,value)
def overwrite(self,path,value):
self.client.set(path,value)
def exists(self,path):
return self.client.exists(path)
class HHFrontierWorker(FrontierWorker):
def __init__(self, settings, no_batches, no_scoring, no_incoming):
super(HHFrontierWorker, self).__init__(settings, no_batches, no_scoring, no_incoming)
self.init_zookeeper()
def init_zookeeper(self):
self._zk = KazooClient(hosts=settings.get('ZOOKEEPER_LOCATION'))
self._zk.add_listener(self.zookeeper_listener)
self._zk.start()
self.znode_path = self._zk.create("/frontera/hh-f-worker", ephemeral=True, sequence=True, makepath=True)
def zookeeper_listener(self, state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
pass
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
pass
else:
# Handle being connected/reconnected to Zookeeper
pass
def set_process_info(self, process_info):
self.process_info = process_info
self._zk.set(self.znode_path, self.process_info)
def set_job_id(self, job_id):
self._backend.set_job_id(job_id)
self.job_id = job_id
class zkClient():
def __init__(self):
self.zk = KazooClient(hosts='10.0.0.2:2181')
self.zk.start()
def get_hostname_ip(self):
host_name = socket.getfqdn(socket.gethostname())
host_ip = socket.gethostbyname(host_name)
return "/" + host_name + "_gateway", host_ip
def register(self):
name, ip = self.get_hostname_ip()
if not self.zk.exists(name):
self.zk.create(name, bytes(ip, 'utf-8'))
else:
self.zk.set(name, bytes(ip, 'utf-8'))
def get_kvs_ips(self):
device_ips = []
for device_name in self.zk.get_children('/'):
if device_name.endswith('_kvs'):
data, stat = self.zk.get(device_name)
device_ips.append(data.decode("utf-8"))
return device_ips
def get_nodes_ips(self, device_list):
device_ips = []
for device_name in self.zk.get_children('/'):
if device_name in device_list:
data, stat = self.zk.get(device_name)
device_ips.append(data.decode("utf-8"))
return device_ips
def test_zookeeper(zookeeper_server):
test_value = str(uuid.uuid4()).encode("utf-8")
zk_client1 = KazooClient(
hosts="127.0.0.1:{}".format(zookeeper_server.port))
zk_client2 = KazooClient(
hosts="127.0.0.1:{}".format(zookeeper_server.port))
zk_client1.start()
zk_client2.start()
zk_client1.ensure_path("/my")
assert zk_client1.exists("/you") is None
assert zk_client2.exists("/you") is None
assert zk_client1.exists("/my") is not None
assert zk_client2.exists("/my") is not None
zk_client1.create("/my/testplan", test_value)
data, _ = zk_client1.get("/my/testplan")
assert data == test_value
data, _ = zk_client2.get("/my/testplan")
assert data == test_value
zk_client2.set("/my/testplan", str(uuid.uuid4()).encode("utf-8"))
data, _ = zk_client1.get("/my/testplan")
assert data != test_value
data, _ = zk_client2.get("/my/testplan")
assert data != test_value
def SvcDoRun(self):
try:
master_hosts = os.getenv("zookeeper")
zk = KazooClient(hosts=master_hosts)
zk.start()
node_data = zk.get(path.server('node'))
# For desktop, we add a 'windows' label, in order to schedule better later.
desktop_data = node_data[0].decode().replace('~', 'windows', 1)
while True:
# update info
remain_cpu, remain_mem, remain_disk = monitorResources()
update_info = 'cpu: {cpuinfo}%\ndisk: {diskinfo}M\nlabel: windows\nmemory: {meminfo}M\n'.format(
cpuinfo=remain_cpu,
diskinfo=remain_disk,
meminfo=remain_mem)
desktop_data = update_info + desktop_data[desktop_data.
find('parent'):]
if zk.exists(path.server(_HOSTNAME)):
zk.set(path.server(_HOSTNAME),
desktop_data.encode('utf-8'))
logging.info("Update resources infomation %s", _HOSTNAME)
if zk.exists(path.server_presence(_HOSTNAME)):
zk.set(path.server_presence(_HOSTNAME),
desktop_data.encode('utf-8'))
logging.info("Update resources infomation %s", _HOSTNAME)
if win32event.WaitForSingleObject(
self.hWaitStop,
int(os.getenv("updateResourcesInterval"))
) == win32event.WAIT_OBJECT_0:
break
except:
pass
class KazooService():
zk = None
rootPath = None
def __init__(self, rootPath):
qs = Zookeeper.objects.filter().first()
if qs is None:
raise LookupError("Unable to find Zookeeper configuration!")
self.zk = KazooClient(hosts=qs.url)
self.rootPath = rootPath
self.zk.start(timeout=qs.timeout)
self.zk.ensure_path(rootPath)
def createOrUpdate(self, pathAffix, nodeValue):
if isinstance(nodeValue, str):
bValue = bytes(nodeValue, 'utf-8')
else:
bValue = bytes(str(nodeValue), 'utf-8')
path = self.rootPath + pathAffix
if self.zk.exists(path) is None:
self.zk.create(path, bValue)
else:
self.zk.set(path, bValue)
def __del__(self):
if self.zk is not None:
self.zk.stop()
class ZookeeperClient(object):
def __init__(self, server_list):
self._retry = KazooRetry(max_tries=None,
max_delay=300,
sleep_func=gevent.sleep)
self._zk_client = KazooClient(hosts=','.join(server_list),
timeout=400,
handler=SequentialGeventHandler(),
logger=logger,
connection_retry=self._retry,
command_retry=self._retry)
def connect(self):
self._zk_client.start()
def disconnect(self):
self._zk_client.stop()
self._zk_client.close()
def create_node(self, path, value=None):
if value is None:
value = uuid.uuid4()
try:
self._zk_client.create(path, str(value), makepath=True)
except NodeExistsError:
self._zk_client.set(path, str(value))
class ActorAddressBook(object):
def __init__(self, zk_hosts, timeout=60.0):
self.retry = KazooRetry(max_tries=10)
self.zk = KazooClient(hosts=zk_hosts, timeout=timeout)
self.zk.start()
def lookup(self, path):
return self.retry(self._lookup, path)
def _lookup(self, path):
actor_url, stat = self.zk.get(path)
return RemoteActor(actor_url.decode('utf-8'))
def register(self, path, actor_url):
return self.retry(self._register, path, actor_url)
def _register(self, path, actor_url):
self.zk.ensure_path(path)
self.zk.set(path, actor_url.encode('utf-8'))
def delete(self, path):
self.zk.delete(path, recursive=True)
def __del__(self):
self.zk.stop()
def write_file_to_zookeeper(zookeeper: KazooClient, source_file: BinaryIO,
dest_path: str) -> bool:
logger.info("Writing to %s in ZooKeeper...", dest_path)
try:
current_data, stat = zookeeper.get(dest_path)
current_version = stat.version
except NoNodeError:
raise NodeDoesNotExistError
new_data = source_file.read()
if current_data == new_data:
logger.info("No changes detected. Not writing.")
return False
try:
current_text = current_data.decode("utf8")
new_text = new_data.decode("utf8")
except UnicodeDecodeError:
logger.info("Skipping diff, data appears to be binary.")
else:
diff = difflib.unified_diff(current_text.splitlines(),
new_text.splitlines())
for line in diff:
logger.info(line)
try:
zookeeper.set(dest_path, new_data, version=current_version)
except BadVersionError:
raise UnexpectedChangeError
logger.info("Wrote data to Zookeeper.")
return True
class ZookeeperClient(object):
def __init__(self, server_list):
self._retry = KazooRetry(max_tries=None, max_delay=300,
sleep_func=gevent.sleep)
self._zk_client = KazooClient(
hosts=','.join(server_list),
timeout=400,
handler=SequentialGeventHandler(),
logger=logger,
connection_retry=self._retry,
command_retry=self._retry)
def connect(self):
self._zk_client.start()
def disconnect(self):
self._zk_client.stop()
self._zk_client.close()
def create_node(self, path, value=None):
if value is None:
value = uuid.uuid4()
try:
self._zk_client.create(path, str(value), makepath=True)
except NodeExistsError:
self._zk_client.set(path, str(value))
def remove_cluster_membership(zk: KazooClient, zk_path: str,
ip: str) -> List[str]:
"""
Remove `ip` from the list of cluster members registered in ZooKeeper.
The ZK lock must be held around the call to this function.
Args:
zk:
The client to use to communicate with ZooKeeper.
zk_path:
The path of the ZNode to use for node registration.
ip:
The ip to add to the list of cluster member IPs in ZooKeeper.
"""
log.info("Removing cluster membership for `%s`", ip)
# Get the latest list of cluster members.
nodes = get_registered_nodes(zk=zk, zk_path=zk_path)
if ip not in nodes:
# We're already registered with ZK.
log.info(
"Cluster member `%s` already removed from Zookeeper. Skipping.",
ip)
return nodes
log.info("Removing `%s` to list of nodes `%s`", ip, nodes)
nodes.remove(ip)
zk.set(zk_path, json.dumps({"nodes": nodes}).encode("ascii"))
zk.sync(zk_path)
log.info("Successfully removed %s from the cluster", ip)
return nodes
class ZKClient:
def __init__(self, zk_url, zk_timeout=5):
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
self.logger = logging.getLogger()
try:
self.client = KazooClient(hosts=zk_url)
self.client.start(timeout=zk_timeout)
except handlers.threading.KazooTimeoutError:
self.logger.error("5 seconds connection timeout")
raise
def store_value(self, value, path):
fmt_value = str(value).encode('utf-8')
self.client.ensure_path(path)
self.client.set(path, fmt_value)
self.client.stop()
def retrieve_value(self, path):
self.client.ensure_path(path)
data, stat = self.client.get(path)
self.client.stop()
if not data:
raise ValueError
else:
return int(data.decode('utf-8'))
def register_server(self, path):
self.client.create(path, ephemeral=True, makepath=True)
class ZookeeperSession(object):
def __init__(self, locations, name_prefix, root_prefix='/frontera'):
self._zk = KazooClient(hosts=locations)
self._zk.add_listener(self.zookeeper_listener)
self._zk.start()
self.root_prefix = root_prefix
self.znode_path = self._zk.create("%s/%s" % (self.root_prefix, name_prefix),
ephemeral=True,
sequence=True,
makepath=True)
def zookeeper_listener(self, state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
pass
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
pass
else:
# Handle being connected/reconnected to Zookeeper
pass
def set(self, value):
self._zk.set(self.znode_path, value)
def get_workers(self, prefix='', exclude_prefix=''):
for znode_name in self._zk.get_children(self.root_prefix):
if prefix and not znode_name.startswith(prefix):
continue
if exclude_prefix and znode_name.startswith(exclude_prefix):
continue
location, _ = self._zk.get(self.root_prefix+"/"+znode_name)
yield location
class HHFrontierWorker(FrontierWorker):
def __init__(self, settings, no_batches, no_scoring, no_incoming):
super(HHFrontierWorker, self).__init__(settings, no_batches,
no_scoring, no_incoming)
self.init_zookeeper()
def init_zookeeper(self):
self._zk = KazooClient(hosts=settings.get('ZOOKEEPER_LOCATION'))
self._zk.add_listener(self.zookeeper_listener)
self._zk.start()
self.znode_path = self._zk.create("/frontera/hh-f-worker",
ephemeral=True,
sequence=True,
makepath=True)
def zookeeper_listener(self, state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
pass
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
pass
else:
# Handle being connected/reconnected to Zookeeper
pass
def set_process_info(self, process_info):
self.process_info = process_info
self._zk.set(self.znode_path, self.process_info)
def set_job_id(self, job_id):
self._backend.set_job_id(job_id)
self.job_id = job_id
class ZooKeeper(object):
def __init__(self, hosts="127.0.0.1:2181", prefix="/"):
prefix = prefix.rstrip("/")
if prefix and not prefix.startswith("/"):
raise ValueError("prefix must start with /")
self._prefix = prefix
self._zk = KazooClient(hosts=hosts, handler=_get_zk_handler())
self._zk.start()
def __del__(self):
self.close()
def _path(self, path):
if self._prefix:
return "{0}/{1}".format(self._prefix, path)
return path
def close(self):
if not self._zk:
return
try:
self._zk.stop()
self._zk.close()
except Exception:
pass
finally:
self._zk = None
def create(self, path, value="", makepath=True):
self._zk.create(self._path(path), value, makepath=makepath)
def delete(self, path, recursive=True):
self._zk.delete(self._path(path), recursive=recursive)
def set(self, path, value, makepath=True):
path = self._path(path)
try:
self._zk.set(path, value)
except zkexc.NoNodeError:
if not makepath:
raise
self._zk.create(path, value, makepath=True)
def get(self, path, none=True):
try:
return self._zk.get(self._path(path))
except zkexc.NoNodeError:
if none:
return None
raise
def ls(self, path, none=True):
try:
return self._zk.get_children(self._path(path))
except zkexc.NoNodeError:
if none:
return None
raise
class Zookeeper(KeyManager): def __init__(self,hosts): self._hosts = hosts self.zk = KazooClient(hosts=hosts) def get(self,key): return self.zk.get(key)[0] def set(self,key,data): self.zk.set(key, data)
def createZkNode(zk_host, zk_port, host_group, host, serverset_member):
zk_connection_string = zk_host + ":" + zk_port
zk = KazooClient(hosts=zk_connection_string)
zk.start()
zk.ensure_path("inventory")
zk_node = "inventory/" + host_group + "/" + host
zk.ensure_path(zk_node)
zk.set(zk_node, str(serverset_member).encode())
class LiveDataWriterTests(unittest.TestCase):
def setUp(self):
self.zookeeper = KazooClient(hosts="%s:%d" %
zookeeper_endpoint.address)
self.zookeeper.start()
try:
self.zookeeper.delete(TEST_NODE_PATH)
except NoNodeError:
pass
self.zookeeper.create(TEST_NODE_PATH, b"")
def tearDown(self):
self.zookeeper.stop()
def test_exits_when_node_does_not_exist(self):
input = BytesIO()
with self.assertRaises(NodeDoesNotExistError):
write_file_to_zookeeper(self.zookeeper, input, "/does_not_exist")
def test_current_data_matches_new_data(self):
data = b"data"
self.zookeeper.set(TEST_NODE_PATH, data)
input = BytesIO(data)
did_write = write_file_to_zookeeper(self.zookeeper, input,
TEST_NODE_PATH)
self.assertFalse(did_write)
def test_successful_set(self):
self.assertEqual(self.zookeeper.get(TEST_NODE_PATH)[0], b"")
input = BytesIO(b"new_data")
did_write = write_file_to_zookeeper(self.zookeeper, input,
TEST_NODE_PATH)
self.assertTrue(did_write)
self.assertEqual(self.zookeeper.get(TEST_NODE_PATH)[0], b"new_data")
def test_version_changed(self):
# this is a horrible hack. we want to test what happens if the contents
# of the znode change between when write_file_to_zookeeper reads and
# when it writes (so that the diff is trustworthy). this mock side
# effect allows us to inject some behavior into the middle of the
# function.
mock_file = mock.Mock()
def mock_read():
self.zookeeper.set(TEST_NODE_PATH, b"I changed!")
return b"contents of file"
mock_file.read.side_effect = mock_read
with self.assertRaises(UnexpectedChangeError):
write_file_to_zookeeper(self.zookeeper, mock_file, TEST_NODE_PATH)
class TimeStampLeader(PublicationLeader):
def __init__(self, zk_hosts, bdii_path):
self.zk_hosts = zk_hosts
self.zk = KazooClient(self.zk_hosts)
self.bdii_path = bdii_path
self.is_leader = False
def pack_ts(self, input_dt):
return struct.pack('f', self.gen_ts(input_dt))
def gen_ts(self, input_dt):
return time.mktime(input_dt.timetuple())
def does_exist(self):
if self.zk.exists(self.bdii_path) is not None:
return True
else:
return False
def is_stale(self, current_time):
data, stat = self.zk.get(self.bdii_path)
if data == '':
return True
last_updated_timestamp = struct.unpack('f',data)[0]
if last_updated_timestamp <= (self.gen_ts(current_time) - 120):
return True
else:
return False
def should_publish(self):
self.zk.start()
current_time = datetime.datetime.utcnow()
if not self.does_exist():
self.zk.create(self.bdii_path, self.pack_ts(current_time))
self.is_leader = True
return self.is_leader
bdii_lock = self.zk.Lock(self.bdii_path, socket.getfqdn())
try:
lock_acquired = bdii_lock.acquire(5.0)
if lock_acquired:
self.is_leader = self.is_stale(current_time)
bdii_lock.release()
self.zk.stop()
return self.is_leader
except LockTimeout:
# Another Compute Element has the lock
pass
return False
def update_ts(self):
if self.is_leader:
self.zk.start()
current_ts = self.gen_ts(datetime.datetime.utcnow())
self.zk.set(self.bdii_path, struct.pack('f', current_ts))
self.zk.stop()
class ZKDatasourceImp:
def __init__(self):
self.zk = KazooClient(ZOOKEEPER)
self.zk.start()
def update_or_create(self, path, node, data):
if self.zk.exists("%s/%s" % (path, node)) is not None:
self.zk.set("%s/%s" % (path, node), data.encode("utf-8"))
else:
self.zk.create("%s/%s" % (path, node), data.encode("utf-8"))
class MidKazooUpdate(object):
def __init__(self):
self.zk_update = KazooClient(
hosts=config.config["kazoo"]["KAZOO_HOST"])
def kazoo_info_update(self, node, data):
self.zk_update.start()
da = json.dumps(data).encode("utf-8")
self.zk_update.set(node, da)
self.zk_update.stop()
def test_search_valid(self):
# curl http and get response
print("test zk valid")
zk = KazooClient(hosts='localhost:2181')
zk.start()
data, stat = zk.get("/autoconfig/config/x")
print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
zk.set("/autoconfig/config/x", b"some data")
print("test zk valid end")
zk.stop()
class ZookeeperContext:
def __init__(self, zookeeper_url, test_id, node_id):
self.assertNotNone(zookeeper_url)
self.assertNotNone(test_id)
self.assertNotNone(node_id)
self.zookeeper_url = zookeeper_url
self.test_id = test_id
self.node_id = node_id
self.zk_path = '/bolt/testing/%s/%s' % (self.test_id, self.node_id)
logger.info("Initialized decorator: %s " % self.__str__())
self.__connect_zookeeper()
def assertNotNone(self,thing):
if thing == None:
logger.fatal("Invalid key in initializer")
def __connect_zookeeper(self):
try:
logger.info('Attempting to connect to: %s' % self.zookeeper_url)
self.zk = KazooClient(hosts=self.zookeeper_url)
self.zk.start(timeout=10)
logger.info('initialized zookeeper')
except Exception as exception:
logger.error('Failed to connect to zk')
logger.fatal(exception)
def __str__(self):
return json.dumps(self, default=lambda o: o.__dict__, indent=4)
def __del__(self):
self.zk.stop()
def publish(self, key, data):
self.assertNotNone(key)
self.assertNotNone(data)
path = self.zk_path + "/" + key
logger.info("Creating data in: %s" % path)
try:
bytes = pickle.dumps(data);
if not self.zk.exists(path):
logger.info("Path does not exist in zk, creating... %s" % path)
self.zk.create(path, value=bytes, makepath=True)
else:
self.zk.set(path, value=bytes)
except Exception as exception:
logger.error('Error setting data in ' % path)
logger.fatal(exception)
def test_zip_response(pub_node, host = '127.0.0.1', port = 2181):
zookeeper = KazooClient('%s:%s' % (host, port,))
zookeeper.start()
node_value = {
'status' : 'ok',
'update_time' : time.time(),
}
if zookeeper.exists('/test/to_zip_result/%s' % pub_node) is not None:
zookeeper.set('/test/to_zip_result/%s' % pub_node, json.dumps(node_value))
else:
print 'Not found node `/test/to_zip_result/%s`' % pub_node
def main():
zk = KazooClient(hosts="127.0.0.1:2181", timeout=2.0)
zk.add_listener(my_listener)
zk.start()
if zk.exists("/ELECTION") == None:
zk.ensure_path("/ELECTION")
c = 1
node_pathes = []
while c < 10:
c += 1
node_path = zk.create("/ELECTION/guid-n_", b"a value", ephemeral=True, sequence=True)
node_pathes.append(node_path)
my_path = random.choice(node_pathes)
my_path = my_path.replace("/ELECTION/", "")
# print "print my_path", my_path
children = zk.get_children("/ELECTION/", watch=election_child_watcher)
get_next_path = False
prev_path = None
for child_path in sorted(children):
if child_path == my_path:
break
prev_path = child_path
# I'm the leader
if prev_path == None:
print "OK I'm leader don't have to watch"
return
# fires twice, once on creation ignore
@zk.DataWatch("/ELECTION/" + prev_path)
def watch_node(data, stat):
# only watch for first change
if stat.version == 1:
print ("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
print "setting watch on " + prev_path
print "my", my_path
zk.set("/ELECTION/" + prev_path, b"some data")
print "boom. watch triggered?"
# time.sleep(10)
print "bye"
zk.stop()
def reset(bpath, new_val):
zk = KazooClient(hosts=args.zkhost)
zk.start()
if zk.exists(bpath): # /offsets
for child in zk.get_children(bpath): # offsets/topic
c = '{0}/{1}'.format(bpath, child)
print('Topic: {0}'.format(c))
for c2 in zk.get_children(c): # offsets/topic/partition
c2 = '{0}/{1}'.format(c, c2)
print('Set {0} to {1}'.format(c2, new_val))
zk.set(c2, new_val)
else:
print('Path <{0}> not exists'.format(bpath))
zk.stop()
def main():
'''
A manual configuration file pusher for the crawlers. This will update
Zookeeper with the contents of the file specified in the args.
'''
import argparse
from kazoo.client import KazooClient
parser = argparse.ArgumentParser(
description="Crawler config file pusher to Zookeeper")
parser.add_argument('-f', '--file', action='store', required=True,
help="The yaml file to use")
parser.add_argument('-i', '--id', action='store', default="all",
help="The crawler id to use in zookeeper")
parser.add_argument('-p', '--path', action='store',
default="/scrapy-cluster/crawler/",
help="The zookeeper path to use")
parser.add_argument('-w', '--wipe', action='store_const', const=True,
help="Remove the current config")
parser.add_argument('-z', '--zoo-keeper', action='store', required=True,
help="The Zookeeper connection <host>:<port>")
args = vars(parser.parse_args())
filename = args['file']
id = args['id']
wipe = args['wipe']
zoo = args['zoo_keeper']
path = args['path']
zk = KazooClient(hosts=zoo)
zk.start()
# ensure path exists
zk.ensure_path(path)
bytes = open(filename, 'rb').read()
if zk.exists(path):
# push the conf file
if not zk.exists(path + id) and not wipe:
print "creaing conf node"
zk.create(path + id, bytes)
elif not wipe:
print "updating conf file"
zk.set(path + id, bytes)
if wipe:
zk.set(path + id, None)
zk.stop()
def put_topology(topology, config=config):
zk = KazooClient(**config)
zk.start()
for service, links in topology.iteritems():
if zk.exists("/sdn/services/{name}".format(name=service)):
zk.set("/sdn/services/{name}".format(name=service), json.dumps(links))
else:
zk.ensure_path("/sdn/services")
zk.create("/sdn/services/{name}".format(name=service), json.dumps(links))
ret_check = zk.get_children("/sdn/services")
zk.stop()
return ret_check
def zkset():
file = "/tmp/redis_config.properties"
zk = KazooClient(hosts='172.29.22.15:2181')
zk.start()
f=open(file,'rb')
m=f.read()
#Ensure a path , Create if necessary.
zk.ensure_path("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes")
#Create a node with data.
try:
zk.create("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)
except kazoo.exceptions.NodeExistsError:
zk.set("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)
zk.stop()
class ZookeeperClient(object):
def __init__(self, zk_hosts):
self.zk = KazooClient(hosts=zk_hosts)
self.zk.start()
def put(self, entity, id, data):
path = '/{}/{}'.format(entity, id)
binary_value = json.dumps(data).encode('utf-8')
try:
self.zk.create(path, binary_value, makepath=True)
return True
except NodeExistsError:
self.zk.set(path, binary_value)
return False
def get(self, entity, id):
path = '/{}/{}'.format(entity, id)
try:
binary_data, _ = self.zk.get(path)
except NoNodeError:
return None
return json.loads(binary_data.decode('utf-8'))
def delete(self, entity, id):
path = '/{}/{}'.format(entity, id)
try:
self.zk.delete(path)
return True
except NoNodeError:
return False
def list(self, entity):
path = '/{}'.format(entity)
try:
children = self.zk.get_children(path)
for child in children:
value = self.get(entity, child)
yield (child, value)
except NoNodeError:
yield []
class ZookClient(object):
def __init__(self, api_client=None):
logging.basicConfig()
# Create a client and start it
self.zk = KazooClient()
self.zk.start()
def create_accounts_path(self, name, **kwargs):
path = "/dso/" + name
self.zk.ensure_path(path)
self.zk.set(path, b"id: 7b4235ca-00fb-4dca-ad3e-8b6e3662631a\ngroupname: hr\ndescription: 人力资源")
def create_accountinfo_path(self, account_id, **kwargs):
self.zk.ensure_path("/app/someservice")
def create_path(self, path, **kwargs):
self.zk.ensure_path(path)
def get_data(self, path):
return self.zk.get(path)
def test_tran(self):
self.zk.delete("/app22")
self.zk.create("/app22", b"" + '{"12": "12"}')
tran = self.zk.transaction()
tran.delete("/app22")
tran.create("/app22", b"" + '{"22": "22"}')
tran.commit()
print "commit"
def stop(self):
# In the end, stop it
self.zk.stop()
def main(args):
zk = KazooClient(hosts=zk_host)
zk.start()
register_emergency_signal_handler(zk)
try:
zk.retry(zk.create(path=test_znode, makepath=True))
except NodeExistsError:
logging.info('{} already exists, no need to create'.format(test_znode))
start_command = 'start({},{})'.format(args.message_size, args.topics)
zk.retry(lambda: zk.set(test_znode, start_command))
t_start = time.time() # in seconds
t_end = t_start + args.duration
while(time.time() < t_end):
time.sleep(t_end - time.time()) # shouldn't introduce error larger than 10-15 ms
zk.retry(lambda: zk.set(test_znode, 'stop'))
zk.stop()
class MicroService(object):
"""docstring for MicroService"""
def __init__(self, hosts, app):
from kazoo.client import KazooClient
self.zk = KazooClient(hosts=hosts)
self.app = "/" + app
def zkStart(self):
self.zk.start()
def zkStop(self):
self.zk.stop()
def registerService(self, service, url):
if self.zk.exists(self.app + "/" + service):
self.zk.delete(self.app + "/" + service, recursive=True)
self.zk.delete(self.app, recursive=True)
self.zk.ensure_path(self.app + "/" + service)
self.zk.set(self.app + "/" + service, bytes(url, encoding="utf-8"))
self.__watchServiceChanged(service)
return 'Service registed'
def unregisterService(self, service):
self.zk.delete(self.app + "/" + service, recursive=True)
return "Service deleted"
def findService(self, service):
data, stat = self.zk.get(self.app + "/" + service)
return data.decode("utf-8")
def __watchServiceChanged(self, service):
@self.zk.DataWatch(self.app + "/" + service)
def changed(data, stat):
print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
class ZooDict(object):
def __init__(self, path_root=None):
self.path_root = path_root or default_path_root
def __enter__(self):
self.zk = KazooClient(hosts=zk_hosts)
self.zk.start(timeout=5)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.zk.stop()
def start(self):
return self.__enter__()
def stop(self):
return self.__exit__()
def get(self, key):
try:
r = self.zk.get(self.path_root + '/' + key)
if r and r[0]:
return json.loads(r[0])
except NoNodeError as no_node:
return None
def set(self, key, data_dict):
self.zk.ensure_path(self.path_root + '/' + key)
self.zk.set(self.path_root + '/' + key, json.dumps(data_dict))
def delete(self, key):
self.zk.delete(self.path_root + '/' + key)
def items(self):
return [ (child, self.get(child)) for child in self.zk.get_children(self.path_root) ]
class KazooCommandProxy():
def __init__(self, module):
self.module = module
self.zk = KazooClient(module.params['hosts'])
def absent(self):
return self._absent(self.module.params['name'])
def exists(self, znode):
return self.zk.exists(znode)
def list(self):
children = self.zk.get_children(self.module.params['name'])
return True, {'count': len(children), 'items': children, 'msg': 'Retrieved znodes in path.',
'znode': self.module.params['name']}
def present(self):
return self._present(self.module.params['name'], self.module.params['value'])
def get(self):
return self._get(self.module.params['name'])
def shutdown(self):
self.zk.stop()
self.zk.close()
def start(self):
self.zk.start()
def wait(self):
return self._wait(self.module.params['name'], self.module.params['timeout'])
def _absent(self, znode):
if self.exists(znode):
self.zk.delete(znode, recursive=self.module.params['recursive'])
return True, {'changed': True, 'msg': 'The znode was deleted.'}
else:
return True, {'changed': False, 'msg': 'The znode does not exist.'}
def _get(self, path):
if self.exists(path):
value, zstat = self.zk.get(path)
stat_dict = {}
for i in dir(zstat):
if not i.startswith('_'):
attr = getattr(zstat, i)
if isinstance(attr, (int, str)):
stat_dict[i] = attr
result = True, {'msg': 'The node was retrieved.', 'znode': path, 'value': value,
'stat': stat_dict}
else:
result = False, {'msg': 'The requested node does not exist.'}
return result
def _present(self, path, value):
if self.exists(path):
(current_value, zstat) = self.zk.get(path)
if value != current_value:
self.zk.set(path, to_bytes(value))
return True, {'changed': True, 'msg': 'Updated the znode value.', 'znode': path,
'value': value}
else:
return True, {'changed': False, 'msg': 'No changes were necessary.', 'znode': path, 'value': value}
else:
self.zk.create(path, to_bytes(value), makepath=True)
return True, {'changed': True, 'msg': 'Created a new znode.', 'znode': path, 'value': value}
def _wait(self, path, timeout, interval=5):
lim = time.time() + timeout
while time.time() < lim:
if self.exists(path):
return True, {'msg': 'The node appeared before the configured timeout.',
'znode': path, 'timeout': timeout}
else:
time.sleep(interval)
return False, {'msg': 'The node did not appear before the operation timed out.', 'timeout': timeout,
'znode': path}
import sys
import os
from kazoo.client import KazooClient
import logging
logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.INFO)
kz = KazooClient('zookeeper', int(os.environ['ZOO_CLIENT_PORT']))
path = '/' + sys.argv[1]
value = sys.argv[2]
kz.start()
kz.set(path, value)
kz.stop()
class ZookeeperClient(object):
def __init__(self, hosts=None, read_only=True):
self.hosts = hosts if hosts else ENSEMBLE.get()
self.read_only = read_only
hdfs = cluster.get_hdfs()
if hdfs is None:
raise ZookeeperConfigurationException('No [hdfs] configured in hue.ini.')
if hdfs.security_enabled:
self.sasl_server_principal = PRINCIPAL_NAME.get()
else:
self.sasl_server_principal = None
self.zk = KazooClient(hosts=self.hosts,
read_only=self.read_only,
sasl_server_principal=self.sasl_server_principal)
def start(self):
"""Start the zookeeper session."""
self.zk.start()
def stop(self):
"""Stop the zookeeper session, but leaves the socket open."""
self.zk.stop()
def close(self):
"""Closes a stopped zookeeper socket."""
self.zk.close()
def get_children_data(self, namespace):
children = self.zk.get_children(namespace)
children_data = []
for node in children:
data, stat = self.zk.get("%s/%s" % (namespace, node))
children_data.append(data)
return children_data
def path_exists(self, namespace):
return self.zk.exists(namespace) is not None
def set(self, path, value, version=-1):
return self.zk.set(path, value, version)
def copy_path(self, namespace, filepath):
if self.read_only:
raise ReadOnlyClientException('Cannot execute copy_path when read_only is set to True.')
self.zk.ensure_path(namespace)
for dir, subdirs, files in os.walk(filepath):
path = dir.replace(filepath, '').strip('/')
if path:
node_path = '%s/%s' % (namespace, path)
self.zk.create(path=node_path, value='', makepath=True)
for filename in files:
node_path = '%s/%s/%s' % (namespace, path, filename)
with open(os.path.join(dir, filename), 'r') as f:
file_content = f.read()
self.zk.create(path=node_path, value=file_content, makepath=True)
def delete_path(self, namespace):
if self.read_only:
raise ReadOnlyClientException('Cannot execute delete_path when read_only is set to True.')
self.zk.delete(namespace, recursive=True)
def __enter__(self):
"""Start a zookeeper session and return a `with` context."""
self.zk.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Stops and closes zookeeper session at the end of the `with` context."""
try:
self.stop()
finally:
self.close()
class zookeeper:
def __init__(self, address):
self.address = address
self.zk = KazooClient(address)
self.zk.start()
self.arcus_cache_map = {}
self.arcus_node_map = {}
self.force = False
self.meta = ('', None)
self.meta_mtime = None
def __repr__(self):
repr = '[ZooKeeper: %s] %s, %s' % (self.address, self.meta[0], str(self.meta[1]))
for code, cache in self.arcus_cache_map.items():
repr = '%s\n\n%s' % (repr, cache)
return repr
def set_force(self):
self.force = True
def zk_read(self, path):
data, stat = self.zk.get(path)
children = self.zk.get_children(path)
return data, stat, children
def zk_children(self, path, watch=None):
if watch != None:
return self.zk.get_children(path, watch = watch)
else:
return self.zk.get_children(path)
def zk_children_if_exists(self, path, watch=None):
if self.zk_exists(path) == False:
return []
return self.zk_children(path, watch)
def zk_exists(self, path):
if self.zk.exists(path) == None:
return False
return True
def zk_create(self, path, value):
try:
self.zk.create(path, bytes(value, 'utf-8'))
except NodeExistsError:
if self.force == False:
raise NodeExistsError
def zk_delete(self, path):
try:
self.zk.delete(path)
except NoNodeError:
if self.force == False:
raise NoNodeError
def zk_delete_tree(self, path):
try:
self.zk.delete(path, recursive=True)
except NoNodeError:
if self.force == False:
raise NoNodeError
def zk_update(self, path, value):
try:
self.zk.set(path, bytes(value, 'utf-8'))
except NoNodeError:
if self.force == False:
raise NoNodeError
def get_arcus_cache_list(self):
children = self.zk_children_if_exists('/arcus/cache_list/')
children += self.zk_children_if_exists('/arcus_repl/cache_list/')
return children
def get_arcus_node_of_code(self, code, server):
# repl case
children = self.zk_children_if_exists('/arcus_repl/cache_list/' + code)
children += self.zk_children_if_exists('/arcus/cache_list/' + code)
ret = []
for child in children:
tmp = child.split('^', 2) # remove repl info
if len(tmp) == 3:
child = tmp[2]
addr, name = child.split('-', 1)
ip, port = addr.split(':', 1)
if server != '' and (server != ip and server != name):
continue # skip this
node = arcus_node(ip, port)
node.name = name
ret.append(node)
return ret
def get_arcus_node_of_server(self, addr):
ip = socket.gethostbyname(addr)
children = self.zk_children_if_exists('/arcus_repl/cache_server_mapping/')
children += self.zk_children_if_exists('/arcus/cache_server_mapping/')
ret = []
for child in children:
l = len(ip)
if child[:l] == ip:
code = self.zk_children_if_exists('/arcus_repl/cache_server_mapping/' + child)
if len(code) == 0:
code = self.zk_children_if_exists('/arcus/cache_server_mapping/' + child)
code = code[0]
tmp = code.split('^') # remove repl info
code = tmp[0]
try:
ip, port = child.split(':')
except ValueError:
print('No port defined in cache_server_mapping: %s' % child)
continue
node = arcus_node(ip, port)
node.code = code
ret.append(node)
return ret
def _get_arcus_node(self, child, results):
code = self.zk_children_if_exists('/arcus_repl/cache_server_mapping/' + child)
if len(code) == 0:
code = self.zk_children_if_exists('/arcus/cache_server_mapping/' + child)
if len(code) == 0:
print('no childrens in cache_server_mapping error: %s' % child)
print(code)
return
code = code[0]
tmp = code.split('^') # remove repl info
code = tmp[0]
try:
ip, port = child.split(':')
except ValueError:
print('No port defined in cache_server_mapping: %s' % child)
ip = child
port = '0'
node = arcus_node(ip, port)
node.code = code
results.append(node)
def get_arcus_node_all(self):
children = self.zk_children_if_exists('/arcus_repl/cache_server_mapping/')
children += self.zk_children_if_exists('/arcus/cache_server_mapping/')
ret = []
threads = []
#print(children)
for child in children:
th = threading.Thread(target = self._get_arcus_node, args = (child, ret))
th.start()
threads.append(th)
for th in threads:
th.join()
return ret
def _get_arcus_meta(self, child, results):
data, stat, children = self.zk_read('/arcus/meta/' + child)
results[child] = [data.decode('utf-8'), stat]
def get_arcus_meta_all(self):
if self.zk.exists('/arcus/meta') == None:
self.zk.create('/arcus/meta', b'arcus meta info')
children = self.zk.get_children('/arcus/meta')
print('# children')
print(children)
threads = []
ret = {}
#print(children)
for child in children:
th = threading.Thread(target = self._get_arcus_meta, args = (child, ret))
th.start()
threads.append(th)
for th in threads:
th.join()
return ret
def _match_code_and_nodes(self, code, cache, meta):
#repl case
children = self.zk_children_if_exists('/arcus_repl/cache_list/' + code)
children += self.zk_children_if_exists('/arcus/cache_list/' + code)
for child in children:
tmp = child.split('^', 2) # remove repl info
if len(tmp) == 3:
child = tmp[2]
addr, name = child.split('-')
try:
node = self.arcus_node_map[addr]
except KeyError:
print('%s of %s is not defined in cache_server_mapping' % (addr, code))
ip, port = addr.split(':')
node = arcus_node(ip, port)
node.noport = True
node.active = True
cache.active_node.append(node)
for node in cache.node:
if node.active == False:
cache.dead_node.append(node)
if code in meta:
cache.meta = meta[code]
def load_all(self):
codes = self.get_arcus_cache_list()
for code in codes:
cache = arcus_cache(self.address, code)
self.arcus_cache_map[code] = cache
print('# get_arcus_node_all()')
nodes = self.get_arcus_node_all()
print('# done')
for node in nodes:
self.arcus_node_map[node.ip + ":" + node.port] = node
self.arcus_cache_map[node.code].node.append(node)
# meta info
print('# get_arcus_meta_all()')
meta = self.get_arcus_meta_all()
print('# done')
print('# match code & nodes')
threads = []
for code, cache in self.arcus_cache_map.items():
th = threading.Thread(target = self._match_code_and_nodes, args = (code, cache, meta))
th.start()
threads.append(th)
for th in threads:
th.join()
print('#done')
if 'zookeeper' in meta:
self.meta = meta['zookeeper']
def _callback(self, event):
child_list = self.zk.get_children(event.path)
cloud = os.path.basename(event.path)
cache = self.arcus_cache_map[cloud]
event_list = { 'created':[], 'deleted':[] }
current = {}
print('##### active node')
print(cache.active_node)
children = []
for child in child_list:
addr = child.split('-')[0]
children.append(addr)
print('#### children')
print(children)
for node in cache.active_node:
current[node.ip + ':' + node.port] = True
print('##### current')
print(current)
for node in cache.active_node:
addr = node.ip + ':' + node.port
if addr not in children:
event_list['deleted'].append(addr)
cache.active_node.remove(node)
for child in children:
if child not in current:
event_list['created'].append(child)
ip, port = child.split(':')
node = arcus_node(ip, port)
cache.active_node.append(node)
print('####### result')
print(cache.active_node)
self.callback(event, event_list)
children = self.zk.get_children(event.path, watch = self._callback)
def watch(self, callback):
self.callback = callback
for code, cache in self.arcus_cache_map.items():
children = self.zk_children_if_exists('/arcus/cache_list/' + code, watch=self._callback)
children += self.zk_children_if_exists('/arcus_repl/cache_list/' + code, watch=self._callback)
class Store:
"""
Uses zoo keeper as a key vaue store. This store is as thread safe as the underlying client.
"""
def __init__(self, host_list):
hosts = ",".join(host_list)
print("zk, connecting to: '%s'" % hosts)
self.zk = KazooClient(hosts)
def __enter__(self):
self.zk.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.zk.stop()
def get(self, uid, ns, key):
"""
Get the value associated with the key.
:param uid: The user id to use.
:param ns: The namespace to use.
:param key: The key to look up.
:return: A tuple of (version, value) stored.
"""
path = "/jwl/sync/%s/%s/%s" % (uid, ns, key)
print("zk, fetching: '%s'" % path)
try:
r = self.zk.get(path)
print("zk, value at: '%s'='%s'" % (path, r))
return r[1].version, r[0]
except NoNodeError:
print("zk, no value at: '%s'" % path)
return None, None
def put(self, uid, ns, expected_version, key, value):
"""
If the version of the value in the system is what we expect, then update the key with
the value given. Otherwise return a tuple of (False, cur_serial, cur_value).
:param uid: The user id to use.
:param ns: The namespace to use.
:param expected_version: The version of the record we expect.
:param key: The key to update.
:param value: The value to associate with the key.
:return: A tuple of (True, new_version, new_value) if it works, (False, cur_version, cur_value) on failure.
"""
path = "/jwl/sync/%s/%s/%s" % (uid, ns, key)
self.zk.ensure_path(path)
if expected_version is None:
self.zk.set(path, value)
return True, 1, value
try:
print("zk, set '%s'='%s'" % (path, value))
self.zk.set(path, value, version=expected_version)
return True, expected_version + 1, value
except BadVersionError:
print("zk, set failed '%s'='%s'" % (path, value))
r = self.zk.get(path)
return False, r[1].version, r[0]
class mock_zip(object):
_root_node = ''
_zookeeper = None
_shell_path = ''
def __init__(self, host = '127.0.0.1', port = 2181, root_node = '/test', shell_path = './'):
self._zookeeper = KazooClient('%s:%s' % (host, port,))
self._root_node = root_node
self._shell_path = shell_path
def run(self):
self._zookeeper.start()
self.init()
def init(self):
zip_node = '%s/to_zip_notice' % self._root_node
default_node_value = json.dumps({'update_time' : time.time()})
try:
if self._zookeeper.exists(zip_node) is None:
self._zookeeper.create(zip_node, default_node_value, makepath = True)
except kazoo.exceptions.NodeExistsError:
pass
@self._zookeeper.ChildrenWatch('%s/to_zip_notice' % (self._root_node, ))
def to_zip_node(zip_node_list):
for zip_node_id in zip_node_list:
LOG.info('watch_zip children %s/to_zip_notice/%s' % (self._root_node, zip_node_id, ))
self.to_zip(zip_node_id)
return self
def to_zip(self, zip_node_id):
@self._zookeeper.DataWatch('%s/to_zip_notice/%s' % (self._root_node, zip_node_id, ))
def to_zip_execute(data, stat, event):
if event is not None and event.type == 'DELETED':
return
if 0 == len(data):
return
LOG.info('watch_zip execute %s/to_zip_notice/%s %s' % (self._root_node, zip_node_id, data, ))
node_detail = json.loads(data)
if node_detail.get('config_version', None) is None or \
node_detail.get('game_version', None) is None or \
node_detail.get('status', None) == 'ok' or \
node_detail.get('status', None) == 'failed':
return
if self.zip_execute(node_detail['config_version'], node_detail['game_version']) is True:
LOG.info('zip node %s/to_zip_notice/%s zip success' % (self._root_node, zip_node_id, ))
node_detail['status'] = 'ok'
node_detail['finish_time'] = time.time()
self._zookeeper.set('%s/to_zip_notice/%s' % (self._root_node, zip_node_id, ), json.dumps(node_detail))
self._zookeeper.set('%s/to_zip_result/%s' % (self._root_node, zip_node_id, ), json.dumps({
'update_time' : time.time(),
'status' : 'ok'
}))
else:
LOG.info('zip node %s/to_zip_notice/%s zip failed' % (self._root_node, zip_node_id, ))
self._zookeeper.set('%s/to_zip_result/%s' % (self._root_node, zip_node_id, ), json.dumps({
'update_time' : time.time(),
'status' : 'failed'
}))
def zip_execute(self, config_version, game_version):
'''
to execute shell to zip resource
'''
LOG.info('start to execute shell %s/zip.sh %s %s' % (self._shell_path, config_version, game_version, ))
result = subprocess.call('%s/zip.sh %s %s > /dev/null 2>&1' % (self._shell_path, config_version, game_version, ), shell = True)
return True if result == 0 else False
class SimpleSwitch13(app_manager.RyuApp):
OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]
def __init__(self, *args, **kwargs):
super(SimpleSwitch13, self).__init__(*args, **kwargs)
self.mac_to_port = {}
self.zkConf = {'root':'/multicontroller', 'topo':'/topology',
'swstat':'/swstat', 'counter': '/counter'}
self.zk = KazooClient('127.0.0.1:2181')
self.zk.start()
self.ip = '202.201.3.51'
self.sws = {}
self.gid = random.randint(0, 10000)
self.dps = {}
self.links = []
self.interval = 5
self.role = OFPCR_ROLE_EQUAL
self.topoThread = hub.spawn(self._topoThread)
self.linkThread = hub.spawn(self._linkDiscover)
self.clearLinkThread = hub.spawn(self._cleanLinks)
self.clearLinkThread = hub.spawn(self._cleanSwitches)
def _cleanSwitches(self):
while True:
self.sws = {k:self.sws[k] for k in self.sws if self.sws[k]}
hub.sleep(self.interval)
def _topoThread(self):
linkNode = self.zkConf['root'] + self.zkConf['topo'] + self.ip
if self.zk.exists(linkNode):
self.zk.set(linkNode, json.dumps(self.links))
else:
self.zk.create(linkNode, json.dumps(self.links))
hub.sleep(self.interval)
def _linkDiscover(self):
while True:
for dpid in self.dps:
self.sendSlldp(dpid)
hub.sleep(self.interval)
def sendSlldp(self, dpid):
dp = self.dps.get(dpid)
if dp is None:
return
actions = [dp.ofproto_parser.OFPActionOutput(dp.ofproto.OFPP_FLOOD)]
pkt = packet.Packet()
pkt.add_protocol(ethernet.ethernet(ethertype=ETH_TYPE_SLLDP,
dst=SLLDP_MAC_DST, src=SLLDP_MAC_SRC))
pkt.add_protocol(slldp(dp.id))
pkt.serialize()
slldpPacket = pkt.data
out = dp.ofproto_parser.OFPPacketOut(
datapath=dp, in_port=dp.ofproto.OFPP_CONTROLLER,
buffer_id=dp.ofproto.OFP_NO_BUFFER, actions=actions,
data=slldpPacket)
dp.send_msg(out)
def getLinks(self):
topoNode = self.zkConf['root'] + self.zkConf['topo']
ips = self.zk.get_children(topoNode)
res = []
for ip in ips:
links = self.zk.get(topoNode + ip)[0]
for link in links:
res.append(link)
return res
@set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
def _packet_in_handler(self, ev):
msg = ev.msg
pkt = packet.Packet(msg.data)
eth = pkt.get_protocols(ethernet.ethernet)[0]
dst = eth.dst
# SLLDP packet
if dst == SLLDP_MAC_DST:
self.handleSlldp(ev)
return
# process packet_in message in subclass
self.packet_in_process(ev)
def handleSlldp(self, ev):
msg = ev.msg
datapath = msg.datapath
dpid = datapath.id
inPort = msg.match['in_port']
pkt = packet.Packet(msg.data)
slldpBuff = pkt.get_protocols(ethernet.ethernet)[2]
dpidSrc, _ = slldp.parser(slldpBuff)
self.links.append({'srcdpid': dpidSrc,
'dst': {'dpid': dpid, 'port': inPort},
'time': time.time()})
def _cleanLinks(self):
while True:
now = time.time()
self.links = [l for l in self.links if now - l['time'] < self.interval]
hub.sleep(self.interval)
@abc.abstractmethod
def packet_in_process(self, ev):
pass
@set_ev_cls(event.EventSwitchEnter)
def switch_enter(self, ev):
dpid = ev.datapath.id
self.sws[dpid] = True
self.dps[dpid] = ev.datapath
dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid)
self.zk.ensure_path(dpNode)
if self.election(dpid):
self.role = OFPCR_ROLE_MASTER
else:
self.role = OFPCR_ROLE_SLAVE
self.countUp(dpid)
self.roleRequest(dpid, self.role)
mflag = dpNode + '/' + 'master'
DataWatch(self.zk, mflag, self.masterWatcher)
def masterWatcher(self, data, stat, ev):
if ev and ev.type == 'DELETED':
_, _, dpid, _ = ev.path.split('/')
dpid = str_to_dpid(dpid)
if self.sws.get(dpid):
if self.election(dpid):
self.role = OFPCR_ROLE_MASTER
self.roleRequest(dpid, self.role)
return self.sws.get(dpid)
def election(self, dpid):
dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid)
mflag = dpNode + '/' + 'master'
while not self.zk.exists(mflag):
mlock = self.zk.Lock(dpNode + '/' + 'mlock', self.ip)
with mlock:
if not self.zk.exists(mflag):
self.zk.create(mflag, self.ip, ephemeral=True)
if self.zk.exists(mflag):
if self.zk.get(mflag) == self.ip:
return True
else:
return False
else:
time.sleep(random.randint(0, 100)/500.0)
return False
def roleRequest(self, dp, role):
msg = dp.ofproto_parser.OFPRoleRequest(dp, role, self.gid)
dp.send_msg(msg)
def getCount(self, dpid):
dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid)
countNode = dpNode + self.zkConf['counter']
counters = self.zk.get_children(countNode)
return len(counters)
def countUp(self, dpid):
countNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid) + self.zkConf['counter']
self.zk.ensure_path(countNode)
self.zk.create(countNode+uuid4().hex, 'alive', ephemeral=True)
@set_ev_cls(event.EventSwitchLeave)
def switch_levave(self, ev):
dpid = ev.datapath
count = self.getCount(dpid)
self.sws[dpid] = False
if count == 0:
dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
+ '/' + dpid_to_str(dpid)
self.zk.delete(dpNode, recursive=True)
