def __init__(self, chairman_list, chairman_clients_num, host_num, agent_host, agent_port, datastores, availability_zones, networks, sleep_min=0, sleep_max=500): self.clients = [] self.hosts = {} self.agent_host = agent_host self.agent_port = agent_port self.host_num = host_num self.handler = AgentHandler(self.hosts) self.datastores = datastores self.availability_zones = availability_zones self.networks = networks self.sleep_min = sleep_min self.sleep_max = sleep_max self.servers = [] self.log = [] # Create clients for x in xrange(chairman_clients_num): chairman = random.choice(chairman_list) chairman = chairman.split(":") client = create_chairman_client(chairman[0], int(chairman[1])) self.clients.append(client[1])
def setUp(self): self.set_up_kazoo_base() self.procs = [] self.thrift_server = None self.transports = [] self.runtime = RuntimeUtils(self.id()) self.runtime.start_cloud_store() host, port = '127.0.0.1', 13000 self.runtime.start_chairman(host, port) (transport, self.chairman_client) = create_chairman_client(host, port) self.transports.append(transport) # Wait for chairman to finish their elections _wait_on_code(self.chairman_client.get_schedulers, GetSchedulersResultCode.OK, GetSchedulersRequest)
def setUp(self): self.set_up_kazoo_base() self.zk_client = self._get_nonchroot_client() self.zk_client.start() self.runtime = RuntimeUtils(self.id()) # Create zk paths self.zk_client.create(MISSING_PREFIX) self.zk_client.create(HOSTS_PREFIX) self.zk_client.create(ROLES_PREFIX) self.root_conf = {} self.root_conf['healthcheck'] = {} self.root_conf['zookeeper'] = {} self.root_conf['zookeeper']['quorum'] = ("localhost:%i" % (DEFAULT_ZK_PORT, )) self.root_conf['healthcheck']['timeout_ms'] = ROOT_SCHEDULER_TIME_OUT self.root_conf['healthcheck']['period_ms'] = ROOT_SCHEDULER_PERIOD # start root scheduler self.root_host = "localhost" self.root_port = 15000 self.root_conf['bind'] = self.root_host self.root_conf['port'] = self.root_port self.runtime.start_root_scheduler(self.root_conf) (self.root_transport, self.root_sch_client) = create_root_client(self.root_port, self.root_host) # start chairman self.chairman_host = 'localhost' self.chairman_port = 13000 self.leaf_fanout = 2 self.runtime.start_chairman(self.chairman_host, self.chairman_port, self.leaf_fanout) (self.chairman_transport, self.chairman_client) = \ create_chairman_client(self.chairman_host, self.chairman_port) # Wait for chairman and root scheduler to finish their elections _wait_on_code(self.root_sch_client.get_schedulers, GetSchedulersResultCode.OK) _wait_on_code(self.chairman_client.get_schedulers, GetSchedulersResultCode.OK, GetSchedulersRequest)
def get_hierarchy_from_chairman(chairman_address, chairman_port, root_address, root_port): """This function will will attempt to build the hierarchy from chairman's in-memory view of the tree. """ try: (_, chairman_client) = create_chairman_client(chairman_address, chairman_port) req = GetSchedulersRequest() resp = chairman_client.get_schedulers(req) if resp.result != GetSchedulersResultCode.OK or not resp.schedulers: log.error("get_schedulers failed, for chairman on %s:%s" % (chairman_address, chairman_port)) return root = None leaves = [] # get the root scheduler for sch in resp.schedulers: if sch.role.id == ROOT_SCHEDULER_ID: # Todo(Maithem): add address/port in SchedulerRole owner = Host(ROOT_SCHEDULER_ID, root_address, root_port) root = Scheduler(ROOT_SCHEDULER_ID, ROOT_SCHEDULER_TYPE, owner) else: leaves.append(sch) if root is None: log.error("Couldn't find a root scheduler") return None # process for schEntry in leaves: leaf_owner_host = schEntry.agent leaf = schEntry.role leafSch = Scheduler(leaf.id, LEAF_SCHEDULER_TYPE) for childHost in leaf.host_children: host = Host(childHost.id, childHost.address, childHost.port, leafSch) if childHost.id == leaf_owner_host: leafSch.owner = host leafSch.add_child(host) root.add_child(leafSch) return root except Exception, e: log.exception(e)
def get_hierarchy_from_chairman(chairman_address, chairman_port, root_address, root_port): """This function will will attempt to build the hierarchy from chairman's in-memory view of the tree. """ try: (_, chairman_client) = create_chairman_client(chairman_address, chairman_port) req = GetSchedulersRequest() resp = chairman_client.get_schedulers(req) if resp.result != GetSchedulersResultCode.OK or not resp.schedulers: log.error("get_schedulers failed, for chairman on %s:%s" % (chairman_address, chairman_port)) return root = None leaves = [] # get the root scheduler for sch in resp.schedulers: if sch.role.id == ROOT_SCHEDULER_ID: # Todo(Maithem): add address/port in SchedulerRole owner = Host(ROOT_SCHEDULER_ID, root_address, root_port) root = Scheduler(ROOT_SCHEDULER_ID, ROOT_SCHEDULER_TYPE, owner) else: leaves.append(sch) if root is None: log.error("Couldn't find a root scheduler") return None # process for schEntry in leaves: leaf_owner_host = schEntry.agent leaf = schEntry.role leafSch = Scheduler(leaf.id, LEAF_SCHEDULER_TYPE) for childHost in leaf.host_children: host = Host(childHost.id, childHost.address, childHost.port, leafSch) if childHost.id == leaf_owner_host: leafSch.owner = host leafSch.add_child(host) root.add_child(leafSch) return root except Exception, e: log.exception(e)
def setUp(self): if not CHAIRMAN_SIM: raise nose.plugins.skip.SkipTest("Skipping Chairman" "simulator test!") atexit.register(self.dump_sim_state) self.runtime = RuntimeUtils(self.id()) self.set_up_kazoo_base() self.zk_client = self._get_nonchroot_client() self.zk_client.start() # start chairman self.chairman_host = '127.0.0.1' self.chairman_port = 13000 self.leaf_fanout = 32 self.runtime.start_chairman(self.chairman_host, self.chairman_port, self.leaf_fanout) (_, self.client1) = create_chairman_client(self.chairman_host, self.chairman_port) # Wait for chairman to finish their elections _wait_on_code(self.client1.get_schedulers, GetSchedulersResultCode.OK, GetSchedulersRequest)
def __init__(self, chairman_list, chairman_clients_num, host_num, agent_host, agent_port, datastores, availability_zones, networks, sleep_min=0, sleep_max=500): self.clients = [] self.hosts = {} self.agent_host = agent_host self.agent_port = agent_port self.host_num = host_num self.handler = AgentHandler(self.hosts) self.datastores = datastores self.availability_zones = availability_zones self.networks = networks self.sleep_min = sleep_min self.sleep_max = sleep_max self.servers = [] self.log = [] # Create clients for x in xrange(chairman_clients_num): chairman = random.choice(chairman_list) chairman = chairman.split(":") client = create_chairman_client(chairman[0], int(chairman[1])) self.clients.append(client[1])
def setUp(self): if not CHAIRMAN_SIM: raise nose.plugins.skip.SkipTest("Skipping Chairman" "simulator test!") atexit.register(self.dump_sim_state) self.runtime = RuntimeUtils(self.id()) self.set_up_kazoo_base() self.zk_client = self._get_nonchroot_client() self.zk_client.start() # start chairman self.chairman_host = '127.0.0.1' self.chairman_port = 13000 self.leaf_fanout = 32 self.runtime.start_chairman(self.chairman_host, self.chairman_port, self.leaf_fanout) (_, self.client1) = create_chairman_client(self.chairman_host, self.chairman_port) # Wait for chairman to finish their elections _wait_on_code(self.client1.get_schedulers, GetSchedulersResultCode.OK, GetSchedulersRequest)
def setUp(self): self.set_up_kazoo_base() self.zk_client = self._get_nonchroot_client() self.zk_client.start() self.runtime = RuntimeUtils(self.id()) # Create zk paths self.zk_client.create(MISSING_PREFIX) self.zk_client.create(HOSTS_PREFIX) self.zk_client.create(ROLES_PREFIX) self.root_conf = {} self.root_conf["healthcheck"] = {} self.root_conf["zookeeper"] = {} self.root_conf["zookeeper"]["quorum"] = "localhost:%i" % (DEFAULT_ZK_PORT,) self.root_conf["healthcheck"]["timeout_ms"] = ROOT_SCHEDULER_TIME_OUT self.root_conf["healthcheck"]["period_ms"] = ROOT_SCHEDULER_PERIOD # start root scheduler self.root_host = "localhost" self.root_port = 15000 self.root_conf["bind"] = self.root_host self.root_conf["port"] = self.root_port self.runtime.start_root_scheduler(self.root_conf) (self.root_transport, self.root_sch_client) = create_root_client(self.root_port, self.root_host) # start chairman self.chairman_host = "localhost" self.chairman_port = 13000 self.leaf_fanout = 2 self.runtime.start_chairman(self.chairman_host, self.chairman_port, self.leaf_fanout) (self.chairman_transport, self.chairman_client) = create_chairman_client(self.chairman_host, self.chairman_port) # Wait for chairman and root scheduler to finish their elections _wait_on_code(self.root_sch_client.get_schedulers, GetSchedulersResultCode.OK) _wait_on_code(self.chairman_client.get_schedulers, GetSchedulersResultCode.OK, GetSchedulersRequest)
def test_get_status(self): chairman_client_leader = self.chairman_client # starts 2 more chairman instances host = '127.0.0.1' port_1 = 13001 port_2 = 13002 self.runtime.start_chairman(host, port_1) self.runtime.start_chairman(host, port_2) (transport_1, chairman_client_non_leader_1) = \ create_chairman_client(host, port_1) (transport_2, chairman_client_non_leader_2) = \ create_chairman_client(host, port_2) self.transports.append(transport_1) self.transports.append(transport_2) h_id_config = {} # Register two hosts with the chairman leader. reg_host_req_1 = self.get_register_host_request() server_address = reg_host_req_1.config.address.host server_port = reg_host_req_1.config.address.port h_id_config[reg_host_req_1.id] = reg_host_req_1.config host_handler = AgentHandler(2) self.thrift_server = ThriftServer(server_address, server_port, host_handler) self.thrift_server.start_server() rc = chairman_client_leader.register_host(reg_host_req_1) self.assertEqual(rc.result, RegisterHostResultCode.OK) reg_host_req_2 = self.get_register_host_request() h_id_config[reg_host_req_2.id] = reg_host_req_2.config rc = self.chairman_client.register_host(reg_host_req_2) self.assertEqual(rc.result, RegisterHostResultCode.OK) # verify that all the agents received configurations host_handler.received_all.wait(20) assert_that(len(host_handler.configs), is_(len(h_id_config))) # verify chairman leader is in READY status get_status_request = GetStatusRequest() rc = chairman_client_leader.get_status(get_status_request) self.assertEqual(rc.type, StatusType.READY) # verify the second chairman which is not leader is in READY status # while it notices that there is a leader exist rc = chairman_client_non_leader_1.get_status(get_status_request) self.assertEqual(rc.type, StatusType.READY) # verify the third chairman which is not leader is in READY status # while it notices that there is a leader exist rc = chairman_client_non_leader_2.get_status(get_status_request) self.assertEqual(rc.type, StatusType.READY) client = self._get_nonchroot_client() client.start() read_chairman_leader = client.get_children(CHAIRMAN_SERVICE)[0] # normalize from unicode to str read_chairman_leader = \ normalize('NFKD', read_chairman_leader).encode('ascii', 'ignore') # expecting chairman leader being deleted leader_deleted_event = async_wait_for(EventType.DELETED, CHAIRMAN_SERVICE, read_chairman_leader, client) # deleting chairman leader from service client.delete(CHAIRMAN_SERVICE + "/" + read_chairman_leader) leader_deleted_event.wait(20) self.assertTrue(leader_deleted_event.isSet()) def wait_for_status(chairman_client, status): retries = 0 while (retries < 10): try: rc = chairman_client.get_status(get_status_request) if rc.type != status: break except: logger.exception("get_status() failed") retries += 1 time.sleep(1) return rc