def get_all_zk_state_managers(conf): """ Connects to all the zookeeper state_managers and returns the connected state_managers instances. """ state_managers = [] state_locations = conf.get_state_locations_of_type("zookeeper") for location in state_locations: name = location['name'] hostport = location['hostport'] host = None port = None if ':' in hostport: hostportlist = hostport.split(':') if len(hostportlist) == 2: host = hostportlist[0] port = int(hostportlist[1]) if not host or not port: raise Exception("Hostport for %s must be of the format 'host:port'." % (name)) tunnelhost = location['tunnelhost'] rootpath = location['rootpath'] LOG.info("Connecting to zk hostport: " + host + ":" + str(port) + " rootpath: " + rootpath) state_manager = ZkStateManager(name, host, port, rootpath, tunnelhost) try: state_manager.start() except Exception as e: LOG.error("Exception while connecting to state_manager.") traceback.print_exc() state_managers.append(state_manager) return state_managers
def get_all_zk_state_managers(conf): """ Connects to all the zookeeper state_managers and returns the connected state_managers instances. """ state_managers = [] state_locations = conf.get_state_locations_of_type("zookeeper") for location in state_locations: name = location['name'] hostport = location['hostport'] host = None port = None if ':' in hostport: hostportlist = hostport.split(':') if len(hostportlist) == 2: host = hostportlist[0] port = int(hostportlist[1]) if not host or not port: raise Exception( "Hostport for %s must be of the format 'host:port'." % (name)) tunnelhost = location['tunnelhost'] rootpath = location['rootpath'] LOG.info("Connecting to zk hostport: " + host + ":" + str(port) + " rootpath: " + rootpath) state_manager = ZkStateManager(name, host, port, rootpath, tunnelhost) try: state_manager.start() except Exception: LOG.error("Exception while connecting to state_manager.") LOG.debug(traceback.format_exc()) state_managers.append(state_manager) return state_managers
def setUp(self): # Create a a ZkStateManager that we will test with self.statemanager = ZkStateManager('zk', [('localhost', 2181), ('localhost', 2281)], 'heron', 'reachable.host') # replace creation of a KazooClient self.mock_kazoo = ZkStateManagerTest.MockKazooClient() self.opened_host_ports = [] def kazoo_client(hostport): self.opened_host_ports.append(hostport) return self.mock_kazoo self.statemanager._kazoo_client = kazoo_client
def get_all_zk_state_managers(conf): """ Creates all the zookeeper state_managers and returns them in a list """ state_managers = [] state_locations = conf.get_state_locations_of_type("zookeeper") for location in state_locations: name = location['name'] hostport = location['hostport'] hostportlist = [] for hostportpair in hostport.split(','): host = None port = None if ':' in hostport: hostandport = hostportpair.split(':') if len(hostandport) == 2: host = hostandport[0] port = int(hostandport[1]) if not host or not port: raise Exception( "Hostport for %s must be of the format 'host:port'." % (name)) hostportlist.append((host, port)) tunnelhost = location['tunnelhost'] rootpath = location['rootpath'] LOG.info("Connecting to zk hostports: " + str(hostportlist) + " rootpath: " + rootpath) state_manager = ZkStateManager(name, hostportlist, rootpath, tunnelhost) state_managers.append(state_manager) return state_managers
def get_all_zk_state_managers(conf): """ Connects to all the zookeeper state_managers and returns the connected state_managers instances. """ state_managers = [] state_locations = conf.get_state_locations_of_type("zookeeper") for location in state_locations: name = location['name'] host = location['host'] port = location['port'] tunnelhost = location['tunnelhost'] rootpath = location['rootpath'] LOG.info("Connecting to zk hostport: " + host + ":" + str(port) + " rootpath: " + rootpath) state_manager = ZkStateManager(name, host, port, rootpath, tunnelhost) try: state_manager.start() except Exception as e: LOG.error("Exception while connecting to state_manager.") traceback.print_exc() state_managers.append(state_manager) return state_managers
def _load_state_mgr(self, cluster): state_mgr_config = configloader.load_state_manager_locations(cluster, os.getenv("HOME") +'/.heron/conf/'+cluster + '/statemgr.yaml') if state_mgr_config[0]["type"] == 'file': return FileStateManager(self.topology_name, os.getenv("HOME") +'/.herondata/repository/state/local') elif state_mgr_config[0]["type"] == 'zookeeper': host_port = state_mgr_config[0]["hostport"].split(':') return ZkStateManager(state_mgr_config[0]["type"], [(host_port[0], int(host_port[1]))], state_mgr_config[0]["rootpath"], state_mgr_config[0]["tunnelhost"]) else: raise status.TestFailure("Unrecognized state manager type: %s" % state_mgr_config["type"])
class ZkStateManagerTest(unittest.TestCase): """Unittest for ZkStateManager""" class MockKazooClient: def __init__(self): self.start_calls = 0 self.stop_calls = 0 def start(self): self.start_calls = self.start_calls + 1 def stop(self): self.stop_calls = self.stop_calls + 1 def add_listener(self, listener): pass def setUp(self): # Create a a ZkStateManager that we will test with self.statemanager = ZkStateManager('zk', [('127.0.0.1', 2181), ('127.0.0.1', 2281)], 'heron', 'reachable.host') # replace creation of a KazooClient self.mock_kazoo = ZkStateManagerTest.MockKazooClient() self.opened_host_ports = [] def kazoo_client(hostport): self.opened_host_ports.append(hostport) return self.mock_kazoo self.statemanager._kazoo_client = kazoo_client def test_start_checks_for_connection(self): global did_connection_check did_connection_check = False def connection_check(): global did_connection_check did_connection_check = True return True self.statemanager.is_host_port_reachable = connection_check self.statemanager.start() self.assertTrue(did_connection_check) def test_start_uses_host_ports(self): def connection_check(): return True self.statemanager.is_host_port_reachable = connection_check self.statemanager.start() self.assertEqual('127.0.0.1:2181,127.0.0.1:2281', self.opened_host_ports[0]) def test_start_opens_proxy_if_no_connection(self): def connection_check(): return False global did_open_proxy did_open_proxy = False def open_proxy(): global did_open_proxy did_open_proxy = True return [('proxy', 2181), ('proxy-2', 2281)] self.statemanager.is_host_port_reachable = connection_check self.statemanager.establish_ssh_tunnel = open_proxy self.statemanager.start() self.assertTrue(did_open_proxy) def test_proxied_start_uses_connection(self): def connection_check(): return False def open_proxy(): return [('smorgasboard', 2200), ('smorgasboard', 2201)] self.statemanager.is_host_port_reachable = connection_check self.statemanager.establish_ssh_tunnel = open_proxy self.statemanager.start() self.assertEqual('smorgasboard:2200,smorgasboard:2201', self.opened_host_ports[0])
class ZkStateManagerTest(unittest.TestCase): """Unittest for ZkStateManager""" class MockKazooClient: def __init__(self): self.start_calls = 0 self.stop_calls = 0 def start(self): self.start_calls = self.start_calls + 1 def stop(self): self.stop_calls = self.stop_calls + 1 def add_listener(self,listener): pass def setUp(self): # Create a a ZkStateManager that we will test with self.statemanager = ZkStateManager('zk', [('localhost', 2181), ('localhost', 2281)], 'heron', 'reachable.host') # replace creation of a KazooClient self.mock_kazoo = ZkStateManagerTest.MockKazooClient() self.opened_host_ports = [] def kazoo_client(hostport): self.opened_host_ports.append(hostport) return self.mock_kazoo self.statemanager._kazoo_client = kazoo_client def test_start_checks_for_connection(self): global did_connection_check did_connection_check = False def connection_check(): global did_connection_check did_connection_check = True return True self.statemanager.is_host_port_reachable = connection_check self.statemanager.start() self.assertTrue(did_connection_check) def test_start_uses_host_ports(self): def connection_check(): return True self.statemanager.is_host_port_reachable = connection_check self.statemanager.start() self.assertEqual('localhost:2181,localhost:2281',self.opened_host_ports[0]) def test_start_opens_proxy_if_no_connection(self): def connection_check(): return False global did_open_proxy did_open_proxy = False def open_proxy(): global did_open_proxy did_open_proxy = True return [('proxy', 2181), ('proxy-2', 2281)] self.statemanager.is_host_port_reachable = connection_check self.statemanager.establish_ssh_tunnel = open_proxy self.statemanager.start() self.assertTrue(did_open_proxy) def test_proxied_start_uses_connection(self): def connection_check(): return False def open_proxy(): return [('smorgasboard',2200),('smorgasboard',2201)] self.statemanager.is_host_port_reachable = connection_check self.statemanager.establish_ssh_tunnel = open_proxy self.statemanager.start() self.assertEqual('smorgasboard:2200,smorgasboard:2201',self.opened_host_ports[0])