def create_execution_state(self, topologyName, executionState): """ create execution state """ if not executionState or not executionState.IsInitialized(): raise_( StateException("Execution State protobuf not init properly", StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[2]) path = self.get_execution_state_path(topologyName) LOG.info("Adding topology: {0} to path: {1}".format( topologyName, path)) executionStateString = executionState.SerializeToString() try: self.client.create(path, value=executionStateString, makepath=True) return True except NoNodeError: raise_( StateException("NoNodeError while creating execution state", StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2]) except NodeExistsError: raise_( StateException( "NodeExistsError while creating execution state", StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[2]) except ZookeeperError: raise_( StateException("Zookeeper while creating execution state", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) except Exception: # Just re raise the exception. raise
def _get_execution_state_with_watch(self, topologyName, callback, isWatching): """ Helper function to get execution state with a callback. The future watch is placed only if isWatching is True. """ path = self.get_execution_state_path(topologyName) if isWatching: LOG.info("Adding data watch for path: " + path) # pylint: disable=unused-variable, unused-argument @self.client.DataWatch(path) def watch_execution_state(data, stats): """ invoke callback to watch execute state """ if data: executionState = ExecutionState() executionState.ParseFromString(data) callback(executionState) else: callback(None) # Returning False will result in no future watches # being triggered. If isWatching is True, then # the future watches will be triggered. return isWatching
def create_pplan(self, topologyName, pplan): """ create physical plan """ if not pplan or not pplan.IsInitialized(): raise_( StateException("Physical Plan protobuf not init properly", StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[2]) path = self.get_pplan_path(topologyName) LOG.info("Adding topology: {0} to path: {1}".format( topologyName, path)) pplanString = pplan.SerializeToString() try: self.client.create(path, value=pplanString, makepath=True) return True except NoNodeError: raise_( StateException("NoNodeError while creating pplan", StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2]) except NodeExistsError: raise_( StateException("NodeExistsError while creating pplan", StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[2]) except ZookeeperError: raise_( StateException("Zookeeper while creating pplan", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) except Exception: # Just re raise the exception. raise
def get_all_zk_state_managers(conf): """ Connects to all the zookeeper state_managers and returns the connected state_managers instances. """ state_managers = [] state_locations = conf.get_state_locations_of_type("zookeeper") for location in state_locations: name = location['name'] hostport = location['hostport'] host = None port = None if ':' in hostport: hostportlist = hostport.split(':') if len(hostportlist) == 2: host = hostportlist[0] port = int(hostportlist[1]) if not host or not port: raise Exception( "Hostport for %s must be of the format 'host:port'." % (name)) tunnelhost = location['tunnelhost'] rootpath = location['rootpath'] LOG.info("Connecting to zk hostport: " + host + ":" + str(port) + " rootpath: " + rootpath) state_manager = ZkStateManager(name, host, port, rootpath, tunnelhost) try: state_manager.start() except Exception as e: LOG.error("Exception while connecting to state_manager.") traceback.print_exc() state_managers.append(state_manager) return state_managers
def delete_execution_state(self, topologyName): """ delete execution state """ path = self.get_execution_state_path(topologyName) LOG.info("Removing topology: {0} from path: {1}".format( topologyName, path)) try: self.client.delete(path) return True except NoNodeError: raise_( StateException("NoNodeError while deleting execution state", StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2]) except NotEmptyError: raise_( StateException("NotEmptyError while deleting execution state", StateException.EX_TYPE_NOT_EMPTY_ERROR), sys.exc_info()[2]) except ZookeeperError: raise_( StateException("Zookeeper while deleting execution state", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) except Exception: # Just re raise the exception. raise
def create_execution_state(self, topologyName, executionState): """ create execution state """ if not executionState or not executionState.IsInitialized(): raise StateException("Execution State protobuf not init properly", StateException.EX_TYPE_PROTOBUF_ERROR), None, sys.exc_info()[2] path = self.get_execution_state_path(topologyName) LOG.info("Adding topology: {0} to path: {1}".format( topologyName, path)) executionStateString = executionState.SerializeToString() try: self.client.create(path, value=executionStateString, makepath=True) return True except NoNodeError: raise StateException("NoNodeError while creating execution state", StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2] except NodeExistsError: raise StateException("NodeExistsError while creating execution state", StateException.EX_TYPE_NODE_EXISTS_ERROR), None, sys.exc_info()[2] except ZookeeperError: raise StateException("Zookeeper while creating execution state", StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2] except Exception: # Just re raise the exception. raise
def _get_topology_with_watch(self, topologyName, callback, isWatching): """ Helper function to get pplan with a callback. The future watch is placed only if isWatching is True. """ path = self.get_topology_path(topologyName) if isWatching: LOG.info("Adding data watch for path: " + path) # pylint: disable=unused-variable, unused-argument @self.client.DataWatch(path) def watch_topology(data, stats, event): """ watch topology """ if data: topology = Topology() topology.ParseFromString(data) callback(topology) else: callback(None) # Returning False will result in no future watches # being triggered. If isWatching is True, then # the future watches will be triggered. return isWatching
def create_topology(self, topologyName, topology): """ crate topology """ if not topology or not topology.IsInitialized(): raise_(StateException("Topology protobuf not init properly", StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[2]) path = self.get_topology_path(topologyName) LOG.info("Adding topology: {0} to path: {1}".format( topologyName, path)) topologyString = topology.SerializeToString() try: self.client.create(path, value=topologyString, makepath=True) return True except NoNodeError: raise_(StateException("NoNodeError while creating topology", StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2]) except NodeExistsError: raise_(StateException("NodeExistsError while creating topology", StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[2]) except ZookeeperError: raise_(StateException("Zookeeper while creating topology", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) except Exception: # Just re raise the exception. raise
def get_all_zk_state_managers(conf): """ Creates all the zookeeper state_managers and returns them in a list """ state_managers = [] state_locations = conf.get_state_locations_of_type("zookeeper") for location in state_locations: name = location['name'] hostport = location['hostport'] hostportlist = [] for hostportpair in hostport.split(','): host = None port = None if ':' in hostport: hostandport = hostportpair.split(':') if len(hostandport) == 2: host = hostandport[0] port = int(hostandport[1]) if not host or not port: raise Exception( "Hostport for %s must be of the format 'host:port'." % (name)) hostportlist.append((host, port)) tunnelhost = location['tunnelhost'] rootpath = location['rootpath'] LOG.info("Connecting to zk hostports: " + str(hostportlist) + " rootpath: " + rootpath) state_manager = ZkStateManager(name, hostportlist, rootpath, tunnelhost) state_managers.append(state_manager) return state_managers
def get_all_zk_state_managers(conf): """ Connects to all the zookeeper state_managers and returns the connected state_managers instances. """ state_managers = [] state_locations = conf.get_state_locations_of_type("zookeeper") for location in state_locations: name = location['name'] hostport = location['hostport'] host = None port = None if ':' in hostport: hostportlist = hostport.split(':') if len(hostportlist) == 2: host = hostportlist[0] port = int(hostportlist[1]) if not host or not port: raise Exception("Hostport for %s must be of the format 'host:port'." % (name)) tunnelhost = location['tunnelhost'] rootpath = location['rootpath'] LOG.info("Connecting to zk hostport: " + host + ":" + str(port) + " rootpath: " + rootpath) state_manager = ZkStateManager(name, host, port, rootpath, tunnelhost) try: state_manager.start() except Exception as e: LOG.error("Exception while connecting to state_manager.") traceback.print_exc() state_managers.append(state_manager) return state_managers
def create_pplan(self, topologyName, pplan): """ create physical plan """ if not pplan or not pplan.IsInitialized(): raise StateException("Physical Plan protobuf not init properly", StateException.EX_TYPE_PROTOBUF_ERROR), None, sys.exc_info()[2] path = self.get_pplan_path(topologyName) LOG.info("Adding topology: {0} to path: {1}".format( topologyName, path)) pplanString = pplan.SerializeToString() try: self.client.create(path, value=pplanString, makepath=True) return True except NoNodeError: raise StateException("NoNodeError while creating pplan", StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2] except NodeExistsError: raise StateException("NodeExistsError while creating pplan", StateException.EX_TYPE_NODE_EXISTS_ERROR), None, sys.exc_info()[2] except ZookeeperError: raise StateException("Zookeeper while creating pplan", StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2] except Exception: # Just re raise the exception. raise
def _get_topology_with_watch(self, topologyName, callback, isWatching): """ Helper function to get pplan with a callback. The future watch is placed only if isWatching is True. """ path = self.get_topology_path(topologyName) if isWatching: LOG.info("Adding data watch for path: " + path) # pylint: disable=unused-variable, unused-argument @self.client.DataWatch(path) def watch_topology(data, stats): """ watch topology """ if data: topology = Topology() topology.ParseFromString(data) callback(topology) else: callback(None) # Returning False will result in no future watches # being triggered. If isWatching is True, then # the future watches will be triggered. return isWatching
def _get_scheduler_location_with_watch(self, topologyName, callback, isWatching): """ Helper function to get scheduler location with a callback. The future watch is placed only if isWatching is True. """ path = self.get_scheduler_location_path(topologyName) if isWatching: LOG.info("Adding data watch for path: " + path) # pylint: disable=unused-variable, unused-argument @self.client.DataWatch(path) def watch_scheduler_location(data, stats): """ invoke callback to watch scheduler location """ if data: scheduler_location = SchedulerLocation() scheduler_location.ParseFromString(data) callback(scheduler_location) else: callback(None) # Returning False will result in no future watches # being triggered. If isWatching is True, then # the future watches will be triggered. return isWatching
def _get_pplan_with_watch(self, topologyName, callback, isWatching): """ Helper function to get pplan with a callback. The future watch is placed only if isWatching is True. """ path = self.get_pplan_path(topologyName) if isWatching: LOG.info("Adding data watch for path: " + path) # pylint: disable=unused-variable, unused-argument @self.client.DataWatch(path) def watch_pplan(data, stats): """ invoke callback to watch physical plan """ if data: pplan = PhysicalPlan() pplan.ParseFromString(data) callback(pplan) else: callback(None) # Returning False will result in no future watches # being triggered. If isWatching is True, then # the future watches will be triggered. return isWatching
def create_topology(self, topologyName, topology): """ crate topology """ if not topology or not topology.IsInitialized(): raise StateException("Topology protobuf not init properly", StateException.EX_TYPE_PROTOBUF_ERROR), None, sys.exc_info()[2] path = self.get_topology_path(topologyName) LOG.info("Adding topology: {0} to path: {1}".format( topologyName, path)) topologyString = topology.SerializeToString() try: self.client.create(path, value=topologyString, makepath=True) return True except NoNodeError: raise StateException("NoNodeError while creating topology", StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2] except NodeExistsError: raise StateException("NodeExistsError while creating topology", StateException.EX_TYPE_NODE_EXISTS_ERROR), None, sys.exc_info()[2] except ZookeeperError: raise StateException("Zookeeper while creating topology", StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2] except Exception: # Just re raise the exception. raise
def get_all_zk_state_managers(conf): """ Creates all the zookeeper state_managers and returns them in a list """ state_managers = [] state_locations = conf.get_state_locations_of_type("zookeeper") for location in state_locations: name = location['name'] hostport = location['hostport'] hostportlist = [] for hostportpair in hostport.split(','): host = None port = None if ':' in hostport: hostandport = hostportpair.split(':') if len(hostandport) == 2: host = hostandport[0] port = int(hostandport[1]) if not host or not port: raise Exception("Hostport for %s must be of the format 'host:port'." % (name)) hostportlist.append((host, port)) tunnelhost = location['tunnelhost'] rootpath = location['rootpath'] LOG.info("Connecting to zk hostports: " + str(hostportlist) + " rootpath: " + rootpath) state_manager = ZkStateManager(name, hostportlist, rootpath, tunnelhost) state_managers.append(state_manager) return state_managers
def delete_execution_state(self, topologyName): """ delete execution state """ path = self.get_execution_state_path(topologyName) LOG.info("Removing topology: {0} from path: {1}".format( topologyName, path)) with reraise_from_zk_exceptions("deleting execution state"): self.client.delete(path) return True
def delete_pplan(self, topologyName): """ delete physical plan info """ path = self.get_pplan_path(topologyName) LOG.info("Removing topology: {0} from path: {1}".format( topologyName, path)) with reraise_from_zk_exceptions("deleting pplan"): self.client.delete(path) return True
def create_topology(self, topologyName, topology): """ crate topology """ if not topology or not topology.IsInitialized(): raise StateException("Topology protobuf not init properly", StateException.EX_TYPE_PROTOBUF_ERROR) path = self.get_topology_path(topologyName) LOG.info(f"Adding topology: {topologyName} to path: {path}") topologyString = topology.SerializeToString() with reraise_from_zk_exceptions("creating topology"): self.client.create(path, value=topologyString, makepath=True) return True
def create_execution_state(self, topologyName, executionState): """ create execution state """ if not executionState or not executionState.IsInitialized(): raise StateException("Execution State protobuf not init properly", StateException.EX_TYPE_PROTOBUF_ERROR) path = self.get_execution_state_path(topologyName) LOG.info(f"Adding topology: {topologyName} to path: {path}") executionStateString = executionState.SerializeToString() with reraise_from_zk_exceptions("creating execution state"): self.client.create(path, value=executionStateString, makepath=True) return True
def create_pplan(self, topologyName, pplan): """ create physical plan """ if not pplan or not pplan.IsInitialized(): raise StateException("Physical Plan protobuf not init properly", StateException.EX_TYPE_PROTOBUF_ERROR) path = self.get_pplan_path(topologyName) LOG.info(f"Adding topology: {topologyName} to path: {path}") pplanString = pplan.SerializeToString() with reraise_from_zk_exceptions("creating pplan"): self.client.create(path, value=pplanString, makepath=True) return True
def get_all_file_state_managers(conf): """ Returns all the file state_managers. """ state_managers = [] state_locations = conf.get_state_locations_of_type("file") for location in state_locations: name = location['name'] rootpath = os.path.expanduser(location['rootpath']) LOG.info("Connecting to file state with rootpath: " + rootpath) state_manager = FileStateManager(name, rootpath) state_managers.append(state_manager) return state_managers
def is_host_port_reachable(self): """ Returns true if the host is reachable. In some cases, it may not be reachable a tunnel must be used. """ for hostport in self.hostportlist: try: socket.create_connection(hostport, StateManager.TIMEOUT_SECONDS) return True except: LOG.info("StateManager %s Unable to connect to host: %s port %i" % (self.name, hostport[0], hostport[1])) continue return False
def get_all_state_managers(conf): """ @param conf - An instance of Config class Reads the config for requested state managers. Instantiates them, start and then return them. """ state_managers = [] try: state_managers.extend(get_all_zk_state_managers(conf)) state_managers.extend(get_all_file_state_managers(conf)) return state_managers except Exception as ex: LOG.error("Exception while getting state_managers.") raise ex
def _get_topologies_with_watch(self, callback, isWatching): """ Helper function to get topologies with a callback. The future watch is placed only if isWatching is True. """ path = self.get_topologies_path() if isWatching: LOG.info("Adding children watch for path: " + path) @self.client.ChildrenWatch(path) def watch_topologies(topologies): callback(topologies) # Returning False will result in no future watches # being triggered. If isWatching is True, then # the future watches will be triggered. return isWatching
def delete_pplan(self, topologyName): path = self.get_pplan_path(topologyName) LOG.info("Removing topology: {0} from path: {1}".format( topologyName, path)) try: self.client.delete(path) return True except NoNodeError as e: raise StateException("NoNodeError while deleting pplan", StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2] except NotEmptyError as e: raise StateException("NotEmptyError while deleting pplan", StateException.EX_TYPE_NOT_EMPTY_ERROR), None, sys.exc_info()[2] except ZookeeperError as e: raise StateException("Zookeeper while deleting pplan", StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2] except Exception as e: # Just re raise the exception. raise
def delete_execution_state(self, topologyName): path = self.get_execution_state_path(topologyName) LOG.info("Removing topology: {0} from path: {1}".format( topologyName, path)) try: self.client.delete(path) return True except NoNodeError as e: raise StateException("NoNodeError while deleting execution state", StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2] except NotEmptyError as e: raise StateException("NotEmptyError while deleting execution state", StateException.EX_TYPE_NOT_EMPTY_ERROR), None, sys.exc_info()[2] except ZookeeperError as e: raise StateException("Zookeeper while deleting execution state", StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2] except Exception as e: # Just re raise the exception. raise
def get_all_file_state_managers(conf): """ Returns all the file state_managers. """ state_managers = [] state_locations = conf.get_state_locations_of_type("file") for location in state_locations: name = location['name'] rootpath = os.path.expanduser(location['rootpath']) LOG.info("Connecting to file state with rootpath: " + rootpath) state_manager = FileStateManager(name, rootpath) try: state_manager.start() except Exception: LOG.error("Exception while connecting to state_manager.") traceback.print_exc() state_managers.append(state_manager) return state_managers
def get_all_file_state_managers(conf): """ Returns all the file state_managers. """ state_managers = [] state_locations = conf.get_state_locations_of_type("file") for location in state_locations: name = location['name'] rootpath = os.path.expanduser(location['rootpath']) LOG.info("Connecting to file state with rootpath: " + rootpath) state_manager = FileStateManager(name, rootpath) try: state_manager.start() except Exception as e: LOG.error("Exception while connecting to state_manager.") traceback.print_exc() state_managers.append(state_manager) return state_managers
def delete_pplan(self, topologyName): """ delete physical plan info """ path = self.get_pplan_path(topologyName) LOG.info("Removing topology: {0} from path: {1}".format( topologyName, path)) try: self.client.delete(path) return True except NoNodeError: raise StateException("NoNodeError while deleting pplan", StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2] except NotEmptyError: raise StateException("NotEmptyError while deleting pplan", StateException.EX_TYPE_NOT_EMPTY_ERROR), None, sys.exc_info()[2] except ZookeeperError: raise StateException("Zookeeper while deleting pplan", StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2] except Exception: # Just re raise the exception. raise
def _get_tmaster_with_watch(self, topologyName, callback, isWatching): """ Helper function to get pplan with a callback. The future watch is placed only if isWatching is True. """ path = self.get_tmaster_path(topologyName) if isWatching: LOG.info("Adding data watch for path: " + path) @self.client.DataWatch(path) def watch_tmaster(data, stats): if data: tmaster = TMasterLocation() tmaster.ParseFromString(data) callback(tmaster) else: callback(None) # Returning False will result in no future watches # being triggered. If isWatching is True, then # the future watches will be triggered. return isWatching
def _get_scheduler_location_with_watch(self, topologyName, callback, isWatching): """ Helper function to get scheduler location with a callback. The future watch is placed only if isWatching is True. """ path = self.get_scheduler_location_path(topologyName) if isWatching: LOG.info("Adding data watch for path: " + path) @self.client.DataWatch(path) def watch_scheduler_location(data, stats): if data: scheduler_location = SchedulerLocation() scheduler_location.ParseFromString(data) callback(scheduler_location) else: callback(None) # Returning False will result in no future watches # being triggered. If isWatching is True, then # the future watches will be triggered. return isWatching
def get_all_zk_state_managers(conf): """ Connects to all the zookeeper state_managers and returns the connected state_managers instances. """ state_managers = [] state_locations = conf.get_state_locations_of_type("zookeeper") for location in state_locations: name = location['name'] host = location['host'] port = location['port'] tunnelhost = location['tunnelhost'] rootpath = location['rootpath'] LOG.info("Connecting to zk hostport: " + host + ":" + str(port) + " rootpath: " + rootpath) state_manager = ZkStateManager(name, host, port, rootpath, tunnelhost) try: state_manager.start() except Exception as e: LOG.error("Exception while connecting to state_manager.") traceback.print_exc() state_managers.append(state_manager) return state_managers
def delete_pplan(self, topologyName): """ delete physical plan info """ path = self.get_pplan_path(topologyName) LOG.info("Removing topology: {0} from path: {1}".format( topologyName, path)) try: self.client.delete(path) return True except NoNodeError: raise_( StateException("NoNodeError while deleting pplan", StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2]) except NotEmptyError: raise_( StateException("NotEmptyError while deleting pplan", StateException.EX_TYPE_NOT_EMPTY_ERROR), sys.exc_info()[2]) except ZookeeperError: raise_( StateException("Zookeeper while deleting pplan", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2])
def on_connection_change(state): """ callback to log """ LOG.info("Connection state changed to: " + state)
def on_connection_change(state): LOG.info("Connection state changed to: " + state)