def __init__(self, name, conf, client=None): super(ZookeeperJobBoard, self).__init__(name) self._conf = conf if client is not None: self._client = client self._owned = False else: self._client = kazoo_utils.make_client(self._conf) self._owned = True path = str(conf.get("path", "/taskflow/jobs")) if not path: raise ValueError("Empty zookeeper path is disallowed") if not k_paths.isabs(path): raise ValueError("Zookeeper path must be absolute") self._path = path # The backend to load the full logbooks from, since whats sent over # the zookeeper data connection is only the logbook uuid and name, and # not currently the full logbook (later when a zookeeper backend # appears we can likely optimize for that backend usage by directly # reading from the path where the data is stored, if we want). self._persistence = self._conf.get("persistence") # Misc. internal details self._known_jobs = {} self._job_mutate = self._client.handler.rlock_object() self._open_close_lock = self._client.handler.rlock_object() self._client.add_listener(self._state_change_listener) self._bad_paths = frozenset([path]) self._job_watcher = None
def __init__(self, name, conf, client=None, persistence=None, emit_notifications=True): super(ZookeeperJobBoard, self).__init__(name, conf) if client is not None: self._client = client self._owned = False else: self._client = kazoo_utils.make_client(self._conf) self._owned = True path = str(conf.get("path", "/taskflow/jobs")) if not path: raise ValueError("Empty zookeeper path is disallowed") if not k_paths.isabs(path): raise ValueError("Zookeeper path must be absolute") self._path = path # The backend to load the full logbooks from, since whats sent over # the zookeeper data connection is only the logbook uuid and name, and # not currently the full logbook (later when a zookeeper backend # appears we can likely optimize for that backend usage by directly # reading from the path where the data is stored, if we want). self._persistence = persistence # Misc. internal details self._known_jobs = {} self._job_lock = threading.RLock() self._job_cond = threading.Condition(self._job_lock) self._open_close_lock = threading.RLock() self._client.add_listener(self._state_change_listener) self._bad_paths = frozenset([path]) self._job_watcher = None # Since we use sequenced ids this will be the path that the sequences # are prefixed with, for example, job0000000001, job0000000002, ... self._job_base = k_paths.join(path, JOB_PREFIX) self._worker = None self._emit_notifications = bool(emit_notifications)
def __init__(self, name, conf, client=None, persistence=None, emit_notifications=True): super(ZookeeperJobBoard, self).__init__(name, conf) if client is not None: self._client = client self._owned = False else: self._client = kazoo_utils.make_client(self._conf) self._owned = True path = str(conf.get("path", self.DEFAULT_PATH)) if not path: raise ValueError("Empty zookeeper path is disallowed") if not k_paths.isabs(path): raise ValueError("Zookeeper path must be absolute") self._path = path self._trash_path = self._path.replace(k_paths.basename(self._path), self.TRASH_FOLDER) # The backend to load the full logbooks from, since what is sent over # the data connection is only the logbook uuid and name, and not the # full logbook. self._persistence = persistence # Misc. internal details self._known_jobs = {} self._job_cond = threading.Condition() self._open_close_lock = threading.RLock() self._client.add_listener(self._state_change_listener) self._bad_paths = frozenset([path]) self._job_watcher = None # Since we use sequenced ids this will be the path that the sequences # are prefixed with, for example, job0000000001, job0000000002, ... self._job_base = k_paths.join(path, self.JOB_PREFIX) self._worker = None self._emit_notifications = bool(emit_notifications) self._connected = False
def __init__(self, conf, client=None): super(ZkBackend, self).__init__(conf) if not paths.isabs(self._path): raise ValueError("Zookeeper path must be absolute") if client is not None: self._client = client self._owned = False else: self._client = k_utils.make_client(self._conf) self._owned = True self._validated = False
def main(): conf = {} if len(sys.argv) > 1: conf['hosts'] = sys.argv[1:] with finalize_client(kazoo_utils.make_client(conf)) as client: client.start(timeout=1.0) children = list(iter_children(client, "/taskflow")) for child_path in reversed(children): if not re.match(r"^/taskflow/(.*?)-test/(.*)$", child_path): continue print("Deleting %s" % child_path) client.delete(child_path)
def clean_backend(backend, conf): with contextlib.closing(backend.get_connection()) as conn: try: conn.clear_all() except exc.NotFound: pass client = kazoo_utils.make_client(conf) client.start() try: client.delete(conf['path'], recursive=True) except kazoo_exceptions.NoNodeError: pass finally: kazoo_utils.finalize_client(client)
def __init__(self, conf, client=None): super(ZkBackend, self).__init__(conf) path = str(conf.get("path", "/taskflow")) if not path: raise ValueError("Empty zookeeper path is disallowed") if not paths.isabs(path): raise ValueError("Zookeeper path must be absolute") self._path = path if client is not None: self._client = client self._owned = False else: self._client = k_utils.make_client(conf) self._owned = True self._validated = False
def create_board(self, persistence=None): def cleanup_path(client, path): if not client.connected: return client.delete(path, recursive=True) client = kazoo_utils.make_client(test_utils.ZK_TEST_CONFIG.copy()) path = TEST_PATH_TPL % (uuidutils.generate_uuid()) board = impl_zookeeper.ZookeeperJobBoard('test-board', {'path': path}, client=client, persistence=persistence) self.addCleanup(self.close_client, client) self.addCleanup(cleanup_path, client, path) self.addCleanup(board.close) return (client, board)
def _create_board(self, persistence=None): def cleanup_path(client, path): if not client.connected: return client.delete(path, recursive=True) client = kazoo_utils.make_client(test_utils.ZK_TEST_CONFIG.copy()) path = TEST_PATH_TPL % (uuidutils.generate_uuid()) board = impl_zookeeper.ZookeeperJobBoard('test-board', {'path': path}, client=client, persistence=persistence) self.addCleanup(kazoo_utils.finalize_client, client) self.addCleanup(cleanup_path, client, path) self.addCleanup(board.close) return (client, board)
def zookeeper_available(min_version, timeout=3): client = kazoo_utils.make_client(ZK_TEST_CONFIG.copy()) try: # NOTE(imelnikov): 3 seconds we should be enough for localhost client.start(timeout=float(timeout)) if min_version: zk_ver = client.server_version() if zk_ver >= min_version: return True else: return False else: return True except Exception: return False finally: kazoo_utils.finalize_client(client)
def _zookeeper_available(): client = kazoo_utils.make_client(TEST_CONFIG) try: client.start() zk_ver = client.server_version() if zk_ver >= impl_zookeeper.MIN_ZK_VERSION: return True else: return False except Exception: return False finally: try: client.stop() client.close() except Exception: pass
def __init__(self, name, conf, client=None, persistence=None, emit_notifications=True): super(ZookeeperJobBoard, self).__init__(name, conf) if client is not None: self._client = client self._owned = False else: self._client = kazoo_utils.make_client(self._conf) self._owned = True path = str(conf.get("path", self.DEFAULT_PATH)) if not path: raise ValueError("Empty zookeeper path is disallowed") if not k_paths.isabs(path): raise ValueError("Zookeeper path must be absolute") self._path = path self._trash_path = self._path.replace(k_paths.basename(self._path), self.TRASH_FOLDER) self._entity_path = self._path.replace(k_paths.basename(self._path), self.ENTITY_FOLDER) # The backend to load the full logbooks from, since what is sent over # the data connection is only the logbook uuid and name, and not the # full logbook. self._persistence = persistence # Misc. internal details self._known_jobs = {} self._job_cond = threading.Condition() self._open_close_lock = threading.RLock() self._client.add_listener(self._state_change_listener) self._bad_paths = frozenset([path]) self._job_watcher = None # Since we use sequenced ids this will be the path that the sequences # are prefixed with, for example, job0000000001, job0000000002, ... self._job_base = k_paths.join(path, self.JOB_PREFIX) self._worker = None self._emit_notifications = bool(emit_notifications) self._connected = False self._suspended = False self._closing = False self._last_states = collections.deque(maxlen=self.STATE_HISTORY_LENGTH)