def __init__(self, name, conf, client=None):
super(ZookeeperJobBoard, self).__init__(name)
self._conf = conf
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", "/taskflow/jobs"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
# The backend to load the full logbooks from, since whats sent over
# the zookeeper data connection is only the logbook uuid and name, and
# not currently the full logbook (later when a zookeeper backend
# appears we can likely optimize for that backend usage by directly
# reading from the path where the data is stored, if we want).
self._persistence = self._conf.get("persistence")
# Misc. internal details
self._known_jobs = {}
self._job_mutate = self._client.handler.rlock_object()
self._open_close_lock = self._client.handler.rlock_object()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
def __init__(self, name, conf, client=None):
super(ZookeeperJobBoard, self).__init__(name)
self._conf = conf
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", "/taskflow/jobs"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
# The backend to load the full logbooks from, since whats sent over
# the zookeeper data connection is only the logbook uuid and name, and
# not currently the full logbook (later when a zookeeper backend
# appears we can likely optimize for that backend usage by directly
# reading from the path where the data is stored, if we want).
self._persistence = self._conf.get("persistence")
# Misc. internal details
self._known_jobs = {}
self._job_mutate = self._client.handler.rlock_object()
self._open_close_lock = self._client.handler.rlock_object()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
def __init__(self, name, conf,
client=None, persistence=None, emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", "/taskflow/jobs"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
# The backend to load the full logbooks from, since whats sent over
# the zookeeper data connection is only the logbook uuid and name, and
# not currently the full logbook (later when a zookeeper backend
# appears we can likely optimize for that backend usage by directly
# reading from the path where the data is stored, if we want).
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_lock = threading.RLock()
self._job_cond = threading.Condition(self._job_lock)
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
def __init__(self, name, conf,
client=None, persistence=None, emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", "/taskflow/jobs"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
# The backend to load the full logbooks from, since whats sent over
# the zookeeper data connection is only the logbook uuid and name, and
# not currently the full logbook (later when a zookeeper backend
# appears we can likely optimize for that backend usage by directly
# reading from the path where the data is stored, if we want).
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_lock = threading.RLock()
self._job_cond = threading.Condition(self._job_lock)
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
def __init__(self, name, conf,
client=None, persistence=None, emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", self.DEFAULT_PATH))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
self._trash_path = self._path.replace(k_paths.basename(self._path),
self.TRASH_FOLDER)
# The backend to load the full logbooks from, since what is sent over
# the data connection is only the logbook uuid and name, and not the
# full logbook.
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_cond = threading.Condition()
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, self.JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
self._connected = False
def __init__(self, conf, client=None):
super(ZkBackend, self).__init__(conf)
if not paths.isabs(self._path):
raise ValueError("Zookeeper path must be absolute")
if client is not None:
self._client = client
self._owned = False
else:
self._client = k_utils.make_client(self._conf)
self._owned = True
self._validated = False
def __init__(self, conf, client=None):
super(ZkBackend, self).__init__(conf)
if not paths.isabs(self._path):
raise ValueError("Zookeeper path must be absolute")
if client is not None:
self._client = client
self._owned = False
else:
self._client = k_utils.make_client(self._conf)
self._owned = True
self._validated = False
def main():
conf = {}
if len(sys.argv) > 1:
conf['hosts'] = sys.argv[1:]
with finalize_client(kazoo_utils.make_client(conf)) as client:
client.start(timeout=1.0)
children = list(iter_children(client, "/taskflow"))
for child_path in reversed(children):
if not re.match(r"^/taskflow/(.*?)-test/(.*)$", child_path):
continue
print("Deleting %s" % child_path)
client.delete(child_path)
def clean_backend(backend, conf):
with contextlib.closing(backend.get_connection()) as conn:
try:
conn.clear_all()
except exc.NotFound:
pass
client = kazoo_utils.make_client(conf)
client.start()
try:
client.delete(conf['path'], recursive=True)
except kazoo_exceptions.NoNodeError:
pass
finally:
kazoo_utils.finalize_client(client)
def __init__(self, conf, client=None):
super(ZkBackend, self).__init__(conf)
path = str(conf.get("path", "/taskflow"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
if client is not None:
self._client = client
self._owned = False
else:
self._client = k_utils.make_client(conf)
self._owned = True
self._validated = False
def create_board(self, persistence=None):
def cleanup_path(client, path):
if not client.connected:
return
client.delete(path, recursive=True)
client = kazoo_utils.make_client(test_utils.ZK_TEST_CONFIG.copy())
path = TEST_PATH_TPL % (uuidutils.generate_uuid())
board = impl_zookeeper.ZookeeperJobBoard('test-board', {'path': path},
client=client,
persistence=persistence)
self.addCleanup(self.close_client, client)
self.addCleanup(cleanup_path, client, path)
self.addCleanup(board.close)
return (client, board)
def __init__(self, conf, client=None):
super(ZkBackend, self).__init__(conf)
path = str(conf.get("path", "/taskflow"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
if client is not None:
self._client = client
self._owned = False
else:
self._client = k_utils.make_client(conf)
self._owned = True
self._validated = False
def _create_board(self, persistence=None):
def cleanup_path(client, path):
if not client.connected:
return
client.delete(path, recursive=True)
client = kazoo_utils.make_client(test_utils.ZK_TEST_CONFIG.copy())
path = TEST_PATH_TPL % (uuidutils.generate_uuid())
board = impl_zookeeper.ZookeeperJobBoard('test-board', {'path': path},
client=client,
persistence=persistence)
self.addCleanup(kazoo_utils.finalize_client, client)
self.addCleanup(cleanup_path, client, path)
self.addCleanup(board.close)
return (client, board)
def zookeeper_available(min_version, timeout=3):
client = kazoo_utils.make_client(ZK_TEST_CONFIG.copy())
try:
# NOTE(imelnikov): 3 seconds we should be enough for localhost
client.start(timeout=float(timeout))
if min_version:
zk_ver = client.server_version()
if zk_ver >= min_version:
return True
else:
return False
else:
return True
except Exception:
return False
finally:
kazoo_utils.finalize_client(client)
def _zookeeper_available():
client = kazoo_utils.make_client(TEST_CONFIG)
try:
client.start()
zk_ver = client.server_version()
if zk_ver >= impl_zookeeper.MIN_ZK_VERSION:
return True
else:
return False
except Exception:
return False
finally:
try:
client.stop()
client.close()
except Exception:
pass
def zookeeper_available(min_version, timeout=3):
client = kazoo_utils.make_client(ZK_TEST_CONFIG.copy())
try:
# NOTE(imelnikov): 3 seconds we should be enough for localhost
client.start(timeout=float(timeout))
if min_version:
zk_ver = client.server_version()
if zk_ver >= min_version:
return True
else:
return False
else:
return True
except Exception:
return False
finally:
kazoo_utils.finalize_client(client)
def __init__(self,
name,
conf,
client=None,
persistence=None,
emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", self.DEFAULT_PATH))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
self._trash_path = self._path.replace(k_paths.basename(self._path),
self.TRASH_FOLDER)
self._entity_path = self._path.replace(k_paths.basename(self._path),
self.ENTITY_FOLDER)
# The backend to load the full logbooks from, since what is sent over
# the data connection is only the logbook uuid and name, and not the
# full logbook.
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_cond = threading.Condition()
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, self.JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
self._connected = False
self._suspended = False
self._closing = False
self._last_states = collections.deque(maxlen=self.STATE_HISTORY_LENGTH)