def __init__(self, name, conf, client=None):
super(ZookeeperJobBoard, self).__init__(name)
self._conf = conf
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", "/taskflow/jobs"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
# The backend to load the full logbooks from, since whats sent over
# the zookeeper data connection is only the logbook uuid and name, and
# not currently the full logbook (later when a zookeeper backend
# appears we can likely optimize for that backend usage by directly
# reading from the path where the data is stored, if we want).
self._persistence = self._conf.get("persistence")
# Misc. internal details
self._known_jobs = {}
self._job_mutate = self._client.handler.rlock_object()
self._open_close_lock = self._client.handler.rlock_object()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
def __init__(self, name, conf,
client=None, persistence=None, emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", "/taskflow/jobs"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
# The backend to load the full logbooks from, since whats sent over
# the zookeeper data connection is only the logbook uuid and name, and
# not currently the full logbook (later when a zookeeper backend
# appears we can likely optimize for that backend usage by directly
# reading from the path where the data is stored, if we want).
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_lock = threading.RLock()
self._job_cond = threading.Condition(self._job_lock)
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
def __init__(self, name, conf, client=None):
super(ZookeeperJobBoard, self).__init__(name)
self._conf = conf
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", "/taskflow/jobs"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
# The backend to load the full logbooks from, since whats sent over
# the zookeeper data connection is only the logbook uuid and name, and
# not currently the full logbook (later when a zookeeper backend
# appears we can likely optimize for that backend usage by directly
# reading from the path where the data is stored, if we want).
self._persistence = self._conf.get("persistence")
# Misc. internal details
self._known_jobs = {}
self._job_mutate = self._client.handler.rlock_object()
self._open_close_lock = self._client.handler.rlock_object()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
def __init__(self, name, conf,
client=None, persistence=None, emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", "/taskflow/jobs"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
# The backend to load the full logbooks from, since whats sent over
# the zookeeper data connection is only the logbook uuid and name, and
# not currently the full logbook (later when a zookeeper backend
# appears we can likely optimize for that backend usage by directly
# reading from the path where the data is stored, if we want).
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_lock = threading.RLock()
self._job_cond = threading.Condition(self._job_lock)
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
def __init__(self, name, conf,
client=None, persistence=None, emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", self.DEFAULT_PATH))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
self._trash_path = self._path.replace(k_paths.basename(self._path),
self.TRASH_FOLDER)
# The backend to load the full logbooks from, since what is sent over
# the data connection is only the logbook uuid and name, and not the
# full logbook.
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_cond = threading.Condition()
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, self.JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
self._connected = False
def __init__(self, conf, client=None):
super(ZkBackend, self).__init__(conf)
if not paths.isabs(self._path):
raise ValueError("Zookeeper path must be absolute")
if client is not None:
self._client = client
self._owned = False
else:
self._client = k_utils.make_client(self._conf)
self._owned = True
self._validated = False
def __init__(self, conf, client=None):
super(ZkBackend, self).__init__(conf)
if not paths.isabs(self._path):
raise ValueError("Zookeeper path must be absolute")
if client is not None:
self._client = client
self._owned = False
else:
self._client = k_utils.make_client(self._conf)
self._owned = True
self._validated = False
def __init__(self, conf, client=None):
super(ZkBackend, self).__init__(conf)
path = str(conf.get("path", "/taskflow"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
if client is not None:
self._client = client
self._owned = False
else:
self._client = k_utils.make_client(conf)
self._owned = True
self._validated = False
def __init__(self, conf, client=None):
super(ZkBackend, self).__init__(conf)
path = str(conf.get("path", "/taskflow"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
if client is not None:
self._client = client
self._owned = False
else:
self._client = k_utils.make_client(conf)
self._owned = True
self._validated = False
def __init__(self,
name,
conf,
client=None,
persistence=None,
emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", self.DEFAULT_PATH))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
self._trash_path = self._path.replace(k_paths.basename(self._path),
self.TRASH_FOLDER)
self._entity_path = self._path.replace(k_paths.basename(self._path),
self.ENTITY_FOLDER)
# The backend to load the full logbooks from, since what is sent over
# the data connection is only the logbook uuid and name, and not the
# full logbook.
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_cond = threading.Condition()
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, self.JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
self._connected = False
self._suspended = False
self._closing = False
self._last_states = collections.deque(maxlen=self.STATE_HISTORY_LENGTH)
def test_isabs_false(self):
self.assertFalse(paths.isabs(''))
self.assertFalse(paths.isabs('a/'))
self.assertFalse(paths.isabs('a/../'))
def test_isabs(self):
self.assertTrue(paths.isabs('/'))
self.assertTrue(paths.isabs('/a'))
self.assertTrue(paths.isabs('/a//b/c'))
self.assertTrue(paths.isabs('//a/b'))
def test_isabs_false(self):
self.assertFalse(paths.isabs(''))
self.assertFalse(paths.isabs('a/'))
self.assertFalse(paths.isabs('a/../'))
def test_isabs(self):
self.assertTrue(paths.isabs('/'))
self.assertTrue(paths.isabs('/a'))
self.assertTrue(paths.isabs('/a//b/c'))
self.assertTrue(paths.isabs('//a/b'))
def test_isabs_false(self):
assert paths.isabs('') is False
assert paths.isabs('a/') is False
assert paths.isabs('a/../') is False
def test_isabs(self):
assert paths.isabs('/') is True
assert paths.isabs('/a') is True
assert paths.isabs('/a//b/c') is True
assert paths.isabs('//a/b') is True