def __init__(self,
name,
board,
client,
backend,
path,
uuid=None,
details=None,
book=None,
book_data=None,
created_on=None):
super(ZookeeperJob, self).__init__(name, uuid=uuid, details=details)
self._board = board
self._book = book
if not book_data:
book_data = {}
self._book_data = book_data
self._client = client
self._backend = backend
if all((self._book, self._book_data)):
raise ValueError("Only one of 'book_data' or 'book'"
" can be provided")
self._path = k_paths.normpath(path)
self._lock_path = path + LOCK_POSTFIX
self._created_on = created_on
self._node_not_found = False
basename = k_paths.basename(self._path)
self._root = self._path[0:-len(basename)]
self._sequence = int(basename[len(JOB_PREFIX):])
def __init__(self,
board,
name,
client,
path,
uuid=None,
details=None,
book=None,
book_data=None,
created_on=None,
backend=None,
priority=base.JobPriority.NORMAL):
super(ZookeeperJob, self).__init__(board,
name,
uuid=uuid,
details=details,
backend=backend,
book=book,
book_data=book_data)
self._client = client
self._path = k_paths.normpath(path)
self._lock_path = self._path + board.LOCK_POSTFIX
self._created_on = created_on
self._node_not_found = False
basename = k_paths.basename(self._path)
self._root = self._path[0:-len(basename)]
self._sequence = int(basename[len(board.JOB_PREFIX):])
self._priority = priority
def __init__(self, name, conf,
client=None, persistence=None, emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", self.DEFAULT_PATH))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
self._trash_path = self._path.replace(k_paths.basename(self._path),
self.TRASH_FOLDER)
# The backend to load the full logbooks from, since what is sent over
# the data connection is only the logbook uuid and name, and not the
# full logbook.
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_cond = threading.Condition()
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, self.JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
self._connected = False
def __init__(self,
name,
conf,
client=None,
persistence=None,
emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", self.DEFAULT_PATH))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
self._trash_path = self._path.replace(k_paths.basename(self._path),
self.TRASH_FOLDER)
self._entity_path = self._path.replace(k_paths.basename(self._path),
self.ENTITY_FOLDER)
# The backend to load the full logbooks from, since what is sent over
# the data connection is only the logbook uuid and name, and not the
# full logbook.
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_cond = threading.Condition()
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, self.JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
self._connected = False
self._suspended = False
self._closing = False
self._last_states = collections.deque(maxlen=self.STATE_HISTORY_LENGTH)
def __init__(self, board, name, client, path,
uuid=None, details=None, book=None, book_data=None,
created_on=None, backend=None):
super(ZookeeperJob, self).__init__(board, name,
uuid=uuid, details=details,
backend=backend,
book=book, book_data=book_data)
self._client = client
self._path = k_paths.normpath(path)
self._lock_path = self._path + board.LOCK_POSTFIX
self._created_on = created_on
self._node_not_found = False
basename = k_paths.basename(self._path)
self._root = self._path[0:-len(basename)]
self._sequence = int(basename[len(board.JOB_PREFIX):])
def __init__(self, name, board, client, backend, path,
uuid=None, details=None, book=None, book_data=None,
created_on=None):
super(ZookeeperJob, self).__init__(name, uuid=uuid, details=details)
self._board = board
self._book = book
if not book_data:
book_data = {}
self._book_data = book_data
self._client = client
self._backend = backend
if all((self._book, self._book_data)):
raise ValueError("Only one of 'book_data' or 'book'"
" can be provided")
self._path = k_paths.normpath(path)
self._lock_path = path + LOCK_POSTFIX
self._created_on = created_on
self._node_not_found = False
basename = k_paths.basename(self._path)
self._root = self._path[0:-len(basename)]
self._sequence = int(basename[len(JOB_PREFIX):])
def __init__(self,
name,
conf,
client=None,
persistence=None,
emit_notifications=True):
super(ZookeeperJobBoard, self).__init__(name, conf)
if client is not None:
self._client = client
self._owned = False
else:
self._client = kazoo_utils.make_client(self._conf)
self._owned = True
path = str(conf.get("path", "/taskflow/jobs"))
if not path:
raise ValueError("Empty zookeeper path is disallowed")
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
self._trash_path = self._path.replace(k_paths.basename(self._path),
TRASH_FOLDER)
# The backend to load the full logbooks from, since whats sent over
# the zookeeper data connection is only the logbook uuid and name, and
# not currently the full logbook (later when a zookeeper backend
# appears we can likely optimize for that backend usage by directly
# reading from the path where the data is stored, if we want).
self._persistence = persistence
# Misc. internal details
self._known_jobs = {}
self._job_cond = threading.Condition()
self._open_close_lock = threading.RLock()
self._client.add_listener(self._state_change_listener)
self._bad_paths = frozenset([path])
self._job_watcher = None
# Since we use sequenced ids this will be the path that the sequences
# are prefixed with, for example, job0000000001, job0000000002, ...
self._job_base = k_paths.join(path, JOB_PREFIX)
self._worker = None
self._emit_notifications = bool(emit_notifications)
self._connected = False
def test_basename(self):
self.assertEquals(paths.basename(''), '')
self.assertEquals(paths.basename('/'), '')
self.assertEquals(paths.basename('//a'), 'a')
self.assertEquals(paths.basename('//a/'), '')
self.assertEquals(paths.basename('/a/b.//c..'), 'c..')
def test_basename(self):
self.assertEquals(paths.basename(''), '')
self.assertEquals(paths.basename('/'), '')
self.assertEquals(paths.basename('//a'), 'a')
self.assertEquals(paths.basename('//a/'), '')
self.assertEquals(paths.basename('/a/b.//c..'), 'c..')
def test_basename(self):
assert paths.basename('') == ''
assert paths.basename('/') == ''
assert paths.basename('//a') == 'a'
assert paths.basename('//a/') == ''
assert paths.basename('/a/b.//c..') == 'c..'