Example #1
0
 def __init__(self,
              name,
              board,
              client,
              backend,
              path,
              uuid=None,
              details=None,
              book=None,
              book_data=None,
              created_on=None):
     super(ZookeeperJob, self).__init__(name, uuid=uuid, details=details)
     self._board = board
     self._book = book
     if not book_data:
         book_data = {}
     self._book_data = book_data
     self._client = client
     self._backend = backend
     if all((self._book, self._book_data)):
         raise ValueError("Only one of 'book_data' or 'book'"
                          " can be provided")
     self._path = k_paths.normpath(path)
     self._lock_path = path + LOCK_POSTFIX
     self._created_on = created_on
     self._node_not_found = False
     basename = k_paths.basename(self._path)
     self._root = self._path[0:-len(basename)]
     self._sequence = int(basename[len(JOB_PREFIX):])
Example #2
0
 def __init__(self,
              board,
              name,
              client,
              path,
              uuid=None,
              details=None,
              book=None,
              book_data=None,
              created_on=None,
              backend=None,
              priority=base.JobPriority.NORMAL):
     super(ZookeeperJob, self).__init__(board,
                                        name,
                                        uuid=uuid,
                                        details=details,
                                        backend=backend,
                                        book=book,
                                        book_data=book_data)
     self._client = client
     self._path = k_paths.normpath(path)
     self._lock_path = self._path + board.LOCK_POSTFIX
     self._created_on = created_on
     self._node_not_found = False
     basename = k_paths.basename(self._path)
     self._root = self._path[0:-len(basename)]
     self._sequence = int(basename[len(board.JOB_PREFIX):])
     self._priority = priority
Example #3
0
 def __init__(self, name, conf,
              client=None, persistence=None, emit_notifications=True):
     super(ZookeeperJobBoard, self).__init__(name, conf)
     if client is not None:
         self._client = client
         self._owned = False
     else:
         self._client = kazoo_utils.make_client(self._conf)
         self._owned = True
     path = str(conf.get("path", self.DEFAULT_PATH))
     if not path:
         raise ValueError("Empty zookeeper path is disallowed")
     if not k_paths.isabs(path):
         raise ValueError("Zookeeper path must be absolute")
     self._path = path
     self._trash_path = self._path.replace(k_paths.basename(self._path),
                                           self.TRASH_FOLDER)
     # The backend to load the full logbooks from, since what is sent over
     # the data connection is only the logbook uuid and name, and not the
     # full logbook.
     self._persistence = persistence
     # Misc. internal details
     self._known_jobs = {}
     self._job_cond = threading.Condition()
     self._open_close_lock = threading.RLock()
     self._client.add_listener(self._state_change_listener)
     self._bad_paths = frozenset([path])
     self._job_watcher = None
     # Since we use sequenced ids this will be the path that the sequences
     # are prefixed with, for example, job0000000001, job0000000002, ...
     self._job_base = k_paths.join(path, self.JOB_PREFIX)
     self._worker = None
     self._emit_notifications = bool(emit_notifications)
     self._connected = False
Example #4
0
 def __init__(self,
              name,
              conf,
              client=None,
              persistence=None,
              emit_notifications=True):
     super(ZookeeperJobBoard, self).__init__(name, conf)
     if client is not None:
         self._client = client
         self._owned = False
     else:
         self._client = kazoo_utils.make_client(self._conf)
         self._owned = True
     path = str(conf.get("path", self.DEFAULT_PATH))
     if not path:
         raise ValueError("Empty zookeeper path is disallowed")
     if not k_paths.isabs(path):
         raise ValueError("Zookeeper path must be absolute")
     self._path = path
     self._trash_path = self._path.replace(k_paths.basename(self._path),
                                           self.TRASH_FOLDER)
     self._entity_path = self._path.replace(k_paths.basename(self._path),
                                            self.ENTITY_FOLDER)
     # The backend to load the full logbooks from, since what is sent over
     # the data connection is only the logbook uuid and name, and not the
     # full logbook.
     self._persistence = persistence
     # Misc. internal details
     self._known_jobs = {}
     self._job_cond = threading.Condition()
     self._open_close_lock = threading.RLock()
     self._client.add_listener(self._state_change_listener)
     self._bad_paths = frozenset([path])
     self._job_watcher = None
     # Since we use sequenced ids this will be the path that the sequences
     # are prefixed with, for example, job0000000001, job0000000002, ...
     self._job_base = k_paths.join(path, self.JOB_PREFIX)
     self._worker = None
     self._emit_notifications = bool(emit_notifications)
     self._connected = False
     self._suspended = False
     self._closing = False
     self._last_states = collections.deque(maxlen=self.STATE_HISTORY_LENGTH)
Example #5
0
 def __init__(self, board, name, client, path,
              uuid=None, details=None, book=None, book_data=None,
              created_on=None, backend=None):
     super(ZookeeperJob, self).__init__(board, name,
                                        uuid=uuid, details=details,
                                        backend=backend,
                                        book=book, book_data=book_data)
     self._client = client
     self._path = k_paths.normpath(path)
     self._lock_path = self._path + board.LOCK_POSTFIX
     self._created_on = created_on
     self._node_not_found = False
     basename = k_paths.basename(self._path)
     self._root = self._path[0:-len(basename)]
     self._sequence = int(basename[len(board.JOB_PREFIX):])
Example #6
0
 def __init__(self, name, board, client, backend, path,
              uuid=None, details=None, book=None, book_data=None,
              created_on=None):
     super(ZookeeperJob, self).__init__(name, uuid=uuid, details=details)
     self._board = board
     self._book = book
     if not book_data:
         book_data = {}
     self._book_data = book_data
     self._client = client
     self._backend = backend
     if all((self._book, self._book_data)):
         raise ValueError("Only one of 'book_data' or 'book'"
                          " can be provided")
     self._path = k_paths.normpath(path)
     self._lock_path = path + LOCK_POSTFIX
     self._created_on = created_on
     self._node_not_found = False
     basename = k_paths.basename(self._path)
     self._root = self._path[0:-len(basename)]
     self._sequence = int(basename[len(JOB_PREFIX):])
Example #7
0
 def __init__(self,
              name,
              conf,
              client=None,
              persistence=None,
              emit_notifications=True):
     super(ZookeeperJobBoard, self).__init__(name, conf)
     if client is not None:
         self._client = client
         self._owned = False
     else:
         self._client = kazoo_utils.make_client(self._conf)
         self._owned = True
     path = str(conf.get("path", "/taskflow/jobs"))
     if not path:
         raise ValueError("Empty zookeeper path is disallowed")
     if not k_paths.isabs(path):
         raise ValueError("Zookeeper path must be absolute")
     self._path = path
     self._trash_path = self._path.replace(k_paths.basename(self._path),
                                           TRASH_FOLDER)
     # The backend to load the full logbooks from, since whats sent over
     # the zookeeper data connection is only the logbook uuid and name, and
     # not currently the full logbook (later when a zookeeper backend
     # appears we can likely optimize for that backend usage by directly
     # reading from the path where the data is stored, if we want).
     self._persistence = persistence
     # Misc. internal details
     self._known_jobs = {}
     self._job_cond = threading.Condition()
     self._open_close_lock = threading.RLock()
     self._client.add_listener(self._state_change_listener)
     self._bad_paths = frozenset([path])
     self._job_watcher = None
     # Since we use sequenced ids this will be the path that the sequences
     # are prefixed with, for example, job0000000001, job0000000002, ...
     self._job_base = k_paths.join(path, JOB_PREFIX)
     self._worker = None
     self._emit_notifications = bool(emit_notifications)
     self._connected = False
Example #8
0
 def test_basename(self):
     self.assertEquals(paths.basename(''), '')
     self.assertEquals(paths.basename('/'), '')
     self.assertEquals(paths.basename('//a'), 'a')
     self.assertEquals(paths.basename('//a/'), '')
     self.assertEquals(paths.basename('/a/b.//c..'), 'c..')
Example #9
0
 def test_basename(self):
     self.assertEquals(paths.basename(''), '')
     self.assertEquals(paths.basename('/'), '')
     self.assertEquals(paths.basename('//a'), 'a')
     self.assertEquals(paths.basename('//a/'), '')
     self.assertEquals(paths.basename('/a/b.//c..'), 'c..')
Example #10
0
 def test_basename(self):
     assert paths.basename('') == ''
     assert paths.basename('/') == ''
     assert paths.basename('//a') == 'a'
     assert paths.basename('//a/') == ''
     assert paths.basename('/a/b.//c..') == 'c..'