def __init__(self, persistImpl, logger=None, lock=SharedData()): """ create an empty queue @param logger a logger to use for warnings and failures @param lock a lock to use for locking multiple queues together. """ # confirm with caller this we are not instantiating this "abstract" # class directly BlackboardItemQueue.__init__(self, True) # we will make certain methods thread-safe LockProtected.__init__(self, lock) # the logger to use self._log = logger # the in-memory copy of the queue self._memq = InMemoryBlackboardQueue() # the disk-persisted copy of the queue self._dskq = persistImpl # load the disk queue into memory self._syncWithDisk() # the list of actions in the current transaction self._pending = None # the "roll-back queue" representing the pre-transaction state self._rbq = None
def __init__(self, dbdir, logger=None, lock=SharedData()): """ create an empty queue @param dbdir the directory to persist the queue items to @param logger a logger to use for warnings and failures @param lock a lock to use for locking multiple queues together. """ persistq = _PolicyBlackboardQueue(dbdir, logger) TransactionalBlackboardQueue.__init__(self, persistq, logger, lock)
class MultiThreadTestCase(unittest.TestCase): def setUp(self): self.sd = SharedData(False, { "c": 0 }) def tearDown(self): pass def testThreads(self): t = TstThread(self.sd) # pdb.set_trace() self.assertEquals(self.sd.c, 0) # print "c = ", self.sd.c with self.sd: t.start() # now take a turn self.sd.wait() # print "c = ", self.sd.c self.assertEquals(self.sd.c, 1) # print "WILL NOTIFY" self.sd.notifyAll() # time.sleep(1.01) with self.sd: # print "WILL WAIT" self.sd.wait(2.0) self.assertEquals(self.sd.c, 0)
class MultiThreadTestCase(unittest.TestCase): def setUp(self): self.sd = SharedData(False, {"c": 0}) def testThreads(self): t = TstThread(self.sd) # pdb.set_trace() self.assertEqual(self.sd.c, 0) # print "c = ", self.sd.c with self.sd: t.start() # now take a turn self.sd.wait() # print "c = ", self.sd.c self.assertEqual(self.sd.c, 1) # print "WILL NOTIFY" self.sd.notifyAll() # time.sleep(1.01) with self.sd: # print "WILL WAIT" self.sd.wait(2.0) self.assertEqual(self.sd.c, 0)
def __init__(self, dbdir, formatter, logger=None): """ a directory containing the queue items """ BlackboardItemQueue.__init__(self, True) # the logger to send messages to (if provided) self._log = logger # the parent directory where items are stored self._dbdir = dbdir parent = os.path.dirname(self._dbdir) if not os.path.isdir(parent) or not os.path.exists(parent): raise BlackboardPersistError( "Unable to create queue directory: %s: directory not found" % parent) if not os.path.exists(self._dbdir): os.mkdir(self._dbdir) elif not os.path.isdir(self._dbdir): raise BlackboardAccessError( "Queue directory: %s: not a directory" % self._dbdir) if not os.path.isdir(self._dbdir) or not os.path.exists(self._dbdir): raise BlackboardAccessError( "Persistence directory does not exist: " + self._dbdir) # the file that keeps the order self._orderfile = os.path.join(self._dbdir, self.orderFilename) # the object that handles reading and writing item files self._formatter = formatter # setup up a lockable data object self._sd = SharedData(False, {"files": []}) # a cache of the ordered list of files with self._sd: self._sd.files = self._loadOrder() self._syncOrder()
def setUp(self): self.sd = SharedData()
class ShareDataTestCase(unittest.TestCase): def setUp(self): self.sd = SharedData() def tearDown(self): pass def testCtor(self): pass def testAcquire(self): self.assert_(not self.sd._is_owned(), "lock without acquire") self.sd.acquire() self.assert_(self.sd._is_owned(), "lock not acquired") self.sd.acquire() self.assert_(self.sd._is_owned(), "lock not kept") self.sd.release() self.assert_(self.sd._is_owned(),"lock not kept after partial release") self.sd.release() self.assert_(not self.sd._is_owned(), "lock not released") def testNoData(self): attrs = self.sd.dir() self.assertEquals(len(attrs), 0, "Non-empty protected dir: " + str(attrs)) self.sd.acquire() self.assert_(self.sd.__, "__ not set") self.sd.release() def testWith(self): with self.sd: self.assert_(self.sd._is_owned(), "lock not acquired") self.assert_(not self.sd._is_owned(), "lock not released") def _initData(self): self.sd.initData({ "name": "Ray", "test": True, "config": {} }) def testNoLockRead(self): self._initData() try: self.sd.name self.fail("AttributeError not raised for reading w/o lock ") except AttributeError, e: pass self.sd.dir()
def setUp(self): self.sd = SharedData(False, { "c": 0 })
class ShareDataTestCase(unittest.TestCase): def setUp(self): self.sd = SharedData() def testAcquire(self): self.assertFalse(self.sd._is_owned(), "lock without acquire") self.sd.acquire() self.assertTrue(self.sd._is_owned(), "lock not acquired") self.sd.acquire() self.assertTrue(self.sd._is_owned(), "lock not kept") self.sd.release() self.assertTrue(self.sd._is_owned(), "lock not kept after partial release") self.sd.release() self.assertFalse(self.sd._is_owned(), "lock not released") def testNoData(self): attrs = self.sd.dir() self.assertEqual(len(attrs), 0, "Non-empty protected dir: " + str(attrs)) self.sd.acquire() self.assertTrue(self.sd.__, "__ not set") self.sd.release() def testWith(self): with self.sd: self.assertTrue(self.sd._is_owned(), "lock not acquired") self.assertFalse(self.sd._is_owned(), "lock not released") def _initData(self): self.sd.initData({"name": "Ray", "test": True, "config": {}}) def testNoLockRead(self): self._initData() try: self.sd.name self.fail("AttributeError not raised for reading w/o lock ") except AttributeError: pass self.sd.dir() def testInit(self): self._initData() attrs = self.sd.dir() self.assertEqual(len(attrs), 3, "Wrong number of items: " + str(attrs)) for key in "name test config".split(): self.assertIn(key, attrs, "Missing attr: " + key) def testAccess(self): self._initData() protected = None with self.sd: self.assertEqual(self.sd.name, "Ray") self.assertIsInstance(self.sd.test, bool) self.assertTrue(self.sd.test) protected = self.sd.config self.assertIsInstance(protected, dict) self.assertEqual(len(protected), 0) # test unsafe access protected["goob"] = "gurn" with self.sd: self.assertEqual(self.sd.config["goob"], "gurn") def testUpdate(self): self._initData() with self.sd: self.sd.name = "Plante" self.assertEqual(self.sd.name, "Plante") attrs = self.sd.dir() self.assertEqual(len(attrs), 3, "Wrong number of items: " + str(attrs)) def testAdd(self): self._initData() with self.sd: self.sd.lname = "Plante" attrs = self.sd.dir() self.assertEqual(len(attrs), 4, "Wrong number of items: " + str(attrs)) self.assertEqual(self.sd.lname, "Plante")
def setUp(self): self.sd = SharedData(False, {"c": 0})
def __init__(self, persistDir, logger=None, lock=SharedData()): """ create an empty blackboard @param persistDir a path to a directory where the blackboard's state can persisted. @param logger the logger to gather error messages @param lock a SharedData instance to be used to protect this instance from simultaneous updates by different threads. If not provided, a default is used. """ LockProtected.__init__(self, lock) # the encompassing persistence directory self._persistDir = persistDir parent = os.path.dirname(self._persistDir) if not os.path.isdir(parent) or not os.path.exists(parent): raise BlackboardPersistError( "Unable to create queue directory: %s: directory not found" % parent) if not os.path.exists(self._persistDir): os.mkdir(self._persistDir) elif not os.path.isdir(self._persistDir): raise BlackboardAccessError( "Queue directory: %s: not a directory" % self._persistDir) # the logger to use if not logger: logger = Log.getDefaultLog() self._log = Log(logger, "blackboard") # prep the queues self.queues = lock with self.queues: # a queue representing available datasets to be processed. dir = os.path.join(self._persistDir, "dataAvailable") self.queues.dataAvailable = DataQueue(dir, self._log, lock) # a queue of datasets that have beend bundled into jobs and queued # for processing. dir = os.path.join(self._persistDir, "jobsPossible") self.queues.jobsPossible = JobQueue(dir, self._log, lock) # a queue of jobs that are ready to be processed dir = os.path.join(self._persistDir, "jobsAvailable") self.queues.jobsAvailable = JobQueue(dir, self._log, lock) # a queue of jobs that have been taken up by a pipeline and are # currently being processed. dir = os.path.join(self._persistDir, "jobsInProgress") self.queues.jobsInProgress = JobQueue(dir, self._log, lock) # a queue for jobs that have been completed by a pipeline dir = os.path.join(self._persistDir, "jobsDone") self.queues.jobsDone = JobQueue(dir, self._log, lock) # a queue of pipelines that are ready to accept a job dir = os.path.join(self._persistDir, "pipelinesReady") self.queues.pipelinesReady = JobQueue(dir, self._log, lock) self._dbfail = 0