def __init__(self, persistImpl, logger=None, lock=SharedData()):
"""
create an empty queue
@param logger a logger to use for warnings and failures
@param lock a lock to use for locking multiple queues together.
"""
# confirm with caller this we are not instantiating this "abstract"
# class directly
BlackboardItemQueue.__init__(self, True)
# we will make certain methods thread-safe
LockProtected.__init__(self, lock)
# the logger to use
self._log = logger
# the in-memory copy of the queue
self._memq = InMemoryBlackboardQueue()
# the disk-persisted copy of the queue
self._dskq = persistImpl
# load the disk queue into memory
self._syncWithDisk()
# the list of actions in the current transaction
self._pending = None
# the "roll-back queue" representing the pre-transaction state
self._rbq = None
def __init__(self, dbdir, logger=None, lock=SharedData()):
"""
create an empty queue
@param dbdir the directory to persist the queue items to
@param logger a logger to use for warnings and failures
@param lock a lock to use for locking multiple queues together.
"""
persistq = _PolicyBlackboardQueue(dbdir, logger)
TransactionalBlackboardQueue.__init__(self, persistq, logger, lock)
class MultiThreadTestCase(unittest.TestCase):
def setUp(self):
self.sd = SharedData(False, { "c": 0 })
def tearDown(self):
pass
def testThreads(self):
t = TstThread(self.sd)
# pdb.set_trace()
self.assertEquals(self.sd.c, 0)
# print "c = ", self.sd.c
with self.sd:
t.start()
# now take a turn
self.sd.wait()
# print "c = ", self.sd.c
self.assertEquals(self.sd.c, 1)
# print "WILL NOTIFY"
self.sd.notifyAll()
# time.sleep(1.01)
with self.sd:
# print "WILL WAIT"
self.sd.wait(2.0)
self.assertEquals(self.sd.c, 0)
class MultiThreadTestCase(unittest.TestCase):
def setUp(self):
self.sd = SharedData(False, {"c": 0})
def testThreads(self):
t = TstThread(self.sd)
# pdb.set_trace()
self.assertEqual(self.sd.c, 0)
# print "c = ", self.sd.c
with self.sd:
t.start()
# now take a turn
self.sd.wait()
# print "c = ", self.sd.c
self.assertEqual(self.sd.c, 1)
# print "WILL NOTIFY"
self.sd.notifyAll()
# time.sleep(1.01)
with self.sd:
# print "WILL WAIT"
self.sd.wait(2.0)
self.assertEqual(self.sd.c, 0)
def __init__(self, dbdir, formatter, logger=None):
"""
a directory containing the queue items
"""
BlackboardItemQueue.__init__(self, True)
# the logger to send messages to (if provided)
self._log = logger
# the parent directory where items are stored
self._dbdir = dbdir
parent = os.path.dirname(self._dbdir)
if not os.path.isdir(parent) or not os.path.exists(parent):
raise BlackboardPersistError(
"Unable to create queue directory: %s: directory not found" %
parent)
if not os.path.exists(self._dbdir):
os.mkdir(self._dbdir)
elif not os.path.isdir(self._dbdir):
raise BlackboardAccessError(
"Queue directory: %s: not a directory" % self._dbdir)
if not os.path.isdir(self._dbdir) or not os.path.exists(self._dbdir):
raise BlackboardAccessError(
"Persistence directory does not exist: " + self._dbdir)
# the file that keeps the order
self._orderfile = os.path.join(self._dbdir, self.orderFilename)
# the object that handles reading and writing item files
self._formatter = formatter
# setup up a lockable data object
self._sd = SharedData(False, {"files": []})
# a cache of the ordered list of files
with self._sd:
self._sd.files = self._loadOrder()
self._syncOrder()
def setUp(self):
self.sd = SharedData()
class ShareDataTestCase(unittest.TestCase):
def setUp(self):
self.sd = SharedData()
def tearDown(self):
pass
def testCtor(self):
pass
def testAcquire(self):
self.assert_(not self.sd._is_owned(), "lock without acquire")
self.sd.acquire()
self.assert_(self.sd._is_owned(), "lock not acquired")
self.sd.acquire()
self.assert_(self.sd._is_owned(), "lock not kept")
self.sd.release()
self.assert_(self.sd._is_owned(),"lock not kept after partial release")
self.sd.release()
self.assert_(not self.sd._is_owned(), "lock not released")
def testNoData(self):
attrs = self.sd.dir()
self.assertEquals(len(attrs), 0,
"Non-empty protected dir: " + str(attrs))
self.sd.acquire()
self.assert_(self.sd.__, "__ not set")
self.sd.release()
def testWith(self):
with self.sd:
self.assert_(self.sd._is_owned(), "lock not acquired")
self.assert_(not self.sd._is_owned(), "lock not released")
def _initData(self):
self.sd.initData({ "name": "Ray", "test": True, "config": {} })
def testNoLockRead(self):
self._initData()
try:
self.sd.name
self.fail("AttributeError not raised for reading w/o lock ")
except AttributeError, e:
pass
self.sd.dir()
def setUp(self):
self.sd = SharedData(False, { "c": 0 })
def setUp(self):
self.sd = SharedData()
class ShareDataTestCase(unittest.TestCase):
def setUp(self):
self.sd = SharedData()
def testAcquire(self):
self.assertFalse(self.sd._is_owned(), "lock without acquire")
self.sd.acquire()
self.assertTrue(self.sd._is_owned(), "lock not acquired")
self.sd.acquire()
self.assertTrue(self.sd._is_owned(), "lock not kept")
self.sd.release()
self.assertTrue(self.sd._is_owned(),
"lock not kept after partial release")
self.sd.release()
self.assertFalse(self.sd._is_owned(), "lock not released")
def testNoData(self):
attrs = self.sd.dir()
self.assertEqual(len(attrs), 0,
"Non-empty protected dir: " + str(attrs))
self.sd.acquire()
self.assertTrue(self.sd.__, "__ not set")
self.sd.release()
def testWith(self):
with self.sd:
self.assertTrue(self.sd._is_owned(), "lock not acquired")
self.assertFalse(self.sd._is_owned(), "lock not released")
def _initData(self):
self.sd.initData({"name": "Ray", "test": True, "config": {}})
def testNoLockRead(self):
self._initData()
try:
self.sd.name
self.fail("AttributeError not raised for reading w/o lock ")
except AttributeError:
pass
self.sd.dir()
def testInit(self):
self._initData()
attrs = self.sd.dir()
self.assertEqual(len(attrs), 3, "Wrong number of items: " + str(attrs))
for key in "name test config".split():
self.assertIn(key, attrs, "Missing attr: " + key)
def testAccess(self):
self._initData()
protected = None
with self.sd:
self.assertEqual(self.sd.name, "Ray")
self.assertIsInstance(self.sd.test, bool)
self.assertTrue(self.sd.test)
protected = self.sd.config
self.assertIsInstance(protected, dict)
self.assertEqual(len(protected), 0)
# test unsafe access
protected["goob"] = "gurn"
with self.sd:
self.assertEqual(self.sd.config["goob"], "gurn")
def testUpdate(self):
self._initData()
with self.sd:
self.sd.name = "Plante"
self.assertEqual(self.sd.name, "Plante")
attrs = self.sd.dir()
self.assertEqual(len(attrs), 3, "Wrong number of items: " + str(attrs))
def testAdd(self):
self._initData()
with self.sd:
self.sd.lname = "Plante"
attrs = self.sd.dir()
self.assertEqual(len(attrs), 4,
"Wrong number of items: " + str(attrs))
self.assertEqual(self.sd.lname, "Plante")
def setUp(self):
self.sd = SharedData(False, {"c": 0})
def __init__(self, persistDir, logger=None, lock=SharedData()):
"""
create an empty blackboard
@param persistDir a path to a directory where the blackboard's
state can persisted.
@param logger the logger to gather error messages
@param lock a SharedData instance to be used to protect this
instance from simultaneous updates by different
threads. If not provided, a default is used.
"""
LockProtected.__init__(self, lock)
# the encompassing persistence directory
self._persistDir = persistDir
parent = os.path.dirname(self._persistDir)
if not os.path.isdir(parent) or not os.path.exists(parent):
raise BlackboardPersistError(
"Unable to create queue directory: %s: directory not found" %
parent)
if not os.path.exists(self._persistDir):
os.mkdir(self._persistDir)
elif not os.path.isdir(self._persistDir):
raise BlackboardAccessError(
"Queue directory: %s: not a directory" % self._persistDir)
# the logger to use
if not logger:
logger = Log.getDefaultLog()
self._log = Log(logger, "blackboard")
# prep the queues
self.queues = lock
with self.queues:
# a queue representing available datasets to be processed.
dir = os.path.join(self._persistDir, "dataAvailable")
self.queues.dataAvailable = DataQueue(dir, self._log, lock)
# a queue of datasets that have beend bundled into jobs and queued
# for processing.
dir = os.path.join(self._persistDir, "jobsPossible")
self.queues.jobsPossible = JobQueue(dir, self._log, lock)
# a queue of jobs that are ready to be processed
dir = os.path.join(self._persistDir, "jobsAvailable")
self.queues.jobsAvailable = JobQueue(dir, self._log, lock)
# a queue of jobs that have been taken up by a pipeline and are
# currently being processed.
dir = os.path.join(self._persistDir, "jobsInProgress")
self.queues.jobsInProgress = JobQueue(dir, self._log, lock)
# a queue for jobs that have been completed by a pipeline
dir = os.path.join(self._persistDir, "jobsDone")
self.queues.jobsDone = JobQueue(dir, self._log, lock)
# a queue of pipelines that are ready to accept a job
dir = os.path.join(self._persistDir, "pipelinesReady")
self.queues.pipelinesReady = JobQueue(dir, self._log, lock)
self._dbfail = 0