def testAddManyTasks(self): ctx = CountingContext() wp = workerpool.WorkerPool("Test", 3, CountingBaseWorker) try: self._CheckWorkerCount(wp, 3) wp.AddManyTasks([( ctx, "Hello world %s" % i, ) for i in range(10)]) wp.AddTask((ctx, "A separate hello")) wp.AddTask((ctx, "Once more, hi!")) wp.AddManyTasks([( ctx, "Hello world %s" % i, ) for i in range(10)]) wp.Quiesce() self._CheckNoTasks(wp) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) self.assertEquals(ctx.GetDoneTasks(), 22)
def testManyTasksSequence(self): ctx = CountingContext() wp = workerpool.WorkerPool("Test", 3, CountingBaseWorker) try: self._CheckWorkerCount(wp, 3) self.assertRaises(AssertionError, wp.AddManyTasks, ["Hello world %s" % i for i in range(10)]) self.assertRaises(AssertionError, wp.AddManyTasks, [i for i in range(10)]) self.assertRaises(AssertionError, wp.AddManyTasks, [], task_id=0) wp.AddManyTasks([( ctx, "Hello world %s" % i, ) for i in range(10)]) wp.AddTask((ctx, "A separate hello")) wp.Quiesce() self._CheckNoTasks(wp) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) self.assertEquals(ctx.GetDoneTasks(), 11)
def testChecksum(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order. wp = workerpool.WorkerPool("Test", 1, ChecksumBaseWorker) try: self._CheckWorkerCount(wp, 1) ctx = ChecksumContext() checksum = ChecksumContext.CHECKSUM_START for i in range(1, 100): checksum = ChecksumContext.UpdateChecksum(checksum, i) wp.AddTask((ctx, i)) wp.Quiesce() self._CheckNoTasks(wp) # Check sum ctx.lock.acquire() try: self.assertEqual(checksum, ctx.checksum) finally: ctx.lock.release() finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0)
def testNoTasks(self): wp = workerpool.WorkerPool("Test", 3, CountingBaseWorker) try: self._CheckWorkerCount(wp, 3) self._CheckNoTasks(wp) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0)
def testPriorityListManyTasks(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order and respects the priority wp = workerpool.WorkerPool("Test", 1, ListBuilderWorker) try: self._CheckWorkerCount(wp, 1) ctx = ListBuilderContext() # Use static seed for this test rnd = random.Random(0) data = {} tasks = [] priorities = [] for i in range(1, 333): prio = int(rnd.random() * 10) tasks.append((ctx, i)) priorities.append(prio) data.setdefault(prio, []).append((prio, i)) wp.AddManyTasks(tasks, priority=priorities) self.assertRaises(errors.ProgrammerError, wp.AddManyTasks, [("x", ), ("y", )], priority=[1] * 5) self.assertRaises(errors.ProgrammerError, wp.AddManyTasks, [("x", ), ("y", )], task_id=[1] * 5) wp.Quiesce() self._CheckNoTasks(wp) # Check result ctx.lock.acquire() try: expresult = [] for priority in sorted(data.keys()): expresult.extend(data[priority]) self.assertEqual(expresult, ctx.result) finally: ctx.lock.release() self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0)
def testCounting(self): ctx = CountingContext() wp = workerpool.WorkerPool("Test", 3, CountingBaseWorker) try: self._CheckWorkerCount(wp, 3) for i in range(10): wp.AddTask((ctx, "Hello world %s" % i)) wp.Quiesce() finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) self.assertEquals(ctx.GetDoneTasks(), 10)
def testActive(self): ctx = CountingContext() wp = workerpool.WorkerPool("TestActive", 5, CountingBaseWorker) try: self._CheckWorkerCount(wp, 5) self.assertTrue(wp._active) # Process some tasks for _ in range(10): wp.AddTask((ctx, None)) wp.Quiesce() self._CheckNoTasks(wp) self.assertEquals(ctx.GetDoneTasks(), 10) # Repeat a few times for count in range(10): # Deactivate pool wp.SetActive(False) self._CheckNoTasks(wp) # Queue some more tasks for _ in range(10): wp.AddTask((ctx, None)) for _ in range(5): # Short delays to give other threads a chance to cause breakage time.sleep(.01) wp.AddTask((ctx, "Hello world %s" % 999)) self.assertFalse(wp._active) self.assertEquals(ctx.GetDoneTasks(), 10 + (count * 15)) # Start processing again wp.SetActive(True) self.assertTrue(wp._active) # Wait for tasks to finish wp.Quiesce() self._CheckNoTasks(wp) self.assertEquals(ctx.GetDoneTasks(), 10 + (count * 15) + 15) self._CheckWorkerCount(wp, 5) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0)
def testPriorityChecksum(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order and respects the priority wp = workerpool.WorkerPool("Test", 1, ChecksumBaseWorker) try: self._CheckWorkerCount(wp, 1) ctx = ChecksumContext() data = {} tasks = [] priorities = [] for i in range(1, 333): prio = i % 7 tasks.append((ctx, i)) priorities.append(prio) data.setdefault(prio, []).append(i) wp.AddManyTasks(tasks, priority=priorities) wp.Quiesce() self._CheckNoTasks(wp) # Check sum ctx.lock.acquire() try: checksum = ChecksumContext.CHECKSUM_START for priority in sorted(data.keys()): for i in data[priority]: checksum = ChecksumContext.UpdateChecksum(checksum, i) self.assertEqual(checksum, ctx.checksum) finally: ctx.lock.release() self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0)
def testPriorityListSingleTasks(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order and respects the priority wp = workerpool.WorkerPool("Test", 1, ListBuilderWorker) try: self._CheckWorkerCount(wp, 1) ctx = ListBuilderContext() # Use static seed for this test rnd = random.Random(26279) data = {} for i in range(1, 333): prio = int(rnd.random() * 30) wp.AddTask((ctx, i), priority=prio) data.setdefault(prio, []).append(i) # Cause some distortion if i % 11 == 0: time.sleep(.001) if i % 41 == 0: wp.Quiesce() wp.Quiesce() self._CheckNoTasks(wp) # Check result ctx.lock.acquire() try: self.assertEqual(data, ctx.prioresult) finally: ctx.lock.release() self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0)
def testDeferTask(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order and respects the priority wp = workerpool.WorkerPool("Test", 1, DeferringWorker) try: self._CheckWorkerCount(wp, 1) ctx = DeferringTaskContext() # Use static seed for this test rnd = random.Random(14921) data = {} num2taskid = {} for i in range(1, 333): ctx.lock.acquire() try: if i % 5 == 0: ctx.samepriodefer[i] = True finally: ctx.lock.release() prio = int(rnd.random() * 30) num2taskid[i] = 1000 * i wp.AddTask((ctx, i, prio), priority=50, task_id=num2taskid[i]) data.setdefault(prio, set()).add(i) # Cause some distortion if i % 24 == 0: time.sleep(.001) if i % 31 == 0: wp.Quiesce() wp.Quiesce() self._CheckNoTasks(wp) # Check result ctx.lock.acquire() try: self.assertEqual(data, ctx.prioresult) all_order_ids = [] for (num, numordertaskid) in ctx.num2ordertaskid.items(): order_ids = map(compat.fst, numordertaskid) self.assertFalse(utils.FindDuplicates(order_ids), msg="Order ID has been reused") all_order_ids.extend(order_ids) for task_id in map(compat.snd, numordertaskid): self.assertEqual( task_id, num2taskid[num], msg=("Task %s used different task IDs" % num)) self.assertFalse(utils.FindDuplicates(all_order_ids), msg="Order ID has been reused") finally: ctx.lock.release() self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0)
def setup_context(self): self.context = GanetiContext() self.request_workers = workerpool.WorkerPool("ClientReq", CLIENT_REQUEST_WORKERS, ClientRequestWorker)
def testChangeTaskPriorityInteralStructures(self): wp = workerpool.WorkerPool("Test", 1, NotImplementedWorker) try: self._CheckWorkerCount(wp, 1) # Use static seed for this test rnd = random.Random(643) (num1, num2) = rnd.sample(range(1000), 2) # Disable processing of tasks wp.SetActive(False) self.assertFalse(wp._tasks) self.assertFalse(wp._taskdata) # No priority or task ID wp.AddTask(()) self.assertEqual(wp._tasks, [ [workerpool._DEFAULT_PRIORITY, 0, None, ()], ]) self.assertFalse(wp._taskdata) # No task ID wp.AddTask((), priority=7413) self.assertEqual(wp._tasks, [ [workerpool._DEFAULT_PRIORITY, 0, None, ()], [7413, 1, None, ()], ]) self.assertFalse(wp._taskdata) # Start adding real tasks wp.AddTask((), priority=10267659, task_id=num1) self.assertEqual(wp._tasks, [ [workerpool._DEFAULT_PRIORITY, 0, None, ()], [7413, 1, None, ()], [10267659, 2, num1, ()], ]) self.assertEqual(wp._taskdata, { num1: [10267659, 2, num1, ()], }) wp.AddTask((), priority=123, task_id=num2) self.assertEqual(sorted(wp._tasks), [ [workerpool._DEFAULT_PRIORITY, 0, None, ()], [123, 3, num2, ()], [7413, 1, None, ()], [10267659, 2, num1, ()], ]) self.assertEqual(wp._taskdata, { num1: [10267659, 2, num1, ()], num2: [123, 3, num2, ()], }) wp.ChangeTaskPriority(num1, 100) self.assertEqual(sorted(wp._tasks), [ [workerpool._DEFAULT_PRIORITY, 0, None, ()], [100, 2, num1, ()], [123, 3, num2, ()], [7413, 1, None, ()], [10267659, 2, num1, None], ]) self.assertEqual(wp._taskdata, { num1: [100, 2, num1, ()], num2: [123, 3, num2, ()], }) wp.ChangeTaskPriority(num2, 91337) self.assertEqual(sorted(wp._tasks), [ [workerpool._DEFAULT_PRIORITY, 0, None, ()], [100, 2, num1, ()], [123, 3, num2, None], [7413, 1, None, ()], [91337, 3, num2, ()], [10267659, 2, num1, None], ]) self.assertEqual(wp._taskdata, { num1: [100, 2, num1, ()], num2: [91337, 3, num2, ()], }) wp.ChangeTaskPriority(num1, 10139) self.assertEqual(sorted(wp._tasks), [ [workerpool._DEFAULT_PRIORITY, 0, None, ()], [100, 2, num1, None], [123, 3, num2, None], [7413, 1, None, ()], [10139, 2, num1, ()], [91337, 3, num2, ()], [10267659, 2, num1, None], ]) self.assertEqual(wp._taskdata, { num1: [10139, 2, num1, ()], num2: [91337, 3, num2, ()], }) # Change to the same priority once again wp.ChangeTaskPriority(num1, 10139) self.assertEqual(sorted(wp._tasks), [ [workerpool._DEFAULT_PRIORITY, 0, None, ()], [100, 2, num1, None], [123, 3, num2, None], [7413, 1, None, ()], [10139, 2, num1, None], [10139, 2, num1, ()], [91337, 3, num2, ()], [10267659, 2, num1, None], ]) self.assertEqual(wp._taskdata, { num1: [10139, 2, num1, ()], num2: [91337, 3, num2, ()], }) self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0)
def testChangeTaskPriority(self): wp = workerpool.WorkerPool("Test", 1, PriorityWorker) try: self._CheckWorkerCount(wp, 1) ctx = PriorityContext() # Use static seed for this test rnd = random.Random(4727) # Disable processing of tasks wp.SetActive(False) # No task ID self.assertRaises(workerpool.NoSuchTask, wp.ChangeTaskPriority, None, 0) # Pre-generate task IDs and priorities count = 100 task_ids = range(0, count) priorities = range(200, 200 + count) * 2 rnd.shuffle(task_ids) rnd.shuffle(priorities) # Make sure there are some duplicate priorities, but not all priorities[count * 2 - 10:count * 2 - 1] = \ priorities[count - 10: count - 1] assert len(priorities) == 2 * count assert priorities[0:(count - 1)] != priorities[count:(2 * count - 1)] # Add some tasks; this loop consumes the first half of all previously # generated priorities for (idx, task_id) in enumerate(task_ids): wp.AddTask((ctx, idx), priority=priorities.pop(), task_id=task_id) self.assertEqual(len(wp._tasks), len(task_ids)) self.assertEqual(len(wp._taskdata), len(task_ids)) # Tasks have been added, so half of the priorities should have been # consumed assert len(priorities) == len(task_ids) # Change task priority expected = [] for ((idx, task_id), prio) in zip(enumerate(task_ids), priorities): wp.ChangeTaskPriority(task_id, prio) expected.append((prio, idx)) self.assertEqual(len(wp._taskdata), len(task_ids)) # Half the entries are now abandoned tasks self.assertEqual(len(wp._tasks), len(task_ids) * 2) assert len(priorities) == count assert len(task_ids) == count # Start processing wp.SetActive(True) # Wait for tasks to finish wp.Quiesce() self._CheckNoTasks(wp) for task_id in task_ids: # All tasks are done self.assertRaises(workerpool.NoSuchTask, wp.ChangeTaskPriority, task_id, 0) # Check result ctx.lock.acquire() try: self.assertEqual(ctx.result, sorted(expected)) finally: ctx.lock.release() self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0)