def _init_queue_from_name(self, name): # Create the queue first. element = self.cfgtree.find('queue[@name="%s"]' % name) max_threads = element.find('max-threads').text queue = Queue(verbose=0, max_threads=max_threads) # Assign account pools to the queue. def match_cb(condition, host): return eval(condition, host.get_dict()) for pool_elem in element.iterfind('account-pool'): pname = pool_elem.text pool = self._init_account_pool_from_name(pname) cond = pool_elem.get('for') if cond is None: print 'Assigning default account pool "%s" to "%s"...' % ( pname, name) queue.add_account_pool(pool) continue print 'Assigning account pool "%s" to "%s"...' % (pname, name) condition = compile(cond, 'config', 'eval') queue.add_account_pool(pool, partial(match_cb, condition)) return queue
def _init_queue_from_name(self, name): # Create the queue first. element = self.cfgtree.find('queue[@name="%s"]' % name) max_threads = element.find('max-threads').text queue = Queue(verbose = 0, max_threads = max_threads) # Assign account pools to the queue. for pool_elem in element.iterfind('account-pool'): pool = self._init_account_pool_from_name(pool_elem.text) cond = pool_elem.get('for') if cond is None: queue.add_account_pool(pool) continue condition = compile(cond, 'config', 'eval') def match_cb(host): return eval(condition, host.get_dict()) queue.add_account_pool(pool, match_cb) return queue
def _init_queue_from_name(self, name): # Create the queue first. element = self.cfgtree.find('queue[@name="%s"]' % name) max_threads = element.find('max-threads').text queue = Queue(verbose=0, max_threads=max_threads) # Assign account pools to the queue. for pool_elem in element.iterfind('account-pool'): pool = self._init_account_pool_from_name(pool_elem.text) cond = pool_elem.get('for') if cond is None: queue.add_account_pool(pool) continue condition = compile(cond, 'config', 'eval') def match_cb(host): return eval(condition, host.get_dict()) queue.add_account_pool(pool, match_cb) return queue
def _init_queue_from_name(self, name): # Create the queue first. element = self.cfgtree.find('queue[@name="%s"]' % name) max_threads = element.find('max-threads').text queue = Queue(verbose = 0, max_threads = max_threads) # Assign account pools to the queue. def match_cb(condition, host): return eval(condition, host.get_dict()) for pool_elem in element.iterfind('account-pool'): pname = pool_elem.text pool = self._init_account_pool_from_name(pname) cond = pool_elem.get('for') if cond is None: print 'Assigning default account pool "%s" to "%s"...' % (pname, name) queue.add_account_pool(pool) continue print 'Assigning account pool "%s" to "%s"...' % (pname, name) condition = compile(cond, 'config', 'eval') queue.add_account_pool(pool, partial(match_cb, condition)) return queue
class QueueTest(unittest.TestCase): CORRELATE = Queue mode = 'threading' def createQueue(self, logdir=None, **kwargs): if self.queue: self.queue.destroy() self.out = self.manager.Log() self.err = self.manager.Log() self.queue = Queue(mode=self.mode, stdout=self.out, stderr=self.err, **kwargs) self.accm = self.queue.account_manager if logdir is not None: self.logger = FileLogger(logdir) def setUp(self): self.tempdir = mkdtemp() self.queue = None self.logger = None self.manager = LogManager() self.manager.start() self.createQueue(verbose=-1, logdir=self.tempdir) def tearDown(self): shutil.rmtree(self.tempdir) try: self.queue.destroy() except: pass # queue already destroyed self.manager.shutdown() def assertVerbosity(self, channel, expected): data = channel.read() if expected == 'no_tb': self.assert_('error' in data, data) self.assert_('Traceback' not in data) elif expected == 'tb': self.assert_('error' in data, data) self.assert_('Traceback' in data) elif expected == '': self.assertEqual(data, '') else: msg = repr(expected) + ' not in ' + repr(data) self.assert_(expected in data, msg) def testConstructor(self): self.assertEqual(1, self.queue.get_max_threads()) # Test all verbosity levels. levels = ( (-1, 1, ('', ''), ('', ''), ('', 'tb')), (-1, 2, ('', ''), ('', ''), ('', 'tb')), (0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')), (0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')), (1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')), (1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')), (2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), ) for level, max_threads, with_simple, with_error, with_fatal in levels: #print "S:", level, max_threads, with_simple, with_error, with_fatal stdout, stderr = with_simple self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', say_hello) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) #print "E:", level, max_threads, with_simple, with_error, with_fatal stdout, stderr = with_error self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', error) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) #print "F:", level, max_threads, with_simple, with_error, with_fatal stdout, stderr = with_fatal self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', fatal_error) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) def testCreatePipe(self): account = Account('user', 'test') self.accm.add_account(account) pipe = self.queue._create_pipe() pipe.send(('acquire-account', None)) response = pipe.recv() expected = (account.__hash__(), account.get_name(), account.get_password(), account.get_authorization_password(), account.get_key()) self.assertEqual(response, expected) pipe.send(('release-account', account.__hash__())) response = pipe.recv() self.assertEqual(response, 'ok') pipe.close() def testSetMaxThreads(self): self.assertEqual(1, self.queue.get_max_threads()) self.queue.set_max_threads(2) self.assertEqual(2, self.queue.get_max_threads()) def testGetMaxThreads(self): pass # Already tested in testSetMaxThreads(). def testGetProgress(self): self.assertEqual(0.0, self.queue.get_progress()) self.testIsCompleted() self.assertEqual(100.0, self.queue.get_progress()) def testAddAccount(self): self.assertEqual(0, self.accm.default_pool.n_accounts()) account = Account('user', 'test') self.queue.add_account(account) self.assertEqual(1, self.accm.default_pool.n_accounts()) def testAddAccountPool(self): self.assertEqual(0, self.accm.default_pool.n_accounts()) account = Account('user', 'test') self.queue.add_account(account) self.assertEqual(1, self.accm.default_pool.n_accounts()) def match_cb(data, host): data['match-called'].value = True return True def start_cb(data, job, host, conn): account = conn.account_factory(None) data['start-called'].value = True data['account-hash'].value = account.__hash__() account.release() # Replace the default pool. pool1 = AccountPool() self.queue.add_account_pool(pool1) self.assertEqual(self.accm.default_pool, pool1) # Add another pool, making sure that it does not replace # the default pool. pool2 = AccountPool() account2 = Account('user', 'test') pool2.add_account(account2) match_called = Value(ctypes.c_bool, False) start_called = Value(ctypes.c_bool, False) account_hash = Value(ctypes.c_long, 0) data = { 'match-called': match_called, 'start-called': start_called, 'account-hash': account_hash } self.queue.add_account_pool(pool2, partial(match_cb, data)) self.assertEqual(self.accm.default_pool, pool1) # Make sure that pool2 is chosen (because the match function # returns True). self.queue.run('dummy://dummy', partial(start_cb, data)) self.queue.shutdown() data = dict((k, v.value) for (k, v) in data.iteritems()) self.assertEqual( data, { 'match-called': True, 'start-called': True, 'account-hash': account2.__hash__() }) def startTask(self): self.testAddAccount() hosts = ['dummy://dummy1', 'dummy://dummy2'] task = self.queue.run(hosts, log_to(self.logger)(do_nothing)) self.assert_(task is not None) return task def testIsCompleted(self): self.assert_(self.queue.is_completed()) task = self.startTask() self.failIf(self.queue.is_completed()) task.wait() self.assert_(task.is_completed()) self.assert_(self.queue.is_completed()) def testJoin(self): task = self.startTask() self.queue.join() self.assert_(task.is_completed()) self.assert_(self.queue.is_completed()) def testShutdown(self): task = self.startTask() # this also adds an account self.queue.shutdown() self.assert_(task.is_completed()) self.assert_(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 1) def testDestroy(self): task = self.startTask() # this also adds an account self.queue.destroy() self.assert_(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 0) def testReset(self): self.testAddAccount() self.queue.reset() self.assertEqual(self.accm.default_pool.n_accounts(), 0) def testExceptionCallback(self): self.exc = {} def my_exc_cb(jobname, exc_info): self.exc[jobname] = exc_info self.createQueue(exc_cb=my_exc_cb) self.queue.run('dummy://mytest', error) self.queue.join() self.assert_("mytest" in self.exc) self.assert_(isinstance(self.exc["mytest"][1], FailException)) def testRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] func = bind(count_calls2, data, testarg=1) self.queue.run(hosts, func) self.queue.run('dummy://dummy3', func) self.queue.shutdown() self.assertEqual(data.value, 3) self.queue.run('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 4) def testRunOrIgnore(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() self.queue.run_or_ignore(hosts, func) self.queue.run_or_ignore('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.run_or_ignore('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 3) def testPriorityRun(self): def write(data, value, *args): data.value = value data = Value('i', 0) self.queue.workqueue.pause() self.queue.enqueue(partial(write, data, 1)) self.queue.priority_run('dummy://dummy', partial(write, data, 2)) self.queue.workqueue.unpause() self.queue.destroy() # The 'dummy' job should run first, so the value must # be overwritten by the other process. self.assertEqual(data.value, 1) def testPriorityRunOrRaise(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() self.queue.priority_run_or_raise(hosts, func) self.queue.priority_run_or_raise('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.priority_run_or_raise('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 3) def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] func = bind(count_calls2, data, testarg=1) # By setting max_threads to 0 we ensure that the 'force' part is # actually tested; the thread should run regardless. self.queue.set_max_threads(0) self.queue.force_run(hosts, func) self.queue.destroy() self.assertEqual(data.value, 2) def testEnqueue(self): data = Value('i', 0) func = bind(count_calls, data, testarg=1) self.queue.enqueue(func) self.queue.enqueue(func) self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.enqueue(func) self.queue.shutdown() self.assertEqual(data.value, 3) func = bind(count_and_fail, data, testarg=1) self.queue.enqueue(func, attempts=7) self.queue.destroy() self.assertEqual(data.value, 10) #FIXME: Not a method test; this should probably be elsewhere. def testLogging(self): task = self.startTask() while not task.is_completed(): time.sleep(.1) # The following function is not synchronous with the above, so add # a timeout to avoid races. time.sleep(.1) self.assert_(self.queue.is_completed()) logfiles = os.listdir(self.tempdir) self.assertEqual(2, len(logfiles)) self.assert_('dummy1.log' in logfiles) self.assert_('dummy2.log' in logfiles) for file in logfiles: content = open(os.path.join(self.tempdir, file)).read()
class QueueTest(unittest.TestCase): CORRELATE = Queue mode = 'threading' def createQueue(self, logdir=None, **kwargs): if self.queue: self.queue.destroy() self.out = self.manager.Log() self.err = self.manager.Log() self.queue = Queue(mode=self.mode, stdout=self.out, stderr=self.err, **kwargs) self.accm = self.queue.account_manager if logdir is not None: self.logger = FileLogger(logdir) def setUp(self): self.tempdir = mkdtemp() self.queue = None self.logger = None self.manager = LogManager() self.manager.start() self.createQueue(verbose=-1, logdir=self.tempdir) def tearDown(self): shutil.rmtree(self.tempdir) try: self.queue.destroy() except: pass # queue already destroyed self.manager.shutdown() def assertVerbosity(self, channel, expected): data = channel.read() if expected == 'no_tb': self.assertTrue('error' in data, data) self.assertNotIn('Traceback', data) elif expected == 'tb': self.assertTrue('error' in data, data) self.assertIn('Traceback', data) elif expected == '': self.assertEqual(data, '') else: msg = repr(expected) + ' not in ' + repr(data) self.assertTrue(expected in data, msg) def testConstructor(self): self.assertEqual(1, self.queue.get_max_threads()) # Test all verbosity levels. levels = ( (-1, 1, ('', ''), ('', ''), ('', 'tb')), (-1, 2, ('', ''), ('', ''), ('', 'tb')), (0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')), (0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')), (1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')), (1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')), (2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), ) for level, max_threads, with_simple, with_error, with_fatal in levels: # print("S:", level, max_threads, with_simple, with_error,) # with_fatal stdout, stderr = with_simple self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', say_hello) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) # print("E:", level, max_threads, with_simple, with_error,) # with_fatal stdout, stderr = with_error self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', error) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) # print("F:", level, max_threads, with_simple, with_error,) # with_fatal stdout, stderr = with_fatal self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', fatal_error) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) def testCreatePipe(self): account = Account('user', 'test') self.accm.add_account(account) pipe = self.queue._create_pipe() pipe.send(('acquire-account', None)) response = pipe.recv() expected = (account.__hash__(), account.get_name(), account.get_password(), account.get_authorization_password(), account.get_key()) self.assertEqual(response, expected) pipe.send(('release-account', account.__hash__())) response = pipe.recv() self.assertEqual(response, 'ok') pipe.close() def testSetMaxThreads(self): self.assertEqual(1, self.queue.get_max_threads()) self.queue.set_max_threads(2) self.assertEqual(2, self.queue.get_max_threads()) def testGetMaxThreads(self): pass # Already tested in testSetMaxThreads(). def testGetProgress(self): self.assertEqual(0.0, self.queue.get_progress()) self.testIsCompleted() self.assertEqual(100.0, self.queue.get_progress()) def testAddAccount(self): self.assertEqual(0, self.accm.default_pool.n_accounts()) account = Account('user', 'test') self.queue.add_account(account) self.assertEqual(1, self.accm.default_pool.n_accounts()) def testAddAccountPool(self): self.assertEqual(0, self.accm.default_pool.n_accounts()) account = Account('user', 'test') self.queue.add_account(account) self.assertEqual(1, self.accm.default_pool.n_accounts()) def match_cb(data, host): data['match-called'].value = True return True def start_cb(data, job, host, conn): account = conn.account_factory(None) data['start-called'].value = True data['account-hash'].value = account.__hash__() account.release() # Replace the default pool. pool1 = AccountPool() self.queue.add_account_pool(pool1) self.assertEqual(self.accm.default_pool, pool1) # Add another pool, making sure that it does not replace # the default pool. pool2 = AccountPool() account2 = Account('user', 'test') pool2.add_account(account2) match_called = Value(ctypes.c_bool, False) start_called = Value(ctypes.c_bool, False) account_hash = Value(ctypes.c_long, 0) data = {'match-called': match_called, 'start-called': start_called, 'account-hash': account_hash} self.queue.add_account_pool(pool2, partial(match_cb, data)) self.assertEqual(self.accm.default_pool, pool1) # Make sure that pool2 is chosen (because the match function # returns True). self.queue.run('dummy://dummy', partial(start_cb, data)) self.queue.shutdown() data = dict((k, v.value) for (k, v) in list(data.items())) self.assertEqual(data, {'match-called': True, 'start-called': True, 'account-hash': account2.__hash__()}) def startTask(self): self.testAddAccount() hosts = ['dummy://dummy1', 'dummy://dummy2'] task = self.queue.run(hosts, log_to(self.logger)(do_nothing)) self.assertTrue(task is not None) return task def testIsCompleted(self): self.assertTrue(self.queue.is_completed()) task = self.startTask() self.assertFalse(self.queue.is_completed()) task.wait() self.assertTrue(task.is_completed()) self.assertTrue(self.queue.is_completed()) def testJoin(self): task = self.startTask() self.queue.join() self.assertTrue(task.is_completed()) self.assertTrue(self.queue.is_completed()) def testShutdown(self): task = self.startTask() # this also adds an account self.queue.shutdown() self.assertTrue(task.is_completed()) self.assertTrue(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 1) def testDestroy(self): task = self.startTask() # this also adds an account self.queue.destroy() self.assertTrue(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 0) def testReset(self): self.testAddAccount() self.queue.reset() self.assertEqual(self.accm.default_pool.n_accounts(), 0) def testExceptionCallback(self): self.exc = {} def my_exc_cb(jobname, exc_info): self.exc[jobname] = exc_info self.createQueue(exc_cb=my_exc_cb) self.queue.run('dummy://mytest', error) self.queue.join() self.assertIn("mytest", self.exc) self.assertIsInstance(self.exc["mytest"][1], FailException) def testRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] func = bind(count_calls2, data, testarg=1) self.queue.run(hosts, func) self.queue.run('dummy://dummy3', func) self.queue.shutdown() self.assertEqual(data.value, 3) self.queue.run('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 4) def testRunOrIgnore(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() self.queue.run_or_ignore(hosts, func) self.queue.run_or_ignore('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.run_or_ignore('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 3) def testPriorityRun(self): def write(data, value, *args): data.value = value data = Value('i', 0) self.queue.workqueue.pause() self.queue.enqueue(partial(write, data, 1)) self.queue.priority_run('dummy://dummy', partial(write, data, 2)) self.queue.workqueue.unpause() self.queue.destroy() # The 'dummy' job should run first, so the value must # be overwritten by the other process. self.assertEqual(data.value, 1) def testPriorityRunOrRaise(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() self.queue.priority_run_or_raise(hosts, func) self.queue.priority_run_or_raise('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.priority_run_or_raise('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 3) def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] func = bind(count_calls2, data, testarg=1) # By setting max_threads to 0 we ensure that the 'force' part is # actually tested; the thread should run regardless. self.queue.set_max_threads(0) self.queue.force_run(hosts, func) self.queue.destroy() self.assertEqual(data.value, 2) def testEnqueue(self): data = Value('i', 0) func = bind(count_calls, data, testarg=1) self.queue.enqueue(func) self.queue.enqueue(func) self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.enqueue(func) self.queue.shutdown() self.assertEqual(data.value, 3) func = bind(count_and_fail, data, testarg=1) self.queue.enqueue(func, attempts=7) self.queue.destroy() self.assertEqual(data.value, 10) # FIXME: Not a method test; this should probably be elsewhere. def testLogging(self): task = self.startTask() while not task.is_completed(): time.sleep(.1) # The following function is not synchronous with the above, so add # a timeout to avoid races. time.sleep(.1) self.assertTrue(self.queue.is_completed()) logfiles = os.listdir(self.tempdir) self.assertEqual(2, len(logfiles)) self.assertIn('dummy1.log', logfiles) self.assertIn('dummy2.log', logfiles) for file in logfiles: with open(os.path.join(self.tempdir, file)) as fp: content = fp.read()