class TemplateTest(unittest.TestCase):
def setUp(self):
account = Account('sab', '')
self.queue = Queue(verbose = 0, max_threads = 1)
self.logger = Logger()
self.queue.add_account(account)
def tearDown(self):
self.queue.destroy()
def testTemplates(self):
callback = bind(log_to(self.logger)(dummy_cb), self)
for test in os.listdir(test_dir):
pseudo = os.path.join(test_dir, test, 'pseudodev.py')
if os.path.exists(pseudo):
self.queue.run('pseudo://' + pseudo, callback)
else:
self.queue.run('ios://' + test, callback)
self.queue.shutdown()
# Unfortunately, unittest.TestCase does not fail if self.assert()
# was called from a subthread, so this is our workaround...
failed = self.logger.get_aborted_logs()
report = format(self.logger, show_successful = False)
self.assert_(not failed, report)
def run(users, hosts, func, **kwargs):
"""
Convenience function that creates an Exscript.Queue instance, adds
the given accounts, and calls Queue.run() with the given
hosts and function as an argument.
If you also want to pass arguments to the given function, you may use
util.decorator.bind() like this::
def my_callback(job, host, conn, my_arg, **kwargs):
print my_arg, kwargs.get('foo')
run(account,
host,
bind(my_callback, 'hello', foo = 'world'),
max_threads = 10)
@type users: Account|list[Account]
@param users: The account(s) to use for logging in.
@type hosts: Host|list[Host]
@param hosts: A list of Host objects.
@type func: function
@param func: The callback function.
@type kwargs: dict
@param kwargs: Passed to the Exscript.Queue constructor.
"""
queue = Queue(**kwargs)
queue.add_account(users)
queue.run(hosts, func)
queue.destroy()
class TemplateTest(unittest.TestCase):
def setUp(self):
account = Account('sab', '')
self.queue = Queue(verbose=0, max_threads=1)
self.logger = Logger()
self.queue.add_account(account)
def tearDown(self):
self.queue.destroy()
def testTemplates(self):
callback = bind(log_to(self.logger)(dummy_cb), self)
for test in os.listdir(test_dir):
pseudo = os.path.join(test_dir, test, 'pseudodev.py')
if os.path.exists(pseudo):
self.queue.run('pseudo://' + pseudo, callback)
else:
self.queue.run('ios://' + test, callback)
self.queue.shutdown()
# Unfortunately, unittest.TestCase does not fail if self.assert()
# was called from a subthread, so this is our workaround...
failed = self.logger.get_aborted_logs()
report = format(self.logger, show_successful=False)
self.assert_(not failed, report)
def run(users, hosts, func, **kwargs):
"""
Convenience function that creates an Exscript.Queue instance, adds
the given accounts, and calls Queue.run() with the given
hosts and function as an argument.
If you also want to pass arguments to the given function, you may use
util.decorator.bind() like this::
def my_callback(job, host, conn, my_arg, **kwargs):
print my_arg, kwargs.get('foo')
run(account,
host,
bind(my_callback, 'hello', foo = 'world'),
max_threads = 10)
@type users: Account|list[Account]
@param users: The account(s) to use for logging in.
@type hosts: Host|list[Host]
@param hosts: A list of Host objects.
@type func: function
@param func: The callback function.
@type kwargs: dict
@param kwargs: Passed to the Exscript.Queue constructor.
"""
queue = Queue(**kwargs)
queue.add_account(users)
queue.run(hosts, func)
queue.destroy()
objnames = ('count_calls',)
follow_modules = False
def count_calls(conn, thedata, **kwargs):
thedata['n_calls'] += 1
def foobar():
pass
queue = Queue()
data = {'n_calls': 0}
func = bind(count_calls, data)
task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func)
task.wait()
queue.shutdown()
queue.destroy()
del func
# Test memory consumption.
from meliae import scanner
gc.collect()
scanner.dump_all_objects("test.dump")
from meliae import loader
om = loader.load('test.dump')
om.remove_expensive_references()
om.collapse_instance_dicts()
om.compute_referrers()
om.compute_total_size()
#print om.summarize()
class QueueTest(unittest.TestCase):
CORRELATE = Queue
mode = 'threading'
def createQueue(self, logdir=None, **kwargs):
if self.queue:
self.queue.destroy()
self.out = self.manager.Log()
self.err = self.manager.Log()
self.queue = Queue(mode=self.mode,
stdout=self.out,
stderr=self.err,
**kwargs)
self.accm = self.queue.account_manager
if logdir is not None:
self.logger = FileLogger(logdir)
def setUp(self):
self.tempdir = mkdtemp()
self.queue = None
self.logger = None
self.manager = LogManager()
self.manager.start()
self.createQueue(verbose=-1, logdir=self.tempdir)
def tearDown(self):
shutil.rmtree(self.tempdir)
try:
self.queue.destroy()
except:
pass # queue already destroyed
self.manager.shutdown()
def assertVerbosity(self, channel, expected):
data = channel.read()
if expected == 'no_tb':
self.assert_('error' in data, data)
self.assert_('Traceback' not in data)
elif expected == 'tb':
self.assert_('error' in data, data)
self.assert_('Traceback' in data)
elif expected == '':
self.assertEqual(data, '')
else:
msg = repr(expected) + ' not in ' + repr(data)
self.assert_(expected in data, msg)
def testConstructor(self):
self.assertEqual(1, self.queue.get_max_threads())
# Test all verbosity levels.
levels = (
(-1, 1, ('', ''), ('', ''), ('', 'tb')),
(-1, 2, ('', ''), ('', ''), ('', 'tb')),
(0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')),
(0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')),
(1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')),
(1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')),
(2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
)
for level, max_threads, with_simple, with_error, with_fatal in levels:
#print "S:", level, max_threads, with_simple, with_error, with_fatal
stdout, stderr = with_simple
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', say_hello)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
#print "E:", level, max_threads, with_simple, with_error, with_fatal
stdout, stderr = with_error
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
#print "F:", level, max_threads, with_simple, with_error, with_fatal
stdout, stderr = with_fatal
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', fatal_error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
def testCreatePipe(self):
account = Account('user', 'test')
self.accm.add_account(account)
pipe = self.queue._create_pipe()
pipe.send(('acquire-account', None))
response = pipe.recv()
expected = (account.__hash__(), account.get_name(),
account.get_password(),
account.get_authorization_password(), account.get_key())
self.assertEqual(response, expected)
pipe.send(('release-account', account.__hash__()))
response = pipe.recv()
self.assertEqual(response, 'ok')
pipe.close()
def testSetMaxThreads(self):
self.assertEqual(1, self.queue.get_max_threads())
self.queue.set_max_threads(2)
self.assertEqual(2, self.queue.get_max_threads())
def testGetMaxThreads(self):
pass # Already tested in testSetMaxThreads().
def testGetProgress(self):
self.assertEqual(0.0, self.queue.get_progress())
self.testIsCompleted()
self.assertEqual(100.0, self.queue.get_progress())
def testAddAccount(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def testAddAccountPool(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def match_cb(data, host):
data['match-called'].value = True
return True
def start_cb(data, job, host, conn):
account = conn.account_factory(None)
data['start-called'].value = True
data['account-hash'].value = account.__hash__()
account.release()
# Replace the default pool.
pool1 = AccountPool()
self.queue.add_account_pool(pool1)
self.assertEqual(self.accm.default_pool, pool1)
# Add another pool, making sure that it does not replace
# the default pool.
pool2 = AccountPool()
account2 = Account('user', 'test')
pool2.add_account(account2)
match_called = Value(ctypes.c_bool, False)
start_called = Value(ctypes.c_bool, False)
account_hash = Value(ctypes.c_long, 0)
data = {
'match-called': match_called,
'start-called': start_called,
'account-hash': account_hash
}
self.queue.add_account_pool(pool2, partial(match_cb, data))
self.assertEqual(self.accm.default_pool, pool1)
# Make sure that pool2 is chosen (because the match function
# returns True).
self.queue.run('dummy://dummy', partial(start_cb, data))
self.queue.shutdown()
data = dict((k, v.value) for (k, v) in data.iteritems())
self.assertEqual(
data, {
'match-called': True,
'start-called': True,
'account-hash': account2.__hash__()
})
def startTask(self):
self.testAddAccount()
hosts = ['dummy://dummy1', 'dummy://dummy2']
task = self.queue.run(hosts, log_to(self.logger)(do_nothing))
self.assert_(task is not None)
return task
def testIsCompleted(self):
self.assert_(self.queue.is_completed())
task = self.startTask()
self.failIf(self.queue.is_completed())
task.wait()
self.assert_(task.is_completed())
self.assert_(self.queue.is_completed())
def testJoin(self):
task = self.startTask()
self.queue.join()
self.assert_(task.is_completed())
self.assert_(self.queue.is_completed())
def testShutdown(self):
task = self.startTask() # this also adds an account
self.queue.shutdown()
self.assert_(task.is_completed())
self.assert_(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 1)
def testDestroy(self):
task = self.startTask() # this also adds an account
self.queue.destroy()
self.assert_(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testReset(self):
self.testAddAccount()
self.queue.reset()
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testExceptionCallback(self):
self.exc = {}
def my_exc_cb(jobname, exc_info):
self.exc[jobname] = exc_info
self.createQueue(exc_cb=my_exc_cb)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assert_("mytest" in self.exc)
self.assert_(isinstance(self.exc["mytest"][1], FailException))
def testRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
self.queue.run(hosts, func)
self.queue.run('dummy://dummy3', func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
self.queue.run('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 4)
def testRunOrIgnore(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.run_or_ignore(hosts, func)
self.queue.run_or_ignore('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.run_or_ignore('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testPriorityRun(self):
def write(data, value, *args):
data.value = value
data = Value('i', 0)
self.queue.workqueue.pause()
self.queue.enqueue(partial(write, data, 1))
self.queue.priority_run('dummy://dummy', partial(write, data, 2))
self.queue.workqueue.unpause()
self.queue.destroy()
# The 'dummy' job should run first, so the value must
# be overwritten by the other process.
self.assertEqual(data.value, 1)
def testPriorityRunOrRaise(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.priority_run_or_raise(hosts, func)
self.queue.priority_run_or_raise('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.priority_run_or_raise('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testForceRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
# By setting max_threads to 0 we ensure that the 'force' part is
# actually tested; the thread should run regardless.
self.queue.set_max_threads(0)
self.queue.force_run(hosts, func)
self.queue.destroy()
self.assertEqual(data.value, 2)
def testEnqueue(self):
data = Value('i', 0)
func = bind(count_calls, data, testarg=1)
self.queue.enqueue(func)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
func = bind(count_and_fail, data, testarg=1)
self.queue.enqueue(func, attempts=7)
self.queue.destroy()
self.assertEqual(data.value, 10)
#FIXME: Not a method test; this should probably be elsewhere.
def testLogging(self):
task = self.startTask()
while not task.is_completed():
time.sleep(.1)
# The following function is not synchronous with the above, so add
# a timeout to avoid races.
time.sleep(.1)
self.assert_(self.queue.is_completed())
logfiles = os.listdir(self.tempdir)
self.assertEqual(2, len(logfiles))
self.assert_('dummy1.log' in logfiles)
self.assert_('dummy2.log' in logfiles)
for file in logfiles:
content = open(os.path.join(self.tempdir, file)).read()
class QueueTest(unittest.TestCase):
CORRELATE = Queue
mode = 'threading'
def createQueue(self, logdir=None, **kwargs):
if self.queue:
self.queue.destroy()
self.out = self.manager.Log()
self.err = self.manager.Log()
self.queue = Queue(mode=self.mode,
stdout=self.out,
stderr=self.err,
**kwargs)
self.accm = self.queue.account_manager
if logdir is not None:
self.logger = FileLogger(logdir)
def setUp(self):
self.tempdir = mkdtemp()
self.queue = None
self.logger = None
self.manager = LogManager()
self.manager.start()
self.createQueue(verbose=-1, logdir=self.tempdir)
def tearDown(self):
shutil.rmtree(self.tempdir)
try:
self.queue.destroy()
except:
pass # queue already destroyed
self.manager.shutdown()
def assertVerbosity(self, channel, expected):
data = channel.read()
if expected == 'no_tb':
self.assertTrue('error' in data, data)
self.assertNotIn('Traceback', data)
elif expected == 'tb':
self.assertTrue('error' in data, data)
self.assertIn('Traceback', data)
elif expected == '':
self.assertEqual(data, '')
else:
msg = repr(expected) + ' not in ' + repr(data)
self.assertTrue(expected in data, msg)
def testConstructor(self):
self.assertEqual(1, self.queue.get_max_threads())
# Test all verbosity levels.
levels = (
(-1, 1, ('', ''), ('', ''), ('', 'tb')),
(-1, 2, ('', ''), ('', ''), ('', 'tb')),
(0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')),
(0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')),
(1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')),
(1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')),
(2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
)
for level, max_threads, with_simple, with_error, with_fatal in levels:
# print("S:", level, max_threads, with_simple, with_error,)
# with_fatal
stdout, stderr = with_simple
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', say_hello)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
# print("E:", level, max_threads, with_simple, with_error,)
# with_fatal
stdout, stderr = with_error
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
# print("F:", level, max_threads, with_simple, with_error,)
# with_fatal
stdout, stderr = with_fatal
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', fatal_error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
def testCreatePipe(self):
account = Account('user', 'test')
self.accm.add_account(account)
pipe = self.queue._create_pipe()
pipe.send(('acquire-account', None))
response = pipe.recv()
expected = (account.__hash__(),
account.get_name(),
account.get_password(),
account.get_authorization_password(),
account.get_key())
self.assertEqual(response, expected)
pipe.send(('release-account', account.__hash__()))
response = pipe.recv()
self.assertEqual(response, 'ok')
pipe.close()
def testSetMaxThreads(self):
self.assertEqual(1, self.queue.get_max_threads())
self.queue.set_max_threads(2)
self.assertEqual(2, self.queue.get_max_threads())
def testGetMaxThreads(self):
pass # Already tested in testSetMaxThreads().
def testGetProgress(self):
self.assertEqual(0.0, self.queue.get_progress())
self.testIsCompleted()
self.assertEqual(100.0, self.queue.get_progress())
def testAddAccount(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def testAddAccountPool(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def match_cb(data, host):
data['match-called'].value = True
return True
def start_cb(data, job, host, conn):
account = conn.account_factory(None)
data['start-called'].value = True
data['account-hash'].value = account.__hash__()
account.release()
# Replace the default pool.
pool1 = AccountPool()
self.queue.add_account_pool(pool1)
self.assertEqual(self.accm.default_pool, pool1)
# Add another pool, making sure that it does not replace
# the default pool.
pool2 = AccountPool()
account2 = Account('user', 'test')
pool2.add_account(account2)
match_called = Value(ctypes.c_bool, False)
start_called = Value(ctypes.c_bool, False)
account_hash = Value(ctypes.c_long, 0)
data = {'match-called': match_called,
'start-called': start_called,
'account-hash': account_hash}
self.queue.add_account_pool(pool2, partial(match_cb, data))
self.assertEqual(self.accm.default_pool, pool1)
# Make sure that pool2 is chosen (because the match function
# returns True).
self.queue.run('dummy://dummy', partial(start_cb, data))
self.queue.shutdown()
data = dict((k, v.value) for (k, v) in list(data.items()))
self.assertEqual(data, {'match-called': True,
'start-called': True,
'account-hash': account2.__hash__()})
def startTask(self):
self.testAddAccount()
hosts = ['dummy://dummy1', 'dummy://dummy2']
task = self.queue.run(hosts, log_to(self.logger)(do_nothing))
self.assertTrue(task is not None)
return task
def testIsCompleted(self):
self.assertTrue(self.queue.is_completed())
task = self.startTask()
self.assertFalse(self.queue.is_completed())
task.wait()
self.assertTrue(task.is_completed())
self.assertTrue(self.queue.is_completed())
def testJoin(self):
task = self.startTask()
self.queue.join()
self.assertTrue(task.is_completed())
self.assertTrue(self.queue.is_completed())
def testShutdown(self):
task = self.startTask() # this also adds an account
self.queue.shutdown()
self.assertTrue(task.is_completed())
self.assertTrue(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 1)
def testDestroy(self):
task = self.startTask() # this also adds an account
self.queue.destroy()
self.assertTrue(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testReset(self):
self.testAddAccount()
self.queue.reset()
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testExceptionCallback(self):
self.exc = {}
def my_exc_cb(jobname, exc_info):
self.exc[jobname] = exc_info
self.createQueue(exc_cb=my_exc_cb)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assertIn("mytest", self.exc)
self.assertIsInstance(self.exc["mytest"][1], FailException)
def testRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
self.queue.run(hosts, func)
self.queue.run('dummy://dummy3', func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
self.queue.run('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 4)
def testRunOrIgnore(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.run_or_ignore(hosts, func)
self.queue.run_or_ignore('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.run_or_ignore('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testPriorityRun(self):
def write(data, value, *args):
data.value = value
data = Value('i', 0)
self.queue.workqueue.pause()
self.queue.enqueue(partial(write, data, 1))
self.queue.priority_run('dummy://dummy', partial(write, data, 2))
self.queue.workqueue.unpause()
self.queue.destroy()
# The 'dummy' job should run first, so the value must
# be overwritten by the other process.
self.assertEqual(data.value, 1)
def testPriorityRunOrRaise(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.priority_run_or_raise(hosts, func)
self.queue.priority_run_or_raise('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.priority_run_or_raise('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testForceRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
# By setting max_threads to 0 we ensure that the 'force' part is
# actually tested; the thread should run regardless.
self.queue.set_max_threads(0)
self.queue.force_run(hosts, func)
self.queue.destroy()
self.assertEqual(data.value, 2)
def testEnqueue(self):
data = Value('i', 0)
func = bind(count_calls, data, testarg=1)
self.queue.enqueue(func)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
func = bind(count_and_fail, data, testarg=1)
self.queue.enqueue(func, attempts=7)
self.queue.destroy()
self.assertEqual(data.value, 10)
# FIXME: Not a method test; this should probably be elsewhere.
def testLogging(self):
task = self.startTask()
while not task.is_completed():
time.sleep(.1)
# The following function is not synchronous with the above, so add
# a timeout to avoid races.
time.sleep(.1)
self.assertTrue(self.queue.is_completed())
logfiles = os.listdir(self.tempdir)
self.assertEqual(2, len(logfiles))
self.assertIn('dummy1.log', logfiles)
self.assertIn('dummy2.log', logfiles)
for file in logfiles:
with open(os.path.join(self.tempdir, file)) as fp:
content = fp.read()
def count_calls(conn, thedata, **kwargs):
thedata['n_calls'] += 1
def foobar():
pass
queue = Queue()
data = {'n_calls': 0}
func = bind(count_calls, data)
task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func)
task.wait()
queue.shutdown()
queue.destroy()
del func
# Test memory consumption.
from meliae import scanner
gc.collect()
scanner.dump_all_objects("test.dump")
from meliae import loader
om = loader.load('test.dump')
om.remove_expensive_references()
om.collapse_instance_dicts()
om.compute_referrers()
om.compute_total_size()
# print(om.summarize())
