def test_timeouting_cluster(self): """Tests behaviour when Cluster.connect raises cassandra.OperationTimedOut.""" with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod class OTOExc(Exception): pass class VeryFaultyCluster(object): def __init__(self, *args, **kwargs): pass def connect(self, *args, **kwargs): raise OTOExc() def shutdown(self): pass mod.cassandra = Mock() mod.cassandra.OperationTimedOut = OTOExc mod.cassandra.cluster = Mock() mod.cassandra.cluster.Cluster = VeryFaultyCluster x = mod.CassandraBackend(app=self.app) with self.assertRaises(OTOExc): x._store_result('task_id', 'result', states.SUCCESS) self.assertIsNone(x._connection) self.assertIsNone(x._session) x.process_cleanup() # should not raise
def test_sched(self): with mock_module(*gevent_modules): with patch_many('gevent.greenlet', 'gevent.greenlet.GreenletExit') as (greenlet, GreenletExit): greenlet.Greenlet = object x = Schedule() greenlet.Greenlet = Mock() x._Greenlet.spawn_later = Mock() x._GreenletExit = KeyError entry = Mock() g = x._enter(1, 0, entry) self.assertTrue(x.queue) x._entry_exit(g) g.kill.assert_called_with() self.assertFalse(x._queue) x._queue.add(g) x.clear() x._queue.add(g) g.kill.side_effect = KeyError() x.clear() g = x._Greenlet() g.cancel()
def test_please_free_memory(self): """Ensure that Cluster object IS shut down.""" with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod class RAMHoggingCluster(object): objects_alive = 0 def __init__(self, *args, **kwargs): pass def connect(self, *args, **kwargs): RAMHoggingCluster.objects_alive += 1 return Mock() def shutdown(self): RAMHoggingCluster.objects_alive -= 1 mod.cassandra = Mock() mod.cassandra.cluster = Mock() mod.cassandra.cluster.Cluster = RAMHoggingCluster for x in range(0, 10): x = mod.CassandraBackend(app=self.app) x._store_result('task_id', 'result', states.SUCCESS) x.process_cleanup() self.assertEquals(RAMHoggingCluster.objects_alive, 0)
def test_auth_provider(self): """Ensure valid auth_provider works properly, and invalid one raises ImproperlyConfigured exception.""" class DummyAuth(object): ValidAuthProvider = Mock() with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod mod.cassandra = Mock() mod.cassandra.auth = DummyAuth # Valid auth_provider self.app.conf.cassandra_auth_provider = 'ValidAuthProvider' self.app.conf.cassandra_auth_kwargs = { 'username': '******' } mod.CassandraBackend(app=self.app) # Invalid auth_provider self.app.conf.cassandra_auth_provider = 'SpiderManAuth' self.app.conf.cassandra_auth_kwargs = { 'username': '******' } with self.assertRaises(ImproperlyConfigured): mod.CassandraBackend(app=self.app)
def test_timeouting_cluster(self): """ Tests behaviour when Cluster.connect raises cassandra.OperationTimedOut """ with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod class OTOExc(Exception): pass class VeryFaultyCluster(object): def __init__(self, *args, **kwargs): pass def connect(self, *args, **kwargs): raise OTOExc() mod.cassandra = Mock() mod.cassandra.OperationTimedOut = OTOExc mod.cassandra.cluster = Mock() mod.cassandra.cluster.Cluster = VeryFaultyCluster x = mod.CassandraBackend(app=self.app) with self.assertRaises(OTOExc): x._store_result('task_id', 'result', states.SUCCESS) self.assertIsNone(x._connection) self.assertIsNone(x._session) x.process_cleanup() # should not raise
def test_get_task_meta_for(self): with mock_module("pycassa"): from celery.backends import cassandra as mod mod.pycassa = Mock() install_exceptions(mod.pycassa) mod.Thrift = Mock() install_exceptions(mod.Thrift) x = mod.CassandraBackend(app=self.app) Get_Column = x._get_column_family = Mock() get_column = Get_Column.return_value = Mock() get = get_column.get META = get.return_value = { "task_id": "task_id", "status": states.SUCCESS, "result": "1", "date_done": "date", "traceback": "", "children": None, } x.decode = Mock() x.detailed_mode = False meta = x._get_task_meta_for("task_id") self.assertEqual(meta["status"], states.SUCCESS) x.detailed_mode = True row = get.return_value = Mock() row.values.return_value = [Mock()] x.decode.return_value = META meta = x._get_task_meta_for("task_id") self.assertEqual(meta["status"], states.SUCCESS) x.decode.return_value = Mock() x.detailed_mode = False get.side_effect = KeyError() meta = x._get_task_meta_for("task_id") self.assertEqual(meta["status"], states.PENDING) calls = [0] end = [10] def work_eventually(*arg): try: if calls[0] > end[0]: return META raise socket.error() finally: calls[0] += 1 get.side_effect = work_eventually x._retry_timeout = 10 x._retry_wait = 0.01 meta = x._get_task_meta_for("task") self.assertEqual(meta["status"], states.SUCCESS) x._retry_timeout = 0.1 calls[0], end[0] = 0, 100 with self.assertRaises(socket.error): x._get_task_meta_for("task")
def test_get_task_meta_for(self): with mock_module('pycassa'): from celery.backends import cassandra as mod mod.pycassa = Mock() install_exceptions(mod.pycassa) mod.Thrift = Mock() install_exceptions(mod.Thrift) app = self.get_app() x = mod.CassandraBackend(app=app) Get_Column = x._get_column_family = Mock() get_column = Get_Column.return_value = Mock() get = get_column.get META = get.return_value = { 'task_id': 'task_id', 'status': states.SUCCESS, 'result': '1', 'date_done': 'date', 'traceback': '', 'children': None, } x.decode = Mock() x.detailed_mode = False meta = x._get_task_meta_for('task_id') self.assertEqual(meta['status'], states.SUCCESS) x.detailed_mode = True row = get.return_value = Mock() row.values.return_value = [Mock()] x.decode.return_value = META meta = x._get_task_meta_for('task_id') self.assertEqual(meta['status'], states.SUCCESS) x.decode.return_value = Mock() x.detailed_mode = False get.side_effect = KeyError() meta = x._get_task_meta_for('task_id') self.assertEqual(meta['status'], states.PENDING) calls = [0] end = [10] def work_eventually(*arg): try: if calls[0] > end[0]: return META raise socket.error() finally: calls[0] += 1 get.side_effect = work_eventually x._retry_timeout = 10 x._retry_wait = 0.01 meta = x._get_task_meta_for('task') self.assertEqual(meta['status'], states.SUCCESS) x._retry_timeout = 0.1 calls[0], end[0] = 0, 100 with self.assertRaises(socket.error): x._get_task_meta_for('task')
def test_is_patched(self): with mock_module(*gevent_modules): with patch('gevent.monkey.patch_all', create=True) as patch_all: import gevent gevent.version_info = (1, 0, 0) from celery import maybe_patch_concurrency maybe_patch_concurrency(['x', '-P', 'gevent']) self.assertTrue(patch_all.called)
def test_process_cleanup(self): with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod x = mod.CassandraBackend(app=self.app) x.process_cleanup() self.assertIsNone(x._connection) self.assertIsNone(x._session)
def test_timer(self): with mock_module(*gevent_modules): x = Timer() x.ensure_started() x.schedule = Mock() x.start() x.stop() x.schedule.clear.assert_called_with()
def test_process_cleanup(self): with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod x = mod.CassandraBackend(app=self.app) x.process_cleanup() self.assertIsNone(x._connection) self.assertIsNone(x._session)
def test_init_no_pycassa(self): with mock_module('pycassa'): from celery.backends import cassandra as mod prev, mod.pycassa = mod.pycassa, None try: with self.assertRaises(ImproperlyConfigured): mod.CassandraBackend(app=self.app) finally: mod.pycassa = prev
def test_get_column_family(self): with mock_module('pycassa'): from celery.backends import cassandra as mod mod.pycassa = Mock() install_exceptions(mod.pycassa) x = mod.CassandraBackend(app=self.app) self.assertTrue(x._get_column_family()) self.assertIsNotNone(x._column_family) self.assertIs(x._get_column_family(), x._column_family)
def test_store_result(self): with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod mod.cassandra = Mock() x = mod.CassandraBackend(app=self.app) x._connection = True session = x._session = Mock() session.execute = Mock() x._store_result('task_id', 'result', states.SUCCESS)
def test_process_cleanup(self): with mock_module('pycassa'): from celery.backends import cassandra as mod x = mod.CassandraBackend(app=self.app) x._column_family = None x.process_cleanup() x._column_family = True x.process_cleanup() self.assertIsNone(x._column_family)
def test_store_result(self): with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod mod.cassandra = Mock() x = mod.CassandraBackend(app=self.app) x._connection = True session = x._session = Mock() session.execute = Mock() x._store_result('task_id', 'result', states.SUCCESS)
def test_pool(self): with mock_module(*eventlet_modules): with patch_many("eventlet.greenpool.GreenPool", "eventlet.greenthread") as (GreenPool, greenthread): x = TaskPool() x.on_start() x.on_stop() x.on_apply(Mock()) x._pool = None x.on_stop() self.assertTrue(x.getpid())
def test_init_no_cassandra(self): """should raise ImproperlyConfigured when no python-driver installed.""" with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod prev, mod.cassandra = mod.cassandra, None try: with self.assertRaises(ImproperlyConfigured): mod.CassandraBackend(app=self.app) finally: mod.cassandra = prev
def test_init_no_cassandra(self): """should raise ImproperlyConfigured when no python-driver installed.""" with mock_module(*CASSANDRA_MODULES): from celery.backends import cassandra as mod prev, mod.cassandra = mod.cassandra, None try: with self.assertRaises(ImproperlyConfigured): mod.CassandraBackend(app=self.app) finally: mod.cassandra = prev
def test_pool(self): with mock_module(*eventlet_modules): with patch_many('eventlet.greenpool.GreenPool', 'eventlet.greenthread') as (GreenPool, greenthread): x = TaskPool() x.on_start() x.on_stop() x.on_apply(Mock()) x._pool = None x.on_stop() self.assertTrue(x.getpid())
def test_on_apply(self): with mock_module('threadpool'): x = TaskPool() x.on_start() callback = Mock() accept_callback = Mock() target = Mock() req = x.on_apply(target, args=(1, 2), kwargs={'a': 10}, callback=callback, accept_callback=accept_callback) x.WorkRequest.assert_called_with( apply_target, (target, (1, 2), {'a': 10}, callback, accept_callback), ) x._pool.putRequest.assert_called_with(req) x._pool._results_queue.queue.clear.assert_called_with()
def test_on_apply(self): with mock_module('threadpool'): x = TaskPool(app=self.app) x.on_start() callback = Mock() accept_callback = Mock() target = Mock() req = x.on_apply(target, args=(1, 2), kwargs={'a': 10}, callback=callback, accept_callback=accept_callback) x.WorkRequest.assert_called_with( apply_target, (target, (1, 2), {'a': 10}, callback, accept_callback), ) x._pool.putRequest.assert_called_with(req) x._pool._results_queue.queue.clear.assert_called_with()
def test_is_patched(self): with mock_module(*gevent_modules): monkey_patched = [] import gevent from gevent import monkey gevent.version_info = (1, 0, 0) prev_monkey_patch = monkey.patch_all monkey.patch_all = lambda: monkey_patched.append(True) prev_gevent = sys.modules.pop('celery.concurrency.gevent', None) os.environ.pop('GEVENT_NOPATCH') try: import celery.concurrency.gevent # noqa self.assertTrue(monkey_patched) finally: sys.modules['celery.concurrency.gevent'] = prev_gevent os.environ['GEVENT_NOPATCH'] = 'yes' monkey.patch_all = prev_monkey_patch
def test_store_result(self): with mock_module('pycassa'): from celery.backends import cassandra as mod mod.pycassa = Mock() install_exceptions(mod.pycassa) mod.Thrift = Mock() install_exceptions(mod.Thrift) x = mod.CassandraBackend(app=self.app) Get_Column = x._get_column_family = Mock() cf = Get_Column.return_value = Mock() x.detailed_mode = False x._store_result('task_id', 'result', states.SUCCESS) self.assertTrue(cf.insert.called) cf.insert.reset() x.detailed_mode = True x._store_result('task_id', 'result', states.SUCCESS) self.assertTrue(cf.insert.called)
def test_get_task_meta_for(self): with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod mod.cassandra = Mock() x = mod.CassandraBackend(app=self.app) x._connection = True session = x._session = Mock() execute = session.execute = Mock() execute.return_value = [ [states.SUCCESS, '1', datetime.now(), b'', b''] ] x.decode = Mock() meta = x._get_task_meta_for('task_id') self.assertEqual(meta['status'], states.SUCCESS) x._session.execute.return_value = [] meta = x._get_task_meta_for('task_id') self.assertEqual(meta['status'], states.PENDING)
def test_init_with_and_without_LOCAL_QUROM(self): with mock_module(*CASSANDRA_MODULES): from celery.backends import new_cassandra as mod mod.cassandra = Mock() cons = mod.cassandra.ConsistencyLevel = Object() cons.LOCAL_QUORUM = 'foo' self.app.conf.cassandra_read_consistency = 'LOCAL_FOO' self.app.conf.cassandra_write_consistency = 'LOCAL_FOO' mod.CassandraBackend(app=self.app) cons.LOCAL_FOO = 'bar' mod.CassandraBackend(app=self.app) # no servers raises ImproperlyConfigured with self.assertRaises(ImproperlyConfigured): self.app.conf.cassandra_servers = None mod.CassandraBackend( app=self.app, keyspace='b', column_family='c', )
def test_init_with_and_without_LOCAL_QUROM(self): with mock_module('pycassa'): from celery.backends import cassandra as mod mod.pycassa = Mock() install_exceptions(mod.pycassa) cons = mod.pycassa.ConsistencyLevel = Object() cons.LOCAL_QUORUM = 'foo' app = self.get_app() app.conf.CASSANDRA_READ_CONSISTENCY = 'LOCAL_FOO' app.conf.CASSANDRA_WRITE_CONSISTENCY = 'LOCAL_FOO' mod.CassandraBackend(app=app) cons.LOCAL_FOO = 'bar' mod.CassandraBackend(app=app) # no servers raises ImproperlyConfigured with self.assertRaises(ImproperlyConfigured): app.conf.CASSANDRA_SERVERS = None mod.CassandraBackend(app=app, keyspace='b', column_family='c')
def test_init_with_and_without_LOCAL_QUROM(self): with mock_module("pycassa"): from celery.backends import cassandra as mod mod.pycassa = Mock() install_exceptions(mod.pycassa) cons = mod.pycassa.ConsistencyLevel = Object() cons.LOCAL_QUORUM = "foo" self.app.conf.CASSANDRA_READ_CONSISTENCY = "LOCAL_FOO" self.app.conf.CASSANDRA_WRITE_CONSISTENCY = "LOCAL_FOO" mod.CassandraBackend(app=self.app) cons.LOCAL_FOO = "bar" mod.CassandraBackend(app=self.app) # no servers raises ImproperlyConfigured with self.assertRaises(ImproperlyConfigured): self.app.conf.CASSANDRA_SERVERS = None mod.CassandraBackend(app=self.app, keyspace="b", column_family="c")
def test_sched(self): with mock_module(*eventlet_modules): with patch_many("eventlet.greenthread.spawn_after", "greenlet.GreenletExit") as (spawn_after, GreenletExit): x = Schedule() x.GreenletExit = KeyError entry = Mock() g = x._enter(1, 0, entry) self.assertTrue(x.queue) x._entry_exit(g, entry) g.wait.side_effect = KeyError() x._entry_exit(g, entry) entry.cancel.assert_called_with() self.assertFalse(x._queue) x._queue.add(g) x.clear() x._queue.add(g) g.cancel.side_effect = KeyError() x.clear()
def test_sched(self): with mock_module(*eventlet_modules): with patch_many('eventlet.greenthread.spawn_after', 'greenlet.GreenletExit') as (spawn_after, GreenletExit): x = Schedule() x.GreenletExit = KeyError entry = Mock() g = x._enter(1, 0, entry) self.assertTrue(x.queue) x._entry_exit(g, entry) g.wait.side_effect = KeyError() x._entry_exit(g, entry) entry.cancel.assert_called_with() self.assertFalse(x._queue) x._queue.add(g) x.clear() x._queue.add(g) g.cancel.side_effect = KeyError() x.clear()
def test_pool(self): with mock_module(*gevent_modules): with patch_many('gevent.spawn_raw', 'gevent.pool.Pool') as ( spawn_raw, Pool): x = TaskPool() x.on_start() x.on_stop() x.on_apply(Mock()) x._pool = None x.on_stop() x._pool = Mock() x._pool._semaphore.counter = 1 x._pool.size = 1 x.grow() self.assertEqual(x._pool.size, 2) self.assertEqual(x._pool._semaphore.counter, 2) x.shrink() self.assertEqual(x._pool.size, 1) self.assertEqual(x._pool._semaphore.counter, 1) x._pool = [4, 5, 6] self.assertEqual(x.num_processes, 3)
def test_pool(self): with mock_module(*gevent_modules): with patch_many('gevent.spawn_raw', 'gevent.pool.Pool') as (spawn_raw, Pool): x = TaskPool() x.on_start() x.on_stop() x.on_apply(Mock()) x._pool = None x.on_stop() x._pool = Mock() x._pool._semaphore.counter = 1 x._pool.size = 1 x.grow() self.assertEqual(x._pool.size, 2) self.assertEqual(x._pool._semaphore.counter, 2) x.shrink() self.assertEqual(x._pool.size, 1) self.assertEqual(x._pool._semaphore.counter, 1) x._pool = [4, 5, 6] self.assertEqual(x.num_processes, 3)
def test_reduce(self): with mock_module(*CASSANDRA_MODULES): from celery.backends.cassandra import CassandraBackend self.assertTrue(loads(dumps(CassandraBackend(app=self.app))))
def test_reduce(self): with mock_module('pycassa'): from celery.backends.cassandra import CassandraBackend self.assertTrue(loads(dumps(CassandraBackend(app=self.get_app()))))
def test_on_stop(self): with mock_module('threadpool'): x = TaskPool() x.on_start() x.on_stop() x._pool.dismissWorkers.assert_called_with(x.limit, do_join=True)
def test_with_threadpool(self): with mock_module('threadpool'): x = TaskPool() self.assertTrue(x.ThreadPool) self.assertTrue(x.WorkRequest)
def test_on_stop(self): with mock_module('threadpool'): x = TaskPool(app=self.app) x.on_start() x.on_stop() x._pool.dismissWorkers.assert_called_with(x.limit, do_join=True)
def test_reduce(self): with mock_module(*CASSANDRA_MODULES): from celery.backends.new_cassandra import CassandraBackend self.assertTrue(loads(dumps(CassandraBackend(app=self.app))))
def test_with_threadpool(self): with mock_module('threadpool'): x = TaskPool(app=self.app) self.assertTrue(x.ThreadPool) self.assertTrue(x.WorkRequest)
def test_on_start(self): with mock_module('threadpool'): x = TaskPool(app=self.app) x.on_start() self.assertTrue(x._pool) self.assertIsInstance(x._pool.workRequests, NullDict)
def test_on_start(self): with mock_module('threadpool'): x = TaskPool() x.on_start() self.assertTrue(x._pool) self.assertIsInstance(x._pool.workRequests, NullDict)