def test_get_task_meta_for(self):
with mock_module('pycassa'):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
install_exceptions(mod.pycassa)
mod.Thrift = Mock()
install_exceptions(mod.Thrift)
app = self.get_app()
x = mod.CassandraBackend(app=app)
Get_Column = x._get_column_family = Mock()
get_column = Get_Column.return_value = Mock()
get = get_column.get
META = get.return_value = {
'task_id': 'task_id',
'status': states.SUCCESS,
'result': '1',
'date_done': 'date',
'traceback': '',
'children': None,
}
x.decode = Mock()
x.detailed_mode = False
meta = x._get_task_meta_for('task_id')
self.assertEqual(meta['status'], states.SUCCESS)
x.detailed_mode = True
row = get.return_value = Mock()
row.values.return_value = [Mock()]
x.decode.return_value = META
meta = x._get_task_meta_for('task_id')
self.assertEqual(meta['status'], states.SUCCESS)
x.decode.return_value = Mock()
x.detailed_mode = False
get.side_effect = KeyError()
meta = x._get_task_meta_for('task_id')
self.assertEqual(meta['status'], states.PENDING)
calls = [0]
end = [10]
def work_eventually(*arg):
try:
if calls[0] > end[0]:
return META
raise socket.error()
finally:
calls[0] += 1
get.side_effect = work_eventually
x._retry_timeout = 10
x._retry_wait = 0.01
meta = x._get_task_meta_for('task')
self.assertEqual(meta['status'], states.SUCCESS)
x._retry_timeout = 0.1
calls[0], end[0] = 0, 100
with self.assertRaises(socket.error):
x._get_task_meta_for('task')
def test_timer(self):
with mock_module(*gevent_modules):
x = Timer()
x.ensure_started()
x.schedule = Mock()
x.start()
x.stop()
x.schedule.clear.assert_called_with()
def test_timer(self):
with mock_module(*gevent_modules):
x = Timer()
x.ensure_started()
x.schedule = Mock()
x.start()
x.stop()
x.schedule.clear.assert_called_with()
def test_get_task_meta_for(self):
with mock_module('pycassa'):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
install_exceptions(mod.pycassa)
mod.Thrift = Mock()
install_exceptions(mod.Thrift)
app = self.get_app()
x = mod.CassandraBackend(app=app)
Get_Column = x._get_column_family = Mock()
get_column = Get_Column.return_value = Mock()
get = get_column.get
META = get.return_value = {
'task_id': 'task_id',
'status': states.SUCCESS,
'result': '1',
'date_done': 'date',
'traceback': '',
'children': None,
}
x.decode = Mock()
x.detailed_mode = False
meta = x._get_task_meta_for('task_id')
self.assertEqual(meta['status'], states.SUCCESS)
x.detailed_mode = True
row = get.return_value = Mock()
row.values.return_value = [Mock()]
x.decode.return_value = META
meta = x._get_task_meta_for('task_id')
self.assertEqual(meta['status'], states.SUCCESS)
x.decode.return_value = Mock()
x.detailed_mode = False
get.side_effect = KeyError()
meta = x._get_task_meta_for('task_id')
self.assertEqual(meta['status'], states.PENDING)
calls = [0]
end = [10]
def work_eventually(*arg):
try:
if calls[0] > end[0]:
return META
raise socket.error()
finally:
calls[0] += 1
get.side_effect = work_eventually
x._retry_timeout = 10
x._retry_wait = 0.01
meta = x._get_task_meta_for('task')
self.assertEqual(meta['status'], states.SUCCESS)
x._retry_timeout = 0.1
calls[0], end[0] = 0, 100
with self.assertRaises(socket.error):
x._get_task_meta_for('task')
def test_init_no_pycassa(self):
with mock_module('pycassa'):
from celery.backends import cassandra as mod
prev, mod.pycassa = mod.pycassa, None
try:
with self.assertRaises(ImproperlyConfigured):
mod.CassandraBackend(app=self.app)
finally:
mod.pycassa = prev
def test_init_no_pycassa(self):
with mock_module('pycassa'):
from celery.backends import cassandra as mod
prev, mod.pycassa = mod.pycassa, None
try:
with self.assertRaises(ImproperlyConfigured):
mod.CassandraBackend(app=self.app)
finally:
mod.pycassa = prev
def test_get_column_family(self):
with mock_module("pycassa"):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
app = self.get_app()
x = mod.CassandraBackend(app=app)
self.assertTrue(x._get_column_family())
self.assertIsNotNone(x._column_family)
self.assertIs(x._get_column_family(), x._column_family)
def test_get_column_family(self):
with mock_module("pycassa"):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
app = self.get_app()
x = mod.CassandraBackend(app=app)
self.assertTrue(x._get_column_family())
self.assertIsNotNone(x._column_family)
self.assertIs(x._get_column_family(), x._column_family)
def test_get_task_meta_for(self):
with mock_module("pycassa"):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
mod.Thrift = Mock()
app = self.get_app()
x = mod.CassandraBackend(app=app)
Get_Column = x._get_column_family = Mock()
get_column = Get_Column.return_value = Mock()
get = get_column.get
META = get.return_value = {
"task_id": "task_id",
"status": states.SUCCESS,
"result": "1",
"date_done": "date",
"traceback": ""
}
x.decode = Mock()
x.detailed_mode = False
meta = x._get_task_meta_for("task_id")
self.assertEqual(meta["status"], states.SUCCESS)
x.detailed_mode = True
row = get.return_value = Mock()
row.values.return_value = [Mock()]
x.decode.return_value = META
meta = x._get_task_meta_for("task_id")
self.assertEqual(meta["status"], states.SUCCESS)
x.decode.return_value = Mock()
x.detailed_mode = False
get.side_effect = KeyError()
meta = x._get_task_meta_for("task_id")
self.assertEqual(meta["status"], states.PENDING)
calls = [0]
end = [10]
def work_eventually(*arg):
try:
if calls[0] > end[0]:
return META
raise socket.error()
finally:
calls[0] += 1
get.side_effect = work_eventually
x._retry_timeout = 10
x._retry_wait = 0.01
meta = x._get_task_meta_for("task")
self.assertEqual(meta["status"], states.SUCCESS)
x._retry_timeout = 0.1
calls[0], end[0] = 0, 100
with self.assertRaises(socket.error):
x._get_task_meta_for("task")
def test_process_cleanup(self):
with mock_module('pycassa'):
from celery.backends import cassandra as mod
app = self.get_app()
x = mod.CassandraBackend(app=app)
x._column_family = None
x.process_cleanup()
x._column_family = True
x.process_cleanup()
self.assertIsNone(x._column_family)
def test_get_task_meta_for(self):
with mock_module("pycassa"):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
mod.Thrift = Mock()
app = self.get_app()
x = mod.CassandraBackend(app=app)
Get_Column = x._get_column_family = Mock()
get_column = Get_Column.return_value = Mock()
get = get_column.get
META = get.return_value = {"task_id": "task_id",
"status": states.SUCCESS,
"result": "1",
"date_done": "date",
"traceback": "",
"children": None}
x.decode = Mock()
x.detailed_mode = False
meta = x._get_task_meta_for("task_id")
self.assertEqual(meta["status"], states.SUCCESS)
x.detailed_mode = True
row = get.return_value = Mock()
row.values.return_value = [Mock()]
x.decode.return_value = META
meta = x._get_task_meta_for("task_id")
self.assertEqual(meta["status"], states.SUCCESS)
x.decode.return_value = Mock()
x.detailed_mode = False
get.side_effect = KeyError()
meta = x._get_task_meta_for("task_id")
self.assertEqual(meta["status"], states.PENDING)
calls = [0]
end = [10]
def work_eventually(*arg):
try:
if calls[0] > end[0]:
return META
raise socket.error()
finally:
calls[0] += 1
get.side_effect = work_eventually
x._retry_timeout = 10
x._retry_wait = 0.01
meta = x._get_task_meta_for("task")
self.assertEqual(meta["status"], states.SUCCESS)
x._retry_timeout = 0.1
calls[0], end[0] = 0, 100
with self.assertRaises(socket.error):
x._get_task_meta_for("task")
def test_process_cleanup(self):
with mock_module('pycassa'):
from celery.backends import cassandra as mod
app = self.get_app()
x = mod.CassandraBackend(app=app)
x._column_family = None
x.process_cleanup()
x._column_family = True
x.process_cleanup()
self.assertIsNone(x._column_family)
def test_pool(self):
with mock_module(*eventlet_modules):
with patch_many('eventlet.greenpool.GreenPool',
'eventlet.greenthread') as (GreenPool,
greenthread):
x = TaskPool()
x.on_start()
x.on_stop()
x.on_apply(Mock())
x._pool = None
x.on_stop()
self.assertTrue(x.getpid())
def test_on_apply(self):
with mock_module('threadpool'):
x = TaskPool()
x.on_start()
callback = Mock()
accept_callback = Mock()
target = Mock()
req = x.on_apply(target, args=(1, 2), kwargs={'a': 10},
callback=callback, accept_callback=accept_callback)
x.WorkRequest.assert_called_with(apply_target, (
target, (1, 2), {'a': 10}, callback, accept_callback))
x._pool.putRequest.assert_called_with(req)
x._pool._results_queue.queue.clear.assert_called_with()
def test_is_patched(self):
with mock_module(*gevent_modules):
monkey_patched = []
from gevent import monkey
prev_monkey_patch = monkey.patch_all
monkey.patch_all = lambda: monkey_patched.append(True)
prev_gevent = sys.modules.pop("celery.concurrency.gevent", None)
os.environ.pop("GEVENT_NOPATCH")
try:
import celery.concurrency.gevent # noqa
self.assertTrue(monkey_patched)
finally:
sys.modules["celery.concurrency.gevent"] = prev_gevent
os.environ["GEVENT_NOPATCH"] = "yes"
monkey.patch_all = prev_monkey_patch
def test_is_patched(self):
with mock_module(*gevent_modules):
monkey_patched = []
from gevent import monkey
prev_monkey_patch = monkey.patch_all
monkey.patch_all = lambda: monkey_patched.append(True)
prev_gevent = sys.modules.pop('celery.concurrency.gevent', None)
os.environ.pop('GEVENT_NOPATCH')
try:
import celery.concurrency.gevent # noqa
self.assertTrue(monkey_patched)
finally:
sys.modules['celery.concurrency.gevent'] = prev_gevent
os.environ['GEVENT_NOPATCH'] = 'yes'
monkey.patch_all = prev_monkey_patch
def test_on_apply(self):
with mock_module('threadpool'):
x = TaskPool()
x.on_start()
callback = Mock()
accept_callback = Mock()
target = Mock()
req = x.on_apply(target, args=(1, 2), kwargs={'a': 10},
callback=callback,
accept_callback=accept_callback)
x.WorkRequest.assert_called_with(
apply_target,
(target, (1, 2), {'a': 10}, callback, accept_callback),
)
x._pool.putRequest.assert_called_with(req)
x._pool._results_queue.queue.clear.assert_called_with()
def test_store_result(self):
with mock_module("pycassa"):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
mod.Thrift = Mock()
app = self.get_app()
x = mod.CassandraBackend(app=app)
Get_Column = x._get_column_family = Mock()
cf = Get_Column.return_value = Mock()
x.detailed_mode = False
x._store_result("task_id", "result", states.SUCCESS)
self.assertTrue(cf.insert.called)
cf.insert.reset()
x.detailed_mode = True
x._store_result("task_id", "result", states.SUCCESS)
self.assertTrue(cf.insert.called)
def test_is_patched(self):
with mock_module(*gevent_modules):
monkey_patched = []
import gevent
from gevent import monkey
gevent.version_info = (1, 0, 0)
prev_monkey_patch = monkey.patch_all
monkey.patch_all = lambda: monkey_patched.append(True)
prev_gevent = sys.modules.pop('celery.concurrency.gevent', None)
os.environ.pop('GEVENT_NOPATCH')
try:
import celery.concurrency.gevent # noqa
self.assertTrue(monkey_patched)
finally:
sys.modules['celery.concurrency.gevent'] = prev_gevent
os.environ['GEVENT_NOPATCH'] = 'yes'
monkey.patch_all = prev_monkey_patch
def test_store_result(self):
with mock_module("pycassa"):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
mod.Thrift = Mock()
app = self.get_app()
x = mod.CassandraBackend(app=app)
Get_Column = x._get_column_family = Mock()
cf = Get_Column.return_value = Mock()
x.detailed_mode = False
x._store_result("task_id", "result", states.SUCCESS)
self.assertTrue(cf.insert.called)
cf.insert.reset()
x.detailed_mode = True
x._store_result("task_id", "result", states.SUCCESS)
self.assertTrue(cf.insert.called)
def test_init_with_and_without_LOCAL_QUROM(self):
with mock_module("pycassa"):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
cons = mod.pycassa.ConsistencyLevel = Object()
cons.LOCAL_QUORUM = "foo"
app = self.get_app()
app.conf.CASSANDRA_READ_CONSISTENCY = "LOCAL_FOO"
app.conf.CASSANDRA_WRITE_CONSISTENCY = "LOCAL_FOO"
mod.CassandraBackend(app=app)
cons.LOCAL_FOO = "bar"
mod.CassandraBackend(app=app)
# no servers raises ImproperlyConfigured
with self.assertRaises(ImproperlyConfigured):
app.conf.CASSANDRA_SERVERS = None
mod.CassandraBackend(app=app, keyspace="b", column_family="c")
def test_init_with_and_without_LOCAL_QUROM(self):
with mock_module("pycassa"):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
cons = mod.pycassa.ConsistencyLevel = Object()
cons.LOCAL_QUORUM = "foo"
app = self.get_app()
app.conf.CASSANDRA_READ_CONSISTENCY = "LOCAL_FOO"
app.conf.CASSANDRA_WRITE_CONSISTENCY = "LOCAL_FOO"
mod.CassandraBackend(app=app)
cons.LOCAL_FOO = "bar"
mod.CassandraBackend(app=app)
# no servers raises ImproperlyConfigured
with self.assertRaises(ImproperlyConfigured):
app.conf.CASSANDRA_SERVERS = None
mod.CassandraBackend(app=app, keyspace="b",
column_family="c")
def test_init_with_and_without_LOCAL_QUROM(self):
with mock_module('pycassa'):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
install_exceptions(mod.pycassa)
cons = mod.pycassa.ConsistencyLevel = Object()
cons.LOCAL_QUORUM = 'foo'
app = self.get_app()
app.conf.CASSANDRA_READ_CONSISTENCY = 'LOCAL_FOO'
app.conf.CASSANDRA_WRITE_CONSISTENCY = 'LOCAL_FOO'
mod.CassandraBackend(app=app)
cons.LOCAL_FOO = 'bar'
mod.CassandraBackend(app=app)
# no servers raises ImproperlyConfigured
with self.assertRaises(ImproperlyConfigured):
app.conf.CASSANDRA_SERVERS = None
mod.CassandraBackend(app=app, keyspace='b', column_family='c')
def test_init_with_and_without_LOCAL_QUROM(self):
with mock_module('pycassa'):
from celery.backends import cassandra as mod
mod.pycassa = Mock()
install_exceptions(mod.pycassa)
cons = mod.pycassa.ConsistencyLevel = Object()
cons.LOCAL_QUORUM = 'foo'
app = self.get_app()
app.conf.CASSANDRA_READ_CONSISTENCY = 'LOCAL_FOO'
app.conf.CASSANDRA_WRITE_CONSISTENCY = 'LOCAL_FOO'
mod.CassandraBackend(app=app)
cons.LOCAL_FOO = 'bar'
mod.CassandraBackend(app=app)
# no servers raises ImproperlyConfigured
with self.assertRaises(ImproperlyConfigured):
app.conf.CASSANDRA_SERVERS = None
mod.CassandraBackend(app=app, keyspace='b', column_family='c')
def test_sched(self):
with mock_module(*eventlet_modules):
with patch_many('eventlet.greenthread.spawn_after',
'greenlet.GreenletExit') as (spawn_after,
GreenletExit):
x = Schedule()
x.GreenletExit = KeyError
entry = Mock()
g = x._enter(1, 0, entry)
self.assertTrue(x.queue)
x._entry_exit(g, entry)
g.wait.side_effect = KeyError()
x._entry_exit(g, entry)
entry.cancel.assert_called_with()
self.assertFalse(x._queue)
x._queue.add(g)
x.clear()
x._queue.add(g)
g.cancel.side_effect = KeyError()
x.clear()
def test_pool(self):
with mock_module(*gevent_modules):
with patch_many('gevent.spawn_raw',
'gevent.pool.Pool') as (spawn_raw, Pool):
x = TaskPool()
x.on_start()
x.on_stop()
x.on_apply(Mock())
x._pool = None
x.on_stop()
x._pool = Mock()
x._pool._semaphore.counter = 1
x._pool.size = 1
x.grow()
self.assertEqual(x._pool.size, 2)
self.assertEqual(x._pool._semaphore.counter, 2)
x.shrink()
self.assertEqual(x._pool.size, 1)
self.assertEqual(x._pool._semaphore.counter, 1)
x._pool = [4, 5, 6]
self.assertEqual(x.num_processes, 3)
def test_sched(self):
with mock_module(*gevent_modules):
with patch_many('gevent.greenlet',
'gevent.greenlet.GreenletExit') as (greenlet,
GreenletExit):
greenlet.Greenlet = object
x = Schedule()
greenlet.Greenlet = Mock()
x._Greenlet.spawn_later = Mock()
x._GreenletExit = KeyError
entry = Mock()
g = x._enter(1, 0, entry)
self.assertTrue(x.queue)
x._entry_exit(g)
g.kill.assert_called_with()
self.assertFalse(x._queue)
x._queue.add(g)
x.clear()
x._queue.add(g)
g.kill.side_effect = KeyError()
x.clear()
def test_sched(self):
with mock_module(*gevent_modules):
with patch_many('gevent.greenlet',
'gevent.greenlet.GreenletExit') as (greenlet,
GreenletExit):
greenlet.Greenlet = object
x = Schedule()
greenlet.Greenlet = Mock()
x._Greenlet.spawn_later = Mock()
x._GreenletExit = KeyError
entry = Mock()
g = x._enter(1, 0, entry)
self.assertTrue(x.queue)
x._entry_exit(g)
g.kill.assert_called_with()
self.assertFalse(x._queue)
x._queue.add(g)
x.clear()
x._queue.add(g)
g.kill.side_effect = KeyError()
x.clear()
def test_pool(self):
with mock_module(*gevent_modules):
with patch_many('gevent.spawn_raw', 'gevent.pool.Pool') as (
spawn_raw, Pool):
x = TaskPool()
x.on_start()
x.on_stop()
x.on_apply(Mock())
x._pool = None
x.on_stop()
x._pool = Mock()
x._pool._semaphore.counter = 1
x._pool.size = 1
x.grow()
self.assertEqual(x._pool.size, 2)
self.assertEqual(x._pool._semaphore.counter, 2)
x.shrink()
self.assertEqual(x._pool.size, 1)
self.assertEqual(x._pool._semaphore.counter, 1)
x._pool = [4, 5, 6]
self.assertEqual(x.num_processes, 3)
def test_reduce(self):
with mock_module('pycassa'):
from celery.backends.cassandra import CassandraBackend
self.assertTrue(loads(dumps(CassandraBackend(app=self.get_app()))))
def test_reduce(self):
with mock_module('pycassa'):
from celery.backends.cassandra import CassandraBackend
self.assertTrue(loads(dumps(CassandraBackend(app=self.get_app()))))
def test_on_stop(self):
with mock_module('threadpool'):
x = TaskPool()
x.on_start()
x.on_stop()
x._pool.dismissWorkers.assert_called_with(x.limit, do_join=True)
def test_on_start(self):
with mock_module('threadpool'):
x = TaskPool()
x.on_start()
self.assertTrue(x._pool)
self.assertIsInstance(x._pool.workRequests, NullDict)
def test_with_threadpool(self):
with mock_module('threadpool'):
x = TaskPool()
self.assertTrue(x.ThreadPool)
self.assertTrue(x.WorkRequest)
def test_on_stop(self):
with mock_module("threadpool"):
x = TaskPool()
x.on_start()
x.on_stop()
x._pool.dismissWorkers.assert_called_with(x.limit, do_join=True)
def test_on_start(self):
with mock_module("threadpool"):
x = TaskPool()
x.on_start()
self.assertTrue(x._pool)
self.assertIsInstance(x._pool.workRequests, NullDict)
def test_with_threadpool(self):
with mock_module("threadpool"):
x = TaskPool()
self.assertTrue(x.ThreadPool)
self.assertTrue(x.WorkRequest)