def _optparse_callback_to_type(option, callback):
parser = Bunch(values=Bunch())
def _on_arg(value):
callback(option, None, value, parser)
return getattr(parser.values, option.dest)
return _on_arg
def test_setting__broker_transport_options(self):
_args = {'foo': 'bar', 'spam': 'baz'}
self.app.config_from_object(Bunch())
assert self.app.conf.broker_transport_options == {}
self.app.config_from_object(Bunch(broker_transport_options=_args))
assert self.app.conf.broker_transport_options == _args
def __init__(self, c, without_gossip=False, interval=5.0, **kwargs):
self.enabled = not without_gossip and self.compatible_transport(c.app)
self.app = c.app
c.gossip = self
self.Receiver = c.app.events.Receiver
self.hostname = c.hostname
self.full_hostname = '.'.join([self.hostname, str(c.pid)])
self.on = Bunch(
node_join=set(),
node_leave=set(),
node_lost=set(),
)
self.timer = c.timer
if self.enabled:
self.state = c.app.events.State(
on_node_join=self.on_node_join,
on_node_leave=self.on_node_leave,
max_tasks_in_memory=1,
)
if c.hub:
c._mutex = DummyLock()
self.update_state = self.state.event
self.interval = interval
self._tref = None
self.consensus_requests = defaultdict(list)
self.consensus_replies = {}
self.event_handlers = {
'worker.elect': self.on_elect,
'worker.elect.ack': self.on_elect_ack,
}
self.clock = c.app.clock
self.election_handlers = {'task': self.call_task}
def test_init_with_and_without_LOCAL_QUROM(self, module):
from celery.backends import cassandra as mod
mod.cassandra = Mock()
cons = mod.cassandra.ConsistencyLevel = Bunch(LOCAL_QUORUM='foo', )
self.app.conf.cassandra_read_consistency = 'LOCAL_FOO'
self.app.conf.cassandra_write_consistency = 'LOCAL_FOO'
mod.CassandraBackend(app=self.app)
cons.LOCAL_FOO = 'bar'
mod.CassandraBackend(app=self.app)
# no servers and no bundle_path raises ImproperlyConfigured
with pytest.raises(ImproperlyConfigured):
self.app.conf.cassandra_servers = None
self.app.conf.cassandra_secure_bundle_path = None
mod.CassandraBackend(
app=self.app,
keyspace='b',
column_family='c',
)
# both servers no bundle_path raises ImproperlyConfigured
with pytest.raises(ImproperlyConfigured):
self.app.conf.cassandra_servers = ['localhost']
self.app.conf.cassandra_secure_bundle_path = (
'/home/user/secure-connect-bundle.zip')
mod.CassandraBackend(
app=self.app,
keyspace='b',
column_family='c',
)
def on_unknown_task(self, body, message, exc):
error(UNKNOWN_TASK_ERROR, exc, dump_body(message, body), exc_info=True)
try:
id_, name = message.headers['id'], message.headers['task']
root_id = message.headers.get('root_id')
except KeyError: # proto1
id_, name = body['id'], body['task']
root_id = None
request = Bunch(
name=name,
chord=None,
root_id=root_id,
correlation_id=message.properties.get('correlation_id'),
reply_to=message.properties.get('reply_to'),
errbacks=None,
)
message.reject_log_error(logger, self.connection_errors)
self.app.backend.mark_as_failure(
id_,
NotRegistered(name),
request=request,
)
if self.event_dispatcher:
self.event_dispatcher.send(
'task-failed',
uuid=id_,
exception='NotRegistered({0!r})'.format(name),
)
signals.task_unknown.send(
sender=self,
message=message,
exc=exc,
name=name,
id=id_,
)
def __init__(self, *args, **kwargs):
self.started = True
self._timeout_handler = Mock()
self._result_handler = Mock()
self.maintain_pool = Mock()
self._state = mp.RUN
self._processes = kwargs.get('processes')
self._pool = [Bunch(pid=i, inqW_fd=1, outqR_fd=2)
for i in range(self._processes)]
self._current_proc = cycle(range(self._processes))
class MockPersistentScheduler(beat.PersistentScheduler):
sh = shelv
persistence = Bunch(open=lambda *a, **kw: shelv, )
tick_raises_exit = False
shutdown_service = None
def tick(self):
if self.tick_raises_exit:
raise SystemExit()
if self.shutdown_service:
self.shutdown_service._is_shutdown.set()
return 0.0
def test_get_set_keys_values_items(self):
x = DictAttribute(Bunch())
x['foo'] = 'The quick brown fox'
assert x['foo'] == 'The quick brown fox'
assert x['foo'] == x.obj.foo
assert x.get('foo') == 'The quick brown fox'
assert x.get('bar') is None
with pytest.raises(KeyError):
x['bar']
x.foo = 'The quick yellow fox'
assert x['foo'] == 'The quick yellow fox'
assert ('foo', 'The quick yellow fox') in list(x.items())
assert 'foo' in list(x.keys())
assert 'The quick yellow fox' in list(x.values())
def test_pending_configuration__django_settings(self):
with self.Celery(broker='foo://bar', backend='foo') as app:
app.config_from_object(DictAttribute(Bunch(
CELERY_TASK_ALWAYS_EAGER=4,
CELERY_TASK_DEFAULT_DELIVERY_MODE=63,
CELERY_WORKER_AGENT='foo:Barz',
CELERY_RESULT_SERIALIZER='pickle',
)), namespace='CELERY')
assert app.conf.result_serializer == 'pickle'
assert app.conf.CELERY_RESULT_SERIALIZER == 'pickle'
assert app.conf.task_always_eager == 4
assert app.conf.task_default_delivery_mode == 63
assert app.conf.worker_agent == 'foo:Barz'
assert app.conf.broker_url == 'foo://bar'
assert app.conf.result_backend == 'foo'
class MockPersistentScheduler(beat.PersistentScheduler):
sh = shelv
persistence = Bunch(
open=lambda *a, **kw: shelv,
)
def __init__(self, *args, **kwargs):
self.sent = []
beat.PersistentScheduler.__init__(self, *args, **kwargs)
def send_task(self, task=None, args=None, kwargs=None, **options):
self.sent.append({'task': task,
'args': args,
'kwargs': kwargs,
'options': options})
return self.app.AsyncResult(uuid())
def test_get_set_keys_values_items(self):
x = DictAttribute(Bunch())
x['foo'] = 'The quick brown fox'
self.assertEqual(x['foo'], 'The quick brown fox')
self.assertEqual(x['foo'], x.obj.foo)
self.assertEqual(x.get('foo'), 'The quick brown fox')
self.assertIsNone(x.get('bar'))
with self.assertRaises(KeyError):
x['bar']
x.foo = 'The quick yellow fox'
self.assertEqual(x['foo'], 'The quick yellow fox')
self.assertIn(
('foo', 'The quick yellow fox'),
list(x.items()),
)
self.assertIn('foo', list(x.keys()))
self.assertIn('The quick yellow fox', list(x.values()))
def test_info(self):
pool = TaskPool(10)
procs = [Bunch(pid=i) for i in range(pool.limit)]
class _Pool:
_pool = procs
_maxtasksperchild = None
timeout = 10
soft_timeout = 5
def human_write_stats(self, *args, **kwargs):
return {}
pool._pool = _Pool()
info = pool.info
assert info['max-concurrency'] == pool.limit
assert info['max-tasks-per-child'] == 'N/A'
assert info['timeouts'] == (5, 10)
def test_init_with_and_without_LOCAL_QUROM(self, *modules):
from celery.backends import cassandra as mod
mod.cassandra = Mock()
cons = mod.cassandra.ConsistencyLevel = Bunch(
LOCAL_QUORUM='foo',
)
self.app.conf.cassandra_read_consistency = 'LOCAL_FOO'
self.app.conf.cassandra_write_consistency = 'LOCAL_FOO'
mod.CassandraBackend(app=self.app)
cons.LOCAL_FOO = 'bar'
mod.CassandraBackend(app=self.app)
# no servers raises ImproperlyConfigured
with pytest.raises(ImproperlyConfigured):
self.app.conf.cassandra_servers = None
mod.CassandraBackend(
app=self.app, keyspace='b', column_family='c',
)
def test__close_database(self):
with self.fixup_context(self.app) as (f, _, _):
conns = [Mock(), Mock(), Mock()]
conns[1].close.side_effect = KeyError('already closed')
f.database_errors = (KeyError, )
f.interface_errors = ()
f._db.connections = Mock() # ConnectionHandler
f._db.connections.all.side_effect = lambda: conns
f._close_database()
conns[0].close.assert_called_with()
conns[1].close.assert_called_with()
conns[2].close.assert_called_with()
conns[1].close.side_effect = KeyError('omg')
with self.assertRaises(KeyError):
f._close_database()
o = Bunch(close_connection=Mock())
f._db = o
f._close_database()
o.close_connection.assert_called_with()
def test_add_defaults_object(self):
defaults = Bunch(foo=10)
self.view.add_defaults(defaults)
self.assertEqual(self.view.foo, 10)
def test_setdefault(self):
x = DictAttribute(Bunch())
x.setdefault('foo', 'NEW')
self.assertEqual(x['foo'], 'NEW')
x.setdefault('foo', 'XYZ')
self.assertEqual(x['foo'], 'NEW')
def test_items(self):
obj = Bunch(attr1=1)
x = DictAttribute(obj)
x['attr2'] = 2
assert x['attr1'] == 1
assert x['attr2'] == 2
def test_contains(self):
x = DictAttribute(Bunch())
x['foo'] = 1
assert 'foo' in x
assert 'bar' not in x
def test_setdefault(self):
x = DictAttribute(Bunch())
x.setdefault('foo', 'NEW')
assert x['foo'] == 'NEW'
x.setdefault('foo', 'XYZ')
assert x['foo'] == 'NEW'
def test_contains(self):
x = DictAttribute(Bunch())
x['foo'] = 1
self.assertIn('foo', x)
self.assertNotIn('bar', x)
def test_add_defaults_object(self):
defaults = Bunch(foo=10)
self.view.add_defaults(defaults)
assert self.view.foo == 10
def test_items(self):
obj = Bunch(attr1=1)
x = DictAttribute(obj)
x['attr2'] = 2
self.assertEqual(x['attr1'], 1)
self.assertEqual(x['attr2'], 2)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._pool = Bunch(_processes=self.limit)
def test(self):
x = Bunch(foo='foo', bar=2)
assert x.foo == 'foo'
assert x.bar == 2
def parse_options(self, prog_name, arguments, command=None):
options = Bunch(foo='bar', prog_name=prog_name)
return options, self.mock_args
def test(self):
x = Bunch(foo='foo', bar=2)
self.assertEqual(x.foo, 'foo')
self.assertEqual(x.bar, 2)
