class CeleryClient(object):
def __init__(self):
self.celery = Celery(broker='amqp://', backend='amqp://')
self.celery.conf.update(CELERY_TASK_SERIALIZER="json")
def execute_task(self, task_name, task_queue, task_id=None, kwargs=None):
"""
Execute a task
:param task_name: the task name
:param task_queue: the task queue
:param task_id: optional id for the task
:param kwargs: optional kwargs to be passed to the task
:return: the celery task async result
"""
return self.celery.send_task(task_name,
queue=task_queue,
task_id=task_id,
kwargs=kwargs)
def get_task_status(self, task_id):
"""
Gets task's celery status by the task's id
:param task_id: the task id
:return: the task's celery status
"""
async_result = self.celery.AsyncResult(task_id)
return async_result.status
def get_failed_task_error(self, task_id):
"""
Gets a failed task's error by the task's id
:param task_id: the task id
:return: the exception object
"""
async_result = self.celery.AsyncResult(task_id)
return async_result.result
def celery_report(celery_task_id):
app = Celery("assetv2")
app.config_from_object("django.conf:settings")
async_result = app.AsyncResult(celery_task_id)
cache = get_cache('deploy', **{'LOCATION': CACHES['deploy']['LOCATION']+'2'})
task_dict = dict()
task_dict['ready'] = async_result.ready()
task_dict['status'] = async_result.status
task_dict['result'] = str(async_result.result)
log_list = cache.get(celery_task_id, [])
log_list.reverse()
task_dict['logs'] = log_list
return task_dict
def get(self, request, heat_id=None):
# list heat queries
if heat_id == None:
objs = HeatQuery.objects.all()
serializer = HeatQuerySerializer(objs, many=True)
out = serializer.data
else:
app = Celery('metrilyx')
app.config_from_object('metrilyx.heatmapsconfig')
rslt = app.AsyncResult(heat_id)
hquery = get_object_or_404(HeatQuery, _id=heat_id)
serializer = HeatQuerySerializer(hquery)
out = serializer.data
out['data'] = rslt.get()
return Response(out)
def progress(request):
task_id = request.GET["task_id"]
app = Celery('tasks',
backend='redis://redis/',
broker='pyamqp://*****:*****@rabbitmq/app')
progress = app.AsyncResult(task_id)
response_data = {"state": progress.state}
if progress.state == "IN PROGRESS":
if progress.info != 0:
output = progress.result["progress"]
job_lenght = progress.result["total"]
progress_percentage = progress.result["current"] * 100 // job_lenght
response_data.update({
"job_lenght": job_lenght,
"progress_percentage": progress_percentage,
"progress": output
})
return HttpResponse(json.dumps(response_data),
content_type="application/json")
class CeleryClient(object):
def __init__(self):
ssl_settings = self._get_broker_ssl_settings(
ssl_enabled=config.instance().amqp_ssl_enabled,
cert_path=config.instance().amqp_ca_path,
)
# Port not required as currently the address is provided with port and
# vhost included.
amqp_uri = 'amqp://{username}:{password}@{address}'.format(
address=config.instance().amqp_address,
username=config.instance().amqp_username,
password=config.instance().amqp_password,
)
self.celery = Celery(broker=amqp_uri, backend=amqp_uri)
self.celery.conf.update(CELERY_TASK_SERIALIZER="json",
CELERY_TASK_RESULT_EXPIRES=600)
if config.instance().amqp_ssl_enabled:
self.celery.conf.update(BROKER_USE_SSL=ssl_settings)
def close(self):
if self.celery:
self.celery.close()
def execute_task(self, task_queue, task_id=None, kwargs=None):
"""
Execute a task
:param task_queue: the task queue
:param task_id: optional id for the task
:param kwargs: optional kwargs to be passed to the task
:return: the celery task async result
"""
return self.celery.send_task('cloudify.dispatch.dispatch',
queue=task_queue,
task_id=task_id,
kwargs=kwargs)
def get_task_status(self, task_id):
"""
Gets task's celery status by the task's id
:param task_id: the task id
:return: the task's celery status
"""
async_result = self.celery.AsyncResult(task_id)
return async_result.status
def get_failed_task_error(self, task_id):
"""
Gets a failed task's error by the task's id
:param task_id: the task id
:return: the exception object
"""
async_result = self.celery.AsyncResult(task_id)
return async_result.result
@staticmethod
def _get_broker_ssl_settings(ssl_enabled, cert_path):
# Input vars may be None if not set. Explicitly defining defaults.
ssl_enabled = ssl_enabled or False
cert_path = cert_path or ''
if ssl_enabled:
if not cert_path:
raise ValueError(
"Broker SSL enabled but no SSL cert was provided. "
"If rabbitmq_ssl_enabled is True in the inputs, "
"rabbitmq_cert_public (and private) must be populated.")
ssl_options = {
'ca_certs': cert_path,
'cert_reqs': ssl.CERT_REQUIRED,
}
else:
ssl_options = {}
return ssl_options
class test_App(Case):
def setUp(self):
self.app = Celery(set_as_current=False)
self.app.conf.update(test_config)
def test_task(self):
app = Celery('foozibari', set_as_current=False)
def fun():
pass
fun.__module__ = '__main__'
task = app.task(fun)
self.assertEqual(task.name, app.main + '.fun')
def test_with_broker(self):
prev = os.environ.get('CELERY_BROKER_URL')
os.environ.pop('CELERY_BROKER_URL', None)
try:
app = Celery(set_as_current=False, broker='foo://baribaz')
self.assertEqual(app.conf.BROKER_HOST, 'foo://baribaz')
finally:
os.environ['CELERY_BROKER_URL'] = prev
def test_repr(self):
self.assertTrue(repr(self.app))
def test_custom_task_registry(self):
app1 = Celery(set_as_current=False)
app2 = Celery(set_as_current=False, tasks=app1.tasks)
self.assertIs(app2.tasks, app1.tasks)
def test_include_argument(self):
app = Celery(set_as_current=False, include=('foo', 'bar.foo'))
self.assertEqual(app.conf.CELERY_IMPORTS, ('foo', 'bar.foo'))
def test_set_as_current(self):
current = state._tls.current_app
try:
app = Celery(set_as_current=True)
self.assertIs(state._tls.current_app, app)
finally:
state._tls.current_app = current
def test_current_task(self):
app = Celery(set_as_current=False)
@app.task
def foo():
pass
state._task_stack.push(foo)
try:
self.assertEqual(app.current_task.name, foo.name)
finally:
state._task_stack.pop()
def test_task_not_shared(self):
with patch('celery.app.base.shared_task') as shared_task:
app = Celery(set_as_current=False)
@app.task(shared=False)
def foo():
pass
self.assertFalse(shared_task.called)
def test_task_compat_with_filter(self):
app = Celery(set_as_current=False, accept_magic_kwargs=True)
check = Mock()
def filter(task):
check(task)
return task
@app.task(filter=filter)
def foo():
pass
check.assert_called_with(foo)
def test_task_with_filter(self):
app = Celery(set_as_current=False, accept_magic_kwargs=False)
check = Mock()
def filter(task):
check(task)
return task
@app.task(filter=filter)
def foo():
pass
check.assert_called_with(foo)
def test_task_sets_main_name_MP_MAIN_FILE(self):
from celery.app import task as _task
_task.MP_MAIN_FILE = __file__
try:
app = Celery('xuzzy', set_as_current=False)
@app.task
def foo():
pass
self.assertEqual(foo.name, 'xuzzy.foo')
finally:
_task.MP_MAIN_FILE = None
def test_base_task_inherits_magic_kwargs_from_app(self):
from celery.app.task import Task
class timkX(Task):
abstract = True
app = Celery(set_as_current=False, accept_magic_kwargs=True)
timkX.bind(app)
self.assertTrue(timkX.accept_magic_kwargs)
def test_annotate_decorator(self):
from celery.app.task import Task
class adX(Task):
abstract = True
def run(self, y, z, x):
return y, z, x
check = Mock()
def deco(fun):
def _inner(*args, **kwargs):
check(*args, **kwargs)
return fun(*args, **kwargs)
return _inner
app = Celery(set_as_current=False)
app.conf.CELERY_ANNOTATIONS = {adX.name: {'@__call__': deco}}
adX.bind(app)
self.assertIs(adX.app, app)
i = adX()
i(2, 4, x=3)
check.assert_called_with(i, 2, 4, x=3)
i.annotate()
i.annotate()
def test_apply_async_has__self__(self):
app = Celery(set_as_current=False)
@app.task(__self__='hello')
def aawsX():
pass
with patch('celery.app.amqp.TaskProducer.delay_task') as dt:
aawsX.apply_async((4, 5))
args = dt.call_args[0][1]
self.assertEqual(args, ('hello', 4, 5))
def test_apply_async__connection_arg(self):
app = Celery(set_as_current=False)
@app.task()
def aacaX():
pass
connection = app.broker_connection('asd://')
with self.assertRaises(KeyError):
aacaX.apply_async(connection=connection)
def test_apply_async_adds_children(self):
from celery.state import _task_stack
app = Celery(set_as_current=False)
@app.task()
def a3cX1(self):
pass
@app.task()
def a3cX2(self):
pass
_task_stack.push(a3cX1)
try:
a3cX1.push_request(called_directly=False)
try:
res = a3cX2.apply_async(add_to_parent=True)
self.assertIn(res, a3cX1.request.children)
finally:
a3cX1.pop_request()
finally:
_task_stack.pop()
def test_TaskSet(self):
ts = self.app.TaskSet()
self.assertListEqual(ts.tasks, [])
self.assertIs(ts.app, self.app)
def test_pickle_app(self):
changes = dict(THE_FOO_BAR='bars', THE_MII_MAR='jars')
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
self.assertLess(len(saved), 2048)
restored = pickle.loads(saved)
self.assertDictContainsSubset(changes, restored.conf)
def test_worker_main(self):
from celery.bin import celeryd
class WorkerCommand(celeryd.WorkerCommand):
def execute_from_commandline(self, argv):
return argv
prev, celeryd.WorkerCommand = celeryd.WorkerCommand, WorkerCommand
try:
ret = self.app.worker_main(argv=['--version'])
self.assertListEqual(ret, ['--version'])
finally:
celeryd.WorkerCommand = prev
def test_config_from_envvar(self):
os.environ['CELERYTEST_CONFIG_OBJECT'] = 'celery.tests.app.test_app'
self.app.config_from_envvar('CELERYTEST_CONFIG_OBJECT')
self.assertEqual(self.app.conf.THIS_IS_A_KEY, 'this is a value')
def test_config_from_object(self):
class Object(object):
LEAVE_FOR_WORK = True
MOMENT_TO_STOP = True
CALL_ME_BACK = 123456789
WANT_ME_TO = False
UNDERSTAND_ME = True
self.app.config_from_object(Object())
self.assertTrue(self.app.conf.LEAVE_FOR_WORK)
self.assertTrue(self.app.conf.MOMENT_TO_STOP)
self.assertEqual(self.app.conf.CALL_ME_BACK, 123456789)
self.assertFalse(self.app.conf.WANT_ME_TO)
self.assertTrue(self.app.conf.UNDERSTAND_ME)
def test_config_from_cmdline(self):
cmdline = [
'.always_eager=no', '.result_backend=/dev/null',
'.task_error_whitelist=(list)["a", "b", "c"]',
'celeryd.prefetch_multiplier=368', '.foobarstring=(string)300',
'.foobarint=(int)300',
'.result_engine_options=(dict){"foo": "bar"}'
]
self.app.config_from_cmdline(cmdline, namespace='celery')
self.assertFalse(self.app.conf.CELERY_ALWAYS_EAGER)
self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, '/dev/null')
self.assertEqual(self.app.conf.CELERYD_PREFETCH_MULTIPLIER, 368)
self.assertListEqual(self.app.conf.CELERY_TASK_ERROR_WHITELIST,
['a', 'b', 'c'])
self.assertEqual(self.app.conf.CELERY_FOOBARSTRING, '300')
self.assertEqual(self.app.conf.CELERY_FOOBARINT, 300)
self.assertDictEqual(self.app.conf.CELERY_RESULT_ENGINE_OPTIONS,
{'foo': 'bar'})
def test_compat_setting_CELERY_BACKEND(self):
self.app.config_from_object(Object(CELERY_BACKEND='set_by_us'))
self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, 'set_by_us')
def test_setting_BROKER_TRANSPORT_OPTIONS(self):
_args = {'foo': 'bar', 'spam': 'baz'}
self.app.config_from_object(Object())
self.assertEqual(self.app.conf.BROKER_TRANSPORT_OPTIONS, {})
self.app.config_from_object(Object(BROKER_TRANSPORT_OPTIONS=_args))
self.assertEqual(self.app.conf.BROKER_TRANSPORT_OPTIONS, _args)
def test_Windows_log_color_disabled(self):
self.app.IS_WINDOWS = True
self.assertFalse(self.app.log.supports_color())
def test_compat_setting_CARROT_BACKEND(self):
self.app.config_from_object(Object(CARROT_BACKEND='set_by_us'))
self.assertEqual(self.app.conf.BROKER_TRANSPORT, 'set_by_us')
def test_WorkController(self):
x = self.app.WorkController
self.assertIs(x.app, self.app)
def test_Worker(self):
x = self.app.Worker
self.assertIs(x.app, self.app)
def test_AsyncResult(self):
x = self.app.AsyncResult('1')
self.assertIs(x.app, self.app)
r = loads(dumps(x))
# not set as current, so ends up as default app after reduce
self.assertIs(r.app, state.default_app)
@patch('celery.bin.celery.CeleryCommand.execute_from_commandline')
def test_start(self, execute):
self.app.start()
self.assertTrue(execute.called)
def test_mail_admins(self):
class Loader(BaseLoader):
def mail_admins(*args, **kwargs):
return args, kwargs
self.app.loader = Loader()
self.app.conf.ADMINS = None
self.assertFalse(self.app.mail_admins('Subject', 'Body'))
self.app.conf.ADMINS = [('George Costanza', '*****@*****.**')]
self.assertTrue(self.app.mail_admins('Subject', 'Body'))
def test_amqp_get_broker_info(self):
self.assertDictContainsSubset(
{
'hostname': 'localhost',
'userid': 'guest',
'password': '******',
'virtual_host': '/'
},
self.app.broker_connection('amqplib://').info())
self.app.conf.BROKER_PORT = 1978
self.app.conf.BROKER_VHOST = 'foo'
self.assertDictContainsSubset(
{
'port': 1978,
'virtual_host': 'foo'
},
self.app.broker_connection('amqplib://:1978/foo').info())
conn = self.app.broker_connection('amqplib:////value')
self.assertDictContainsSubset({'virtual_host': '/value'}, conn.info())
def test_BROKER_BACKEND_alias(self):
self.assertEqual(self.app.conf.BROKER_BACKEND,
self.app.conf.BROKER_TRANSPORT)
def test_with_default_connection(self):
@self.app.with_default_connection
def handler(connection=None, foo=None):
return connection, foo
connection, foo = handler(foo=42)
self.assertEqual(foo, 42)
self.assertTrue(connection)
def test_after_fork(self):
p = self.app._pool = Mock()
self.app._after_fork(self.app)
p.force_close_all.assert_called_with()
self.assertIsNone(self.app._pool)
self.app._after_fork(self.app)
def test_pool_no_multiprocessing(self):
with mask_modules('multiprocessing.util'):
pool = self.app.pool
self.assertIs(pool, self.app._pool)
def test_bugreport(self):
self.assertTrue(self.app.bugreport())
def test_send_task_sent_event(self):
class Dispatcher(object):
sent = []
def send(self, type, **fields):
self.sent.append((type, fields))
conn = self.app.broker_connection()
chan = conn.channel()
try:
for e in ('foo_exchange', 'moo_exchange', 'bar_exchange'):
chan.exchange_declare(e, 'direct', durable=True)
chan.queue_declare(e, durable=True)
chan.queue_bind(e, e, e)
finally:
chan.close()
assert conn.transport_cls == 'memory'
pub = self.app.amqp.TaskProducer(conn,
exchange=Exchange('foo_exchange'))
dispatcher = Dispatcher()
self.assertTrue(
pub.delay_task('footask', (), {},
exchange='moo_exchange',
routing_key='moo_exchange',
event_dispatcher=dispatcher))
self.assertTrue(dispatcher.sent)
self.assertEqual(dispatcher.sent[0][0], 'task-sent')
self.assertTrue(
pub.delay_task('footask', (), {},
event_dispatcher=dispatcher,
exchange='bar_exchange',
routing_key='bar_exchange'))
def test_error_mail_sender(self):
x = ErrorMail.subject % {
'name': 'task_name',
'id': uuid(),
'exc': 'FOOBARBAZ',
'hostname': 'lana'
}
self.assertTrue(x)
print "total_rented_modelruns_allowed", total_rented_modelruns_allowed
print "budget_period_min", budget_period_min
print "budget_period_sec", budget_period_sec
print "time_interval_sec", time_interval_sec
print "mu_own", mu_own
print "mu_own", mu_rent
print "mu", mu
print "lamda", lamda
# print '\nTotal Money: ', total_budget, '$'
# print 'Rate of Container: ', instance_rate,' $/hr'
# print 'Average time for modelrun : ', tavg_modelrun_secs,' sec'
# print 'Maximum rented modelruns : ', total_rented_modelruns_allowed
# print 'Calculated time interval : ', time_interval_sec,' sec'
print 'Total Models Allowed ', total_rented_modelruns_allowed
thread.start_new_thread(poisson_job_generator, ())
thread.start_new_thread(rentedContainerCreation, ())
thread.start_new_thread(plot_completedjobs_time, ())
# Thread(target = poisson_job_generator).start()
# Thread(target = rentedContainerCreation).start()
# Thread(target = plot_completedjobs_time).start()
while True:
for jobId in taskMap.copy():
if celery.AsyncResult(jobId).state == 'SUCCESS':
if jobId not in completedjobs:
completedjobs.append(jobId)
taskMap.pop(jobId, 0)
class EventListener(object):
"""Listens for celery events.
Server object, to capture events and handle tasks and workers.
Attributes:
_app (Celery): a configured celery app instance
_queue_output (Queue): to send to streaming dispatcher
memory (State): LRU storage object to keep tasks and workers
_use_result_backend (bool): if True, there's a result backend to fetch results from
"""
def __init__(self,
broker,
queue_output,
backend=None,
max_tasks_in_memory=None,
max_workers_in_memory=None):
"""Constructs an event listener instance.
Args:
broker (str): the broker being used by the celery system.
queue_output (Queue): to send to streaming dispatcher.
backend (str): the result backend being used by the celery system.
max_tasks_in_memory (int): max tasks stored
max_workers_in_memory (int): max workers stored
"""
self._app = Celery(broker=broker, backend=backend)
self._queue_output = queue_output
from celery.backends.base import DisabledBackend
self._use_result_backend = not isinstance(self._app.backend,
DisabledBackend)
logger.info('Creating %s: max_tasks=%d; max_workers=%d',
EventListener.__name__, max_tasks_in_memory,
max_workers_in_memory)
logger.info('Celery broker=%s; backend=%s; using_result_backend=%s',
broker, backend, self._use_result_backend)
# events handling: storage and filling missing states.
self.memory = State(
max_tasks_in_memory=max_tasks_in_memory,
max_workers_in_memory=max_workers_in_memory,
) # type: State
# running engine (should be asyncio in the future)
self._listener_thread = None # type:threading.Thread
self._celery_receiver = None # type:EventReceiver
# concurrency control
self._wait_event = threading.Event()
# detect shutdown.
def sigterm_handler(_signo, _stack_frame): # pragma: no cover
self.__stop()
signal.signal(signal.SIGTERM, sigterm_handler)
self.__start()
def __start(self): # pragma: no cover
"""Starts the real-time engine that captures events."""
assert not self._listener_thread
self._listener_thread = threading.Thread(target=self.__run_listener,
name='clearly-listener')
self._listener_thread.daemon = True
self._listener_thread.start()
self._wait_event.wait()
self._wait_event.clear()
def __stop(self): # pragma: no cover
"""Stops the background engine."""
if not self._listener_thread:
return
logger.info('Stopping listener')
self._celery_receiver.should_stop = True
self._listener_thread.join()
self._listener_thread = self._celery_receiver = None
def __run_listener(self): # pragma: no cover
logger.info('Starting listener: %s', threading.current_thread())
with self._app.connection() as connection:
self._celery_receiver = self._app.events.Receiver(
connection, handlers={
'*': self._process_event,
}) # type: EventReceiver
self._wait_event.set()
self._celery_receiver.capture(limit=None,
timeout=None,
wakeup=True)
logger.info('Listener stopped: %s', threading.current_thread())
def _process_event(self, event):
event_type = event['type']
if event_type.startswith('task'):
data = self._process_task_event(event)
elif event_type.startswith('worker'):
data = self._process_worker_event(event)
else:
logger.warning('unknown event: %s', event)
return
self._queue_output.put(data)
def _process_task_event(self, event):
task = self.memory.tasks.get(event['uuid'])
pre_state, created = (task.state, False) if task else (states.PENDING,
True)
(task,
_), _ = self.memory.event(event) # the `created` is broken in celery.
if task.state == states.SUCCESS:
try:
# verify if the celery task result is truncated.
task.result = EventListener.compile_task_result(task)
except SyntaxError:
# use result backend as fallback if allowed and available.
if self._use_result_backend: # pragma: no cover
task.result = repr(self._app.AsyncResult(task.uuid).result)
return immutable_task(task, task.state, pre_state, created)
def _process_worker_event(self, event):
worker = self.memory.workers.get(event['hostname'])
pre_state, created = (worker.status_string, False) \
if worker else (worker_states.OFFLINE, True)
(worker, _), _ = self.memory.event(event)
return immutable_worker(worker, worker.status_string, pre_state,
created)
@staticmethod
def compile_task_result(task):
result = task.result
# celery 4 sends results converted as strings, sometimes truncated (...)
if task.worker.sw_ver < '4':
# celery 3 tasks' results are converted twice.
result = safe_compile_text(result, raises=True)
# compile with `raises`, to detect truncated results.
safe_compile_text(result, raises=True)
# if compilable, returns the same, as the clients will be able to too.
return result
from celery import Celery
# 创建势力对象（生产者）
celery_app = Celery("note")
# 指定任务存放位子（经济人）
celery_app.config_from_object("celery_tasks.config")
# 注册执行的任务
celery_app.AsyncResult([celery_app.sms])
class test_App(Case):
def setUp(self):
self.app = Celery(set_as_current=False)
self.app.conf.update(test_config)
def test_task(self):
app = Celery('foozibari', set_as_current=False)
def fun():
pass
fun.__module__ = '__main__'
task = app.task(fun)
self.assertEqual(task.name, app.main + '.fun')
def test_with_config_source(self):
app = Celery(set_as_current=False, config_source=ObjectConfig)
self.assertEqual(app.conf.FOO, 1)
self.assertEqual(app.conf.BAR, 2)
def test_task_windows_execv(self):
app = Celery(set_as_current=False)
prev, _appbase._EXECV = _appbase._EXECV, True
try:
@app.task()
def foo():
pass
self.assertTrue(foo._get_current_object()) # is proxy
finally:
_appbase._EXECV = prev
assert not _appbase._EXECV
def test_task_takes_no_args(self):
app = Celery(set_as_current=False)
with self.assertRaises(TypeError):
@app.task(1)
def foo():
pass
def test_add_defaults(self):
app = Celery(set_as_current=False)
self.assertFalse(app.configured)
_conf = {'FOO': 300}
conf = lambda: _conf
app.add_defaults(conf)
self.assertIn(conf, app._pending_defaults)
self.assertFalse(app.configured)
self.assertEqual(app.conf.FOO, 300)
self.assertTrue(app.configured)
self.assertFalse(app._pending_defaults)
# defaults not pickled
appr = loads(dumps(app))
with self.assertRaises(AttributeError):
appr.conf.FOO
# add more defaults after configured
conf2 = {'FOO': 'BAR'}
app.add_defaults(conf2)
self.assertEqual(app.conf.FOO, 'BAR')
self.assertIn(_conf, app.conf.defaults)
self.assertIn(conf2, app.conf.defaults)
def test_connection_or_acquire(self):
with self.app.connection_or_acquire(block=True):
self.assertTrue(self.app.pool._dirty)
with self.app.connection_or_acquire(pool=False):
self.assertFalse(self.app.pool._dirty)
def test_maybe_close_pool(self):
cpool = self.app._pool = Mock()
ppool = self.app.amqp._producer_pool = Mock()
self.app._maybe_close_pool()
cpool.force_close_all.assert_called_with()
ppool.force_close_all.assert_called_with()
self.assertIsNone(self.app._pool)
self.assertIsNone(self.app.amqp._producer_pool)
self.app._pool = Mock()
self.app._maybe_close_pool()
self.app._maybe_close_pool()
def test_using_v1_reduce(self):
self.app._using_v1_reduce = True
self.assertTrue(loads(dumps(self.app)))
def test_autodiscover_tasks(self):
self.app.conf.CELERY_FORCE_BILLIARD_LOGGING = True
with patch('celery.app.base.ensure_process_aware_logger') as ep:
self.app.loader.autodiscover_tasks = Mock()
self.app.autodiscover_tasks(['proj.A', 'proj.B'])
ep.assert_called_with()
self.app.loader.autodiscover_tasks.assert_called_with(
['proj.A', 'proj.B'], 'tasks',
)
with patch('celery.app.base.ensure_process_aware_logger') as ep:
self.app.conf.CELERY_FORCE_BILLIARD_LOGGING = False
self.app.autodiscover_tasks(['proj.A', 'proj.B'])
self.assertFalse(ep.called)
def test_with_broker(self):
prev = os.environ.get('CELERY_BROKER_URL')
os.environ.pop('CELERY_BROKER_URL', None)
try:
app = Celery(set_as_current=False, broker='foo://baribaz')
self.assertEqual(app.conf.BROKER_HOST, 'foo://baribaz')
finally:
os.environ['CELERY_BROKER_URL'] = prev
def test_repr(self):
self.assertTrue(repr(self.app))
def test_custom_task_registry(self):
app1 = Celery(set_as_current=False)
app2 = Celery(set_as_current=False, tasks=app1.tasks)
self.assertIs(app2.tasks, app1.tasks)
def test_include_argument(self):
app = Celery(set_as_current=False, include=('foo', 'bar.foo'))
self.assertEqual(app.conf.CELERY_IMPORTS, ('foo', 'bar.foo'))
def test_set_as_current(self):
current = _state._tls.current_app
try:
app = Celery(set_as_current=True)
self.assertIs(_state._tls.current_app, app)
finally:
_state._tls.current_app = current
def test_current_task(self):
app = Celery(set_as_current=False)
@app.task
def foo():
pass
_state._task_stack.push(foo)
try:
self.assertEqual(app.current_task.name, foo.name)
finally:
_state._task_stack.pop()
def test_task_not_shared(self):
with patch('celery.app.base.shared_task') as sh:
app = Celery(set_as_current=False)
@app.task(shared=False)
def foo():
pass
self.assertFalse(sh.called)
def test_task_compat_with_filter(self):
app = Celery(set_as_current=False, accept_magic_kwargs=True)
check = Mock()
def filter(task):
check(task)
return task
@app.task(filter=filter)
def foo():
pass
check.assert_called_with(foo)
def test_task_with_filter(self):
app = Celery(set_as_current=False, accept_magic_kwargs=False)
check = Mock()
def filter(task):
check(task)
return task
assert not _appbase._EXECV
@app.task(filter=filter)
def foo():
pass
check.assert_called_with(foo)
def test_task_sets_main_name_MP_MAIN_FILE(self):
from celery import utils as _utils
_utils.MP_MAIN_FILE = __file__
try:
app = Celery('xuzzy', set_as_current=False)
@app.task
def foo():
pass
self.assertEqual(foo.name, 'xuzzy.foo')
finally:
_utils.MP_MAIN_FILE = None
def test_base_task_inherits_magic_kwargs_from_app(self):
from celery.task import Task as OldTask
class timkX(OldTask):
abstract = True
app = Celery(set_as_current=False, accept_magic_kwargs=True)
timkX.bind(app)
# see #918
self.assertFalse(timkX.accept_magic_kwargs)
from celery import Task as NewTask
class timkY(NewTask):
abstract = True
timkY.bind(app)
self.assertFalse(timkY.accept_magic_kwargs)
def test_annotate_decorator(self):
from celery.app.task import Task
class adX(Task):
abstract = True
def run(self, y, z, x):
return y, z, x
check = Mock()
def deco(fun):
def _inner(*args, **kwargs):
check(*args, **kwargs)
return fun(*args, **kwargs)
return _inner
app = Celery(set_as_current=False)
app.conf.CELERY_ANNOTATIONS = {
adX.name: {'@__call__': deco}
}
adX.bind(app)
self.assertIs(adX.app, app)
i = adX()
i(2, 4, x=3)
check.assert_called_with(i, 2, 4, x=3)
i.annotate()
i.annotate()
def test_apply_async_has__self__(self):
app = Celery(set_as_current=False)
@app.task(__self__='hello')
def aawsX():
pass
with patch('celery.app.amqp.TaskProducer.publish_task') as dt:
aawsX.apply_async((4, 5))
args = dt.call_args[0][1]
self.assertEqual(args, ('hello', 4, 5))
def test_apply_async__connection_arg(self):
app = Celery(set_as_current=False)
@app.task()
def aacaX():
pass
connection = app.connection('asd://')
with self.assertRaises(KeyError):
aacaX.apply_async(connection=connection)
def test_apply_async_adds_children(self):
from celery._state import _task_stack
app = Celery(set_as_current=False)
@app.task()
def a3cX1(self):
pass
@app.task()
def a3cX2(self):
pass
_task_stack.push(a3cX1)
try:
a3cX1.push_request(called_directly=False)
try:
res = a3cX2.apply_async(add_to_parent=True)
self.assertIn(res, a3cX1.request.children)
finally:
a3cX1.pop_request()
finally:
_task_stack.pop()
def test_TaskSet(self):
ts = self.app.TaskSet()
self.assertListEqual(ts.tasks, [])
self.assertIs(ts.app, self.app)
def test_pickle_app(self):
changes = dict(THE_FOO_BAR='bars',
THE_MII_MAR='jars')
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
self.assertLess(len(saved), 2048)
restored = pickle.loads(saved)
self.assertDictContainsSubset(changes, restored.conf)
def test_worker_main(self):
from celery.bin import worker as worker_bin
class worker(worker_bin.worker):
def execute_from_commandline(self, argv):
return argv
prev, worker_bin.worker = worker_bin.worker, worker
try:
ret = self.app.worker_main(argv=['--version'])
self.assertListEqual(ret, ['--version'])
finally:
worker_bin.worker = prev
def test_config_from_envvar(self):
os.environ['CELERYTEST_CONFIG_OBJECT'] = 'celery.tests.app.test_app'
self.app.config_from_envvar('CELERYTEST_CONFIG_OBJECT')
self.assertEqual(self.app.conf.THIS_IS_A_KEY, 'this is a value')
def test_config_from_object(self):
class Object(object):
LEAVE_FOR_WORK = True
MOMENT_TO_STOP = True
CALL_ME_BACK = 123456789
WANT_ME_TO = False
UNDERSTAND_ME = True
self.app.config_from_object(Object())
self.assertTrue(self.app.conf.LEAVE_FOR_WORK)
self.assertTrue(self.app.conf.MOMENT_TO_STOP)
self.assertEqual(self.app.conf.CALL_ME_BACK, 123456789)
self.assertFalse(self.app.conf.WANT_ME_TO)
self.assertTrue(self.app.conf.UNDERSTAND_ME)
def test_config_from_cmdline(self):
cmdline = ['.always_eager=no',
'.result_backend=/dev/null',
'celeryd.prefetch_multiplier=368',
'.foobarstring=(string)300',
'.foobarint=(int)300',
'.result_engine_options=(dict){"foo": "bar"}']
self.app.config_from_cmdline(cmdline, namespace='celery')
self.assertFalse(self.app.conf.CELERY_ALWAYS_EAGER)
self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, '/dev/null')
self.assertEqual(self.app.conf.CELERYD_PREFETCH_MULTIPLIER, 368)
self.assertEqual(self.app.conf.CELERY_FOOBARSTRING, '300')
self.assertEqual(self.app.conf.CELERY_FOOBARINT, 300)
self.assertDictEqual(self.app.conf.CELERY_RESULT_ENGINE_OPTIONS,
{'foo': 'bar'})
def test_compat_setting_CELERY_BACKEND(self):
self.app.config_from_object(Object(CELERY_BACKEND='set_by_us'))
self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, 'set_by_us')
def test_setting_BROKER_TRANSPORT_OPTIONS(self):
_args = {'foo': 'bar', 'spam': 'baz'}
self.app.config_from_object(Object())
self.assertEqual(self.app.conf.BROKER_TRANSPORT_OPTIONS, {})
self.app.config_from_object(Object(BROKER_TRANSPORT_OPTIONS=_args))
self.assertEqual(self.app.conf.BROKER_TRANSPORT_OPTIONS, _args)
def test_Windows_log_color_disabled(self):
self.app.IS_WINDOWS = True
self.assertFalse(self.app.log.supports_color(True))
def test_compat_setting_CARROT_BACKEND(self):
self.app.config_from_object(Object(CARROT_BACKEND='set_by_us'))
self.assertEqual(self.app.conf.BROKER_TRANSPORT, 'set_by_us')
def test_WorkController(self):
x = self.app.WorkController
self.assertIs(x.app, self.app)
def test_Worker(self):
x = self.app.Worker
self.assertIs(x.app, self.app)
def test_AsyncResult(self):
x = self.app.AsyncResult('1')
self.assertIs(x.app, self.app)
r = loads(dumps(x))
# not set as current, so ends up as default app after reduce
self.assertIs(r.app, _state.default_app)
def test_get_active_apps(self):
self.assertTrue(list(_state._get_active_apps()))
app1 = Celery(set_as_current=False)
appid = id(app1)
self.assertIn(app1, _state._get_active_apps())
del(app1)
# weakref removed from list when app goes out of scope.
with self.assertRaises(StopIteration):
next(app for app in _state._get_active_apps() if id(app) == appid)
def test_config_from_envvar_more(self, key='CELERY_HARNESS_CFG1'):
self.assertFalse(self.app.config_from_envvar('HDSAJIHWIQHEWQU',
silent=True))
with self.assertRaises(ImproperlyConfigured):
self.app.config_from_envvar('HDSAJIHWIQHEWQU', silent=False)
os.environ[key] = __name__ + '.object_config'
self.assertTrue(self.app.config_from_envvar(key))
self.assertEqual(self.app.conf['FOO'], 1)
self.assertEqual(self.app.conf['BAR'], 2)
os.environ[key] = 'unknown_asdwqe.asdwqewqe'
with self.assertRaises(ImportError):
self.app.config_from_envvar(key, silent=False)
self.assertFalse(self.app.config_from_envvar(key, silent=True))
os.environ[key] = __name__ + '.dict_config'
self.assertTrue(self.app.config_from_envvar(key))
self.assertEqual(self.app.conf['FOO'], 10)
self.assertEqual(self.app.conf['BAR'], 20)
@patch('celery.bin.celery.CeleryCommand.execute_from_commandline')
def test_start(self, execute):
self.app.start()
self.assertTrue(execute.called)
def test_mail_admins(self):
class Loader(BaseLoader):
def mail_admins(*args, **kwargs):
return args, kwargs
self.app.loader = Loader(app=self.app)
self.app.conf.ADMINS = None
self.assertFalse(self.app.mail_admins('Subject', 'Body'))
self.app.conf.ADMINS = [('George Costanza', '*****@*****.**')]
self.assertTrue(self.app.mail_admins('Subject', 'Body'))
def test_amqp_get_broker_info(self):
self.assertDictContainsSubset(
{'hostname': 'localhost',
'userid': 'guest',
'password': '******',
'virtual_host': '/'},
self.app.connection('pyamqp://').info(),
)
self.app.conf.BROKER_PORT = 1978
self.app.conf.BROKER_VHOST = 'foo'
self.assertDictContainsSubset(
{'port': 1978, 'virtual_host': 'foo'},
self.app.connection('pyamqp://:1978/foo').info(),
)
conn = self.app.connection('pyamqp:////value')
self.assertDictContainsSubset({'virtual_host': '/value'},
conn.info())
def test_BROKER_BACKEND_alias(self):
self.assertEqual(self.app.conf.BROKER_BACKEND,
self.app.conf.BROKER_TRANSPORT)
def test_after_fork(self):
p = self.app._pool = Mock()
self.app._after_fork(self.app)
p.force_close_all.assert_called_with()
self.assertIsNone(self.app._pool)
self.app._after_fork(self.app)
def test_pool_no_multiprocessing(self):
with mask_modules('multiprocessing.util'):
pool = self.app.pool
self.assertIs(pool, self.app._pool)
def test_bugreport(self):
self.assertTrue(self.app.bugreport())
def test_send_task_sent_event(self):
class Dispatcher(object):
sent = []
def publish(self, type, fields, *args, **kwargs):
self.sent.append((type, fields))
conn = self.app.connection()
chan = conn.channel()
try:
for e in ('foo_exchange', 'moo_exchange', 'bar_exchange'):
chan.exchange_declare(e, 'direct', durable=True)
chan.queue_declare(e, durable=True)
chan.queue_bind(e, e, e)
finally:
chan.close()
assert conn.transport_cls == 'memory'
prod = self.app.amqp.TaskProducer(
conn, exchange=Exchange('foo_exchange'),
send_sent_event=True,
)
dispatcher = Dispatcher()
self.assertTrue(prod.publish_task('footask', (), {},
exchange='moo_exchange',
routing_key='moo_exchange',
event_dispatcher=dispatcher))
self.assertTrue(dispatcher.sent)
self.assertEqual(dispatcher.sent[0][0], 'task-sent')
self.assertTrue(prod.publish_task('footask', (), {},
event_dispatcher=dispatcher,
exchange='bar_exchange',
routing_key='bar_exchange'))
def test_error_mail_sender(self):
x = ErrorMail.subject % {'name': 'task_name',
'id': uuid(),
'exc': 'FOOBARBAZ',
'hostname': 'lana'}
self.assertTrue(x)
def test_error_mail_disabled(self):
task = Mock()
x = ErrorMail(task)
x.should_send = Mock()
x.should_send.return_value = False
x.send(Mock(), Mock())
self.assertFalse(task.app.mail_admins.called)
class EventListener:
"""Listen for celery events.
Server object, to capture events and handle tasks and workers.
Attributes:
app: a configured celery app instance
queue_tasks: to send to the streaming dispatcher responsible for tasks
queue_workers: to send to the streaming dispatcher responsible for workers
memory: LRU storage object to keep tasks and workers
use_result_backend: if True, there's a result backend to fetch results from
gen_task_states: object that fills missing tasks' states
"""
def __init__(self,
broker: str,
queue_tasks: Queue,
queue_workers: Queue,
memory: State,
backend: str = None):
"""Construct an event listener instance.
Args:
broker: the broker being used by the celery system.
queue_tasks: to send to the streaming dispatcher responsible for tasks
queue_workers: to send to the streaming dispatcher responsible for workers
backend: the result backend being used by the celery system.
"""
logger.info('Creating %s', EventListener.__name__)
self.queue_tasks, self.queue_workers, self.memory = queue_tasks, queue_workers, memory
self.use_result_backend = bool(backend)
self.app = Celery(broker=broker, backend=backend)
logger.info('broker : %s', self.app.pool.connection.as_uri())
logger.info('backend: %s', self.app.backend.as_uri())
# fill missing gaps in states.
self.gen_task_states: ExpectedStateHandler = setup_task_states()
# running engine (should be asyncio in the future)
self._listener_thread: Optional[threading.Thread] = None
self._celery_receiver: Optional[EventReceiver] = None
# concurrency control
self._wait_event = threading.Event()
# detect shutdown.
def sigterm_handler(_signo, _stack_frame): # pragma: no cover
self.__stop()
signal.signal(signal.SIGTERM, sigterm_handler)
self.__start()
def __start(self) -> None: # pragma: no cover
"""Start the real time engine that captures events."""
assert not self._listener_thread
self._listener_thread = threading.Thread(target=self.__run,
name=THREAD_NAME)
self._listener_thread.daemon = True
self._listener_thread.start()
if not self._wait_event.wait(timeout=BROKER_CONNECT_TIMEOUT):
raise TimeoutError('Could not connect to broker.')
self._wait_event.clear()
def __stop(self) -> None: # pragma: no cover
"""Stop the background engine."""
if not self._listener_thread:
return
logger.info('Stopping %s', THREAD_NAME)
self._celery_receiver.should_stop = True
self._listener_thread.join(1)
self._listener_thread = self._celery_receiver = None
def __run(self) -> None: # pragma: no cover
logger.info('Starting: %r', threading.current_thread())
with self.app.connection() as connection:
self._celery_receiver: EventReceiver = self.app.events.Receiver(
connection, handlers={
'*': self._process_event,
})
self._wait_event.set()
self._celery_receiver.capture(limit=None,
timeout=None,
wakeup=True)
logger.info('Stopped: %r', threading.current_thread())
def _process_event(self, event: dict) -> None:
event_type = event['type']
if event_type.startswith('task'):
gen, queue, to_type = self._set_task_event(
event), self.queue_tasks, TaskMessage
elif event_type.startswith('worker'):
gen, queue, to_type = self._set_worker_event(
event), self.queue_workers, WorkerMessage
else:
self._set_custom_event(event)
return
obj = next(gen)
for state in gen:
queue.put(obj_to_message(obj, to_type, state=state))
def _set_task_event(self, event: dict) -> Iterator[Union[Task, str]]:
task = self.memory.tasks.get(event['uuid'])
pre_state = task and task.state
(task, _), _ = self.memory.event(event)
# fix or insert fields.
task.result_meta = None
if task.state == SUCCESS:
task.result_meta, task.result = self._derive_task_result(task)
yield task
# fix shortcomings of `created` field: a task should be displayed as PENDING if not
# new; if a task is first seen in any other state, it should not be new.
if not pre_state:
yield '' if task.state == PENDING else task.state # empty state will mean new.
return
yield from self.gen_task_states.states_through(pre_state, task.state)
def _set_worker_event(self, event: dict) -> Iterator[Union[Worker, str]]:
(worker, _), _ = self.memory.event(event)
# fix or insert fields.
worker.state = worker_states.TYPES[event['type']]
yield worker
yield worker.state
# noinspection PyMethodMayBeStatic
def _set_custom_event(self, event: dict) -> None: # pragma: no cover
# could pass custom user events here, if the need ever arises.
logger.warning('unknown event: %s', event)
# noinspection PyBroadException
def _derive_task_result(self, task: Task) -> Tuple[str, str]:
try: # verify if the task result is truncated.
return 'event', EventListener.compile_task_result(task)
except SyntaxError: # <== this means the result is truncated.
error = 'no-backend'
except Exception:
logger.exception('Failed to compile task result: %s, worker: %s',
task.result, task.worker)
error = 'compile-failed'
if self.use_result_backend:
try:
return 'backend', repr(self.app.AsyncResult(task.uuid).result)
except Exception: # probably incompatible celery versions in clearly and user code.
logger.exception(
'Failed to fetch task result from result_backend: %s',
task.uuid)
error = 'fetch-failed'
return error, task.result
@staticmethod
def compile_task_result(task: Task) -> Any:
result = task.result
# celery 4 sends results converted as strings, sometimes truncated (...)
if task.worker.sw_ver < '4':
# celery 3 tasks' results are converted twice.
result = safe_compile_text(result, raises=True)
# compile with `raises`, to detect truncated results.
safe_compile_text(result, raises=True)
# if compilable, returns the same, as the clients will be able to too.
return result
class deployment_job_service(object):
'''
Job Service for templates deployment jobs
'''
def __init__(self):
print('Job Service Created')
self.app = Celery('tasks')
self.app.config_from_object(config)
self.mongo_db = MongoClient(
"mongodb://*****:*****@mongo_result_backend:27017/?authSource=TMS_DB"
)['TMS_DB']
# TODO: put pagination here https://www.codementor.io/arpitbhayani/fast-and-efficient-pagination-in-mongodb-9095flbqr
def get_jobs(self):
'''
Get list of jobs
'''
total_records = []
cursor = self.mongo_db['jobs'].find().sort("create_time",
pymongo.DESCENDING)
for record in cursor:
# del record["_id"]
record["_id"] = str(record["_id"])
try:
template_name = self.mongo_db['templates'].find_one(
{'_id': ObjectId(record["task_name"])})['name']
except:
template_name = "SOME TEMPLATE"
record['template_name'] = template_name
total_records.append(record)
return total_records
def create_deployment_job(self,
template_id,
region_id,
target_queue='sample_region_1'):
'''
Create a deployment job
'''
task = self.app.send_task('task.add', [template_id, region_id],
queue=target_queue)
print('Created Task', task.id)
self.mongo_db['jobs'].insert_one({
'task_id': task.id,
'task_name': template_id,
'target_region': region_id,
'create_time': datetime.datetime.now(),
'status': 'PENDING'
})
return task
def revoke_job(self, task_id):
'''
Revoke an ongoing job by task id in the db
'''
self.app.control.revoke(task_id, terminate=True)
self.mongo_db['jobs'].update_one({'task_id': task_id},
{'$set': {
'status': 'REVOKED'
}})
return ({'message': 'Job {} successfully revoked'.format(task_id)})
def get_job_status(self, task_id):
'''
Get status of a deployment job
'''
return self.app.AsyncResult(task_id).status
def get_job_detail(self, task_id):
'''
Get deployment job detail of a job
'''
result = self.mongo_db['jobs'].find_one({'task_id': task_id})
del result['_id']
return result
class test_App(Case):
def setUp(self):
self.app = Celery(set_as_current=False)
self.app.conf.update(test_config)
def test_task(self):
app = Celery("foozibari", set_as_current=False)
def fun():
pass
fun.__module__ = "__main__"
task = app.task(fun)
self.assertEqual(task.name, app.main + ".fun")
def test_with_broker(self):
app = Celery(set_as_current=False, broker="foo://baribaz")
self.assertEqual(app.conf.BROKER_HOST, "foo://baribaz")
def test_repr(self):
self.assertTrue(repr(self.app))
def test_TaskSet(self):
ts = self.app.TaskSet()
self.assertListEqual(ts.tasks, [])
self.assertIs(ts.app, self.app)
def test_pickle_app(self):
changes = dict(THE_FOO_BAR="bars",
THE_MII_MAR="jars")
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
self.assertLess(len(saved), 2048)
restored = pickle.loads(saved)
self.assertDictContainsSubset(changes, restored.conf)
def test_worker_main(self):
from celery.bin import celeryd
class WorkerCommand(celeryd.WorkerCommand):
def execute_from_commandline(self, argv):
return argv
prev, celeryd.WorkerCommand = celeryd.WorkerCommand, WorkerCommand
try:
ret = self.app.worker_main(argv=["--version"])
self.assertListEqual(ret, ["--version"])
finally:
celeryd.WorkerCommand = prev
def test_config_from_envvar(self):
os.environ["CELERYTEST_CONFIG_OBJECT"] = "celery.tests.app.test_app"
self.app.config_from_envvar("CELERYTEST_CONFIG_OBJECT")
self.assertEqual(self.app.conf.THIS_IS_A_KEY, "this is a value")
def test_config_from_object(self):
class Object(object):
LEAVE_FOR_WORK = True
MOMENT_TO_STOP = True
CALL_ME_BACK = 123456789
WANT_ME_TO = False
UNDERSTAND_ME = True
self.app.config_from_object(Object())
self.assertTrue(self.app.conf.LEAVE_FOR_WORK)
self.assertTrue(self.app.conf.MOMENT_TO_STOP)
self.assertEqual(self.app.conf.CALL_ME_BACK, 123456789)
self.assertFalse(self.app.conf.WANT_ME_TO)
self.assertTrue(self.app.conf.UNDERSTAND_ME)
def test_config_from_cmdline(self):
cmdline = [".always_eager=no",
".result_backend=/dev/null",
'.task_error_whitelist=(list)["a", "b", "c"]',
"celeryd.prefetch_multiplier=368",
".foobarstring=(string)300",
".foobarint=(int)300",
'.result_engine_options=(dict){"foo": "bar"}']
self.app.config_from_cmdline(cmdline, namespace="celery")
self.assertFalse(self.app.conf.CELERY_ALWAYS_EAGER)
self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, "/dev/null")
self.assertEqual(self.app.conf.CELERYD_PREFETCH_MULTIPLIER, 368)
self.assertListEqual(self.app.conf.CELERY_TASK_ERROR_WHITELIST,
["a", "b", "c"])
self.assertEqual(self.app.conf.CELERY_FOOBARSTRING, "300")
self.assertEqual(self.app.conf.CELERY_FOOBARINT, 300)
self.assertDictEqual(self.app.conf.CELERY_RESULT_ENGINE_OPTIONS,
{"foo": "bar"})
def test_compat_setting_CELERY_BACKEND(self):
self.app.config_from_object(Object(CELERY_BACKEND="set_by_us"))
self.assertEqual(self.app.conf.CELERY_RESULT_BACKEND, "set_by_us")
def test_setting_BROKER_TRANSPORT_OPTIONS(self):
_args = {'foo': 'bar', 'spam': 'baz'}
self.app.config_from_object(Object())
self.assertEqual(self.app.conf.BROKER_TRANSPORT_OPTIONS, {})
self.app.config_from_object(Object(BROKER_TRANSPORT_OPTIONS=_args))
self.assertEqual(self.app.conf.BROKER_TRANSPORT_OPTIONS, _args)
def test_Windows_log_color_disabled(self):
self.app.IS_WINDOWS = True
self.assertFalse(self.app.log.supports_color())
def test_compat_setting_CARROT_BACKEND(self):
self.app.config_from_object(Object(CARROT_BACKEND="set_by_us"))
self.assertEqual(self.app.conf.BROKER_TRANSPORT, "set_by_us")
def test_WorkController(self):
x = self.app.Worker()
self.assertIs(x.app, self.app)
def test_AsyncResult(self):
x = self.app.AsyncResult("1")
self.assertIs(x.app, self.app)
r = loads(dumps(x))
# not set as current, so ends up as default app after reduce
self.assertIs(r.app, state.default_app)
@patch("celery.bin.celery.CeleryCommand.execute_from_commandline")
def test_start(self, execute):
self.app.start()
self.assertTrue(execute.called)
def test_mail_admins(self):
class Loader(BaseLoader):
def mail_admins(*args, **kwargs):
return args, kwargs
self.app.loader = Loader()
self.app.conf.ADMINS = None
self.assertFalse(self.app.mail_admins("Subject", "Body"))
self.app.conf.ADMINS = [("George Costanza", "*****@*****.**")]
self.assertTrue(self.app.mail_admins("Subject", "Body"))
def test_amqp_get_broker_info(self):
self.assertDictContainsSubset({"hostname": "localhost",
"userid": "guest",
"password": "******",
"virtual_host": "/"},
self.app.broker_connection(
transport="amqplib").info())
self.app.conf.BROKER_PORT = 1978
self.app.conf.BROKER_VHOST = "foo"
self.assertDictContainsSubset({"port": 1978,
"virtual_host": "foo"},
self.app.broker_connection(
transport="amqplib").info())
conn = self.app.broker_connection(virtual_host="/value")
self.assertDictContainsSubset({"virtual_host": "/value"},
conn.info())
def test_BROKER_BACKEND_alias(self):
self.assertEqual(self.app.conf.BROKER_BACKEND,
self.app.conf.BROKER_TRANSPORT)
def test_with_default_connection(self):
@self.app.with_default_connection
def handler(connection=None, foo=None):
return connection, foo
connection, foo = handler(foo=42)
self.assertEqual(foo, 42)
self.assertTrue(connection)
def test_after_fork(self):
p = self.app._pool = Mock()
self.app._after_fork(self.app)
p.force_close_all.assert_called_with()
self.assertIsNone(self.app._pool)
self.app._after_fork(self.app)
def test_pool_no_multiprocessing(self):
with mask_modules("multiprocessing.util"):
pool = self.app.pool
self.assertIs(pool, self.app._pool)
def test_bugreport(self):
self.assertTrue(self.app.bugreport())
def test_send_task_sent_event(self):
class Dispatcher(object):
sent = []
def send(self, type, **fields):
self.sent.append((type, fields))
conn = self.app.broker_connection()
chan = conn.channel()
try:
for e in ("foo_exchange", "moo_exchange", "bar_exchange"):
chan.exchange_declare(e, "direct", durable=True)
chan.queue_declare(e, durable=True)
chan.queue_bind(e, e, e)
finally:
chan.close()
assert conn.transport_cls == "memory"
entities = conn.declared_entities
pub = self.app.amqp.TaskPublisher(conn, exchange="foo_exchange")
self.assertNotIn(pub._get_exchange("foo_exchange"), entities)
dispatcher = Dispatcher()
self.assertTrue(pub.delay_task("footask", (), {},
exchange="moo_exchange",
routing_key="moo_exchange",
event_dispatcher=dispatcher))
self.assertIn(pub._get_exchange("moo_exchange"), entities)
self.assertTrue(dispatcher.sent)
self.assertEqual(dispatcher.sent[0][0], "task-sent")
self.assertTrue(pub.delay_task("footask", (), {},
event_dispatcher=dispatcher,
exchange="bar_exchange",
routing_key="bar_exchange"))
self.assertIn(pub._get_exchange("bar_exchange"), entities)
def test_error_mail_sender(self):
x = ErrorMail.subject % {"name": "task_name",
"id": uuid(),
"exc": "FOOBARBAZ",
"hostname": "lana"}
self.assertTrue(x)
