def __init__(self, connection=None, *args, **kwargs):
if not connection:
connection = celery.broker_connection()
self.clicks = Clicks()
self.actors = [self.clicks]
self.timers = []
super(Agent, self).__init__(connection, *args, **kwargs)
def get_branches(broker=None, limit=None):
from cyme.branch.controller import Branch
from celery import current_app as celery
if limit:
limit = int(limit)
args = [broker] if broker else []
conn = celery.broker_connection(*args)
return Branch(connection=conn).all(limit=limit)
def test_info(self):
l = MyKombuConsumer(self.ready_queue, self.eta_schedule, self.logger, send_events=False)
l.qos = QoS(l.task_consumer, 10, l.logger)
info = l.info
self.assertEqual(info["prefetch_count"], 10)
self.assertFalse(info["broker"])
l.connection = current_app.broker_connection()
info = l.info
self.assertTrue(info["broker"])
def test_info(self):
l = MyKombuConsumer(self.ready_queue, timer=self.timer)
l.qos = QoS(l.task_consumer, 10)
info = l.info
self.assertEqual(info["prefetch_count"], 10)
self.assertFalse(info["broker"])
l.connection = current_app.broker_connection()
info = l.info
self.assertTrue(info["broker"])
def _producer(self):
connection = current_app.broker_connection()
publisher = current_app.amqp.TaskPublisher(connection)
inqueue = self.inqueue
while 1:
task, args, kwargs, options, receipt = inqueue.get()
result = task.apply_async(args, kwargs,
publisher=publisher,
**options)
receipt.finished(result)
def test_active_queues(self):
import kombu
x = kombu.Consumer(current_app.broker_connection(),
[kombu.Queue('foo', kombu.Exchange('foo'), 'foo'),
kombu.Queue('bar', kombu.Exchange('bar'), 'bar')],
auto_declare=False)
consumer = Mock()
consumer.task_consumer = x
panel = self.create_panel(consumer=consumer)
r = panel.handle('active_queues')
self.assertListEqual(list(sorted(q['name'] for q in r)),
['bar', 'foo'])
def test_active_queues(self):
import kombu
x = kombu.Consumer(
current_app.broker_connection(),
[kombu.Queue("foo", kombu.Exchange("foo"), "foo"), kombu.Queue("bar", kombu.Exchange("bar"), "bar")],
auto_declare=False,
)
consumer = Mock()
consumer.task_consumer = x
panel = self.create_panel(consumer=consumer)
r = panel.handle("active_queues")
self.assertListEqual(list(sorted(q["name"] for q in r)), ["bar", "foo"])
def route_for_task(self, *args, **kwargs):
if self.setup:
return
self.setup = True
from celery import current_app, VERSION as celery_version
# will not connect anywhere when using the Django transport
# because declarations happen in memory.
with current_app.broker_connection() as conn:
queues = current_app.amqp.queues
channel = conn.default_channel
if celery_version >= (2, 6):
for queue in queues.itervalues():
queue(channel).declare()
def __init__(self,
addrport='',
id=None,
loglevel=logging.INFO,
logfile=None,
without_httpd=False,
numc=2,
sup_interval=None,
ready_event=None,
colored=None,
**kwargs):
self.id = id or gen_unique_id()
if isinstance(addrport, basestring):
addr, _, port = addrport.partition(':')
addrport = (addr, int(port) if port else 8000)
self.addrport = addrport
self.connection = celery.broker_connection()
self.without_httpd = without_httpd
self.logfile = logfile
self.loglevel = loglevel
self.numc = numc
self.ready_event = ready_event
self.exit_request = Event()
self.colored = colored or term.colored(enabled=False)
self.httpd = None
gSup = find_symbol(self, self.intsup_cls)
if not self.without_httpd:
self.httpd = MockSup(instantiate(self, self.httpd_cls, addrport),
signals.httpd_ready)
self.supervisor = gSup(
instantiate(self, self.supervisor_cls, sup_interval),
signals.supervisor_ready)
self.controllers = [
gSup(
instantiate(self,
self.controller_cls,
id='%s.%s' % (self.id, i),
connection=self.connection,
branch=self), signals.controller_ready)
for i in xrange(1, numc + 1)
]
c = [self.supervisor] + self.controllers + [self.httpd]
c = self.components = list(filter(None, c))
self._components_ready = dict(
zip([z.thread for z in c], [False] * len(c)))
for controller in self.controllers:
if hasattr(controller.thread, 'presence'):
self._components_ready[controller.thread.presence] = False
self._components_shutdown = dict(self._components_ready)
super(Branch, self).__init__()
def test_active_queues(self):
import kombu
x = kombu.Consumer(current_app.broker_connection(), [
kombu.Queue('foo', kombu.Exchange('foo'), 'foo'),
kombu.Queue('bar', kombu.Exchange('bar'), 'bar')
],
auto_declare=False)
consumer = Mock()
consumer.task_consumer = x
panel = self.create_panel(consumer=consumer)
r = panel.handle('active_queues')
self.assertListEqual(list(sorted(q['name'] for q in r)),
['bar', 'foo'])
def test_active_queues(self):
import kombu
x = kombu.Consumer(current_app.broker_connection(), [
kombu.Queue("foo", kombu.Exchange("foo"), "foo"),
kombu.Queue("bar", kombu.Exchange("bar"), "bar")
],
auto_declare=False)
consumer = Mock()
consumer.task_consumer = x
panel = self.create_panel(consumer=consumer)
r = panel.handle("active_queues")
self.assertListEqual(list(sorted(q["name"] for q in r)),
["bar", "foo"])
def main():
emailStats = [0, 0]
#connection pool for the broker
tset = TaskSetResult(gen_unique_id(), [])
connection = current_app.broker_connection()
publisher = current_app.amqp.TaskPublisher(connection)
start = time.time()
try:
for root, dirs, files in os.walk('/big/mails'):
for name in files:
email = join(root,name)
if not 'envelope' in email:
size = getsize(email)
emailStats[0] += 1
emailStats[1] += size
insertData.apply_async((email, ), publisher=publisher)
#checkData.apply_async((email,), publisher=publisher)
#indexData.apply_async((email,), publisher=publisher)
#if emailStats[0] == 10:
# break
#statistics
if emailStats[0] % 3000 == 0:
print emailStats[0]
#if emailStats[0] == 10:
#break
except EnvironmentError:
print 'I/O error'
finally:
publisher.close()
connection.close()
duration = time.time() - start
print "Number of emails: " + str(emailStats[0])
print "Size of emails: " + str(emailStats[1])
print "Duration:" + str(duration/60)
def __init__(self, addrport='', id=None, loglevel=logging.INFO,
logfile=None, without_httpd=False, numc=2, sup_interval=None,
ready_event=None, colored=None, **kwargs):
self.id = id or gen_unique_id()
if isinstance(addrport, basestring):
addr, _, port = addrport.partition(':')
addrport = (addr, int(port) if port else 8000)
self.addrport = addrport
self.connection = celery.broker_connection()
self.without_httpd = without_httpd
self.logfile = logfile
self.loglevel = loglevel
self.numc = numc
self.ready_event = ready_event
self.exit_request = Event()
self.colored = colored or term.colored(enabled=False)
self.httpd = None
gSup = find_symbol(self, self.intsup_cls)
if not self.without_httpd:
self.httpd = MockSup(instantiate(self, self.httpd_cls, addrport),
signals.httpd_ready)
self.supervisor = gSup(instantiate(self, self.supervisor_cls,
sup_interval), signals.supervisor_ready)
self.controllers = [gSup(instantiate(self, self.controller_cls,
id='%s.%s' % (self.id, i),
connection=self.connection,
branch=self),
signals.controller_ready)
for i in xrange(1, numc + 1)]
c = [self.supervisor] + self.controllers + [self.httpd]
c = self.components = list(filter(None, c))
self._components_ready = dict(zip([z.thread for z in c],
[False] * len(c)))
for controller in self.controllers:
if hasattr(controller.thread, 'presence'):
self._components_ready[controller.thread.presence] = False
self._components_shutdown = dict(self._components_ready)
super(Branch, self).__init__()
def all_branches(self):
from cyme.branch.controller import Branch
args = [self.broker] if self.broker else []
conn = celery.broker_connection(*args)
return Branch(connection=conn).all(limit=self.limit)
def setup_pool_limit(self, **kwargs):
from kombu import pools
from celery import current_app as celery
limit = kwargs.get("limit", celery.conf.BROKER_POOL_LIMIT)
pools.set_limit(limit if self.needs_eventlet else 1)
celery._pool = pools.connections[celery.broker_connection()]
def __init__(self, connection=None, *args, **kwargs):
if not connection:
connection = celery.broker_connection()
super(Clicks, self).__init__(connection, *args, **kwargs)
class Connection(current_app.broker_connection().__class__):
obj = None
def drain_events(self, **kwargs):
self.obj.connection = None
def connection(self):
return celery.broker_connection(self.url)
def default_url(self):
return celery.broker_connection().as_uri()
def get_or_create_queue(self):
broker_connection = current_app.broker_connection()
channel = broker_connection.channel()
return channel._queue_for(settings.CELERY_DEFAULT_QUEUE)
def __init__(self, connection=None, *args, **kwargs):
# - use celery's connection by default,
# - means the agent must be started from where it has access
# - to the celery config.
super().__init__(connection or celery.broker_connection(), *args,
**kwargs)
def __init__(self, connection=None, *args, **kwargs):
if not connection:
connection = celery.broker_connection()
super(Clicks, self).__init__(connection, *args, **kwargs)
def setup_pool_limit(self, **kwargs):
from kombu import pools
from celery import current_app as celery
limit = kwargs.get('limit', celery.conf.BROKER_POOL_LIMIT)
pools.set_limit(limit if self.needs_eventlet else 1)
celery._pool = pools.connections[celery.broker_connection()]
def __init__(self, connection=None, *args, **kwargs):
# - use celery's connection by default,
# - means the agent must be started from where it has access
# - to the celery config.
super(BlenderActor, self).__init__(
connection or celery.broker_connection(), *args, **kwargs)
def __init__(self, connection=None, *args, **kwargs):
# - use celery's connection by default
super(Agent, self).__init__(
connection or celery.broker_connection(), *args, **kwargs)
def __init__(self, connection=None, *args, **kwargs):
# - use celery's connection by default
super(Agent, self).__init__(connection or celery.broker_connection(),
*args, **kwargs)
def get_queue_info(queue_name):
with current_app.broker_connection() as conn:
with conn.channel() as channel:
return channel.queue_declare(queue_name, passive=True)
def connection(self):
return celery.broker_connection(self.url)
def all_branches(self):
from cyme.branch.controller import Branch
args = [self.broker] if self.broker else []
conn = celery.broker_connection(*args)
return Branch(connection=conn).all(limit=self.limit)
def default_url(self):
return celery.broker_connection().as_uri()
