def run(self, *args, **kwargs):
if len(args) != 2:
return self.show_help("migrate")
from kombu import Connection
from celery.contrib.migrate import migrate_tasks
migrate_tasks(Connection(args[0]), Connection(args[1]), callback=self.on_migrate_task, **kwargs)
def run(self, source, destination, **kwargs):
from kombu import Connection
from celery.contrib.migrate import migrate_tasks
migrate_tasks(Connection(source),
Connection(destination),
callback=self.on_migrate_task,
**kwargs)
def run(self, source, destination, **kwargs):
from kombu import Connection
from celery.contrib.migrate import migrate_tasks
migrate_tasks(Connection(source),
Connection(destination),
callback=self.on_migrate_task,
**kwargs)
def run(self, *args, **kwargs):
if len(args) != 2:
return self.show_help("migrate")
from kombu import BrokerConnection
from celery.contrib.migrate import migrate_tasks
migrate_tasks(BrokerConnection(args[0]),
BrokerConnection(args[1]),
callback=self.on_migrate_task)
def run(self, *args, **kwargs):
if len(args) != 2:
# this never exits due to OptionParser.parse_options
self.run_from_argv(self.prog_name, ["migrate", "--help"])
raise SystemExit()
from kombu import Connection
from celery.contrib.migrate import migrate_tasks
migrate_tasks(Connection(args[0]), Connection(args[1]), callback=self.on_migrate_task, **kwargs)
def run(self, *args, **kwargs):
if len(args) != 2:
return self.exit_help('migrate')
from kombu import Connection
from celery.contrib.migrate import migrate_tasks
migrate_tasks(Connection(args[0]),
Connection(args[1]),
callback=self.on_migrate_task,
**kwargs)
def run(self, *args, **kwargs):
if len(args) != 2:
# this never exits due to OptionParser.parse_options
self.run_from_argv(self.prog_name, ['migrate', '--help'])
raise SystemExit()
from kombu import Connection
from celery.contrib.migrate import migrate_tasks
migrate_tasks(Connection(args[0]),
Connection(args[1]),
callback=self.on_migrate_task,
**kwargs)
def test_migrate(self, app, name='testcelery'):
connection_kwargs = {'transport_options': {'polling_interval': 0.01}}
x = Connection('memory://foo', **connection_kwargs)
y = Connection('memory://foo', **connection_kwargs)
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, 'direct')
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish('foo', exchange=name, routing_key=name)
Producer(x).publish('bar', exchange=name, routing_key=name)
Producer(x).publish('baz', exchange=name, routing_key=name)
assert x.default_channel.queues
assert not y.default_channel.queues
migrate_tasks(x, y, accept=['text/plain'], app=app)
yq = q(y.default_channel)
assert yq.get().body == ensure_bytes('foo')
assert yq.get().body == ensure_bytes('bar')
assert yq.get().body == ensure_bytes('baz')
Producer(x).publish('foo', exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=app)
callback.assert_called()
migrate = Mock()
Producer(x).publish('baz', exchange=name, routing_key=name)
migrate_tasks(x,
y,
callback=callback,
migrate=migrate,
accept=['text/plain'],
app=app)
migrate.assert_called()
with patch('kombu.transport.virtual.Channel.queue_declare') as qd:
def effect(*args, **kwargs):
if kwargs.get('passive'):
raise ChannelError('some channel error')
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y, app=app)
x = Connection('memory://', **connection_kwargs)
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=app)
callback.assert_not_called()
def test_migrate(self, app, name='testcelery'):
connection_kwargs = {
'transport_options': {'polling_interval': 0.01}
}
x = Connection('memory://foo', **connection_kwargs)
y = Connection('memory://foo', **connection_kwargs)
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, 'direct')
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish('foo', exchange=name, routing_key=name)
Producer(x).publish('bar', exchange=name, routing_key=name)
Producer(x).publish('baz', exchange=name, routing_key=name)
assert x.default_channel.queues
assert not y.default_channel.queues
migrate_tasks(x, y, accept=['text/plain'], app=app)
yq = q(y.default_channel)
assert yq.get().body == ensure_bytes('foo')
assert yq.get().body == ensure_bytes('bar')
assert yq.get().body == ensure_bytes('baz')
Producer(x).publish('foo', exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=app)
callback.assert_called()
migrate = Mock()
Producer(x).publish('baz', exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback,
migrate=migrate, accept=['text/plain'], app=app)
migrate.assert_called()
with patch('kombu.transport.virtual.Channel.queue_declare') as qd:
def effect(*args, **kwargs):
if kwargs.get('passive'):
raise ChannelError('some channel error')
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y, app=app)
x = Connection('memory://', **connection_kwargs)
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=app)
callback.assert_not_called()
def test_migrate(self, name='testcelery'):
x = Connection('memory://foo')
y = Connection('memory://foo')
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, 'direct')
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish('foo', exchange=name, routing_key=name)
Producer(x).publish('bar', exchange=name, routing_key=name)
Producer(x).publish('baz', exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y, accept=['text/plain'], app=self.app)
yq = q(y.default_channel)
self.assertEqual(yq.get().body, ensure_bytes('foo'))
self.assertEqual(yq.get().body, ensure_bytes('bar'))
self.assertEqual(yq.get().body, ensure_bytes('baz'))
Producer(x).publish('foo', exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=self.app)
self.assertTrue(callback.called)
migrate = Mock()
Producer(x).publish('baz', exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback,
migrate=migrate, accept=['text/plain'], app=self.app)
self.assertTrue(migrate.called)
with patch('kombu.transport.virtual.Channel.queue_declare') as qd:
def effect(*args, **kwargs):
if kwargs.get('passive'):
raise ChannelError('some channel error')
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y, app=self.app)
x = Connection('memory://')
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=self.app)
self.assertFalse(callback.called)
def test_migrate(self, name='testcelery'):
x = Connection('memory://foo')
y = Connection('memory://foo')
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, 'direct')
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish('foo', exchange=name, routing_key=name)
Producer(x).publish('bar', exchange=name, routing_key=name)
Producer(x).publish('baz', exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y, accept=['text/plain'], app=self.app)
yq = q(y.default_channel)
self.assertEqual(yq.get().body, ensure_bytes('foo'))
self.assertEqual(yq.get().body, ensure_bytes('bar'))
self.assertEqual(yq.get().body, ensure_bytes('baz'))
Producer(x).publish('foo', exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=self.app)
self.assertTrue(callback.called)
migrate = Mock()
Producer(x).publish('baz', exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback,
migrate=migrate, accept=['text/plain'], app=self.app)
self.assertTrue(migrate.called)
with patch('kombu.transport.virtual.Channel.queue_declare') as qd:
def effect(*args, **kwargs):
if kwargs.get('passive'):
raise StdChannelError()
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y, app=self.app)
x = Connection('memory://')
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=self.app)
self.assertFalse(callback.called)
def test_migrate(self, name="testcelery"):
x = Connection("memory://foo")
y = Connection("memory://foo")
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, "direct")
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish("foo", exchange=name, routing_key=name)
Producer(x).publish("bar", exchange=name, routing_key=name)
Producer(x).publish("baz", exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y, accept=["text/plain"])
yq = q(y.default_channel)
self.assertEqual(yq.get().body, ensure_bytes("foo"))
self.assertEqual(yq.get().body, ensure_bytes("bar"))
self.assertEqual(yq.get().body, ensure_bytes("baz"))
Producer(x).publish("foo", exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=["text/plain"])
self.assertTrue(callback.called)
migrate = Mock()
Producer(x).publish("baz", exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback, migrate=migrate, accept=["text/plain"])
self.assertTrue(migrate.called)
with patch("kombu.transport.virtual.Channel.queue_declare") as qd:
def effect(*args, **kwargs):
if kwargs.get("passive"):
raise StdChannelError()
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y)
x = Connection("memory://")
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=["text/plain"])
self.assertFalse(callback.called)
def test_migrate(self, app, name="testcelery"):
x = Connection("memory://foo")
y = Connection("memory://foo")
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, "direct")
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish("foo", exchange=name, routing_key=name)
Producer(x).publish("bar", exchange=name, routing_key=name)
Producer(x).publish("baz", exchange=name, routing_key=name)
assert x.default_channel.queues
assert not y.default_channel.queues
migrate_tasks(x, y, accept=["text/plain"], app=app)
yq = q(y.default_channel)
assert yq.get().body == ensure_bytes("foo")
assert yq.get().body == ensure_bytes("bar")
assert yq.get().body == ensure_bytes("baz")
Producer(x).publish("foo", exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=["text/plain"], app=app)
callback.assert_called()
migrate = Mock()
Producer(x).publish("baz", exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback, migrate=migrate, accept=["text/plain"], app=app)
migrate.assert_called()
with patch("kombu.transport.virtual.Channel.queue_declare") as qd:
def effect(*args, **kwargs):
if kwargs.get("passive"):
raise ChannelError("some channel error")
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y, app=app)
x = Connection("memory://")
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=["text/plain"], app=app)
callback.assert_not_called()
def test_migrate(self, name="testcelery"):
x = BrokerConnection("memory://foo")
y = BrokerConnection("memory://foo")
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, "direct")
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish("foo", exchange=name, routing_key=name)
Producer(x).publish("bar", exchange=name, routing_key=name)
Producer(x).publish("baz", exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y)
yq = q(y.default_channel)
self.assertEqual(yq.get().body, "foo")
self.assertEqual(yq.get().body, "bar")
self.assertEqual(yq.get().body, "baz")
Producer(x).publish("foo", exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y, callback=callback)
self.assertTrue(callback.called)
migrate = Mock()
Producer(x).publish("baz", exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback, migrate=migrate)
self.assertTrue(migrate.called)
with patch("kombu.transport.virtual.Channel.queue_declare") as qd:
def effect(*args, **kwargs):
if kwargs.get("passive"):
raise StdChannelError()
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y)
x = BrokerConnection("memory://")
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y, callback=callback)
self.assertFalse(callback.called)
def migrate(ctx, source, destination, **kwargs):
"""Migrate tasks from one broker to another.
Warning:
This command is experimental, make sure you have a backup of
the tasks before you continue.
"""
# TODO: Use a progress bar
def on_migrate_task(state, body, message):
ctx.obj.echo(f"Migrating task {state.count}/{state.strtotal}: {body}")
migrate_tasks(Connection(source),
Connection(destination),
callback=on_migrate_task,
**kwargs)
