def run(self, *args, **kwargs): if len(args) != 2: return self.show_help("migrate") from kombu import Connection from celery.contrib.migrate import migrate_tasks migrate_tasks(Connection(args[0]), Connection(args[1]), callback=self.on_migrate_task, **kwargs)
def run(self, source, destination, **kwargs): from kombu import Connection from celery.contrib.migrate import migrate_tasks migrate_tasks(Connection(source), Connection(destination), callback=self.on_migrate_task, **kwargs)
def run(self, *args, **kwargs): if len(args) != 2: return self.show_help("migrate") from kombu import BrokerConnection from celery.contrib.migrate import migrate_tasks migrate_tasks(BrokerConnection(args[0]), BrokerConnection(args[1]), callback=self.on_migrate_task)
def run(self, *args, **kwargs): if len(args) != 2: # this never exits due to OptionParser.parse_options self.run_from_argv(self.prog_name, ["migrate", "--help"]) raise SystemExit() from kombu import Connection from celery.contrib.migrate import migrate_tasks migrate_tasks(Connection(args[0]), Connection(args[1]), callback=self.on_migrate_task, **kwargs)
def run(self, *args, **kwargs): if len(args) != 2: return self.exit_help('migrate') from kombu import Connection from celery.contrib.migrate import migrate_tasks migrate_tasks(Connection(args[0]), Connection(args[1]), callback=self.on_migrate_task, **kwargs)
def run(self, *args, **kwargs): if len(args) != 2: # this never exits due to OptionParser.parse_options self.run_from_argv(self.prog_name, ['migrate', '--help']) raise SystemExit() from kombu import Connection from celery.contrib.migrate import migrate_tasks migrate_tasks(Connection(args[0]), Connection(args[1]), callback=self.on_migrate_task, **kwargs)
def test_migrate(self, app, name='testcelery'): connection_kwargs = {'transport_options': {'polling_interval': 0.01}} x = Connection('memory://foo', **connection_kwargs) y = Connection('memory://foo', **connection_kwargs) # use separate state x.default_channel.queues = {} y.default_channel.queues = {} ex = Exchange(name, 'direct') q = Queue(name, exchange=ex, routing_key=name) q(x.default_channel).declare() Producer(x).publish('foo', exchange=name, routing_key=name) Producer(x).publish('bar', exchange=name, routing_key=name) Producer(x).publish('baz', exchange=name, routing_key=name) assert x.default_channel.queues assert not y.default_channel.queues migrate_tasks(x, y, accept=['text/plain'], app=app) yq = q(y.default_channel) assert yq.get().body == ensure_bytes('foo') assert yq.get().body == ensure_bytes('bar') assert yq.get().body == ensure_bytes('baz') Producer(x).publish('foo', exchange=name, routing_key=name) callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=app) callback.assert_called() migrate = Mock() Producer(x).publish('baz', exchange=name, routing_key=name) migrate_tasks(x, y, callback=callback, migrate=migrate, accept=['text/plain'], app=app) migrate.assert_called() with patch('kombu.transport.virtual.Channel.queue_declare') as qd: def effect(*args, **kwargs): if kwargs.get('passive'): raise ChannelError('some channel error') return 0, 3, 0 qd.side_effect = effect migrate_tasks(x, y, app=app) x = Connection('memory://', **connection_kwargs) x.default_channel.queues = {} y.default_channel.queues = {} callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=app) callback.assert_not_called()
def test_migrate(self, app, name='testcelery'): connection_kwargs = { 'transport_options': {'polling_interval': 0.01} } x = Connection('memory://foo', **connection_kwargs) y = Connection('memory://foo', **connection_kwargs) # use separate state x.default_channel.queues = {} y.default_channel.queues = {} ex = Exchange(name, 'direct') q = Queue(name, exchange=ex, routing_key=name) q(x.default_channel).declare() Producer(x).publish('foo', exchange=name, routing_key=name) Producer(x).publish('bar', exchange=name, routing_key=name) Producer(x).publish('baz', exchange=name, routing_key=name) assert x.default_channel.queues assert not y.default_channel.queues migrate_tasks(x, y, accept=['text/plain'], app=app) yq = q(y.default_channel) assert yq.get().body == ensure_bytes('foo') assert yq.get().body == ensure_bytes('bar') assert yq.get().body == ensure_bytes('baz') Producer(x).publish('foo', exchange=name, routing_key=name) callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=app) callback.assert_called() migrate = Mock() Producer(x).publish('baz', exchange=name, routing_key=name) migrate_tasks(x, y, callback=callback, migrate=migrate, accept=['text/plain'], app=app) migrate.assert_called() with patch('kombu.transport.virtual.Channel.queue_declare') as qd: def effect(*args, **kwargs): if kwargs.get('passive'): raise ChannelError('some channel error') return 0, 3, 0 qd.side_effect = effect migrate_tasks(x, y, app=app) x = Connection('memory://', **connection_kwargs) x.default_channel.queues = {} y.default_channel.queues = {} callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=app) callback.assert_not_called()
def test_migrate(self, name='testcelery'): x = Connection('memory://foo') y = Connection('memory://foo') # use separate state x.default_channel.queues = {} y.default_channel.queues = {} ex = Exchange(name, 'direct') q = Queue(name, exchange=ex, routing_key=name) q(x.default_channel).declare() Producer(x).publish('foo', exchange=name, routing_key=name) Producer(x).publish('bar', exchange=name, routing_key=name) Producer(x).publish('baz', exchange=name, routing_key=name) self.assertTrue(x.default_channel.queues) self.assertFalse(y.default_channel.queues) migrate_tasks(x, y, accept=['text/plain'], app=self.app) yq = q(y.default_channel) self.assertEqual(yq.get().body, ensure_bytes('foo')) self.assertEqual(yq.get().body, ensure_bytes('bar')) self.assertEqual(yq.get().body, ensure_bytes('baz')) Producer(x).publish('foo', exchange=name, routing_key=name) callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=self.app) self.assertTrue(callback.called) migrate = Mock() Producer(x).publish('baz', exchange=name, routing_key=name) migrate_tasks(x, y, callback=callback, migrate=migrate, accept=['text/plain'], app=self.app) self.assertTrue(migrate.called) with patch('kombu.transport.virtual.Channel.queue_declare') as qd: def effect(*args, **kwargs): if kwargs.get('passive'): raise ChannelError('some channel error') return 0, 3, 0 qd.side_effect = effect migrate_tasks(x, y, app=self.app) x = Connection('memory://') x.default_channel.queues = {} y.default_channel.queues = {} callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=self.app) self.assertFalse(callback.called)
def test_migrate(self, name='testcelery'): x = Connection('memory://foo') y = Connection('memory://foo') # use separate state x.default_channel.queues = {} y.default_channel.queues = {} ex = Exchange(name, 'direct') q = Queue(name, exchange=ex, routing_key=name) q(x.default_channel).declare() Producer(x).publish('foo', exchange=name, routing_key=name) Producer(x).publish('bar', exchange=name, routing_key=name) Producer(x).publish('baz', exchange=name, routing_key=name) self.assertTrue(x.default_channel.queues) self.assertFalse(y.default_channel.queues) migrate_tasks(x, y, accept=['text/plain'], app=self.app) yq = q(y.default_channel) self.assertEqual(yq.get().body, ensure_bytes('foo')) self.assertEqual(yq.get().body, ensure_bytes('bar')) self.assertEqual(yq.get().body, ensure_bytes('baz')) Producer(x).publish('foo', exchange=name, routing_key=name) callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=self.app) self.assertTrue(callback.called) migrate = Mock() Producer(x).publish('baz', exchange=name, routing_key=name) migrate_tasks(x, y, callback=callback, migrate=migrate, accept=['text/plain'], app=self.app) self.assertTrue(migrate.called) with patch('kombu.transport.virtual.Channel.queue_declare') as qd: def effect(*args, **kwargs): if kwargs.get('passive'): raise StdChannelError() return 0, 3, 0 qd.side_effect = effect migrate_tasks(x, y, app=self.app) x = Connection('memory://') x.default_channel.queues = {} y.default_channel.queues = {} callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain'], app=self.app) self.assertFalse(callback.called)
def test_migrate(self, name="testcelery"): x = Connection("memory://foo") y = Connection("memory://foo") # use separate state x.default_channel.queues = {} y.default_channel.queues = {} ex = Exchange(name, "direct") q = Queue(name, exchange=ex, routing_key=name) q(x.default_channel).declare() Producer(x).publish("foo", exchange=name, routing_key=name) Producer(x).publish("bar", exchange=name, routing_key=name) Producer(x).publish("baz", exchange=name, routing_key=name) self.assertTrue(x.default_channel.queues) self.assertFalse(y.default_channel.queues) migrate_tasks(x, y, accept=["text/plain"]) yq = q(y.default_channel) self.assertEqual(yq.get().body, ensure_bytes("foo")) self.assertEqual(yq.get().body, ensure_bytes("bar")) self.assertEqual(yq.get().body, ensure_bytes("baz")) Producer(x).publish("foo", exchange=name, routing_key=name) callback = Mock() migrate_tasks(x, y, callback=callback, accept=["text/plain"]) self.assertTrue(callback.called) migrate = Mock() Producer(x).publish("baz", exchange=name, routing_key=name) migrate_tasks(x, y, callback=callback, migrate=migrate, accept=["text/plain"]) self.assertTrue(migrate.called) with patch("kombu.transport.virtual.Channel.queue_declare") as qd: def effect(*args, **kwargs): if kwargs.get("passive"): raise StdChannelError() return 0, 3, 0 qd.side_effect = effect migrate_tasks(x, y) x = Connection("memory://") x.default_channel.queues = {} y.default_channel.queues = {} callback = Mock() migrate_tasks(x, y, callback=callback, accept=["text/plain"]) self.assertFalse(callback.called)
def test_migrate(self, app, name="testcelery"): x = Connection("memory://foo") y = Connection("memory://foo") # use separate state x.default_channel.queues = {} y.default_channel.queues = {} ex = Exchange(name, "direct") q = Queue(name, exchange=ex, routing_key=name) q(x.default_channel).declare() Producer(x).publish("foo", exchange=name, routing_key=name) Producer(x).publish("bar", exchange=name, routing_key=name) Producer(x).publish("baz", exchange=name, routing_key=name) assert x.default_channel.queues assert not y.default_channel.queues migrate_tasks(x, y, accept=["text/plain"], app=app) yq = q(y.default_channel) assert yq.get().body == ensure_bytes("foo") assert yq.get().body == ensure_bytes("bar") assert yq.get().body == ensure_bytes("baz") Producer(x).publish("foo", exchange=name, routing_key=name) callback = Mock() migrate_tasks(x, y, callback=callback, accept=["text/plain"], app=app) callback.assert_called() migrate = Mock() Producer(x).publish("baz", exchange=name, routing_key=name) migrate_tasks(x, y, callback=callback, migrate=migrate, accept=["text/plain"], app=app) migrate.assert_called() with patch("kombu.transport.virtual.Channel.queue_declare") as qd: def effect(*args, **kwargs): if kwargs.get("passive"): raise ChannelError("some channel error") return 0, 3, 0 qd.side_effect = effect migrate_tasks(x, y, app=app) x = Connection("memory://") x.default_channel.queues = {} y.default_channel.queues = {} callback = Mock() migrate_tasks(x, y, callback=callback, accept=["text/plain"], app=app) callback.assert_not_called()
def test_migrate(self, name="testcelery"): x = BrokerConnection("memory://foo") y = BrokerConnection("memory://foo") # use separate state x.default_channel.queues = {} y.default_channel.queues = {} ex = Exchange(name, "direct") q = Queue(name, exchange=ex, routing_key=name) q(x.default_channel).declare() Producer(x).publish("foo", exchange=name, routing_key=name) Producer(x).publish("bar", exchange=name, routing_key=name) Producer(x).publish("baz", exchange=name, routing_key=name) self.assertTrue(x.default_channel.queues) self.assertFalse(y.default_channel.queues) migrate_tasks(x, y) yq = q(y.default_channel) self.assertEqual(yq.get().body, "foo") self.assertEqual(yq.get().body, "bar") self.assertEqual(yq.get().body, "baz") Producer(x).publish("foo", exchange=name, routing_key=name) callback = Mock() migrate_tasks(x, y, callback=callback) self.assertTrue(callback.called) migrate = Mock() Producer(x).publish("baz", exchange=name, routing_key=name) migrate_tasks(x, y, callback=callback, migrate=migrate) self.assertTrue(migrate.called) with patch("kombu.transport.virtual.Channel.queue_declare") as qd: def effect(*args, **kwargs): if kwargs.get("passive"): raise StdChannelError() return 0, 3, 0 qd.side_effect = effect migrate_tasks(x, y) x = BrokerConnection("memory://") x.default_channel.queues = {} y.default_channel.queues = {} callback = Mock() migrate_tasks(x, y, callback=callback) self.assertFalse(callback.called)
def migrate(ctx, source, destination, **kwargs): """Migrate tasks from one broker to another. Warning: This command is experimental, make sure you have a backup of the tasks before you continue. """ # TODO: Use a progress bar def on_migrate_task(state, body, message): ctx.obj.echo(f"Migrating task {state.count}/{state.strtotal}: {body}") migrate_tasks(Connection(source), Connection(destination), callback=on_migrate_task, **kwargs)