class test_MongoBackend(AppCase): def setup(self): if pymongo is None: raise SkipTest('pymongo is not installed.') R = self._reset = {} R['encode'], MongoBackend.encode = MongoBackend.encode, Mock() R['decode'], MongoBackend.decode = MongoBackend.decode, Mock() R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app) def teardown(self): MongoBackend.encode = self._reset['encode'] MongoBackend.decode = self._reset['decode'] module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] def test_Bunch(self): x = Bunch(foo='foo', bar=2) self.assertEqual(x.foo, 'foo') self.assertEqual(x.bar, 2) def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=self.app) def test_restore_group_no_entry(self): x = MongoBackend(app=self.app) x.collection = Mock() fo = x.collection.find_one = Mock() fo.return_value = None self.assertIsNone(x._restore_group('1f3fab')) @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__['database'] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) self.backend._connection = 'not none' self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once() self.assertEqual(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( ['status', 'task_id', 'date_done', 'traceback', 'result', 'children'], list(ret_val.keys())) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({'status': states.PENDING, 'result': None}, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._save_group( sentinel.taskset_id, sentinel.result) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once() self.assertEqual(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertEqual( ['date_done', 'result', 'task_id'], list(ret_val.keys()), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.assert_called_once() def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l')
class test_MongoBackend(AppCase): def setUp(self): if pymongo is None: raise SkipTest('pymongo is not installed.') R = self._reset = {} R['encode'], MongoBackend.encode = MongoBackend.encode, Mock() R['decode'], MongoBackend.decode = MongoBackend.decode, Mock() R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend() def tearDown(self): MongoBackend.encode = self._reset['encode'] MongoBackend.decode = self._reset['decode'] module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] def test_Bunch(self): x = Bunch(foo='foo', bar=2) self.assertEqual(x.foo, 'foo') self.assertEqual(x.bar, 2) def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend() finally: module.pymongo = prev def test_init_no_settings(self): celery = Celery(set_as_current=False) celery.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=celery) def test_init_settings_is_None(self): celery = Celery(set_as_current=False) celery.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=celery) def test_restore_group_no_entry(self): x = MongoBackend() x.collection = Mock() fo = x.collection.find_one = Mock() fo.return_value = None self.assertIsNone(x._restore_group('1f3fab')) def test_reduce(self): x = MongoBackend() self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch('pymongo.connection.Connection') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEquals(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch('pymongo.connection.Connection') as mock_Connection: self.backend._connection = None self.backend.mongodb_host = MONGODB_HOST self.backend.mongodb_port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with(MONGODB_HOST, MONGODB_PORT) self.assertEquals(sentinel.connection, connection) def test_get_connection_no_connection_mongodb_uri(self): with patch('pymongo.connection.Connection') as mock_Connection: mongodb_uri = 'mongodb://%s:%d' % (MONGODB_HOST, MONGODB_PORT) self.backend._connection = None self.backend.mongodb_host = mongodb_uri mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with(mongodb_uri) self.assertEquals(sentinel.connection, connection) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.mongodb_user = MONGODB_USER self.backend.mongodb_password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__['database'] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.mongodb_user = None self.backend.mongodb_password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEquals(self.backend._connection, None) self.backend._connection = 'not none' self.backend.process_cleanup() self.assertEquals(self.backend._connection, None) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once() self.assertEquals(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEquals([ 'status', 'task_id', 'date_done', 'traceback', 'result', 'children' ], ret_val.keys()) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEquals({'status': states.PENDING, 'result': None}, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._save_group(sentinel.taskset_id, sentinel.result) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once() self.assertEquals(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertEquals(['date_done', 'result', 'task_id'], ret_val.keys()) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_forget(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}, safe=True) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.assert_called_once() def test_get_database_authfailure(self): x = MongoBackend() x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.mongodb_database] = Mock() db.authenticate.return_value = False x.mongodb_user = '******' x.mongodb_password = '******' with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l')
class test_MongoBackend(AppCase): def setup(self): if pymongo is None: raise SkipTest('pymongo is not installed.') R = self._reset = {} R['encode'], MongoBackend.encode = MongoBackend.encode, Mock() R['decode'], MongoBackend.decode = MongoBackend.decode, Mock() R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app) def teardown(self): MongoBackend.encode = self._reset['encode'] MongoBackend.decode = self._reset['decode'] module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] def test_Bunch(self): x = Bunch(foo='foo', bar=2) self.assertEqual(x.foo, 'foo') self.assertEqual(x.bar, 2) def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=self.app) @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__['database'] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) self.backend._connection = 'not none' self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( list( sorted([ 'status', 'task_id', 'date_done', 'traceback', 'result', 'children' ])), list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({'status': states.PENDING, 'result': None}, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group( sentinel.taskset_id, res, ) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_GROUP_COLLECTION, ) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(res, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = { '_id': sentinel.taskset_id, 'result': [uuid(), uuid()], 'date_done': 1, } self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertEqual( list(sorted(['date_done', 'result', 'task_id'])), list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD') self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__ = Mock(name='MD.__getitem__') mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l')
class test_MongoBackend(AppCase): def setup(self): if pymongo is None: raise SkipTest('pymongo is not installed.') R = self._reset = {} R['encode'], MongoBackend.encode = MongoBackend.encode, Mock() R['decode'], MongoBackend.decode = MongoBackend.decode, Mock() R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app) def teardown(self): MongoBackend.encode = self._reset['encode'] MongoBackend.decode = self._reset['decode'] module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=self.app) def test_restore_group_no_entry(self): x = MongoBackend(app=self.app) x.collection = Mock() fo = x.collection.find_one = Mock() fo.return_value = None self.assertIsNone(x._restore_group('1f3fab')) @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__['database'] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) self.backend._connection = 'not none' self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( list( sorted([ 'status', 'task_id', 'date_done', 'traceback', 'result', 'children' ])), list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({'status': states.PENDING, 'result': None}, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._save_group(sentinel.taskset_id, sentinel.result) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertItemsEqual( ['date_done', 'result', 'task_id'], list(ret_val.keys()), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l') @patch('celery.backends.mongodb.detect_environment') def test_prepare_client_options_for_ver_2(self, m_detect_env): m_detect_env.return_value = 'default' with patch('pymongo.version_tuple', new=(2, 6, 3)): options = self.backend._prepare_client_options() self.assertDictEqual( options, { 'max_pool_size': self.backend.max_pool_size, 'auto_start_request': False }) @patch('celery.backends.mongodb.detect_environment') def test_prepare_client_options_for_ver_2_with_gevent(self, m_detect_env): m_detect_env.return_value = 'gevent' with patch('pymongo.version_tuple', new=(2, 6, 3)): options = self.backend._prepare_client_options() self.assertDictEqual( options, { 'max_pool_size': self.backend.max_pool_size, 'auto_start_request': False, 'use_greenlets': True }) @patch('celery.backends.mongodb.detect_environment') def test_prepare_client_options_for_ver_3(self, m_detect_env): m_detect_env.return_value = 'default' with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, {'maxPoolSize': self.backend.max_pool_size}) @patch('celery.backends.mongodb.detect_environment') def test_prepare_client_options_for_ver_3_with_gevent(self, m_detect_env): m_detect_env.return_value = 'gevent' with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, {'maxPoolSize': self.backend.max_pool_size})
class test_MongoBackend(AppCase): default_url = 'mongodb://*****:*****@hostname.dom/database' replica_set_url = ('mongodb://*****:*****@hostname.dom,' 'hostname.dom/database?replicaSet=rs') sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database' sanitized_replica_set_url = ('mongodb://*****:*****@hostname.dom/,' 'hostname.dom/database?replicaSet=rs') def setup(self): R = self._reset = {} R['encode'], MongoBackend.encode = MongoBackend.encode, Mock() R['decode'], MongoBackend.decode = MongoBackend.decode, Mock() R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app, url=self.default_url) def teardown(self): MongoBackend.encode = self._reset['encode'] MongoBackend.decode = self._reset['decode'] module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.mongodb_backend_settings = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.mongodb_backend_settings = None MongoBackend(app=self.app) def test_init_with_settings(self): self.app.conf.mongodb_backend_settings = None # empty settings mb = MongoBackend(app=self.app) # uri uri = 'mongodb://*****:*****@' 'mongo1.example.com:27017,' 'mongo2.example.com:27017,' 'mongo3.example.com:27017/' 'celerydatabase?replicaSet=rs0') mb = MongoBackend(app=self.app, url=uri) self.assertEqual(mb.mongo_host, [ 'mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017' ]) self.assertEqual( mb.options, dict(mb._prepare_client_options(), replicaset='rs0'), ) self.assertEqual(mb.user, 'celeryuser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'celerydatabase') # same uri, change some parameters in backend settings self.app.conf.mongodb_backend_settings = { 'replicaset': 'rs1', 'user': '******', 'database': 'another_db', 'options': { 'socketKeepAlive': True, }, } mb = MongoBackend(app=self.app, url=uri) self.assertEqual(mb.mongo_host, [ 'mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017' ]) self.assertEqual( mb.options, dict(mb._prepare_client_options(), replicaset='rs1', socketKeepAlive=True), ) self.assertEqual(mb.user, 'backenduser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'another_db') mb = MongoBackend(app=self.app, url='mongodb://') @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__['database'] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) mock_collection.save.side_effect = InvalidDocument() with self.assertRaises(EncodeError): self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( list( sorted([ 'status', 'task_id', 'date_done', 'traceback', 'result', 'children' ])), list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({'status': states.PENDING, 'result': None}, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group( sentinel.taskset_id, res, ) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_GROUP_COLLECTION, ) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(res, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = { '_id': sentinel.taskset_id, 'result': [uuid(), uuid()], 'date_done': 1, } self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertItemsEqual( ['date_done', 'result', 'task_id'], list(ret_val.keys()), ) mock_collection.find_one.return_value = None self.backend._restore_group(sentinel.taskset_id) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD') self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__ = Mock(name='MD.__getitem__') mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l') def test_prepare_client_options(self): with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, {'maxPoolSize': self.backend.max_pool_size}) def test_as_uri_include_password(self): self.assertEqual(self.backend.as_uri(True), self.default_url) def test_as_uri_exclude_password(self): self.assertEqual(self.backend.as_uri(), self.sanitized_default_url) def test_as_uri_include_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) self.assertEqual(backend.as_uri(True), self.replica_set_url) def test_as_uri_exclude_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) self.assertEqual(backend.as_uri(), self.sanitized_replica_set_url) @override_stdouts def test_regression_worker_startup_info(self): self.app.conf.result_backend = ( 'mongodb://*****:*****@host0.com:43437,host1.com:43437' '/work4us?replicaSet=rs&ssl=true') worker = self.app.Worker() worker.on_start() self.assertTrue(worker.startup_info())
class test_MongoBackend: default_url = 'mongodb://*****:*****@hostname.dom/database' replica_set_url = ('mongodb://*****:*****@hostname.dom,' 'hostname.dom/database?replicaSet=rs') sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database' sanitized_replica_set_url = ('mongodb://*****:*****@hostname.dom/,' 'hostname.dom/database?replicaSet=rs') def setup(self): self.patching('celery.backends.mongodb.MongoBackend.encode') self.patching('celery.backends.mongodb.MongoBackend.decode') self.patching('celery.backends.mongodb.Binary') self.patching('datetime.datetime') self.backend = MongoBackend(app=self.app, url=self.default_url) def test_init_no_mongodb(self, patching): patching('celery.backends.mongodb.pymongo', None) with pytest.raises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_no_settings(self): self.app.conf.mongodb_backend_settings = [] with pytest.raises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.mongodb_backend_settings = None MongoBackend(app=self.app) def test_init_with_settings(self): self.app.conf.mongodb_backend_settings = None # empty settings mb = MongoBackend(app=self.app) # uri uri = 'mongodb://*****:*****@' 'mongo1.example.com:27017,' 'mongo2.example.com:27017,' 'mongo3.example.com:27017/' 'celerydatabase?replicaSet=rs0') mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == [ 'mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017', ] assert mb.options == dict( mb._prepare_client_options(), replicaset='rs0', ) assert mb.user == 'celeryuser' assert mb.password == 'celerypassword' assert mb.database_name == 'celerydatabase' # same uri, change some parameters in backend settings self.app.conf.mongodb_backend_settings = { 'replicaset': 'rs1', 'user': '******', 'database': 'another_db', 'options': { 'socketKeepAlive': True, }, } mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == [ 'mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017', ] assert mb.options == dict( mb._prepare_client_options(), replicaset='rs1', socketKeepAlive=True, ) assert mb.user == 'backenduser' assert mb.password == 'celerypassword' assert mb.database_name == 'another_db' mb = MongoBackend(app=self.app, url='mongodb://') def test_init_mongodb_dns_seedlist(self): Name = pytest.importorskip('dns.name').Name TXT = pytest.importorskip('dns.rdtypes.ANY.TXT').TXT SRV = pytest.importorskip('dns.rdtypes.IN.SRV').SRV self.app.conf.mongodb_backend_settings = None def mock_resolver(_, rdtype, rdclass=None, lifetime=None, **kwargs): if rdtype == 'SRV': return [ SRV(0, 0, 0, 0, 27017, Name(labels=hostname)) for hostname in [ b'mongo1.example.com'.split( b'.'), b'mongo2.example.com'.split(b'.'), b'mongo3.example.com'.split(b'.') ] ] elif rdtype == 'TXT': return [TXT(0, 0, [b'replicaSet=rs0'])] # uri with user, password, database name, replica set, # DNS seedlist format uri = ('srv://' 'celeryuser:celerypassword@' 'dns-seedlist-host.example.com/' 'celerydatabase') with patch('dns.resolver.query', side_effect=mock_resolver): mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == [ 'mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017', ] assert mb.options == dict(mb._prepare_client_options(), replicaset='rs0', ssl=True) assert mb.user == 'celeryuser' assert mb.password == 'celerypassword' assert mb.database_name == 'celerydatabase' def test_ensure_mongodb_uri_compliance(self): mb = MongoBackend(app=self.app, url=None) compliant_uri = mb._ensure_mongodb_uri_compliance assert compliant_uri('mongodb://') == 'mongodb://localhost' assert compliant_uri('mongodb+something://host') == \ 'mongodb+something://host' assert compliant_uri('something://host') == 'mongodb+something://host' @pytest.mark.usefixtures('depends_on_current_app') def test_reduce(self): x = MongoBackend(app=self.app) assert loads(dumps(x)) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() assert sentinel._connection == connection mock_Connection.assert_not_called() def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@' 'localhost:27017/' 'celerydatabase?authMechanism=SCRAM-SHA-256') mb = MongoBackend(app=self.app, url=uri) mock_Connection.return_value = sentinel.connection connection = mb._get_connection() mock_Connection.assert_called_once_with( host=['localhost:27017'], username='******', password='******', authmechanism='SCRAM-SHA-256', **mb._prepare_client_options()) assert sentinel.connection == connection def test_get_connection_with_authmechanism_no_username(self): with patch('pymongo.MongoClient') as mock_Connection: self.app.conf.mongodb_backend_settings = None uri = ('mongodb://' 'localhost:27017/' 'celerydatabase?authMechanism=SCRAM-SHA-256') mb = MongoBackend(app=self.app, url=uri) mock_Connection.side_effect = ConfigurationError( 'SCRAM-SHA-256 requires a username.') with pytest.raises(ConfigurationError): mb._get_connection() mock_Connection.assert_called_once_with( host=['localhost:27017'], authmechanism='SCRAM-SHA-256', **mb._prepare_client_options()) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database assert database is mock_database assert self.backend.__dict__['database'] is mock_database mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD, source=self.backend.database_name) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database assert database is mock_database mock_database.authenticate.assert_not_called() assert self.backend.__dict__['database'] is mock_database @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.replace_one.assert_called_once_with(ANY, ANY, upsert=True) assert sentinel.result == ret_val mock_collection.replace_one.side_effect = InvalidDocument() with pytest.raises(EncodeError): self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result_with_request(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_request = MagicMock(spec=['parent_id']) mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection mock_request.parent_id = sentinel.parent_id ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status, request=mock_request) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) parameters = mock_collection.replace_one.call_args[0][1] assert parameters['parent_id'] == sentinel.parent_id assert sentinel.result == ret_val mock_collection.replace_one.side_effect = InvalidDocument() with pytest.raises(EncodeError): self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) assert list( sorted([ 'status', 'task_id', 'date_done', 'traceback', 'result', 'children', ])) == list(sorted(ret_val.keys())) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) assert {'status': states.PENDING, 'result': None} == ret_val @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group( sentinel.taskset_id, res, ) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_GROUP_COLLECTION, ) mock_collection.replace_one.assert_called_once_with(ANY, ANY, upsert=True) assert res == ret_val @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = { '_id': sentinel.taskset_id, 'result': [uuid(), uuid()], 'date_done': 1, } self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) assert (sorted(['date_done', 'result', 'task_id']) == sorted(list(ret_val.keys()))) mock_collection.find_one.return_value = None self.backend._restore_group(sentinel.taskset_id) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.delete_one.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test__forget(self, mock_get_database): # note: here tested _forget method, not forget method self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.delete_one.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD') self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__ = Mock(name='MD.__getitem__') mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() mock_collection.delete_many.assert_called() def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with pytest.raises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l', source=x.database_name) def test_prepare_client_options(self): with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() assert options == {'maxPoolSize': self.backend.max_pool_size} def test_as_uri_include_password(self): assert self.backend.as_uri(True) == self.default_url def test_as_uri_exclude_password(self): assert self.backend.as_uri() == self.sanitized_default_url def test_as_uri_include_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) assert backend.as_uri(True) == self.replica_set_url def test_as_uri_exclude_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) assert backend.as_uri() == self.sanitized_replica_set_url def test_regression_worker_startup_info(self): self.app.conf.result_backend = ( 'mongodb://*****:*****@host0.com:43437,host1.com:43437' '/work4us?replicaSet=rs&ssl=true') worker = self.app.Worker() with mock.stdouts(): worker.on_start() assert worker.startup_info()
class test_MongoBackend(AppCase): def setup(self): if pymongo is None: raise SkipTest("pymongo is not installed.") R = self._reset = {} R["encode"], MongoBackend.encode = MongoBackend.encode, Mock() R["decode"], MongoBackend.decode = MongoBackend.decode, Mock() R["Binary"], module.Binary = module.Binary, Mock() R["datetime"], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app) def teardown(self): MongoBackend.encode = self._reset["encode"] MongoBackend.decode = self._reset["decode"] module.Binary = self._reset["Binary"] datetime.datetime = self._reset["datetime"] def test_Bunch(self): x = Bunch(foo="foo", bar=2) self.assertEqual(x.foo, "foo") self.assertEqual(x.bar, 2) def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=self.app) def test_restore_group_no_entry(self): x = MongoBackend(app=self.app) x.collection = Mock() fo = x.collection.find_one = Mock() fo.return_value = None self.assertIsNone(x._restore_group("1f3fab")) @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch("pymongo.MongoClient") as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch("pymongo.MongoClient") as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host="mongodb://*****:*****@patch("celery.backends.mongodb.MongoBackend._get_connection") def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=["__getitem__"]) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__["database"] is mock_database) mock_database.authenticate.assert_called_once_with(MONGODB_USER, MONGODB_PASSWORD) @patch("celery.backends.mongodb.MongoBackend._get_connection") def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=["__getitem__"]) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__["database"] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) self.backend._connection = "not none" self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset["datetime"] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( list(sorted(["status", "task_id", "date_done", "traceback", "result", "children"])), list(sorted(ret_val.keys())), ) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({"status": states.PENDING, "result": None}, ret_val) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_save_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._save_group(sentinel.taskset_id, sentinel.result) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_restore_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.find_one.assert_called_once_with({"_id": sentinel.taskset_id}) self.assertItemsEqual(["date_done", "result", "task_id"], list(ret_val.keys())) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with({"_id": sentinel.taskset_id}) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with({"_id": sentinel.task_id}) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_cleanup(self, mock_get_database): datetime.datetime = self._reset["datetime"] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = "******" x.password = "******" with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with("jerry", "cere4l") @patch("celery.backends.mongodb.detect_environment") def test_prepare_client_options_for_ver_2(self, m_detect_env): m_detect_env.return_value = "default" with patch("pymongo.version_tuple", new=(2, 6, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, {"max_pool_size": self.backend.max_pool_size, "auto_start_request": False}) @patch("celery.backends.mongodb.detect_environment") def test_prepare_client_options_for_ver_2_with_gevent(self, m_detect_env): m_detect_env.return_value = "gevent" with patch("pymongo.version_tuple", new=(2, 6, 3)): options = self.backend._prepare_client_options() self.assertDictEqual( options, {"max_pool_size": self.backend.max_pool_size, "auto_start_request": False, "use_greenlets": True}, ) @patch("celery.backends.mongodb.detect_environment") def test_prepare_client_options_for_ver_3(self, m_detect_env): m_detect_env.return_value = "default" with patch("pymongo.version_tuple", new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, {"maxPoolSize": self.backend.max_pool_size}) @patch("celery.backends.mongodb.detect_environment") def test_prepare_client_options_for_ver_3_with_gevent(self, m_detect_env): m_detect_env.return_value = "gevent" with patch("pymongo.version_tuple", new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, {"maxPoolSize": self.backend.max_pool_size})
class test_MongoBackend(AppCase): default_url = 'mongodb://*****:*****@hostname.dom/database' replica_set_url = ( 'mongodb://*****:*****@hostname.dom,' 'hostname.dom/database?replicaSet=rs' ) sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database' sanitized_replica_set_url = ( 'mongodb://*****:*****@hostname.dom/,' 'hostname.dom/database?replicaSet=rs' ) def setup(self): R = self._reset = {} R['encode'], MongoBackend.encode = MongoBackend.encode, Mock() R['decode'], MongoBackend.decode = MongoBackend.decode, Mock() R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app, url=self.default_url) def teardown(self): MongoBackend.encode = self._reset['encode'] MongoBackend.decode = self._reset['decode'] module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.mongodb_backend_settings = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.mongodb_backend_settings = None MongoBackend(app=self.app) def test_init_with_settings(self): self.app.conf.mongodb_backend_settings = None # empty settings mb = MongoBackend(app=self.app) # uri uri = 'mongodb://*****:*****@' 'mongo1.example.com:27017,' 'mongo2.example.com:27017,' 'mongo3.example.com:27017/' 'celerydatabase?replicaSet=rs0') mb = MongoBackend(app=self.app, url=uri) self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017']) self.assertEqual( mb.options, dict(mb._prepare_client_options(), replicaset='rs0'), ) self.assertEqual(mb.user, 'celeryuser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'celerydatabase') # same uri, change some parameters in backend settings self.app.conf.mongodb_backend_settings = { 'replicaset': 'rs1', 'user': '******', 'database': 'another_db', 'options': { 'socketKeepAlive': True, }, } mb = MongoBackend(app=self.app, url=uri) self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017']) self.assertEqual( mb.options, dict(mb._prepare_client_options(), replicaset='rs1', socketKeepAlive=True), ) self.assertEqual(mb.user, 'backenduser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'another_db') mb = MongoBackend(app=self.app, url='mongodb://') @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__['database'] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) mock_collection.save.side_effect = InvalidDocument() with self.assertRaises(EncodeError): self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( list(sorted(['status', 'task_id', 'date_done', 'traceback', 'result', 'children'])), list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({'status': states.PENDING, 'result': None}, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group( sentinel.taskset_id, res, ) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_GROUP_COLLECTION, ) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(res, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = { '_id': sentinel.taskset_id, 'result': [uuid(), uuid()], 'date_done': 1, } self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertItemsEqual( ['date_done', 'result', 'task_id'], list(ret_val.keys()), ) mock_collection.find_one.return_value = None self.backend._restore_group(sentinel.taskset_id) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD') self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__ = Mock(name='MD.__getitem__') mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l') def test_prepare_client_options(self): with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, { 'maxPoolSize': self.backend.max_pool_size }) def test_as_uri_include_password(self): self.assertEqual(self.backend.as_uri(True), self.default_url) def test_as_uri_exclude_password(self): self.assertEqual(self.backend.as_uri(), self.sanitized_default_url) def test_as_uri_include_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) self.assertEqual(backend.as_uri(True), self.replica_set_url) def test_as_uri_exclude_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) self.assertEqual(backend.as_uri(), self.sanitized_replica_set_url) @override_stdouts def test_regression_worker_startup_info(self): self.app.conf.result_backend = ( 'mongodb://*****:*****@host0.com:43437,host1.com:43437' '/work4us?replicaSet=rs&ssl=true' ) worker = self.app.Worker() worker.on_start() self.assertTrue(worker.startup_info())
class test_MongoBackend: default_url = 'mongodb://*****:*****@hostname.dom/database' replica_set_url = ( 'mongodb://*****:*****@hostname.dom,' 'hostname.dom/database?replicaSet=rs' ) sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database' sanitized_replica_set_url = ( 'mongodb://*****:*****@hostname.dom/,' 'hostname.dom/database?replicaSet=rs' ) def setup(self): self.patching('celery.backends.mongodb.MongoBackend.encode') self.patching('celery.backends.mongodb.MongoBackend.decode') self.patching('celery.backends.mongodb.Binary') self.patching('datetime.datetime') self.backend = MongoBackend(app=self.app, url=self.default_url) def test_init_no_mongodb(self, patching): patching('celery.backends.mongodb.pymongo', None) with pytest.raises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_no_settings(self): self.app.conf.mongodb_backend_settings = [] with pytest.raises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.mongodb_backend_settings = None MongoBackend(app=self.app) def test_init_with_settings(self): self.app.conf.mongodb_backend_settings = None # empty settings mb = MongoBackend(app=self.app) # uri uri = 'mongodb://*****:*****@' 'mongo1.example.com:27017,' 'mongo2.example.com:27017,' 'mongo3.example.com:27017/' 'celerydatabase?replicaSet=rs0') mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == [ 'mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017', ] assert mb.options == dict( mb._prepare_client_options(), replicaset='rs0', ) assert mb.user == 'celeryuser' assert mb.password == 'celerypassword' assert mb.database_name == 'celerydatabase' # same uri, change some parameters in backend settings self.app.conf.mongodb_backend_settings = { 'replicaset': 'rs1', 'user': '******', 'database': 'another_db', 'options': { 'socketKeepAlive': True, }, } mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == [ 'mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017', ] assert mb.options == dict( mb._prepare_client_options(), replicaset='rs1', socketKeepAlive=True, ) assert mb.user == 'backenduser' assert mb.password == 'celerypassword' assert mb.database_name == 'another_db' mb = MongoBackend(app=self.app, url='mongodb://') @pytest.mark.usefixtures('depends_on_current_app') def test_reduce(self): x = MongoBackend(app=self.app) assert loads(dumps(x)) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() assert sentinel._connection == connection mock_Connection.assert_not_called() def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database assert database is mock_database assert self.backend.__dict__['database'] is mock_database mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database assert database is mock_database mock_database.authenticate.assert_not_called() assert self.backend.__dict__['database'] is mock_database @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) assert sentinel.result == ret_val mock_collection.save.side_effect = InvalidDocument() with pytest.raises(EncodeError): self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result_with_request(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_request = MagicMock(spec=['parent_id']) mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection mock_request.parent_id = sentinel.parent_id ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status, request=mock_request) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) parameters = mock_collection.save.call_args[0][0] assert parameters['parent_id'] == sentinel.parent_id assert sentinel.result == ret_val mock_collection.save.side_effect = InvalidDocument() with pytest.raises(EncodeError): self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) assert list(sorted([ 'status', 'task_id', 'date_done', 'traceback', 'result', 'children', ])) == list(sorted(ret_val.keys())) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) assert {'status': states.PENDING, 'result': None} == ret_val @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group( sentinel.taskset_id, res, ) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_GROUP_COLLECTION, ) mock_collection.save.assert_called_once_with(ANY) assert res == ret_val @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = { '_id': sentinel.taskset_id, 'result': [uuid(), uuid()], 'date_done': 1, } self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) assert (sorted(['date_done', 'result', 'task_id']) == sorted(list(ret_val.keys()))) mock_collection.find_one.return_value = None self.backend._restore_group(sentinel.taskset_id) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test__forget(self, mock_get_database): # note: here tested _forget method, not forget method self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD') self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__ = Mock(name='MD.__getitem__') mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() mock_collection.remove.assert_called() def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with pytest.raises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l') def test_prepare_client_options(self): with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() assert options == { 'maxPoolSize': self.backend.max_pool_size } def test_as_uri_include_password(self): assert self.backend.as_uri(True) == self.default_url def test_as_uri_exclude_password(self): assert self.backend.as_uri() == self.sanitized_default_url def test_as_uri_include_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) assert backend.as_uri(True) == self.replica_set_url def test_as_uri_exclude_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) assert backend.as_uri() == self.sanitized_replica_set_url def test_regression_worker_startup_info(self): self.app.conf.result_backend = ( 'mongodb://*****:*****@host0.com:43437,host1.com:43437' '/work4us?replicaSet=rs&ssl=true' ) worker = self.app.Worker() with mock.stdouts(): worker.on_start() assert worker.startup_info()
class test_MongoBackend(AppCase): default_url = 'mongodb://*****:*****@hostname.dom/database' replica_set_url = ( 'mongodb://*****:*****@hostname.dom,' 'hostname.dom/database?replicaSet=rs' ) sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database' sanitized_replica_set_url = ( 'mongodb://*****:*****@hostname.dom/,' 'hostname.dom/database?replicaSet=rs' ) def setup(self): if pymongo is None: raise SkipTest('pymongo is not installed.') R = self._reset = {} R['encode'], MongoBackend.encode = MongoBackend.encode, Mock() R['decode'], MongoBackend.decode = MongoBackend.decode, Mock() R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app, url=self.default_url) def teardown(self): MongoBackend.encode = self._reset['encode'] MongoBackend.decode = self._reset['decode'] module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=self.app) def test_restore_group_no_entry(self): x = MongoBackend(app=self.app) x.collection = Mock() fo = x.collection.find_one = Mock() fo.return_value = None self.assertIsNone(x._restore_group('1f3fab')) @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__['database'] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) self.backend._connection = 'not none' self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( list(sorted(['status', 'task_id', 'date_done', 'traceback', 'result', 'children'])), list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({'status': states.PENDING, 'result': None}, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._save_group( sentinel.taskset_id, sentinel.result) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertItemsEqual( ['date_done', 'result', 'task_id'], list(ret_val.keys()), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l') @patch('celery.backends.mongodb.detect_environment') def test_prepare_client_options_for_ver_2(self, m_detect_env): m_detect_env.return_value = 'default' with patch('pymongo.version_tuple', new=(2, 6, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, { 'max_pool_size': self.backend.max_pool_size, 'auto_start_request': False }) def test_as_uri_include_password(self): self.assertEqual(self.backend.as_uri(True), self.default_url) def test_as_uri_exclude_password(self): self.assertEqual(self.backend.as_uri(), self.sanitized_default_url) def test_as_uri_include_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) self.assertEqual(backend.as_uri(True), self.replica_set_url) def test_as_uri_exclude_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) self.assertEqual(backend.as_uri(), self.sanitized_replica_set_url) @disable_stdouts def test_regression_worker_startup_info(self): self.app.conf.result_backend = ( 'mongodb://*****:*****@host0.com:43437,host1.com:43437' '/work4us?replicaSet=rs&ssl=true' ) worker = self.app.Worker() worker.on_start() self.assertTrue(worker.startup_info())
class test_MongoBackend(AppCase): def setUp(self): if pymongo is None: raise SkipTest("pymongo is not installed.") R = self._reset = {} R["encode"], MongoBackend.encode = MongoBackend.encode, Mock() R["decode"], MongoBackend.decode = MongoBackend.decode, Mock() R["Binary"], module.Binary = module.Binary, Mock() R["datetime"], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend() def tearDown(self): MongoBackend.encode = self._reset["encode"] MongoBackend.decode = self._reset["decode"] module.Binary = self._reset["Binary"] datetime.datetime = self._reset["datetime"] def test_Bunch(self): x = Bunch(foo="foo", bar=2) self.assertEqual(x.foo, "foo") self.assertEqual(x.bar, 2) def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend() finally: module.pymongo = prev def test_init_no_settings(self): celery = Celery(set_as_current=False) celery.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=celery) def test_init_settings_is_None(self): celery = Celery(set_as_current=False) celery.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=celery) def test_restore_group_no_entry(self): x = MongoBackend() x.collection = Mock() fo = x.collection.find_one = Mock() fo.return_value = None self.assertIsNone(x._restore_group("1f3fab")) def test_reduce(self): x = MongoBackend() self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch("pymongo.connection.Connection") as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEquals(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch("pymongo.connection.Connection") as mock_Connection: self.backend._connection = None self.backend.mongodb_host = MONGODB_HOST self.backend.mongodb_port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( MONGODB_HOST, MONGODB_PORT) self.assertEquals(sentinel.connection, connection) def test_get_connection_no_connection_mongodb_uri(self): with patch("pymongo.connection.Connection") as mock_Connection: mongodb_uri = "mongodb://%s:%d" % (MONGODB_HOST, MONGODB_PORT) self.backend._connection = None self.backend.mongodb_host = mongodb_uri mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with(mongodb_uri) self.assertEquals(sentinel.connection, connection) @patch("celery.backends.mongodb.MongoBackend._get_connection") def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.mongodb_user = MONGODB_USER self.backend.mongodb_password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__["database"] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch("celery.backends.mongodb.MongoBackend._get_connection") def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.mongodb_user = None self.backend.mongodb_password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__["database"] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEquals(self.backend._connection, None) self.backend._connection = "not none" self.backend.process_cleanup() self.assertEquals(self.backend._connection, None) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_store_result(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once() self.assertEquals(sentinel.result, ret_val) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset["datetime"] self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEquals( ['status', 'task_id', 'date_done', 'traceback', 'result', 'children'], ret_val.keys()) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEquals({"status": states.PENDING, "result": None}, ret_val) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_save_group(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._save_group( sentinel.taskset_id, sentinel.result) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once() self.assertEquals(sentinel.result, ret_val) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_restore_group(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.find_one.assert_called_once_with( {"_id": sentinel.taskset_id}) self.assertEquals(['date_done', 'result', 'task_id'], ret_val.keys()) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_delete_group(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {"_id": sentinel.taskset_id}) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_forget(self, mock_get_database): self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {"_id": sentinel.task_id}, safe=True) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_cleanup(self, mock_get_database): datetime.datetime = self._reset["datetime"] self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.assert_called_once() def test_get_database_authfailure(self): x = MongoBackend() x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.mongodb_database] = Mock() db.authenticate.return_value = False x.mongodb_user = "******" x.mongodb_password = "******" with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with("jerry", "cere4l")
class test_MongoBackend(AppCase): def setup(self): if pymongo is None: raise SkipTest('pymongo is not installed.') R = self._reset = {} R['encode'], MongoBackend.encode = MongoBackend.encode, Mock() R['decode'], MongoBackend.decode = MongoBackend.decode, Mock() R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app) def teardown(self): MongoBackend.encode = self._reset['encode'] MongoBackend.decode = self._reset['decode'] module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] def test_Bunch(self): x = Bunch(foo='foo', bar=2) self.assertEqual(x.foo, 'foo') self.assertEqual(x.bar, 2) def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=self.app) def test_init_with_settings(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None # empty settings mb = MongoBackend(app=self.app) # uri uri = 'mongodb://*****:*****@' 'mongo1.example.com:27017,' 'mongo2.example.com:27017,' 'mongo3.example.com:27017/' 'celerydatabase?replicaSet=rs0') mb = MongoBackend(app=self.app, url=uri) self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017']) self.assertEqual( mb.options, dict(mb._prepare_client_options(), replicaset='rs0'), ) self.assertEqual(mb.user, 'celeryuser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'celerydatabase') # same uri, change some parameters in backend settings self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = { 'replicaset': 'rs1', 'user': '******', 'database': 'another_db', 'options': { 'socketKeepAlive': True, }, } mb = MongoBackend(app=self.app, url=uri) self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017']) self.assertEqual( mb.options, dict(mb._prepare_client_options(), replicaset='rs1', socketKeepAlive=True), ) self.assertEqual(mb.user, 'backenduser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'another_db') @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__['database'] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) self.backend._connection = 'not none' self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( list(sorted(['status', 'task_id', 'date_done', 'traceback', 'result', 'children'])), list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({'status': states.PENDING, 'result': None}, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group( sentinel.taskset_id, res, ) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_GROUP_COLLECTION, ) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(res, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = { '_id': sentinel.taskset_id, 'result': [uuid(), uuid()], 'date_done': 1, } self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertItemsEqual( ['date_done', 'result', 'task_id'], list(ret_val.keys()), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD') self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__ = Mock(name='MD.__getitem__') mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l') @patch('celery.backends.mongodb.detect_environment') def test_prepare_client_options_for_ver_2(self, m_detect_env): m_detect_env.return_value = 'default' with patch('pymongo.version_tuple', new=(2, 6, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, { 'max_pool_size': self.backend.max_pool_size, 'auto_start_request': False }) @patch('celery.backends.mongodb.detect_environment') def test_prepare_client_options_for_ver_2_with_gevent(self, m_detect_env): m_detect_env.return_value = 'gevent' with patch('pymongo.version_tuple', new=(2, 6, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, { 'max_pool_size': self.backend.max_pool_size, 'auto_start_request': False, 'use_greenlets': True }) @patch('celery.backends.mongodb.detect_environment') def test_prepare_client_options_for_ver_3(self, m_detect_env): m_detect_env.return_value = 'default' with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, { 'maxPoolSize': self.backend.max_pool_size }) @patch('celery.backends.mongodb.detect_environment') def test_prepare_client_options_for_ver_3_with_gevent(self, m_detect_env): m_detect_env.return_value = 'gevent' with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, { 'maxPoolSize': self.backend.max_pool_size })
class test_MongoBackend(AppCase): def setup(self): if pymongo is None: raise SkipTest("pymongo is not installed.") R = self._reset = {} R["encode"], MongoBackend.encode = MongoBackend.encode, Mock() R["decode"], MongoBackend.decode = MongoBackend.decode, Mock() R["Binary"], module.Binary = module.Binary, Mock() R["datetime"], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app) def teardown(self): MongoBackend.encode = self._reset["encode"] MongoBackend.decode = self._reset["decode"] module.Binary = self._reset["Binary"] datetime.datetime = self._reset["datetime"] def test_Bunch(self): x = Bunch(foo="foo", bar=2) self.assertEqual(x.foo, "foo") self.assertEqual(x.bar, 2) def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.mongodb_backend_settings = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.mongodb_backend_settings = None MongoBackend(app=self.app) def test_init_with_settings(self): self.app.conf.mongodb_backend_settings = None # empty settings mb = MongoBackend(app=self.app) # uri uri = "mongodb://*****:*****@" "mongo1.example.com:27017," "mongo2.example.com:27017," "mongo3.example.com:27017/" "celerydatabase?replicaSet=rs0" ) mb = MongoBackend(app=self.app, url=uri) self.assertEqual( mb.mongo_host, ["mongo1.example.com:27017", "mongo2.example.com:27017", "mongo3.example.com:27017"] ) self.assertEqual(mb.options, dict(mb._prepare_client_options(), replicaset="rs0")) self.assertEqual(mb.user, "celeryuser") self.assertEqual(mb.password, "celerypassword") self.assertEqual(mb.database_name, "celerydatabase") # same uri, change some parameters in backend settings self.app.conf.mongodb_backend_settings = { "replicaset": "rs1", "user": "******", "database": "another_db", "options": {"socketKeepAlive": True}, } mb = MongoBackend(app=self.app, url=uri) self.assertEqual( mb.mongo_host, ["mongo1.example.com:27017", "mongo2.example.com:27017", "mongo3.example.com:27017"] ) self.assertEqual(mb.options, dict(mb._prepare_client_options(), replicaset="rs1", socketKeepAlive=True)) self.assertEqual(mb.user, "backenduser") self.assertEqual(mb.password, "celerypassword") self.assertEqual(mb.database_name, "another_db") mb = MongoBackend(app=self.app, url="mongodb://") @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch("pymongo.MongoClient") as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch("pymongo.MongoClient") as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host="mongodb://*****:*****@patch("celery.backends.mongodb.MongoBackend._get_connection") def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=["__getitem__"]) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__["database"] is mock_database) mock_database.authenticate.assert_called_once_with(MONGODB_USER, MONGODB_PASSWORD) @patch("celery.backends.mongodb.MongoBackend._get_connection") def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=["__getitem__"]) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__["database"] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) self.backend._connection = "not none" self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) mock_collection.save.side_effect = InvalidDocument() with self.assertRaises(EncodeError): self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset["datetime"] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( list(sorted(["status", "task_id", "date_done", "traceback", "result", "children"])), list(sorted(ret_val.keys())), ) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({"status": states.PENDING, "result": None}, ret_val) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_save_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group(sentinel.taskset_id, res) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_GROUP_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(res, ret_val) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_restore_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_collection.find_one.return_value = {"_id": sentinel.taskset_id, "result": [uuid(), uuid()], "date_done": 1} self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with({"_id": sentinel.taskset_id}) self.assertItemsEqual(["date_done", "result", "task_id"], list(ret_val.keys())) mock_collection.find_one.return_value = None self.backend._restore_group(sentinel.taskset_id) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.remove.assert_called_once_with({"_id": sentinel.taskset_id}) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=["__getitem__", "__setitem__"]) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with({"_id": sentinel.task_id}) @patch("celery.backends.mongodb.MongoBackend._get_database") def test_cleanup(self, mock_get_database): datetime.datetime = self._reset["datetime"] self.backend.taskmeta_collection = MONGODB_COLLECTION self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = Mock(spec=["__getitem__", "__setitem__"], name="MD") self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__ = Mock(name="MD.__getitem__") mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = "******" x.password = "******" with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with("jerry", "cere4l") def test_prepare_client_options(self): with patch("pymongo.version_tuple", new=(3, 0, 3)): options = self.backend._prepare_client_options() self.assertDictEqual(options, {"maxPoolSize": self.backend.max_pool_size})
class test_MongoBackend(AppCase): def setup(self): if pymongo is None: raise SkipTest('pymongo is not installed.') R = self._reset = {} R['encode'], MongoBackend.encode = MongoBackend.encode, Mock() R['decode'], MongoBackend.decode = MongoBackend.decode, Mock() R['Binary'], module.Binary = module.Binary, Mock() R['datetime'], datetime.datetime = datetime.datetime, Mock() self.backend = MongoBackend(app=self.app) def teardown(self): MongoBackend.encode = self._reset['encode'] MongoBackend.decode = self._reset['decode'] module.Binary = self._reset['Binary'] datetime.datetime = self._reset['datetime'] def test_Bunch(self): x = Bunch(foo='foo', bar=2) self.assertEqual(x.foo, 'foo') self.assertEqual(x.bar, 2) def test_init_no_mongodb(self): prev, module.pymongo = module.pymongo, None try: with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) finally: module.pymongo = prev def test_init_no_settings(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = [] with self.assertRaises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None MongoBackend(app=self.app) def test_init_with_settings(self): self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None # empty settings mb = MongoBackend(app=self.app) # uri uri = 'mongodb://*****:*****@' 'mongo1.example.com:27017,' 'mongo2.example.com:27017,' 'mongo3.example.com:27017/' 'celerydatabase?replicaSet=rs0') mb = MongoBackend(app=self.app, url=uri) self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017']) self.assertEqual(mb.options, {'auto_start_request': False, 'max_pool_size': 10, 'replicaset': 'rs0'}) self.assertEqual(mb.user, 'celeryuser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'celerydatabase') # same uri, change some parameters in backend settings self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = { 'replicaset': 'rs1', 'user': '******', 'database': 'another_db', 'options': { 'socketKeepAlive': True, }, } mb = MongoBackend(app=self.app, url=uri) self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017', 'mongo2.example.com:27017', 'mongo3.example.com:27017']) self.assertEqual(mb.options, {'auto_start_request': False, 'max_pool_size': 10, 'replicaset': 'rs1', 'socketKeepAlive': True}) self.assertEqual(mb.user, 'backenduser') self.assertEqual(mb.password, 'celerypassword') self.assertEqual(mb.database_name, 'another_db') @depends_on_current_app def test_reduce(self): x = MongoBackend(app=self.app) self.assertTrue(loads(dumps(x))) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() self.assertEqual(sentinel._connection, connection) self.assertFalse(mock_Connection.called) def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertTrue(self.backend.__dict__['database'] is mock_database) mock_database.authenticate.assert_called_once_with( MONGODB_USER, MONGODB_PASSWORD) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database self.assertTrue(database is mock_database) self.assertFalse(mock_database.authenticate.called) self.assertTrue(self.backend.__dict__['database'] is mock_database) def test_process_cleanup(self): self.backend._connection = None self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) self.backend._connection = 'not none' self.backend.process_cleanup() self.assertEqual(self.backend._connection, None) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(sentinel.result, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual( list(sorted(['status', 'task_id', 'date_done', 'traceback', 'result', 'children'])), list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) self.assertEqual({'status': states.PENDING, 'result': None}, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group( sentinel.taskset_id, res, ) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_GROUP_COLLECTION, ) mock_collection.save.assert_called_once_with(ANY) self.assertEqual(res, ret_val) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = { '_id': sentinel.taskset_id, 'result': [uuid(), uuid()], 'date_done': 1, } self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) self.assertEqual( list(sorted(['date_done', 'result', 'task_id'])), list(sorted(ret_val.keys())), ) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.remove.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_forget(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.remove.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): datetime.datetime = self._reset['datetime'] self.backend.taskmeta_collection = MONGODB_COLLECTION self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD') self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__ = Mock(name='MD.__getitem__') mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() self.assertTrue(mock_collection.remove.called) def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() conn = x._get_connection.return_value = {} db = conn[x.database_name] = Mock() db.authenticate.return_value = False x.user = '******' x.password = '******' with self.assertRaises(ImproperlyConfigured): x._get_database() db.authenticate.assert_called_with('jerry', 'cere4l')
class test_MongoBackend: default_url = 'mongodb://*****:*****@hostname.dom/database' replica_set_url = ( 'mongodb://*****:*****@hostname.dom,' 'hostname.dom/database?replicaSet=rs' ) sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database' sanitized_replica_set_url = ( 'mongodb://*****:*****@hostname.dom/,' 'hostname.dom/database?replicaSet=rs' ) def setup(self): self.patching('celery.backends.mongodb.MongoBackend.encode') self.patching('celery.backends.mongodb.MongoBackend.decode') self.patching('celery.backends.mongodb.Binary') self.backend = MongoBackend(app=self.app, url=self.default_url) def test_init_no_mongodb(self, patching): patching('celery.backends.mongodb.pymongo', None) with pytest.raises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_no_settings(self): self.app.conf.mongodb_backend_settings = [] with pytest.raises(ImproperlyConfigured): MongoBackend(app=self.app) def test_init_settings_is_None(self): self.app.conf.mongodb_backend_settings = None MongoBackend(app=self.app) def test_init_with_settings(self): self.app.conf.mongodb_backend_settings = None # empty settings mb = MongoBackend(app=self.app) # uri uri = 'mongodb://*****:*****@' 'mongo1.example.com:27017,' 'mongo2.example.com:27017,' 'mongo3.example.com:27017/' 'celerydatabase?replicaSet=rs0') mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == MONGODB_BACKEND_HOST assert mb.options == dict( mb._prepare_client_options(), replicaset='rs0', ) assert mb.user == CELERY_USER assert mb.password == CELERY_PASSWORD assert mb.database_name == CELERY_DATABASE # same uri, change some parameters in backend settings self.app.conf.mongodb_backend_settings = { 'replicaset': 'rs1', 'user': '******', 'database': 'another_db', 'options': { 'socketKeepAlive': True, }, } mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == MONGODB_BACKEND_HOST assert mb.options == dict( mb._prepare_client_options(), replicaset='rs1', socketKeepAlive=True, ) assert mb.user == 'backenduser' assert mb.password == CELERY_PASSWORD assert mb.database_name == 'another_db' mb = MongoBackend(app=self.app, url='mongodb://') @pytest.mark.skipif(dns.version.MAJOR > 1, reason="For dnspython version > 1, pymongo's" "srv_resolver calls resolver.resolve") @pytest.mark.skipif(pymongo.version_tuple[0] > 3, reason="For pymongo version > 3, options returns ssl") def test_init_mongodb_dnspython1_pymongo3_seedlist(self): resolver = fake_resolver_dnspython() self.app.conf.mongodb_backend_settings = None with patch('dns.resolver.query', side_effect=resolver): mb = self.perform_seedlist_assertions() assert mb.options == dict( mb._prepare_client_options(), replicaset='rs0', ssl=True ) @pytest.mark.skipif(dns.version.MAJOR <= 1, reason="For dnspython versions 1.X, pymongo's" "srv_resolver calls resolver.query") @pytest.mark.skipif(pymongo.version_tuple[0] > 3, reason="For pymongo version > 3, options returns ssl") def test_init_mongodb_dnspython2_pymongo3_seedlist(self): resolver = fake_resolver_dnspython() self.app.conf.mongodb_backend_settings = None with patch('dns.resolver.resolve', side_effect=resolver): mb = self.perform_seedlist_assertions() assert mb.options == dict( mb._prepare_client_options(), replicaset='rs0', ssl=True ) @pytest.mark.skipif(dns.version.MAJOR > 1, reason="For dnspython version >= 2, pymongo's" "srv_resolver calls resolver.resolve") @pytest.mark.skipif(pymongo.version_tuple[0] <= 3, reason="For pymongo version > 3, options returns tls") def test_init_mongodb_dnspython1_pymongo4_seedlist(self): resolver = fake_resolver_dnspython() self.app.conf.mongodb_backend_settings = None with patch('dns.resolver.query', side_effect=resolver): mb = self.perform_seedlist_assertions() assert mb.options == dict( mb._prepare_client_options(), replicaset='rs0', tls=True ) @pytest.mark.skipif(dns.version.MAJOR <= 1, reason="For dnspython versions 1.X, pymongo's" "srv_resolver calls resolver.query") @pytest.mark.skipif(pymongo.version_tuple[0] <= 3, reason="For pymongo version > 3, options returns tls") def test_init_mongodb_dnspython2_pymongo4_seedlist(self): resolver = fake_resolver_dnspython() self.app.conf.mongodb_backend_settings = None with patch('dns.resolver.resolve', side_effect=resolver): mb = self.perform_seedlist_assertions() assert mb.options == dict( mb._prepare_client_options(), replicaset='rs0', tls=True ) def perform_seedlist_assertions(self): mb = MongoBackend(app=self.app, url=MONGODB_SEEDLIST_URI) assert mb.mongo_host == MONGODB_BACKEND_HOST assert mb.user == CELERY_USER assert mb.password == CELERY_PASSWORD assert mb.database_name == CELERY_DATABASE return mb def test_ensure_mongodb_uri_compliance(self): mb = MongoBackend(app=self.app, url=None) compliant_uri = mb._ensure_mongodb_uri_compliance assert compliant_uri('mongodb://') == 'mongodb://localhost' assert compliant_uri('mongodb+something://host') == \ 'mongodb+something://host' assert compliant_uri('something://host') == 'mongodb+something://host' @pytest.mark.usefixtures('depends_on_current_app') def test_reduce(self): x = MongoBackend(app=self.app) assert loads(dumps(x)) def test_get_connection_connection_exists(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = sentinel._connection connection = self.backend._get_connection() assert sentinel._connection == connection mock_Connection.assert_not_called() def test_get_connection_no_connection_host(self): with patch('pymongo.MongoClient') as mock_Connection: self.backend._connection = None self.backend.host = MONGODB_HOST self.backend.port = MONGODB_PORT mock_Connection.return_value = sentinel.connection connection = self.backend._get_connection() mock_Connection.assert_called_once_with( host='mongodb://*****:*****@' 'localhost:27017/' 'celerydatabase?authMechanism=SCRAM-SHA-256') mb = MongoBackend(app=self.app, url=uri) mock_Connection.return_value = sentinel.connection connection = mb._get_connection() mock_Connection.assert_called_once_with( host=['localhost:27017'], username=CELERY_USER, password=CELERY_PASSWORD, authmechanism='SCRAM-SHA-256', **mb._prepare_client_options() ) assert sentinel.connection == connection def test_get_connection_with_authmechanism_no_username(self): with patch('pymongo.MongoClient') as mock_Connection: self.app.conf.mongodb_backend_settings = None uri = ('mongodb://' 'localhost:27017/' 'celerydatabase?authMechanism=SCRAM-SHA-256') mb = MongoBackend(app=self.app, url=uri) mock_Connection.side_effect = ConfigurationError( 'SCRAM-SHA-256 requires a username.') with pytest.raises(ConfigurationError): mb._get_connection() mock_Connection.assert_called_once_with( host=['localhost:27017'], authmechanism='SCRAM-SHA-256', **mb._prepare_client_options() ) @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = MONGODB_USER self.backend.password = MONGODB_PASSWORD mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database assert database is mock_database assert self.backend.__dict__['database'] is mock_database @patch('celery.backends.mongodb.MongoBackend._get_connection') def test_get_database_no_existing_no_auth(self, mock_get_connection): # Should really check for combinations of these two, to be complete. self.backend.user = None self.backend.password = None mock_database = Mock() mock_connection = MagicMock(spec=['__getitem__']) mock_connection.__getitem__.return_value = mock_database mock_get_connection.return_value = mock_connection database = self.backend.database assert database is mock_database assert self.backend.__dict__['database'] is mock_database @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) mock_collection.replace_one.assert_called_once_with(ANY, ANY, upsert=True) assert sentinel.result == ret_val mock_collection.replace_one.side_effect = InvalidDocument() with pytest.raises(EncodeError): self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_store_result_with_request(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_request = MagicMock(spec=['parent_id']) mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection mock_request.parent_id = sentinel.parent_id ret_val = self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status, request=mock_request) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) parameters = mock_collection.replace_one.call_args[0][1] assert parameters['parent_id'] == sentinel.parent_id assert sentinel.result == ret_val mock_collection.replace_one.side_effect = InvalidDocument() with pytest.raises(EncodeError): self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = MagicMock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) assert list(sorted([ 'status', 'task_id', 'date_done', 'traceback', 'result', 'children', ])) == list(sorted(ret_val.keys())) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_get_task_meta_for_no_result(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = None mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._get_task_meta_for(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION) assert {'status': states.PENDING, 'result': None} == ret_val @patch('celery.backends.mongodb.MongoBackend._get_database') def test_save_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection res = [self.app.AsyncResult(i) for i in range(3)] ret_val = self.backend._save_group( sentinel.taskset_id, res, ) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_GROUP_COLLECTION, ) mock_collection.replace_one.assert_called_once_with(ANY, ANY, upsert=True) assert res == ret_val @patch('celery.backends.mongodb.MongoBackend._get_database') def test_restore_group(self, mock_get_database): self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_collection.find_one.return_value = { '_id': sentinel.taskset_id, 'result': [uuid(), uuid()], 'date_done': 1, } self.backend.decode.side_effect = lambda r: r mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection ret_val = self.backend._restore_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.find_one.assert_called_once_with( {'_id': sentinel.taskset_id}) assert (sorted(['date_done', 'result', 'task_id']) == sorted(list(ret_val.keys()))) mock_collection.find_one.return_value = None self.backend._restore_group(sentinel.taskset_id) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_delete_group(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._delete_group(sentinel.taskset_id) mock_get_database.assert_called_once_with() mock_collection.delete_one.assert_called_once_with( {'_id': sentinel.taskset_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test__forget(self, mock_get_database): # note: here tested _forget method, not forget method self.backend.taskmeta_collection = MONGODB_COLLECTION mock_database = MagicMock(spec=['__getitem__', '__setitem__']) mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__.return_value = mock_collection self.backend._forget(sentinel.task_id) mock_get_database.assert_called_once_with() mock_database.__getitem__.assert_called_once_with( MONGODB_COLLECTION) mock_collection.delete_one.assert_called_once_with( {'_id': sentinel.task_id}) @patch('celery.backends.mongodb.MongoBackend._get_database') def test_cleanup(self, mock_get_database): self.backend.taskmeta_collection = MONGODB_COLLECTION self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD') self.backend.collections = mock_collection = Mock() mock_get_database.return_value = mock_database mock_database.__getitem__ = Mock(name='MD.__getitem__') mock_database.__getitem__.return_value = mock_collection self.backend.app.now = datetime.datetime.utcnow self.backend.cleanup() mock_get_database.assert_called_once_with() mock_collection.delete_many.assert_called() self.backend.collections = mock_collection = Mock() self.backend.expires = None self.backend.cleanup() mock_collection.delete_many.assert_not_called() def test_prepare_client_options(self): with patch('pymongo.version_tuple', new=(3, 0, 3)): options = self.backend._prepare_client_options() assert options == { 'maxPoolSize': self.backend.max_pool_size } def test_as_uri_include_password(self): assert self.backend.as_uri(True) == self.default_url def test_as_uri_exclude_password(self): assert self.backend.as_uri() == self.sanitized_default_url def test_as_uri_include_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) assert backend.as_uri(True) == self.replica_set_url def test_as_uri_exclude_password_replica_set(self): backend = MongoBackend(app=self.app, url=self.replica_set_url) assert backend.as_uri() == self.sanitized_replica_set_url def test_regression_worker_startup_info(self): self.app.conf.result_backend = ( 'mongodb://*****:*****@host0.com:43437,host1.com:43437' '/work4us?replicaSet=rs&ssl=true' ) worker = self.app.Worker() with conftest.stdouts(): worker.on_start() assert worker.startup_info()