class test_MongoBackend(AppCase):
def setup(self):
if pymongo is None:
raise SkipTest('pymongo is not installed.')
R = self._reset = {}
R['encode'], MongoBackend.encode = MongoBackend.encode, Mock()
R['decode'], MongoBackend.decode = MongoBackend.decode, Mock()
R['Binary'], module.Binary = module.Binary, Mock()
R['datetime'], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app)
def teardown(self):
MongoBackend.encode = self._reset['encode']
MongoBackend.decode = self._reset['decode']
module.Binary = self._reset['Binary']
datetime.datetime = self._reset['datetime']
def test_Bunch(self):
x = Bunch(foo='foo', bar=2)
self.assertEqual(x.foo, 'foo')
self.assertEqual(x.bar, 2)
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
MongoBackend(app=self.app)
def test_restore_group_no_entry(self):
x = MongoBackend(app=self.app)
x.collection = Mock()
fo = x.collection.find_one = Mock()
fo.return_value = None
self.assertIsNone(x._restore_group('1f3fab'))
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
self.backend._connection = 'not none'
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once()
self.assertEqual(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
['status', 'task_id', 'date_done', 'traceback', 'result',
'children'],
list(ret_val.keys()))
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({'status': states.PENDING, 'result': None}, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._save_group(
sentinel.taskset_id, sentinel.result)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once()
self.assertEqual(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertEqual(
['date_done', 'result', 'task_id'],
list(ret_val.keys()),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.assert_called_once()
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
class test_MongoBackend(AppCase):
def setUp(self):
if pymongo is None:
raise SkipTest('pymongo is not installed.')
R = self._reset = {}
R['encode'], MongoBackend.encode = MongoBackend.encode, Mock()
R['decode'], MongoBackend.decode = MongoBackend.decode, Mock()
R['Binary'], module.Binary = module.Binary, Mock()
R['datetime'], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend()
def tearDown(self):
MongoBackend.encode = self._reset['encode']
MongoBackend.decode = self._reset['decode']
module.Binary = self._reset['Binary']
datetime.datetime = self._reset['datetime']
def test_Bunch(self):
x = Bunch(foo='foo', bar=2)
self.assertEqual(x.foo, 'foo')
self.assertEqual(x.bar, 2)
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend()
finally:
module.pymongo = prev
def test_init_no_settings(self):
celery = Celery(set_as_current=False)
celery.conf.CELERY_MONGODB_BACKEND_SETTINGS = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=celery)
def test_init_settings_is_None(self):
celery = Celery(set_as_current=False)
celery.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
MongoBackend(app=celery)
def test_restore_group_no_entry(self):
x = MongoBackend()
x.collection = Mock()
fo = x.collection.find_one = Mock()
fo.return_value = None
self.assertIsNone(x._restore_group('1f3fab'))
def test_reduce(self):
x = MongoBackend()
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch('pymongo.connection.Connection') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEquals(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch('pymongo.connection.Connection') as mock_Connection:
self.backend._connection = None
self.backend.mongodb_host = MONGODB_HOST
self.backend.mongodb_port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(MONGODB_HOST, MONGODB_PORT)
self.assertEquals(sentinel.connection, connection)
def test_get_connection_no_connection_mongodb_uri(self):
with patch('pymongo.connection.Connection') as mock_Connection:
mongodb_uri = 'mongodb://%s:%d' % (MONGODB_HOST, MONGODB_PORT)
self.backend._connection = None
self.backend.mongodb_host = mongodb_uri
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(mongodb_uri)
self.assertEquals(sentinel.connection, connection)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.mongodb_user = MONGODB_USER
self.backend.mongodb_password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.mongodb_user = None
self.backend.mongodb_password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEquals(self.backend._connection, None)
self.backend._connection = 'not none'
self.backend.process_cleanup()
self.assertEquals(self.backend._connection, None)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(sentinel.task_id, sentinel.result,
sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once()
self.assertEquals(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEquals([
'status', 'task_id', 'date_done', 'traceback', 'result', 'children'
], ret_val.keys())
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEquals({'status': states.PENDING, 'result': None}, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._save_group(sentinel.taskset_id,
sentinel.result)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once()
self.assertEquals(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertEquals(['date_done', 'result', 'task_id'], ret_val.keys())
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_forget(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id}, safe=True)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.assert_called_once()
def test_get_database_authfailure(self):
x = MongoBackend()
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.mongodb_database] = Mock()
db.authenticate.return_value = False
x.mongodb_user = '******'
x.mongodb_password = '******'
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
class test_MongoBackend(AppCase):
def setup(self):
if pymongo is None:
raise SkipTest('pymongo is not installed.')
R = self._reset = {}
R['encode'], MongoBackend.encode = MongoBackend.encode, Mock()
R['decode'], MongoBackend.decode = MongoBackend.decode, Mock()
R['Binary'], module.Binary = module.Binary, Mock()
R['datetime'], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app)
def teardown(self):
MongoBackend.encode = self._reset['encode']
MongoBackend.decode = self._reset['decode']
module.Binary = self._reset['Binary']
datetime.datetime = self._reset['datetime']
def test_Bunch(self):
x = Bunch(foo='foo', bar=2)
self.assertEqual(x.foo, 'foo')
self.assertEqual(x.bar, 2)
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
MongoBackend(app=self.app)
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
self.backend._connection = 'not none'
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(sentinel.task_id, sentinel.result,
sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
list(
sorted([
'status', 'task_id', 'date_done', 'traceback', 'result',
'children'
])),
list(sorted(ret_val.keys())),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({'status': states.PENDING, 'result': None}, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
res = [self.app.AsyncResult(i) for i in range(3)]
ret_val = self.backend._save_group(
sentinel.taskset_id,
res,
)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_GROUP_COLLECTION, )
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(res, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = {
'_id': sentinel.taskset_id,
'result': [uuid(), uuid()],
'date_done': 1,
}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertEqual(
list(sorted(['date_done', 'result', 'task_id'])),
list(sorted(ret_val.keys())),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD')
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__ = Mock(name='MD.__getitem__')
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
self.assertTrue(mock_collection.remove.called)
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
class test_MongoBackend(AppCase):
def setup(self):
if pymongo is None:
raise SkipTest('pymongo is not installed.')
R = self._reset = {}
R['encode'], MongoBackend.encode = MongoBackend.encode, Mock()
R['decode'], MongoBackend.decode = MongoBackend.decode, Mock()
R['Binary'], module.Binary = module.Binary, Mock()
R['datetime'], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app)
def teardown(self):
MongoBackend.encode = self._reset['encode']
MongoBackend.decode = self._reset['decode']
module.Binary = self._reset['Binary']
datetime.datetime = self._reset['datetime']
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
MongoBackend(app=self.app)
def test_restore_group_no_entry(self):
x = MongoBackend(app=self.app)
x.collection = Mock()
fo = x.collection.find_one = Mock()
fo.return_value = None
self.assertIsNone(x._restore_group('1f3fab'))
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
self.backend._connection = 'not none'
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(sentinel.task_id, sentinel.result,
sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
list(
sorted([
'status', 'task_id', 'date_done', 'traceback', 'result',
'children'
])),
list(sorted(ret_val.keys())),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({'status': states.PENDING, 'result': None}, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._save_group(sentinel.taskset_id,
sentinel.result)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertItemsEqual(
['date_done', 'result', 'task_id'],
list(ret_val.keys()),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertTrue(mock_collection.remove.called)
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
@patch('celery.backends.mongodb.detect_environment')
def test_prepare_client_options_for_ver_2(self, m_detect_env):
m_detect_env.return_value = 'default'
with patch('pymongo.version_tuple', new=(2, 6, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(
options, {
'max_pool_size': self.backend.max_pool_size,
'auto_start_request': False
})
@patch('celery.backends.mongodb.detect_environment')
def test_prepare_client_options_for_ver_2_with_gevent(self, m_detect_env):
m_detect_env.return_value = 'gevent'
with patch('pymongo.version_tuple', new=(2, 6, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(
options, {
'max_pool_size': self.backend.max_pool_size,
'auto_start_request': False,
'use_greenlets': True
})
@patch('celery.backends.mongodb.detect_environment')
def test_prepare_client_options_for_ver_3(self, m_detect_env):
m_detect_env.return_value = 'default'
with patch('pymongo.version_tuple', new=(3, 0, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options,
{'maxPoolSize': self.backend.max_pool_size})
@patch('celery.backends.mongodb.detect_environment')
def test_prepare_client_options_for_ver_3_with_gevent(self, m_detect_env):
m_detect_env.return_value = 'gevent'
with patch('pymongo.version_tuple', new=(3, 0, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options,
{'maxPoolSize': self.backend.max_pool_size})
class test_MongoBackend(AppCase):
default_url = 'mongodb://*****:*****@hostname.dom/database'
replica_set_url = ('mongodb://*****:*****@hostname.dom,'
'hostname.dom/database?replicaSet=rs')
sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database'
sanitized_replica_set_url = ('mongodb://*****:*****@hostname.dom/,'
'hostname.dom/database?replicaSet=rs')
def setup(self):
R = self._reset = {}
R['encode'], MongoBackend.encode = MongoBackend.encode, Mock()
R['decode'], MongoBackend.decode = MongoBackend.decode, Mock()
R['Binary'], module.Binary = module.Binary, Mock()
R['datetime'], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app, url=self.default_url)
def teardown(self):
MongoBackend.encode = self._reset['encode']
MongoBackend.decode = self._reset['decode']
module.Binary = self._reset['Binary']
datetime.datetime = self._reset['datetime']
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.mongodb_backend_settings = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.mongodb_backend_settings = None
MongoBackend(app=self.app)
def test_init_with_settings(self):
self.app.conf.mongodb_backend_settings = None
# empty settings
mb = MongoBackend(app=self.app)
# uri
uri = 'mongodb://*****:*****@'
'mongo1.example.com:27017,'
'mongo2.example.com:27017,'
'mongo3.example.com:27017/'
'celerydatabase?replicaSet=rs0')
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(mb.mongo_host, [
'mongo1.example.com:27017', 'mongo2.example.com:27017',
'mongo3.example.com:27017'
])
self.assertEqual(
mb.options,
dict(mb._prepare_client_options(), replicaset='rs0'),
)
self.assertEqual(mb.user, 'celeryuser')
self.assertEqual(mb.password, 'celerypassword')
self.assertEqual(mb.database_name, 'celerydatabase')
# same uri, change some parameters in backend settings
self.app.conf.mongodb_backend_settings = {
'replicaset': 'rs1',
'user': '******',
'database': 'another_db',
'options': {
'socketKeepAlive': True,
},
}
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(mb.mongo_host, [
'mongo1.example.com:27017', 'mongo2.example.com:27017',
'mongo3.example.com:27017'
])
self.assertEqual(
mb.options,
dict(mb._prepare_client_options(),
replicaset='rs1',
socketKeepAlive=True),
)
self.assertEqual(mb.user, 'backenduser')
self.assertEqual(mb.password, 'celerypassword')
self.assertEqual(mb.database_name, 'another_db')
mb = MongoBackend(app=self.app, url='mongodb://')
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(sentinel.task_id, sentinel.result,
sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
mock_collection.save.side_effect = InvalidDocument()
with self.assertRaises(EncodeError):
self.backend._store_result(sentinel.task_id, sentinel.result,
sentinel.status)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
list(
sorted([
'status', 'task_id', 'date_done', 'traceback', 'result',
'children'
])),
list(sorted(ret_val.keys())),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({'status': states.PENDING, 'result': None}, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
res = [self.app.AsyncResult(i) for i in range(3)]
ret_val = self.backend._save_group(
sentinel.taskset_id,
res,
)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_GROUP_COLLECTION, )
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(res, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = {
'_id': sentinel.taskset_id,
'result': [uuid(), uuid()],
'date_done': 1,
}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertItemsEqual(
['date_done', 'result', 'task_id'],
list(ret_val.keys()),
)
mock_collection.find_one.return_value = None
self.backend._restore_group(sentinel.taskset_id)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD')
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__ = Mock(name='MD.__getitem__')
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
self.assertTrue(mock_collection.remove.called)
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
def test_prepare_client_options(self):
with patch('pymongo.version_tuple', new=(3, 0, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options,
{'maxPoolSize': self.backend.max_pool_size})
def test_as_uri_include_password(self):
self.assertEqual(self.backend.as_uri(True), self.default_url)
def test_as_uri_exclude_password(self):
self.assertEqual(self.backend.as_uri(), self.sanitized_default_url)
def test_as_uri_include_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
self.assertEqual(backend.as_uri(True), self.replica_set_url)
def test_as_uri_exclude_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
self.assertEqual(backend.as_uri(), self.sanitized_replica_set_url)
@override_stdouts
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'mongodb://*****:*****@host0.com:43437,host1.com:43437'
'/work4us?replicaSet=rs&ssl=true')
worker = self.app.Worker()
worker.on_start()
self.assertTrue(worker.startup_info())
class test_MongoBackend:
default_url = 'mongodb://*****:*****@hostname.dom/database'
replica_set_url = ('mongodb://*****:*****@hostname.dom,'
'hostname.dom/database?replicaSet=rs')
sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database'
sanitized_replica_set_url = ('mongodb://*****:*****@hostname.dom/,'
'hostname.dom/database?replicaSet=rs')
def setup(self):
self.patching('celery.backends.mongodb.MongoBackend.encode')
self.patching('celery.backends.mongodb.MongoBackend.decode')
self.patching('celery.backends.mongodb.Binary')
self.patching('datetime.datetime')
self.backend = MongoBackend(app=self.app, url=self.default_url)
def test_init_no_mongodb(self, patching):
patching('celery.backends.mongodb.pymongo', None)
with pytest.raises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_no_settings(self):
self.app.conf.mongodb_backend_settings = []
with pytest.raises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.mongodb_backend_settings = None
MongoBackend(app=self.app)
def test_init_with_settings(self):
self.app.conf.mongodb_backend_settings = None
# empty settings
mb = MongoBackend(app=self.app)
# uri
uri = 'mongodb://*****:*****@'
'mongo1.example.com:27017,'
'mongo2.example.com:27017,'
'mongo3.example.com:27017/'
'celerydatabase?replicaSet=rs0')
mb = MongoBackend(app=self.app, url=uri)
assert mb.mongo_host == [
'mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017',
]
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs0',
)
assert mb.user == 'celeryuser'
assert mb.password == 'celerypassword'
assert mb.database_name == 'celerydatabase'
# same uri, change some parameters in backend settings
self.app.conf.mongodb_backend_settings = {
'replicaset': 'rs1',
'user': '******',
'database': 'another_db',
'options': {
'socketKeepAlive': True,
},
}
mb = MongoBackend(app=self.app, url=uri)
assert mb.mongo_host == [
'mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017',
]
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs1',
socketKeepAlive=True,
)
assert mb.user == 'backenduser'
assert mb.password == 'celerypassword'
assert mb.database_name == 'another_db'
mb = MongoBackend(app=self.app, url='mongodb://')
def test_init_mongodb_dns_seedlist(self):
Name = pytest.importorskip('dns.name').Name
TXT = pytest.importorskip('dns.rdtypes.ANY.TXT').TXT
SRV = pytest.importorskip('dns.rdtypes.IN.SRV').SRV
self.app.conf.mongodb_backend_settings = None
def mock_resolver(_, rdtype, rdclass=None, lifetime=None, **kwargs):
if rdtype == 'SRV':
return [
SRV(0, 0, 0, 0, 27017, Name(labels=hostname))
for hostname in [
b'mongo1.example.com'.split(
b'.'), b'mongo2.example.com'.split(b'.'),
b'mongo3.example.com'.split(b'.')
]
]
elif rdtype == 'TXT':
return [TXT(0, 0, [b'replicaSet=rs0'])]
# uri with user, password, database name, replica set,
# DNS seedlist format
uri = ('srv://'
'celeryuser:celerypassword@'
'dns-seedlist-host.example.com/'
'celerydatabase')
with patch('dns.resolver.query', side_effect=mock_resolver):
mb = MongoBackend(app=self.app, url=uri)
assert mb.mongo_host == [
'mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017',
]
assert mb.options == dict(mb._prepare_client_options(),
replicaset='rs0',
ssl=True)
assert mb.user == 'celeryuser'
assert mb.password == 'celerypassword'
assert mb.database_name == 'celerydatabase'
def test_ensure_mongodb_uri_compliance(self):
mb = MongoBackend(app=self.app, url=None)
compliant_uri = mb._ensure_mongodb_uri_compliance
assert compliant_uri('mongodb://') == 'mongodb://localhost'
assert compliant_uri('mongodb+something://host') == \
'mongodb+something://host'
assert compliant_uri('something://host') == 'mongodb+something://host'
@pytest.mark.usefixtures('depends_on_current_app')
def test_reduce(self):
x = MongoBackend(app=self.app)
assert loads(dumps(x))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
assert sentinel._connection == connection
mock_Connection.assert_not_called()
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@'
'localhost:27017/'
'celerydatabase?authMechanism=SCRAM-SHA-256')
mb = MongoBackend(app=self.app, url=uri)
mock_Connection.return_value = sentinel.connection
connection = mb._get_connection()
mock_Connection.assert_called_once_with(
host=['localhost:27017'],
username='******',
password='******',
authmechanism='SCRAM-SHA-256',
**mb._prepare_client_options())
assert sentinel.connection == connection
def test_get_connection_with_authmechanism_no_username(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.app.conf.mongodb_backend_settings = None
uri = ('mongodb://'
'localhost:27017/'
'celerydatabase?authMechanism=SCRAM-SHA-256')
mb = MongoBackend(app=self.app, url=uri)
mock_Connection.side_effect = ConfigurationError(
'SCRAM-SHA-256 requires a username.')
with pytest.raises(ConfigurationError):
mb._get_connection()
mock_Connection.assert_called_once_with(
host=['localhost:27017'],
authmechanism='SCRAM-SHA-256',
**mb._prepare_client_options())
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
assert database is mock_database
assert self.backend.__dict__['database'] is mock_database
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD, source=self.backend.database_name)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
assert database is mock_database
mock_database.authenticate.assert_not_called()
assert self.backend.__dict__['database'] is mock_database
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(sentinel.task_id, sentinel.result,
sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.replace_one.assert_called_once_with(ANY,
ANY,
upsert=True)
assert sentinel.result == ret_val
mock_collection.replace_one.side_effect = InvalidDocument()
with pytest.raises(EncodeError):
self.backend._store_result(sentinel.task_id, sentinel.result,
sentinel.status)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result_with_request(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_request = MagicMock(spec=['parent_id'])
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
mock_request.parent_id = sentinel.parent_id
ret_val = self.backend._store_result(sentinel.task_id,
sentinel.result,
sentinel.status,
request=mock_request)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
parameters = mock_collection.replace_one.call_args[0][1]
assert parameters['parent_id'] == sentinel.parent_id
assert sentinel.result == ret_val
mock_collection.replace_one.side_effect = InvalidDocument()
with pytest.raises(EncodeError):
self.backend._store_result(sentinel.task_id, sentinel.result,
sentinel.status)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
assert list(
sorted([
'status',
'task_id',
'date_done',
'traceback',
'result',
'children',
])) == list(sorted(ret_val.keys()))
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
assert {'status': states.PENDING, 'result': None} == ret_val
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
res = [self.app.AsyncResult(i) for i in range(3)]
ret_val = self.backend._save_group(
sentinel.taskset_id,
res,
)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_GROUP_COLLECTION, )
mock_collection.replace_one.assert_called_once_with(ANY,
ANY,
upsert=True)
assert res == ret_val
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = {
'_id': sentinel.taskset_id,
'result': [uuid(), uuid()],
'date_done': 1,
}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
assert (sorted(['date_done', 'result',
'task_id']) == sorted(list(ret_val.keys())))
mock_collection.find_one.return_value = None
self.backend._restore_group(sentinel.taskset_id)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.delete_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test__forget(self, mock_get_database):
# note: here tested _forget method, not forget method
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.delete_one.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = Mock(spec=['__getitem__', '__setitem__'], name='MD')
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__ = Mock(name='MD.__getitem__')
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
mock_collection.delete_many.assert_called()
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with pytest.raises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry',
'cere4l',
source=x.database_name)
def test_prepare_client_options(self):
with patch('pymongo.version_tuple', new=(3, 0, 3)):
options = self.backend._prepare_client_options()
assert options == {'maxPoolSize': self.backend.max_pool_size}
def test_as_uri_include_password(self):
assert self.backend.as_uri(True) == self.default_url
def test_as_uri_exclude_password(self):
assert self.backend.as_uri() == self.sanitized_default_url
def test_as_uri_include_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
assert backend.as_uri(True) == self.replica_set_url
def test_as_uri_exclude_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
assert backend.as_uri() == self.sanitized_replica_set_url
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'mongodb://*****:*****@host0.com:43437,host1.com:43437'
'/work4us?replicaSet=rs&ssl=true')
worker = self.app.Worker()
with mock.stdouts():
worker.on_start()
assert worker.startup_info()
class test_MongoBackend(AppCase):
def setup(self):
if pymongo is None:
raise SkipTest("pymongo is not installed.")
R = self._reset = {}
R["encode"], MongoBackend.encode = MongoBackend.encode, Mock()
R["decode"], MongoBackend.decode = MongoBackend.decode, Mock()
R["Binary"], module.Binary = module.Binary, Mock()
R["datetime"], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app)
def teardown(self):
MongoBackend.encode = self._reset["encode"]
MongoBackend.decode = self._reset["decode"]
module.Binary = self._reset["Binary"]
datetime.datetime = self._reset["datetime"]
def test_Bunch(self):
x = Bunch(foo="foo", bar=2)
self.assertEqual(x.foo, "foo")
self.assertEqual(x.bar, 2)
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
MongoBackend(app=self.app)
def test_restore_group_no_entry(self):
x = MongoBackend(app=self.app)
x.collection = Mock()
fo = x.collection.find_one = Mock()
fo.return_value = None
self.assertIsNone(x._restore_group("1f3fab"))
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch("pymongo.MongoClient") as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch("pymongo.MongoClient") as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host="mongodb://*****:*****@patch("celery.backends.mongodb.MongoBackend._get_connection")
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=["__getitem__"])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__["database"] is mock_database)
mock_database.authenticate.assert_called_once_with(MONGODB_USER, MONGODB_PASSWORD)
@patch("celery.backends.mongodb.MongoBackend._get_connection")
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=["__getitem__"])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__["database"] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
self.backend._connection = "not none"
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset["datetime"]
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
list(sorted(["status", "task_id", "date_done", "traceback", "result", "children"])),
list(sorted(ret_val.keys())),
)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({"status": states.PENDING, "result": None}, ret_val)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_save_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._save_group(sentinel.taskset_id, sentinel.result)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_restore_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.find_one.assert_called_once_with({"_id": sentinel.taskset_id})
self.assertItemsEqual(["date_done", "result", "task_id"], list(ret_val.keys()))
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with({"_id": sentinel.taskset_id})
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with({"_id": sentinel.task_id})
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset["datetime"]
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertTrue(mock_collection.remove.called)
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = "******"
x.password = "******"
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with("jerry", "cere4l")
@patch("celery.backends.mongodb.detect_environment")
def test_prepare_client_options_for_ver_2(self, m_detect_env):
m_detect_env.return_value = "default"
with patch("pymongo.version_tuple", new=(2, 6, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {"max_pool_size": self.backend.max_pool_size, "auto_start_request": False})
@patch("celery.backends.mongodb.detect_environment")
def test_prepare_client_options_for_ver_2_with_gevent(self, m_detect_env):
m_detect_env.return_value = "gevent"
with patch("pymongo.version_tuple", new=(2, 6, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(
options,
{"max_pool_size": self.backend.max_pool_size, "auto_start_request": False, "use_greenlets": True},
)
@patch("celery.backends.mongodb.detect_environment")
def test_prepare_client_options_for_ver_3(self, m_detect_env):
m_detect_env.return_value = "default"
with patch("pymongo.version_tuple", new=(3, 0, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {"maxPoolSize": self.backend.max_pool_size})
@patch("celery.backends.mongodb.detect_environment")
def test_prepare_client_options_for_ver_3_with_gevent(self, m_detect_env):
m_detect_env.return_value = "gevent"
with patch("pymongo.version_tuple", new=(3, 0, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {"maxPoolSize": self.backend.max_pool_size})
class test_MongoBackend(AppCase):
default_url = 'mongodb://*****:*****@hostname.dom/database'
replica_set_url = (
'mongodb://*****:*****@hostname.dom,'
'hostname.dom/database?replicaSet=rs'
)
sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database'
sanitized_replica_set_url = (
'mongodb://*****:*****@hostname.dom/,'
'hostname.dom/database?replicaSet=rs'
)
def setup(self):
R = self._reset = {}
R['encode'], MongoBackend.encode = MongoBackend.encode, Mock()
R['decode'], MongoBackend.decode = MongoBackend.decode, Mock()
R['Binary'], module.Binary = module.Binary, Mock()
R['datetime'], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app, url=self.default_url)
def teardown(self):
MongoBackend.encode = self._reset['encode']
MongoBackend.decode = self._reset['decode']
module.Binary = self._reset['Binary']
datetime.datetime = self._reset['datetime']
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.mongodb_backend_settings = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.mongodb_backend_settings = None
MongoBackend(app=self.app)
def test_init_with_settings(self):
self.app.conf.mongodb_backend_settings = None
# empty settings
mb = MongoBackend(app=self.app)
# uri
uri = 'mongodb://*****:*****@'
'mongo1.example.com:27017,'
'mongo2.example.com:27017,'
'mongo3.example.com:27017/'
'celerydatabase?replicaSet=rs0')
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017'])
self.assertEqual(
mb.options, dict(mb._prepare_client_options(), replicaset='rs0'),
)
self.assertEqual(mb.user, 'celeryuser')
self.assertEqual(mb.password, 'celerypassword')
self.assertEqual(mb.database_name, 'celerydatabase')
# same uri, change some parameters in backend settings
self.app.conf.mongodb_backend_settings = {
'replicaset': 'rs1',
'user': '******',
'database': 'another_db',
'options': {
'socketKeepAlive': True,
},
}
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017'])
self.assertEqual(
mb.options, dict(mb._prepare_client_options(),
replicaset='rs1', socketKeepAlive=True),
)
self.assertEqual(mb.user, 'backenduser')
self.assertEqual(mb.password, 'celerypassword')
self.assertEqual(mb.database_name, 'another_db')
mb = MongoBackend(app=self.app, url='mongodb://')
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
mock_collection.save.side_effect = InvalidDocument()
with self.assertRaises(EncodeError):
self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
list(sorted(['status', 'task_id', 'date_done', 'traceback',
'result', 'children'])),
list(sorted(ret_val.keys())),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({'status': states.PENDING, 'result': None}, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
res = [self.app.AsyncResult(i) for i in range(3)]
ret_val = self.backend._save_group(
sentinel.taskset_id, res,
)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_GROUP_COLLECTION,
)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(res, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = {
'_id': sentinel.taskset_id,
'result': [uuid(), uuid()],
'date_done': 1,
}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertItemsEqual(
['date_done', 'result', 'task_id'],
list(ret_val.keys()),
)
mock_collection.find_one.return_value = None
self.backend._restore_group(sentinel.taskset_id)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = Mock(spec=['__getitem__', '__setitem__'],
name='MD')
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__ = Mock(name='MD.__getitem__')
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
self.assertTrue(mock_collection.remove.called)
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
def test_prepare_client_options(self):
with patch('pymongo.version_tuple', new=(3, 0, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {
'maxPoolSize': self.backend.max_pool_size
})
def test_as_uri_include_password(self):
self.assertEqual(self.backend.as_uri(True), self.default_url)
def test_as_uri_exclude_password(self):
self.assertEqual(self.backend.as_uri(), self.sanitized_default_url)
def test_as_uri_include_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
self.assertEqual(backend.as_uri(True), self.replica_set_url)
def test_as_uri_exclude_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
self.assertEqual(backend.as_uri(), self.sanitized_replica_set_url)
@override_stdouts
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'mongodb://*****:*****@host0.com:43437,host1.com:43437'
'/work4us?replicaSet=rs&ssl=true'
)
worker = self.app.Worker()
worker.on_start()
self.assertTrue(worker.startup_info())
class test_MongoBackend:
default_url = 'mongodb://*****:*****@hostname.dom/database'
replica_set_url = (
'mongodb://*****:*****@hostname.dom,'
'hostname.dom/database?replicaSet=rs'
)
sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database'
sanitized_replica_set_url = (
'mongodb://*****:*****@hostname.dom/,'
'hostname.dom/database?replicaSet=rs'
)
def setup(self):
self.patching('celery.backends.mongodb.MongoBackend.encode')
self.patching('celery.backends.mongodb.MongoBackend.decode')
self.patching('celery.backends.mongodb.Binary')
self.patching('datetime.datetime')
self.backend = MongoBackend(app=self.app, url=self.default_url)
def test_init_no_mongodb(self, patching):
patching('celery.backends.mongodb.pymongo', None)
with pytest.raises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_no_settings(self):
self.app.conf.mongodb_backend_settings = []
with pytest.raises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.mongodb_backend_settings = None
MongoBackend(app=self.app)
def test_init_with_settings(self):
self.app.conf.mongodb_backend_settings = None
# empty settings
mb = MongoBackend(app=self.app)
# uri
uri = 'mongodb://*****:*****@'
'mongo1.example.com:27017,'
'mongo2.example.com:27017,'
'mongo3.example.com:27017/'
'celerydatabase?replicaSet=rs0')
mb = MongoBackend(app=self.app, url=uri)
assert mb.mongo_host == [
'mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017',
]
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs0',
)
assert mb.user == 'celeryuser'
assert mb.password == 'celerypassword'
assert mb.database_name == 'celerydatabase'
# same uri, change some parameters in backend settings
self.app.conf.mongodb_backend_settings = {
'replicaset': 'rs1',
'user': '******',
'database': 'another_db',
'options': {
'socketKeepAlive': True,
},
}
mb = MongoBackend(app=self.app, url=uri)
assert mb.mongo_host == [
'mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017',
]
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs1',
socketKeepAlive=True,
)
assert mb.user == 'backenduser'
assert mb.password == 'celerypassword'
assert mb.database_name == 'another_db'
mb = MongoBackend(app=self.app, url='mongodb://')
@pytest.mark.usefixtures('depends_on_current_app')
def test_reduce(self):
x = MongoBackend(app=self.app)
assert loads(dumps(x))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
assert sentinel._connection == connection
mock_Connection.assert_not_called()
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
assert database is mock_database
assert self.backend.__dict__['database'] is mock_database
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
assert database is mock_database
mock_database.authenticate.assert_not_called()
assert self.backend.__dict__['database'] is mock_database
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
assert sentinel.result == ret_val
mock_collection.save.side_effect = InvalidDocument()
with pytest.raises(EncodeError):
self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result_with_request(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_request = MagicMock(spec=['parent_id'])
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
mock_request.parent_id = sentinel.parent_id
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status,
request=mock_request)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
parameters = mock_collection.save.call_args[0][0]
assert parameters['parent_id'] == sentinel.parent_id
assert sentinel.result == ret_val
mock_collection.save.side_effect = InvalidDocument()
with pytest.raises(EncodeError):
self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
assert list(sorted([
'status', 'task_id', 'date_done',
'traceback', 'result', 'children',
])) == list(sorted(ret_val.keys()))
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
assert {'status': states.PENDING, 'result': None} == ret_val
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
res = [self.app.AsyncResult(i) for i in range(3)]
ret_val = self.backend._save_group(
sentinel.taskset_id, res,
)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_GROUP_COLLECTION,
)
mock_collection.save.assert_called_once_with(ANY)
assert res == ret_val
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = {
'_id': sentinel.taskset_id,
'result': [uuid(), uuid()],
'date_done': 1,
}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
assert (sorted(['date_done', 'result', 'task_id']) ==
sorted(list(ret_val.keys())))
mock_collection.find_one.return_value = None
self.backend._restore_group(sentinel.taskset_id)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test__forget(self, mock_get_database):
# note: here tested _forget method, not forget method
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = Mock(spec=['__getitem__', '__setitem__'],
name='MD')
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__ = Mock(name='MD.__getitem__')
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
mock_collection.remove.assert_called()
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with pytest.raises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
def test_prepare_client_options(self):
with patch('pymongo.version_tuple', new=(3, 0, 3)):
options = self.backend._prepare_client_options()
assert options == {
'maxPoolSize': self.backend.max_pool_size
}
def test_as_uri_include_password(self):
assert self.backend.as_uri(True) == self.default_url
def test_as_uri_exclude_password(self):
assert self.backend.as_uri() == self.sanitized_default_url
def test_as_uri_include_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
assert backend.as_uri(True) == self.replica_set_url
def test_as_uri_exclude_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
assert backend.as_uri() == self.sanitized_replica_set_url
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'mongodb://*****:*****@host0.com:43437,host1.com:43437'
'/work4us?replicaSet=rs&ssl=true'
)
worker = self.app.Worker()
with mock.stdouts():
worker.on_start()
assert worker.startup_info()
class test_MongoBackend(AppCase):
default_url = 'mongodb://*****:*****@hostname.dom/database'
replica_set_url = (
'mongodb://*****:*****@hostname.dom,'
'hostname.dom/database?replicaSet=rs'
)
sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database'
sanitized_replica_set_url = (
'mongodb://*****:*****@hostname.dom/,'
'hostname.dom/database?replicaSet=rs'
)
def setup(self):
if pymongo is None:
raise SkipTest('pymongo is not installed.')
R = self._reset = {}
R['encode'], MongoBackend.encode = MongoBackend.encode, Mock()
R['decode'], MongoBackend.decode = MongoBackend.decode, Mock()
R['Binary'], module.Binary = module.Binary, Mock()
R['datetime'], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app, url=self.default_url)
def teardown(self):
MongoBackend.encode = self._reset['encode']
MongoBackend.decode = self._reset['decode']
module.Binary = self._reset['Binary']
datetime.datetime = self._reset['datetime']
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
MongoBackend(app=self.app)
def test_restore_group_no_entry(self):
x = MongoBackend(app=self.app)
x.collection = Mock()
fo = x.collection.find_one = Mock()
fo.return_value = None
self.assertIsNone(x._restore_group('1f3fab'))
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
self.backend._connection = 'not none'
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
list(sorted(['status', 'task_id', 'date_done', 'traceback',
'result', 'children'])),
list(sorted(ret_val.keys())),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({'status': states.PENDING, 'result': None}, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._save_group(
sentinel.taskset_id, sentinel.result)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertItemsEqual(
['date_done', 'result', 'task_id'],
list(ret_val.keys()),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
self.assertTrue(mock_collection.remove.called)
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
@patch('celery.backends.mongodb.detect_environment')
def test_prepare_client_options_for_ver_2(self, m_detect_env):
m_detect_env.return_value = 'default'
with patch('pymongo.version_tuple', new=(2, 6, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {
'max_pool_size': self.backend.max_pool_size,
'auto_start_request': False
})
def test_as_uri_include_password(self):
self.assertEqual(self.backend.as_uri(True), self.default_url)
def test_as_uri_exclude_password(self):
self.assertEqual(self.backend.as_uri(), self.sanitized_default_url)
def test_as_uri_include_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
self.assertEqual(backend.as_uri(True), self.replica_set_url)
def test_as_uri_exclude_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
self.assertEqual(backend.as_uri(), self.sanitized_replica_set_url)
@disable_stdouts
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'mongodb://*****:*****@host0.com:43437,host1.com:43437'
'/work4us?replicaSet=rs&ssl=true'
)
worker = self.app.Worker()
worker.on_start()
self.assertTrue(worker.startup_info())
class test_MongoBackend(AppCase):
def setUp(self):
if pymongo is None:
raise SkipTest("pymongo is not installed.")
R = self._reset = {}
R["encode"], MongoBackend.encode = MongoBackend.encode, Mock()
R["decode"], MongoBackend.decode = MongoBackend.decode, Mock()
R["Binary"], module.Binary = module.Binary, Mock()
R["datetime"], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend()
def tearDown(self):
MongoBackend.encode = self._reset["encode"]
MongoBackend.decode = self._reset["decode"]
module.Binary = self._reset["Binary"]
datetime.datetime = self._reset["datetime"]
def test_Bunch(self):
x = Bunch(foo="foo", bar=2)
self.assertEqual(x.foo, "foo")
self.assertEqual(x.bar, 2)
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend()
finally:
module.pymongo = prev
def test_init_no_settings(self):
celery = Celery(set_as_current=False)
celery.conf.CELERY_MONGODB_BACKEND_SETTINGS = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=celery)
def test_init_settings_is_None(self):
celery = Celery(set_as_current=False)
celery.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
MongoBackend(app=celery)
def test_restore_group_no_entry(self):
x = MongoBackend()
x.collection = Mock()
fo = x.collection.find_one = Mock()
fo.return_value = None
self.assertIsNone(x._restore_group("1f3fab"))
def test_reduce(self):
x = MongoBackend()
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch("pymongo.connection.Connection") as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEquals(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch("pymongo.connection.Connection") as mock_Connection:
self.backend._connection = None
self.backend.mongodb_host = MONGODB_HOST
self.backend.mongodb_port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
MONGODB_HOST, MONGODB_PORT)
self.assertEquals(sentinel.connection, connection)
def test_get_connection_no_connection_mongodb_uri(self):
with patch("pymongo.connection.Connection") as mock_Connection:
mongodb_uri = "mongodb://%s:%d" % (MONGODB_HOST, MONGODB_PORT)
self.backend._connection = None
self.backend.mongodb_host = mongodb_uri
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(mongodb_uri)
self.assertEquals(sentinel.connection, connection)
@patch("celery.backends.mongodb.MongoBackend._get_connection")
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.mongodb_user = MONGODB_USER
self.backend.mongodb_password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__["database"] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch("celery.backends.mongodb.MongoBackend._get_connection")
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.mongodb_user = None
self.backend.mongodb_password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__["database"] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEquals(self.backend._connection, None)
self.backend._connection = "not none"
self.backend.process_cleanup()
self.assertEquals(self.backend._connection, None)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_store_result(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once()
self.assertEquals(sentinel.result, ret_val)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset["datetime"]
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEquals(
['status', 'task_id', 'date_done', 'traceback', 'result',
'children'],
ret_val.keys())
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEquals({"status": states.PENDING, "result": None}, ret_val)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_save_group(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._save_group(
sentinel.taskset_id, sentinel.result)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once()
self.assertEquals(sentinel.result, ret_val)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_restore_group(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.find_one.assert_called_once_with(
{"_id": sentinel.taskset_id})
self.assertEquals(['date_done', 'result', 'task_id'], ret_val.keys())
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_delete_group(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{"_id": sentinel.taskset_id})
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_forget(self, mock_get_database):
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{"_id": sentinel.task_id}, safe=True)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset["datetime"]
self.backend.mongodb_taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.assert_called_once()
def test_get_database_authfailure(self):
x = MongoBackend()
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.mongodb_database] = Mock()
db.authenticate.return_value = False
x.mongodb_user = "******"
x.mongodb_password = "******"
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with("jerry", "cere4l")
class test_MongoBackend(AppCase):
def setup(self):
if pymongo is None:
raise SkipTest('pymongo is not installed.')
R = self._reset = {}
R['encode'], MongoBackend.encode = MongoBackend.encode, Mock()
R['decode'], MongoBackend.decode = MongoBackend.decode, Mock()
R['Binary'], module.Binary = module.Binary, Mock()
R['datetime'], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app)
def teardown(self):
MongoBackend.encode = self._reset['encode']
MongoBackend.decode = self._reset['decode']
module.Binary = self._reset['Binary']
datetime.datetime = self._reset['datetime']
def test_Bunch(self):
x = Bunch(foo='foo', bar=2)
self.assertEqual(x.foo, 'foo')
self.assertEqual(x.bar, 2)
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
MongoBackend(app=self.app)
def test_init_with_settings(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
# empty settings
mb = MongoBackend(app=self.app)
# uri
uri = 'mongodb://*****:*****@'
'mongo1.example.com:27017,'
'mongo2.example.com:27017,'
'mongo3.example.com:27017/'
'celerydatabase?replicaSet=rs0')
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017'])
self.assertEqual(
mb.options, dict(mb._prepare_client_options(), replicaset='rs0'),
)
self.assertEqual(mb.user, 'celeryuser')
self.assertEqual(mb.password, 'celerypassword')
self.assertEqual(mb.database_name, 'celerydatabase')
# same uri, change some parameters in backend settings
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = {
'replicaset': 'rs1',
'user': '******',
'database': 'another_db',
'options': {
'socketKeepAlive': True,
},
}
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017'])
self.assertEqual(
mb.options, dict(mb._prepare_client_options(),
replicaset='rs1', socketKeepAlive=True),
)
self.assertEqual(mb.user, 'backenduser')
self.assertEqual(mb.password, 'celerypassword')
self.assertEqual(mb.database_name, 'another_db')
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
self.backend._connection = 'not none'
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
list(sorted(['status', 'task_id', 'date_done', 'traceback',
'result', 'children'])),
list(sorted(ret_val.keys())),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({'status': states.PENDING, 'result': None}, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
res = [self.app.AsyncResult(i) for i in range(3)]
ret_val = self.backend._save_group(
sentinel.taskset_id, res,
)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_GROUP_COLLECTION,
)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(res, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = {
'_id': sentinel.taskset_id,
'result': [uuid(), uuid()],
'date_done': 1,
}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertItemsEqual(
['date_done', 'result', 'task_id'],
list(ret_val.keys()),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = Mock(spec=['__getitem__', '__setitem__'],
name='MD')
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__ = Mock(name='MD.__getitem__')
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
self.assertTrue(mock_collection.remove.called)
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
@patch('celery.backends.mongodb.detect_environment')
def test_prepare_client_options_for_ver_2(self, m_detect_env):
m_detect_env.return_value = 'default'
with patch('pymongo.version_tuple', new=(2, 6, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {
'max_pool_size': self.backend.max_pool_size,
'auto_start_request': False
})
@patch('celery.backends.mongodb.detect_environment')
def test_prepare_client_options_for_ver_2_with_gevent(self, m_detect_env):
m_detect_env.return_value = 'gevent'
with patch('pymongo.version_tuple', new=(2, 6, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {
'max_pool_size': self.backend.max_pool_size,
'auto_start_request': False,
'use_greenlets': True
})
@patch('celery.backends.mongodb.detect_environment')
def test_prepare_client_options_for_ver_3(self, m_detect_env):
m_detect_env.return_value = 'default'
with patch('pymongo.version_tuple', new=(3, 0, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {
'maxPoolSize': self.backend.max_pool_size
})
@patch('celery.backends.mongodb.detect_environment')
def test_prepare_client_options_for_ver_3_with_gevent(self, m_detect_env):
m_detect_env.return_value = 'gevent'
with patch('pymongo.version_tuple', new=(3, 0, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {
'maxPoolSize': self.backend.max_pool_size
})
class test_MongoBackend(AppCase):
def setup(self):
if pymongo is None:
raise SkipTest("pymongo is not installed.")
R = self._reset = {}
R["encode"], MongoBackend.encode = MongoBackend.encode, Mock()
R["decode"], MongoBackend.decode = MongoBackend.decode, Mock()
R["Binary"], module.Binary = module.Binary, Mock()
R["datetime"], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app)
def teardown(self):
MongoBackend.encode = self._reset["encode"]
MongoBackend.decode = self._reset["decode"]
module.Binary = self._reset["Binary"]
datetime.datetime = self._reset["datetime"]
def test_Bunch(self):
x = Bunch(foo="foo", bar=2)
self.assertEqual(x.foo, "foo")
self.assertEqual(x.bar, 2)
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.mongodb_backend_settings = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.mongodb_backend_settings = None
MongoBackend(app=self.app)
def test_init_with_settings(self):
self.app.conf.mongodb_backend_settings = None
# empty settings
mb = MongoBackend(app=self.app)
# uri
uri = "mongodb://*****:*****@"
"mongo1.example.com:27017,"
"mongo2.example.com:27017,"
"mongo3.example.com:27017/"
"celerydatabase?replicaSet=rs0"
)
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(
mb.mongo_host, ["mongo1.example.com:27017", "mongo2.example.com:27017", "mongo3.example.com:27017"]
)
self.assertEqual(mb.options, dict(mb._prepare_client_options(), replicaset="rs0"))
self.assertEqual(mb.user, "celeryuser")
self.assertEqual(mb.password, "celerypassword")
self.assertEqual(mb.database_name, "celerydatabase")
# same uri, change some parameters in backend settings
self.app.conf.mongodb_backend_settings = {
"replicaset": "rs1",
"user": "******",
"database": "another_db",
"options": {"socketKeepAlive": True},
}
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(
mb.mongo_host, ["mongo1.example.com:27017", "mongo2.example.com:27017", "mongo3.example.com:27017"]
)
self.assertEqual(mb.options, dict(mb._prepare_client_options(), replicaset="rs1", socketKeepAlive=True))
self.assertEqual(mb.user, "backenduser")
self.assertEqual(mb.password, "celerypassword")
self.assertEqual(mb.database_name, "another_db")
mb = MongoBackend(app=self.app, url="mongodb://")
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch("pymongo.MongoClient") as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch("pymongo.MongoClient") as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host="mongodb://*****:*****@patch("celery.backends.mongodb.MongoBackend._get_connection")
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=["__getitem__"])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__["database"] is mock_database)
mock_database.authenticate.assert_called_once_with(MONGODB_USER, MONGODB_PASSWORD)
@patch("celery.backends.mongodb.MongoBackend._get_connection")
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=["__getitem__"])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__["database"] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
self.backend._connection = "not none"
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
mock_collection.save.side_effect = InvalidDocument()
with self.assertRaises(EncodeError):
self.backend._store_result(sentinel.task_id, sentinel.result, sentinel.status)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset["datetime"]
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
list(sorted(["status", "task_id", "date_done", "traceback", "result", "children"])),
list(sorted(ret_val.keys())),
)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({"status": states.PENDING, "result": None}, ret_val)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_save_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
res = [self.app.AsyncResult(i) for i in range(3)]
ret_val = self.backend._save_group(sentinel.taskset_id, res)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_GROUP_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(res, ret_val)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_collection.find_one.return_value = {"_id": sentinel.taskset_id, "result": [uuid(), uuid()], "date_done": 1}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with({"_id": sentinel.taskset_id})
self.assertItemsEqual(["date_done", "result", "task_id"], list(ret_val.keys()))
mock_collection.find_one.return_value = None
self.backend._restore_group(sentinel.taskset_id)
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.remove.assert_called_once_with({"_id": sentinel.taskset_id})
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=["__getitem__", "__setitem__"])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with({"_id": sentinel.task_id})
@patch("celery.backends.mongodb.MongoBackend._get_database")
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset["datetime"]
self.backend.taskmeta_collection = MONGODB_COLLECTION
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = Mock(spec=["__getitem__", "__setitem__"], name="MD")
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__ = Mock(name="MD.__getitem__")
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
self.assertTrue(mock_collection.remove.called)
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = "******"
x.password = "******"
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with("jerry", "cere4l")
def test_prepare_client_options(self):
with patch("pymongo.version_tuple", new=(3, 0, 3)):
options = self.backend._prepare_client_options()
self.assertDictEqual(options, {"maxPoolSize": self.backend.max_pool_size})
class test_MongoBackend(AppCase):
def setup(self):
if pymongo is None:
raise SkipTest('pymongo is not installed.')
R = self._reset = {}
R['encode'], MongoBackend.encode = MongoBackend.encode, Mock()
R['decode'], MongoBackend.decode = MongoBackend.decode, Mock()
R['Binary'], module.Binary = module.Binary, Mock()
R['datetime'], datetime.datetime = datetime.datetime, Mock()
self.backend = MongoBackend(app=self.app)
def teardown(self):
MongoBackend.encode = self._reset['encode']
MongoBackend.decode = self._reset['decode']
module.Binary = self._reset['Binary']
datetime.datetime = self._reset['datetime']
def test_Bunch(self):
x = Bunch(foo='foo', bar=2)
self.assertEqual(x.foo, 'foo')
self.assertEqual(x.bar, 2)
def test_init_no_mongodb(self):
prev, module.pymongo = module.pymongo, None
try:
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
finally:
module.pymongo = prev
def test_init_no_settings(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = []
with self.assertRaises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
MongoBackend(app=self.app)
def test_init_with_settings(self):
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = None
# empty settings
mb = MongoBackend(app=self.app)
# uri
uri = 'mongodb://*****:*****@'
'mongo1.example.com:27017,'
'mongo2.example.com:27017,'
'mongo3.example.com:27017/'
'celerydatabase?replicaSet=rs0')
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017'])
self.assertEqual(mb.options, {'auto_start_request': False,
'max_pool_size': 10,
'replicaset': 'rs0'})
self.assertEqual(mb.user, 'celeryuser')
self.assertEqual(mb.password, 'celerypassword')
self.assertEqual(mb.database_name, 'celerydatabase')
# same uri, change some parameters in backend settings
self.app.conf.CELERY_MONGODB_BACKEND_SETTINGS = {
'replicaset': 'rs1',
'user': '******',
'database': 'another_db',
'options': {
'socketKeepAlive': True,
},
}
mb = MongoBackend(app=self.app, url=uri)
self.assertEqual(mb.mongo_host, ['mongo1.example.com:27017',
'mongo2.example.com:27017',
'mongo3.example.com:27017'])
self.assertEqual(mb.options, {'auto_start_request': False,
'max_pool_size': 10,
'replicaset': 'rs1',
'socketKeepAlive': True})
self.assertEqual(mb.user, 'backenduser')
self.assertEqual(mb.password, 'celerypassword')
self.assertEqual(mb.database_name, 'another_db')
@depends_on_current_app
def test_reduce(self):
x = MongoBackend(app=self.app)
self.assertTrue(loads(dumps(x)))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
self.assertEqual(sentinel._connection, connection)
self.assertFalse(mock_Connection.called)
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
mock_database.authenticate.assert_called_once_with(
MONGODB_USER, MONGODB_PASSWORD)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
self.assertTrue(database is mock_database)
self.assertFalse(mock_database.authenticate.called)
self.assertTrue(self.backend.__dict__['database'] is mock_database)
def test_process_cleanup(self):
self.backend._connection = None
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
self.backend._connection = 'not none'
self.backend.process_cleanup()
self.assertEqual(self.backend._connection, None)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(sentinel.result, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual(
list(sorted(['status', 'task_id', 'date_done', 'traceback',
'result', 'children'])),
list(sorted(ret_val.keys())),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
self.assertEqual({'status': states.PENDING, 'result': None}, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
res = [self.app.AsyncResult(i) for i in range(3)]
ret_val = self.backend._save_group(
sentinel.taskset_id, res,
)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_GROUP_COLLECTION,
)
mock_collection.save.assert_called_once_with(ANY)
self.assertEqual(res, ret_val)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = {
'_id': sentinel.taskset_id,
'result': [uuid(), uuid()],
'date_done': 1,
}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertEqual(
list(sorted(['date_done', 'result', 'task_id'])),
list(sorted(ret_val.keys())),
)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_forget(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.remove.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
datetime.datetime = self._reset['datetime']
self.backend.taskmeta_collection = MONGODB_COLLECTION
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = Mock(spec=['__getitem__', '__setitem__'],
name='MD')
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__ = Mock(name='MD.__getitem__')
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
self.assertTrue(mock_collection.remove.called)
def test_get_database_authfailure(self):
x = MongoBackend(app=self.app)
x._get_connection = Mock()
conn = x._get_connection.return_value = {}
db = conn[x.database_name] = Mock()
db.authenticate.return_value = False
x.user = '******'
x.password = '******'
with self.assertRaises(ImproperlyConfigured):
x._get_database()
db.authenticate.assert_called_with('jerry', 'cere4l')
class test_MongoBackend:
default_url = 'mongodb://*****:*****@hostname.dom/database'
replica_set_url = (
'mongodb://*****:*****@hostname.dom,'
'hostname.dom/database?replicaSet=rs'
)
sanitized_default_url = 'mongodb://*****:*****@hostname.dom/database'
sanitized_replica_set_url = (
'mongodb://*****:*****@hostname.dom/,'
'hostname.dom/database?replicaSet=rs'
)
def setup(self):
self.patching('celery.backends.mongodb.MongoBackend.encode')
self.patching('celery.backends.mongodb.MongoBackend.decode')
self.patching('celery.backends.mongodb.Binary')
self.backend = MongoBackend(app=self.app, url=self.default_url)
def test_init_no_mongodb(self, patching):
patching('celery.backends.mongodb.pymongo', None)
with pytest.raises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_no_settings(self):
self.app.conf.mongodb_backend_settings = []
with pytest.raises(ImproperlyConfigured):
MongoBackend(app=self.app)
def test_init_settings_is_None(self):
self.app.conf.mongodb_backend_settings = None
MongoBackend(app=self.app)
def test_init_with_settings(self):
self.app.conf.mongodb_backend_settings = None
# empty settings
mb = MongoBackend(app=self.app)
# uri
uri = 'mongodb://*****:*****@'
'mongo1.example.com:27017,'
'mongo2.example.com:27017,'
'mongo3.example.com:27017/'
'celerydatabase?replicaSet=rs0')
mb = MongoBackend(app=self.app, url=uri)
assert mb.mongo_host == MONGODB_BACKEND_HOST
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs0',
)
assert mb.user == CELERY_USER
assert mb.password == CELERY_PASSWORD
assert mb.database_name == CELERY_DATABASE
# same uri, change some parameters in backend settings
self.app.conf.mongodb_backend_settings = {
'replicaset': 'rs1',
'user': '******',
'database': 'another_db',
'options': {
'socketKeepAlive': True,
},
}
mb = MongoBackend(app=self.app, url=uri)
assert mb.mongo_host == MONGODB_BACKEND_HOST
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs1',
socketKeepAlive=True,
)
assert mb.user == 'backenduser'
assert mb.password == CELERY_PASSWORD
assert mb.database_name == 'another_db'
mb = MongoBackend(app=self.app, url='mongodb://')
@pytest.mark.skipif(dns.version.MAJOR > 1,
reason="For dnspython version > 1, pymongo's"
"srv_resolver calls resolver.resolve")
@pytest.mark.skipif(pymongo.version_tuple[0] > 3,
reason="For pymongo version > 3, options returns ssl")
def test_init_mongodb_dnspython1_pymongo3_seedlist(self):
resolver = fake_resolver_dnspython()
self.app.conf.mongodb_backend_settings = None
with patch('dns.resolver.query', side_effect=resolver):
mb = self.perform_seedlist_assertions()
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs0',
ssl=True
)
@pytest.mark.skipif(dns.version.MAJOR <= 1,
reason="For dnspython versions 1.X, pymongo's"
"srv_resolver calls resolver.query")
@pytest.mark.skipif(pymongo.version_tuple[0] > 3,
reason="For pymongo version > 3, options returns ssl")
def test_init_mongodb_dnspython2_pymongo3_seedlist(self):
resolver = fake_resolver_dnspython()
self.app.conf.mongodb_backend_settings = None
with patch('dns.resolver.resolve', side_effect=resolver):
mb = self.perform_seedlist_assertions()
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs0',
ssl=True
)
@pytest.mark.skipif(dns.version.MAJOR > 1,
reason="For dnspython version >= 2, pymongo's"
"srv_resolver calls resolver.resolve")
@pytest.mark.skipif(pymongo.version_tuple[0] <= 3,
reason="For pymongo version > 3, options returns tls")
def test_init_mongodb_dnspython1_pymongo4_seedlist(self):
resolver = fake_resolver_dnspython()
self.app.conf.mongodb_backend_settings = None
with patch('dns.resolver.query', side_effect=resolver):
mb = self.perform_seedlist_assertions()
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs0',
tls=True
)
@pytest.mark.skipif(dns.version.MAJOR <= 1,
reason="For dnspython versions 1.X, pymongo's"
"srv_resolver calls resolver.query")
@pytest.mark.skipif(pymongo.version_tuple[0] <= 3,
reason="For pymongo version > 3, options returns tls")
def test_init_mongodb_dnspython2_pymongo4_seedlist(self):
resolver = fake_resolver_dnspython()
self.app.conf.mongodb_backend_settings = None
with patch('dns.resolver.resolve', side_effect=resolver):
mb = self.perform_seedlist_assertions()
assert mb.options == dict(
mb._prepare_client_options(),
replicaset='rs0',
tls=True
)
def perform_seedlist_assertions(self):
mb = MongoBackend(app=self.app, url=MONGODB_SEEDLIST_URI)
assert mb.mongo_host == MONGODB_BACKEND_HOST
assert mb.user == CELERY_USER
assert mb.password == CELERY_PASSWORD
assert mb.database_name == CELERY_DATABASE
return mb
def test_ensure_mongodb_uri_compliance(self):
mb = MongoBackend(app=self.app, url=None)
compliant_uri = mb._ensure_mongodb_uri_compliance
assert compliant_uri('mongodb://') == 'mongodb://localhost'
assert compliant_uri('mongodb+something://host') == \
'mongodb+something://host'
assert compliant_uri('something://host') == 'mongodb+something://host'
@pytest.mark.usefixtures('depends_on_current_app')
def test_reduce(self):
x = MongoBackend(app=self.app)
assert loads(dumps(x))
def test_get_connection_connection_exists(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = sentinel._connection
connection = self.backend._get_connection()
assert sentinel._connection == connection
mock_Connection.assert_not_called()
def test_get_connection_no_connection_host(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.backend._connection = None
self.backend.host = MONGODB_HOST
self.backend.port = MONGODB_PORT
mock_Connection.return_value = sentinel.connection
connection = self.backend._get_connection()
mock_Connection.assert_called_once_with(
host='mongodb://*****:*****@'
'localhost:27017/'
'celerydatabase?authMechanism=SCRAM-SHA-256')
mb = MongoBackend(app=self.app, url=uri)
mock_Connection.return_value = sentinel.connection
connection = mb._get_connection()
mock_Connection.assert_called_once_with(
host=['localhost:27017'],
username=CELERY_USER,
password=CELERY_PASSWORD,
authmechanism='SCRAM-SHA-256',
**mb._prepare_client_options()
)
assert sentinel.connection == connection
def test_get_connection_with_authmechanism_no_username(self):
with patch('pymongo.MongoClient') as mock_Connection:
self.app.conf.mongodb_backend_settings = None
uri = ('mongodb://'
'localhost:27017/'
'celerydatabase?authMechanism=SCRAM-SHA-256')
mb = MongoBackend(app=self.app, url=uri)
mock_Connection.side_effect = ConfigurationError(
'SCRAM-SHA-256 requires a username.')
with pytest.raises(ConfigurationError):
mb._get_connection()
mock_Connection.assert_called_once_with(
host=['localhost:27017'],
authmechanism='SCRAM-SHA-256',
**mb._prepare_client_options()
)
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = MONGODB_USER
self.backend.password = MONGODB_PASSWORD
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
assert database is mock_database
assert self.backend.__dict__['database'] is mock_database
@patch('celery.backends.mongodb.MongoBackend._get_connection')
def test_get_database_no_existing_no_auth(self, mock_get_connection):
# Should really check for combinations of these two, to be complete.
self.backend.user = None
self.backend.password = None
mock_database = Mock()
mock_connection = MagicMock(spec=['__getitem__'])
mock_connection.__getitem__.return_value = mock_database
mock_get_connection.return_value = mock_connection
database = self.backend.database
assert database is mock_database
assert self.backend.__dict__['database'] is mock_database
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
mock_collection.replace_one.assert_called_once_with(ANY, ANY,
upsert=True)
assert sentinel.result == ret_val
mock_collection.replace_one.side_effect = InvalidDocument()
with pytest.raises(EncodeError):
self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_store_result_with_request(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_request = MagicMock(spec=['parent_id'])
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
mock_request.parent_id = sentinel.parent_id
ret_val = self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status,
request=mock_request)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
parameters = mock_collection.replace_one.call_args[0][1]
assert parameters['parent_id'] == sentinel.parent_id
assert sentinel.result == ret_val
mock_collection.replace_one.side_effect = InvalidDocument()
with pytest.raises(EncodeError):
self.backend._store_result(
sentinel.task_id, sentinel.result, sentinel.status)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = MagicMock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
assert list(sorted([
'status', 'task_id', 'date_done',
'traceback', 'result', 'children',
])) == list(sorted(ret_val.keys()))
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_get_task_meta_for_no_result(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = None
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._get_task_meta_for(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
assert {'status': states.PENDING, 'result': None} == ret_val
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_save_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
res = [self.app.AsyncResult(i) for i in range(3)]
ret_val = self.backend._save_group(
sentinel.taskset_id, res,
)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_GROUP_COLLECTION,
)
mock_collection.replace_one.assert_called_once_with(ANY, ANY,
upsert=True)
assert res == ret_val
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = {
'_id': sentinel.taskset_id,
'result': [uuid(), uuid()],
'date_done': 1,
}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
assert (sorted(['date_done', 'result', 'task_id']) ==
sorted(list(ret_val.keys())))
mock_collection.find_one.return_value = None
self.backend._restore_group(sentinel.taskset_id)
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_delete_group(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._delete_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.delete_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test__forget(self, mock_get_database):
# note: here tested _forget method, not forget method
self.backend.taskmeta_collection = MONGODB_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
self.backend._forget(sentinel.task_id)
mock_get_database.assert_called_once_with()
mock_database.__getitem__.assert_called_once_with(
MONGODB_COLLECTION)
mock_collection.delete_one.assert_called_once_with(
{'_id': sentinel.task_id})
@patch('celery.backends.mongodb.MongoBackend._get_database')
def test_cleanup(self, mock_get_database):
self.backend.taskmeta_collection = MONGODB_COLLECTION
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = Mock(spec=['__getitem__', '__setitem__'],
name='MD')
self.backend.collections = mock_collection = Mock()
mock_get_database.return_value = mock_database
mock_database.__getitem__ = Mock(name='MD.__getitem__')
mock_database.__getitem__.return_value = mock_collection
self.backend.app.now = datetime.datetime.utcnow
self.backend.cleanup()
mock_get_database.assert_called_once_with()
mock_collection.delete_many.assert_called()
self.backend.collections = mock_collection = Mock()
self.backend.expires = None
self.backend.cleanup()
mock_collection.delete_many.assert_not_called()
def test_prepare_client_options(self):
with patch('pymongo.version_tuple', new=(3, 0, 3)):
options = self.backend._prepare_client_options()
assert options == {
'maxPoolSize': self.backend.max_pool_size
}
def test_as_uri_include_password(self):
assert self.backend.as_uri(True) == self.default_url
def test_as_uri_exclude_password(self):
assert self.backend.as_uri() == self.sanitized_default_url
def test_as_uri_include_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
assert backend.as_uri(True) == self.replica_set_url
def test_as_uri_exclude_password_replica_set(self):
backend = MongoBackend(app=self.app, url=self.replica_set_url)
assert backend.as_uri() == self.sanitized_replica_set_url
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'mongodb://*****:*****@host0.com:43437,host1.com:43437'
'/work4us?replicaSet=rs&ssl=true'
)
worker = self.app.Worker()
with conftest.stdouts():
worker.on_start()
assert worker.startup_info()
