def test_set_upload_results_attached(self, get_collection):
collection = MagicMock()
get_collection.return_value = collection
args = ("one", 2, {3: 4, 5: 6})
result = set_upload_results_attached(*args)
collection.update_one.assert_called_once_with(
{'_id': args_to_uid(args)},
{"$set": {'attached': True}}
)
self.assertEqual(result, args_to_uid(args))
def test_mongodb_exc(self, get_collection):
"""
in case of mongodb connection error task should be retried
"""
get_collection.return_value.insert_one.side_effect = ConnectionFailure
test_method_procedure = Mock()
@concurrency_lock
def test_method(*args, **kwargs):
test_method_procedure(*args, **kwargs)
dummy_task.retry = Mock(side_effect=Retry)
with patch("celery_worker.locks.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
with self.assertRaises(Retry):
test_method(dummy_task) # run
task_uid = args_to_uid(
(test_method.__module__, test_method.__name__, tuple(), dict()))
get_collection.return_value.insert_one.assert_called_once_with({
"_id":
task_uid,
"expireAt":
datetime_mock.utcnow.return_value + timedelta(seconds=10)
})
dummy_task.retry.assert_called_once_with(max_retries=20)
test_method_procedure.assert_not_called()
def test_duplicate_exc(self, get_collection):
"""
in case of lock retry later after lock timeout
"""
get_collection.return_value.insert_one.side_effect = DuplicateKeyError(
"E11000 duplicate key error collection")
test_method_procedure = Mock()
@concurrency_lock
def test_method(*args, **kwargs):
test_method_procedure(*args, **kwargs)
dummy_task.retry = Mock(side_effect=Retry)
with patch("celery_worker.locks.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
with self.assertRaises(Retry):
test_method(dummy_task) # run
task_uid = args_to_uid(
(test_method.__module__, test_method.__name__, tuple(), dict()))
get_collection.return_value.insert_one.assert_called_once_with({
"_id":
task_uid,
"expireAt":
datetime_mock.utcnow.return_value + timedelta(seconds=10)
})
dummy_task.retry.assert_called_once_with(max_retries=20, countdown=70)
test_method_procedure.assert_not_called()
def test_success(self, get_collection):
get_collection.return_value.insert_one.return_value = None
test_method_procedure = Mock()
@concurrency_lock
def test_method(*args, **kwargs):
test_method_procedure(*args, **kwargs)
with patch("celery_worker.locks.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
test_method(dummy_task, 1, 2, message="Hi") # run
task_uid = args_to_uid(
(test_method.__module__, test_method.__name__, (1, 2),
dict(message="Hi")))
get_collection.return_value.insert_one.assert_called_once_with({
"_id":
task_uid,
"expireAt":
datetime_mock.utcnow.return_value + timedelta(seconds=10)
})
test_method_procedure.assert_called_once_with(dummy_task,
1,
2,
message="Hi")
def test_unique_task_decorator_duplicate_detected(self, get_collection):
get_collection.return_value.find_one.return_value = {"Achtung": "!"}
test_method_procedure = Mock()
# create decorated function
@unique_lock
def test_method(*args, **kwargs):
test_method_procedure(*args, **kwargs)
with patch("celery_worker.locks.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
# run it
test_method(1, 2, message="Hi")
# check that decorator checked the lock and set it
task_uid = "v2_" + args_to_uid(
(test_method.__module__, test_method.__name__,
(1, 2), dict(message="Hi")))
get_collection.return_value.find_one.assert_called_once_with({
"_id":
task_uid,
"name":
test_method.__name__,
"module":
test_method.__module__
})
test_method_procedure.assert_not_called()
get_collection.return_value.insert.assert_not_called()
def test_max_reties_exceed(self, get_collection):
"""
in case of too many reties because of mongodb errors the max_retry limit may exceed
we should run task then
"""
get_collection.return_value.insert_one.side_effect = ConnectionFailure
test_method_procedure = Mock()
@concurrency_lock
def test_method(*args, **kwargs):
test_method_procedure(*args, **kwargs)
dummy_task.retry = Mock(side_effect=MaxRetriesExceededError)
with patch("celery_worker.locks.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
test_method(dummy_task) # run
task_uid = args_to_uid(
(test_method.__module__, test_method.__name__, tuple(), dict()))
get_collection.return_value.insert_one.assert_called_once_with({
"_id":
task_uid,
"expireAt":
datetime_mock.utcnow.return_value + timedelta(seconds=10)
})
dummy_task.retry.assert_called_once_with(max_retries=20)
test_method_procedure.assert_called_once_with(dummy_task)
def test_save_upload_results(self, get_collection):
collection = MagicMock()
get_collection.return_value = collection
file_data = {"data": 1, "meta": "hello"}
args = ("one", 2, [3, 4])
with patch("edr_bot.results_db.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
result = save_upload_results(file_data, *args)
collection.insert.assert_called_once_with(
{
'_id': args_to_uid(args),
'file_data': file_data,
'createdAt': datetime_mock.utcnow.return_value
}
)
self.assertEqual(result, args_to_uid(args))
def set_upload_results_attached(*args):
uid = args_to_uid(args)
collection = get_mongodb_collection()
try:
collection.update_one({'_id': uid}, {"$set": {'attached': True}})
except PyMongoError as exc:
logger.exception(
exc, extra={"MESSAGE_ID": "EDR_UPDATE_RESULTS_MONGODB_EXCEPTION"})
else:
return uid
def get_upload_results(self, *args):
uid = args_to_uid(args)
collection = get_mongodb_collection()
try:
doc = collection.find_one({'_id': uid})
except PyMongoError as exc:
logger.exception(
exc, extra={"MESSAGE_ID": "EDR_GET_RESULTS_MONGODB_EXCEPTION"})
raise self.retry()
else:
return doc
def test_get_upload_results(self, get_collection):
task = MagicMock()
collection = MagicMock()
collection.find_one.return_value = {"hello": "there"}
get_collection.return_value = collection
args = (1, "two", 3)
result = get_upload_results(task, *args)
self.assertEqual(result, {"hello": "there"})
collection.find_one.assert_called_once_with({'_id': args_to_uid(args)})
def get_task_result(self, args):
args = tuple((self.name, *args))
uid = args_to_uid(args)
collection = get_mongodb_collection()
try:
doc = collection.find_one({'_id': uid})
except PyMongoError as exc:
logger.exception(
exc, extra={"MESSAGE_ID": "GET_TASK_RESULTS_MONGODB_EXCEPTION"})
raise self.retry()
else:
return doc and doc["result"]
def test_get_upload_results(self, get_collection):
task = MagicMock()
task.name = "bot.some_task"
collection = MagicMock()
collection.find_one.return_value = {"result": {"hello": "there"}}
get_collection.return_value = collection
args = (1, "two", 3)
result = get_task_result(task, args)
self.assertEqual(result, {"hello": "there"})
collection.find_one.assert_called_once_with(
{'_id': args_to_uid(("bot.some_task", ) + args)})
def test_set_upload_results_attached_raise_exc(self, get_collection):
collection = MagicMock()
collection.update_one.side_effect = NetworkTimeout()
get_collection.return_value = collection
args = ("one", 2, {3: 4, 5: 6})
result = set_upload_results_attached(*args)
collection.update_one.assert_called_once_with(
{'_id': args_to_uid(args)},
{"$set": {'attached': True}}
)
self.assertIs(result, None)
def save_upload_results(response_json, *args):
uid = args_to_uid(args)
collection = get_mongodb_collection()
try:
collection.insert({
'_id': uid,
'file_data': response_json,
'createdAt': datetime.utcnow(),
})
except PyMongoError as exc:
logger.exception(
exc, extra={"MESSAGE_ID": "EDR_POST_RESULTS_MONGODB_EXCEPTION"})
else:
return uid
def test_save_upload_results(self, get_collection):
task = MagicMock()
task.name = "bot.some_task"
collection = MagicMock()
get_collection.return_value = collection
file_data = {"data": 1, "meta": "hello"}
args = ("one", 2, [3, 4])
with patch("tasks_utils.results_db.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
result = save_task_result(task, file_data, args)
collection.insert_one.assert_called_once_with({
'_id':
args_to_uid(("bot.some_task", ) + args),
'result':
file_data,
'createdAt':
datetime_mock.utcnow.return_value
})
self.assertEqual(result, args_to_uid(("bot.some_task", ) + args))
def save_task_result(self, result, args):
args = tuple((self.name, *args))
uid = args_to_uid(args)
collection = get_mongodb_collection()
try:
collection.insert_one({
'_id': uid,
'result': result,
'createdAt': datetime.utcnow(),
})
except PyMongoError as exc:
logger.exception(
exc, extra={"MESSAGE_ID": "POST_TASK_RESULTS_MONGODB_EXCEPTION"})
else:
return uid
def test_unique_task_decorator_for_bind_task(self, get_collection):
get_collection.return_value.find_one.return_value = None
test_method_procedure = Mock()
# create decorated function
@app.task(bind=True, lazy=False)
@unique_lock
def test_method(self, *args, **kwargs):
test_method_procedure(self, *args, **kwargs)
class RetryException(Exception):
pass
test_method.retry = RetryException
with patch("celery_worker.locks.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
# run it
test_method(1, 2, message="Hi")
# check that decorator checked the lock and set it
task_uid = "v2_" + args_to_uid(
(test_method.__module__, test_method.__name__,
(1, 2), dict(message="Hi")))
get_collection.return_value.find_one.assert_called_once_with({
"_id":
task_uid,
"name":
test_method.__name__,
"module":
test_method.__module__
})
test_method_procedure.assert_called_once_with(test_method,
1,
2,
message="Hi")
get_collection.return_value.insert.assert_called_once_with({
"_id":
task_uid,
"name":
test_method.__name__,
"module":
test_method.__module__,
"createdAt":
datetime_mock.utcnow.return_value
})
def test_get_upload_results_raise_exc(self, get_collection):
class RetryException(Exception):
pass
task = MagicMock()
task.retry = RetryException
collection = MagicMock()
collection.find_one.side_effect = NetworkTimeout
get_collection.return_value = collection
args = (1, "two", 3)
with self.assertRaises(RetryException):
get_upload_results(task, *args)
collection.find_one.assert_called_once_with({'_id': args_to_uid(args)})
def test_args_to_uid(self):
result1 = args_to_uid((1, 2, [3, 4], "hello"))
result2 = args_to_uid((1, 2, [3, 4], "hello"))
self.assertEqual(result1, result2)
result3 = args_to_uid((12, [3, 4], "hello"))
self.assertEqual(result1, result3)
result4 = args_to_uid((1, 3, [3, 4], "hello"))
self.assertNotEqual(result1, result4)
result5 = args_to_uid(({"test": 1, "hello": [1, 2]}, ))
result6 = args_to_uid(({"test": 1, "hello": [1, 2]}, ))
self.assertEqual(result5, result6)
result7 = args_to_uid(({"test": 1, "hello": [1, 2]}, ))
result8 = args_to_uid(({"test": 1, "hello": [1, 2, 3]}, ))
self.assertNotEqual(result7, result8)
def test_unique_task_decorator_mongodb_get_exc(self, get_collection):
get_collection.return_value.find_one.side_effect = ConnectionFailure()
test_method_procedure = Mock()
# create decorated function
@unique_lock
def test_method(*args, **kwargs):
test_method_procedure(*args, **kwargs)
with patch("celery_worker.locks.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
# run it
test_method(1, 2, message="Hi")
# check that decorator checked the lock and set it
task_uid = "v2_" + args_to_uid(
(test_method.__module__, test_method.__name__,
(1, 2), dict(message="Hi")))
get_collection.return_value.find_one.assert_called_once_with({
"_id":
task_uid,
"name":
test_method.__name__,
"module":
test_method.__module__
})
test_method_procedure.assert_called_once_with(1, 2, message="Hi")
get_collection.return_value.insert.assert_called_once_with({
"_id":
task_uid,
"name":
test_method.__name__,
"module":
test_method.__module__,
"createdAt":
datetime_mock.utcnow.return_value
})
def data_to_uid(data, keys=None):
return args_to_uid(
sorted([
str(value) if not isinstance(value, dict) else data_to_uid(value)
for key, value in data.items() if keys is None or key in keys
]))
