def test_bulk_write(self):
self.db.test.collection.bulk_write([
DeleteOne({'noCollation': 42}),
DeleteMany({'noCollation': 42}),
DeleteOne({'foo': 42}, collation=self.collation),
DeleteMany({'foo': 42}, collation=self.collation),
ReplaceOne({'noCollation': 24}, {'bar': 42}),
UpdateOne({'noCollation': 84}, {'$set': {'bar': 10}}, upsert=True),
UpdateMany({'noCollation': 45}, {'$set': {'bar': 42}}),
ReplaceOne({'foo': 24}, {'foo': 42}, collation=self.collation),
UpdateOne({'foo': 84}, {'$set': {'foo': 10}}, upsert=True,
collation=self.collation),
UpdateMany({'foo': 45}, {'$set': {'foo': 42}},
collation=self.collation)
])
delete_cmd = self.listener.results['started'][0].command
update_cmd = self.listener.results['started'][1].command
def check_ops(ops):
for op in ops:
if 'noCollation' in op['q']:
self.assertNotIn('collation', op)
else:
self.assertEqual(self.collation.document,
op['collation'])
check_ops(delete_cmd['deletes'])
check_ops(update_cmd['updates'])
def test_batch_splitting(self):
"""Test retry succeeds after failures during batch splitting."""
large = 's' * 1024 * 1024 * 15
coll = self.db.retryable_write_test
coll.delete_many({})
self.listener.results.clear()
bulk_result = coll.bulk_write([
InsertOne({'_id': 1, 'l': large}),
InsertOne({'_id': 2, 'l': large}),
InsertOne({'_id': 3, 'l': large}),
UpdateOne({'_id': 1, 'l': large},
{'$unset': {'l': 1}, '$inc': {'count': 1}}),
UpdateOne({'_id': 2, 'l': large}, {'$set': {'foo': 'bar'}}),
DeleteOne({'l': large}),
DeleteOne({'l': large})])
# Each command should fail and be retried.
# With OP_MSG 3 inserts are one batch. 2 updates another.
# 2 deletes a third.
self.assertEqual(len(self.listener.results['started']), 6)
self.assertEqual(coll.find_one(), {'_id': 1, 'count': 1})
# Assert the final result
expected_result = {
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 3,
"nUpserted": 0,
"nMatched": 2,
"nModified": 2,
"nRemoved": 2,
"upserted": [],
}
self.assertEqual(bulk_result.bulk_api_result, expected_result)
def test_DeleteOne(self):
result = yield self.coll.bulk_write([
DeleteOne({'x': 42}),
DeleteOne({'x': {'$gt': 100}})
])
self.assertEqual((yield self.coll.count()), 1)
self.assertIsInstance(result, BulkWriteResult)
self.assertEqual(result.deleted_count, 2)
def non_retryable_single_statement_ops(coll):
return [
(coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}),
UpdateMany({}, {'$set': {'a': 1}})]], {}),
(coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}),
(coll.update_many, [{}, {'$set': {'a': 1}}], {}),
(coll.delete_many, [{}], {}),
# Deprecated methods.
# Multi remove.
(coll.remove, [{}], {}),
# Multi update.
(coll.update, [{}, {'$set': {'a': 1}}], {'multi': True}),
# Unacknowledged deprecated methods.
(coll.insert, [{}], {'w': 0}),
# Unacknowledged Non-multi update.
(coll.update, [{}, {'$set': {'a': 1}}], {'w': 0}),
# Unacknowledged Non-multi remove.
(coll.remove, [{}], {'multi': False, 'w': 0}),
# Unacknowledged Replace.
(coll.find_and_modify, [{}, {'a': 3}], {'writeConcern': {'w': 0}}),
# Unacknowledged Update.
(coll.find_and_modify, [{}, {'$set': {'a': 1}}],
{'writeConcern': {'w': 0}}),
# Unacknowledged Delete.
(coll.find_and_modify, [{}, {}],
{'remove': True, 'writeConcern': {'w': 0}}),
]
def non_retryable_single_statement_ops(coll):
return [
(coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}),
UpdateMany({}, {'$set': {'a': 1}})]], {}),
(coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}),
(coll.update_many, [{}, {'$set': {'a': 1}}], {}),
(coll.delete_many, [{}], {}),
]
def bulk_operation(self, set_name, data_list, operation_type, ordered=False):
requests = []
if operation_type == "insert":
requests = list(map(lambda doc: InsertOne(doc), data_list))
elif operation_type == "update":
requests = list(map(lambda info: UpdateOne(filter=info[0], update=info[1]), data_list))
elif operation_type == "delete":
requests = list(map(lambda info: DeleteOne(filter=info), data_list))
elif operation_type == "multi_type":
for key in data_list.keys():
if data_list[key]["op_type"] == "insert":
requests.append(InsertOne(data_list[key]["info"]))
elif data_list[key]["op_type"] == "update":
requests.append(UpdateOne(filter=data_list[key]["info"][0], update=data_list[key]["info"][1]))
elif data_list[key]["op_type"] == "delete":
requests.append(DeleteOne(data_list[key]["info"]))
return self.db[set_name].bulk_write(requests, ordered=ordered)
def test_batch_splitting(self):
"""Test retry succeeds after failures during batch splitting."""
large = 's' * 1024 * 1024 * 15
coll = self.db.retryable_write_test
coll.delete_many({})
self.listener.results.clear()
coll.bulk_write([
InsertOne({
'_id': 1,
'l': large
}),
InsertOne({
'_id': 2,
'l': large
}),
InsertOne({
'_id': 3,
'l': large
}),
UpdateOne({
'_id': 1,
'l': large
}, {
'$unset': {
'l': 1
},
'$inc': {
'count': 1
}
}),
UpdateOne({
'_id': 2,
'l': large
}, {'$set': {
'foo': 'bar'
}}),
DeleteOne({'l': large}),
DeleteOne({'l': large})
])
# Each command should fail and be retried.
self.assertEqual(len(self.listener.results['started']), 14)
self.assertEqual(coll.find_one(), {'_id': 1, 'count': 1})
def retryable_single_statement_ops(coll):
return [
(coll.bulk_write, [[InsertOne({}), InsertOne({})]], {}),
(coll.bulk_write, [[InsertOne({}),
InsertOne({})]], {'ordered': False}),
(coll.bulk_write, [[ReplaceOne({}, {})]], {}),
(coll.bulk_write, [[ReplaceOne({}, {}), ReplaceOne({}, {})]], {}),
(coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}),
UpdateOne({}, {'$set': {'a': 1}})]], {}),
(coll.bulk_write, [[DeleteOne({})]], {}),
(coll.bulk_write, [[DeleteOne({}), DeleteOne({})]], {}),
(coll.insert_one, [{}], {}),
(coll.insert_many, [[{}, {}]], {}),
(coll.replace_one, [{}, {}], {}),
(coll.update_one, [{}, {'$set': {'a': 1}}], {}),
(coll.delete_one, [{}], {}),
(coll.find_one_and_replace, [{}, {'a': 3}], {}),
(coll.find_one_and_update, [{}, {'$set': {'a': 1}}], {}),
(coll.find_one_and_delete, [{}, {}], {}),
]
def retryable_single_statement_ops(coll):
return [
(coll.bulk_write, [[InsertOne({}), InsertOne({})]], {}),
(coll.bulk_write, [[InsertOne({}),
InsertOne({})]], {'ordered': False}),
(coll.bulk_write, [[ReplaceOne({}, {})]], {}),
(coll.bulk_write, [[ReplaceOne({}, {}), ReplaceOne({}, {})]], {}),
(coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}),
UpdateOne({}, {'$set': {'a': 1}})]], {}),
(coll.bulk_write, [[DeleteOne({})]], {}),
(coll.bulk_write, [[DeleteOne({}), DeleteOne({})]], {}),
(coll.insert_one, [{}], {}),
(coll.insert_many, [[{}, {}]], {}),
(coll.replace_one, [{}, {}], {}),
(coll.update_one, [{}, {'$set': {'a': 1}}], {}),
(coll.delete_one, [{}], {}),
(coll.find_one_and_replace, [{}, {'a': 3}], {}),
(coll.find_one_and_update, [{}, {'$set': {'a': 1}}], {}),
(coll.find_one_and_delete, [{}, {}], {}),
# Deprecated methods.
# Insert with single or multiple documents.
(coll.insert, [{}], {}),
(coll.insert, [[{}]], {}),
(coll.insert, [[{}, {}]], {}),
# Save with and without an _id.
(coll.save, [{}], {}),
(coll.save, [{'_id': ObjectId()}], {}),
# Non-multi update.
(coll.update, [{}, {'$set': {'a': 1}}], {}),
# Non-multi remove.
(coll.remove, [{}], {'multi': False}),
# Replace.
(coll.find_and_modify, [{}, {'a': 3}], {}),
# Update.
(coll.find_and_modify, [{}, {'$set': {'a': 1}}], {}),
# Delete.
(coll.find_and_modify, [{}, {}], {'remove': True}),
]
def test_DeleteOneNotEqualsDeleteMany(self):
self.assertNotEqual(DeleteOne({'foo': 42}), DeleteMany({'foo': 42}))
def test_DeleteOneNotEquals(self):
self.assertNotEqual(DeleteOne({'foo': 42}), DeleteOne({'foo': 23}))
lambda coll: coll.with_options(write_concern=WriteConcern(
w=0)).bulk_write([
UpdateOne({"_id": 1}, {"$set": {
"new": 1
}}),
UpdateOne({"_id": 2}, {"$set": {
"new": 1
}})
],
ordered=False),
request=OpMsg({"update": "coll"},
flags=OP_MSG_FLAGS['moreToCome']),
reply=None),
WriteOperation('bulk_write_delete',
lambda coll: coll.bulk_write(
[DeleteOne({"_id": 1}),
DeleteOne({"_id": 2})]),
request=OpMsg({"delete": "coll"}, flags=0),
reply={
'ok': 1,
'n': 2
}),
WriteOperation(
'bulk_write_delete-w0',
lambda coll: coll.with_options(write_concern=WriteConcern(
w=0)).bulk_write([DeleteOne({"_id": 1}),
DeleteOne({"_id": 2})]),
request=OpMsg({"delete": "coll"}, flags=0),
reply={
'ok': 1,
'n': 2
def bulk_remove(self, filter, multi):
if multi:
self._requests.append(DeleteMany(filter))
else:
self._requests.append(DeleteOne(filter))
def delete(self):
self._add_operation(DeleteOne({"_id": self.id}))
UpdateOne({"_id": 1}, {"$set": {"new": 1}}),
UpdateOne({"_id": 2}, {"$set": {"new": 1}})]),
request=OpMsg({"update": "coll"}, flags=0),
reply={'ok': 1, 'n': 2, 'nModified': 2}),
Operation(
'bulk_write_update-w0-unordered',
lambda coll: coll.with_options(
write_concern=WriteConcern(w=0)).bulk_write([
UpdateOne({"_id": 1}, {"$set": {"new": 1}}),
UpdateOne({"_id": 2}, {"$set": {"new": 1}})], ordered=False),
request=OpMsg({"update": "coll"}, flags=OP_MSG_FLAGS['moreToCome']),
reply=None),
Operation(
'bulk_write_delete',
lambda coll: coll.bulk_write([
DeleteOne({"_id": 1}), DeleteOne({"_id": 2})]),
request=OpMsg({"delete": "coll"}, flags=0),
reply={'ok': 1, 'n': 2}),
Operation(
'bulk_write_delete-w0',
lambda coll: coll.with_options(
write_concern=WriteConcern(w=0)).bulk_write([
DeleteOne({"_id": 1}), DeleteOne({"_id": 2})]),
request=OpMsg({"delete": "coll"}, flags=0),
reply={'ok': 1, 'n': 2}),
Operation(
'bulk_write_delete-w0-unordered',
lambda coll: coll.with_options(
write_concern=WriteConcern(w=0)).bulk_write([
DeleteOne({"_id": 1}), DeleteOne({"_id": 2})], ordered=False),
request=OpMsg({"delete": "coll"}, flags=OP_MSG_FLAGS['moreToCome']),
