def test_bulk_write(self): self.db.test.collection.bulk_write([ DeleteOne({'noCollation': 42}), DeleteMany({'noCollation': 42}), DeleteOne({'foo': 42}, collation=self.collation), DeleteMany({'foo': 42}, collation=self.collation), ReplaceOne({'noCollation': 24}, {'bar': 42}), UpdateOne({'noCollation': 84}, {'$set': {'bar': 10}}, upsert=True), UpdateMany({'noCollation': 45}, {'$set': {'bar': 42}}), ReplaceOne({'foo': 24}, {'foo': 42}, collation=self.collation), UpdateOne({'foo': 84}, {'$set': {'foo': 10}}, upsert=True, collation=self.collation), UpdateMany({'foo': 45}, {'$set': {'foo': 42}}, collation=self.collation) ]) delete_cmd = self.listener.results['started'][0].command update_cmd = self.listener.results['started'][1].command def check_ops(ops): for op in ops: if 'noCollation' in op['q']: self.assertNotIn('collation', op) else: self.assertEqual(self.collation.document, op['collation']) check_ops(delete_cmd['deletes']) check_ops(update_cmd['updates'])
def test_batch_splitting(self): """Test retry succeeds after failures during batch splitting.""" large = 's' * 1024 * 1024 * 15 coll = self.db.retryable_write_test coll.delete_many({}) self.listener.results.clear() bulk_result = coll.bulk_write([ InsertOne({'_id': 1, 'l': large}), InsertOne({'_id': 2, 'l': large}), InsertOne({'_id': 3, 'l': large}), UpdateOne({'_id': 1, 'l': large}, {'$unset': {'l': 1}, '$inc': {'count': 1}}), UpdateOne({'_id': 2, 'l': large}, {'$set': {'foo': 'bar'}}), DeleteOne({'l': large}), DeleteOne({'l': large})]) # Each command should fail and be retried. # With OP_MSG 3 inserts are one batch. 2 updates another. # 2 deletes a third. self.assertEqual(len(self.listener.results['started']), 6) self.assertEqual(coll.find_one(), {'_id': 1, 'count': 1}) # Assert the final result expected_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 3, "nUpserted": 0, "nMatched": 2, "nModified": 2, "nRemoved": 2, "upserted": [], } self.assertEqual(bulk_result.bulk_api_result, expected_result)
def test_DeleteOne(self): result = yield self.coll.bulk_write([ DeleteOne({'x': 42}), DeleteOne({'x': {'$gt': 100}}) ]) self.assertEqual((yield self.coll.count()), 1) self.assertIsInstance(result, BulkWriteResult) self.assertEqual(result.deleted_count, 2)
def non_retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}), UpdateMany({}, {'$set': {'a': 1}})]], {}), (coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}), (coll.update_many, [{}, {'$set': {'a': 1}}], {}), (coll.delete_many, [{}], {}), # Deprecated methods. # Multi remove. (coll.remove, [{}], {}), # Multi update. (coll.update, [{}, {'$set': {'a': 1}}], {'multi': True}), # Unacknowledged deprecated methods. (coll.insert, [{}], {'w': 0}), # Unacknowledged Non-multi update. (coll.update, [{}, {'$set': {'a': 1}}], {'w': 0}), # Unacknowledged Non-multi remove. (coll.remove, [{}], {'multi': False, 'w': 0}), # Unacknowledged Replace. (coll.find_and_modify, [{}, {'a': 3}], {'writeConcern': {'w': 0}}), # Unacknowledged Update. (coll.find_and_modify, [{}, {'$set': {'a': 1}}], {'writeConcern': {'w': 0}}), # Unacknowledged Delete. (coll.find_and_modify, [{}, {}], {'remove': True, 'writeConcern': {'w': 0}}), ]
def non_retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}), UpdateMany({}, {'$set': {'a': 1}})]], {}), (coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}), (coll.update_many, [{}, {'$set': {'a': 1}}], {}), (coll.delete_many, [{}], {}), ]
def bulk_operation(self, set_name, data_list, operation_type, ordered=False): requests = [] if operation_type == "insert": requests = list(map(lambda doc: InsertOne(doc), data_list)) elif operation_type == "update": requests = list(map(lambda info: UpdateOne(filter=info[0], update=info[1]), data_list)) elif operation_type == "delete": requests = list(map(lambda info: DeleteOne(filter=info), data_list)) elif operation_type == "multi_type": for key in data_list.keys(): if data_list[key]["op_type"] == "insert": requests.append(InsertOne(data_list[key]["info"])) elif data_list[key]["op_type"] == "update": requests.append(UpdateOne(filter=data_list[key]["info"][0], update=data_list[key]["info"][1])) elif data_list[key]["op_type"] == "delete": requests.append(DeleteOne(data_list[key]["info"])) return self.db[set_name].bulk_write(requests, ordered=ordered)
def test_batch_splitting(self): """Test retry succeeds after failures during batch splitting.""" large = 's' * 1024 * 1024 * 15 coll = self.db.retryable_write_test coll.delete_many({}) self.listener.results.clear() coll.bulk_write([ InsertOne({ '_id': 1, 'l': large }), InsertOne({ '_id': 2, 'l': large }), InsertOne({ '_id': 3, 'l': large }), UpdateOne({ '_id': 1, 'l': large }, { '$unset': { 'l': 1 }, '$inc': { 'count': 1 } }), UpdateOne({ '_id': 2, 'l': large }, {'$set': { 'foo': 'bar' }}), DeleteOne({'l': large}), DeleteOne({'l': large}) ]) # Each command should fail and be retried. self.assertEqual(len(self.listener.results['started']), 14) self.assertEqual(coll.find_one(), {'_id': 1, 'count': 1})
def retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[InsertOne({}), InsertOne({})]], {}), (coll.bulk_write, [[InsertOne({}), InsertOne({})]], {'ordered': False}), (coll.bulk_write, [[ReplaceOne({}, {})]], {}), (coll.bulk_write, [[ReplaceOne({}, {}), ReplaceOne({}, {})]], {}), (coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}), UpdateOne({}, {'$set': {'a': 1}})]], {}), (coll.bulk_write, [[DeleteOne({})]], {}), (coll.bulk_write, [[DeleteOne({}), DeleteOne({})]], {}), (coll.insert_one, [{}], {}), (coll.insert_many, [[{}, {}]], {}), (coll.replace_one, [{}, {}], {}), (coll.update_one, [{}, {'$set': {'a': 1}}], {}), (coll.delete_one, [{}], {}), (coll.find_one_and_replace, [{}, {'a': 3}], {}), (coll.find_one_and_update, [{}, {'$set': {'a': 1}}], {}), (coll.find_one_and_delete, [{}, {}], {}), ]
def retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[InsertOne({}), InsertOne({})]], {}), (coll.bulk_write, [[InsertOne({}), InsertOne({})]], {'ordered': False}), (coll.bulk_write, [[ReplaceOne({}, {})]], {}), (coll.bulk_write, [[ReplaceOne({}, {}), ReplaceOne({}, {})]], {}), (coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}), UpdateOne({}, {'$set': {'a': 1}})]], {}), (coll.bulk_write, [[DeleteOne({})]], {}), (coll.bulk_write, [[DeleteOne({}), DeleteOne({})]], {}), (coll.insert_one, [{}], {}), (coll.insert_many, [[{}, {}]], {}), (coll.replace_one, [{}, {}], {}), (coll.update_one, [{}, {'$set': {'a': 1}}], {}), (coll.delete_one, [{}], {}), (coll.find_one_and_replace, [{}, {'a': 3}], {}), (coll.find_one_and_update, [{}, {'$set': {'a': 1}}], {}), (coll.find_one_and_delete, [{}, {}], {}), # Deprecated methods. # Insert with single or multiple documents. (coll.insert, [{}], {}), (coll.insert, [[{}]], {}), (coll.insert, [[{}, {}]], {}), # Save with and without an _id. (coll.save, [{}], {}), (coll.save, [{'_id': ObjectId()}], {}), # Non-multi update. (coll.update, [{}, {'$set': {'a': 1}}], {}), # Non-multi remove. (coll.remove, [{}], {'multi': False}), # Replace. (coll.find_and_modify, [{}, {'a': 3}], {}), # Update. (coll.find_and_modify, [{}, {'$set': {'a': 1}}], {}), # Delete. (coll.find_and_modify, [{}, {}], {'remove': True}), ]
def test_DeleteOneNotEqualsDeleteMany(self): self.assertNotEqual(DeleteOne({'foo': 42}), DeleteMany({'foo': 42}))
def test_DeleteOneNotEquals(self): self.assertNotEqual(DeleteOne({'foo': 42}), DeleteOne({'foo': 23}))
lambda coll: coll.with_options(write_concern=WriteConcern( w=0)).bulk_write([ UpdateOne({"_id": 1}, {"$set": { "new": 1 }}), UpdateOne({"_id": 2}, {"$set": { "new": 1 }}) ], ordered=False), request=OpMsg({"update": "coll"}, flags=OP_MSG_FLAGS['moreToCome']), reply=None), WriteOperation('bulk_write_delete', lambda coll: coll.bulk_write( [DeleteOne({"_id": 1}), DeleteOne({"_id": 2})]), request=OpMsg({"delete": "coll"}, flags=0), reply={ 'ok': 1, 'n': 2 }), WriteOperation( 'bulk_write_delete-w0', lambda coll: coll.with_options(write_concern=WriteConcern( w=0)).bulk_write([DeleteOne({"_id": 1}), DeleteOne({"_id": 2})]), request=OpMsg({"delete": "coll"}, flags=0), reply={ 'ok': 1, 'n': 2
def bulk_remove(self, filter, multi): if multi: self._requests.append(DeleteMany(filter)) else: self._requests.append(DeleteOne(filter))
def delete(self): self._add_operation(DeleteOne({"_id": self.id}))
UpdateOne({"_id": 1}, {"$set": {"new": 1}}), UpdateOne({"_id": 2}, {"$set": {"new": 1}})]), request=OpMsg({"update": "coll"}, flags=0), reply={'ok': 1, 'n': 2, 'nModified': 2}), Operation( 'bulk_write_update-w0-unordered', lambda coll: coll.with_options( write_concern=WriteConcern(w=0)).bulk_write([ UpdateOne({"_id": 1}, {"$set": {"new": 1}}), UpdateOne({"_id": 2}, {"$set": {"new": 1}})], ordered=False), request=OpMsg({"update": "coll"}, flags=OP_MSG_FLAGS['moreToCome']), reply=None), Operation( 'bulk_write_delete', lambda coll: coll.bulk_write([ DeleteOne({"_id": 1}), DeleteOne({"_id": 2})]), request=OpMsg({"delete": "coll"}, flags=0), reply={'ok': 1, 'n': 2}), Operation( 'bulk_write_delete-w0', lambda coll: coll.with_options( write_concern=WriteConcern(w=0)).bulk_write([ DeleteOne({"_id": 1}), DeleteOne({"_id": 2})]), request=OpMsg({"delete": "coll"}, flags=0), reply={'ok': 1, 'n': 2}), Operation( 'bulk_write_delete-w0-unordered', lambda coll: coll.with_options( write_concern=WriteConcern(w=0)).bulk_write([ DeleteOne({"_id": 1}), DeleteOne({"_id": 2})], ordered=False), request=OpMsg({"delete": "coll"}, flags=OP_MSG_FLAGS['moreToCome']),