def insert_many(self, docs, **kwargs): docs_with_id = [add_id(fix_date(d)) for d in docs or []] ordered = kwargs['ordered'] if 'ordered' in kwargs else True success = True results = { 'nInserted': 0, 'nMatched': 0, 'nModified': 0, 'nRemoved': 0, 'nUpserted': 0, 'writeConcernErrors': [], 'writeErrors': [] } for doc in docs_with_id: try: self._check_duplicate(doc) self.data.append(doc) results['nInserted'] += 1 except DuplicateKeyError: success = False if ordered: raise BulkWriteError(results) else: continue if success: return InsertManyResult([d['_id'] for d in docs_with_id]) else: raise BulkWriteError(results)
def _raise_bulk_write_error(full_result): """Raise a BulkWriteError from the full bulk api result. """ if full_result["writeErrors"]: full_result["writeErrors"].sort( key=lambda error: error["index"]) raise BulkWriteError(full_result)
def test_task_mappings_exceptions(self, update_one_mock, exception_mock): mongo_mock = MagicMock() test_mapping_id = {"test_mapping_id": 1} test_file = { "name": "jstests/core/txns/commands_not_allowed_in_txn.js", "test_file_seen_count": 1, } details = {"errorLabels": []} mongo_mock.test_mappings_test_files.return_value.bulk_write.side_effect = BulkWriteError( details) pytest.raises( BulkWriteError, under_test.update_test_mappings_test_files, [test_file], test_mapping_id, mongo_mock, ) exception_mock.assert_called_once_with( "bulk_write error", parent=test_mapping_id, operations=[update_one_mock.return_value], details=details, )
def test_task_mappings_exceptions(self, update_one_mock, exception_mock): mongo_mock = MagicMock() task_mapping_id = {"task_mapping_id": 1} task = { "name": "query_fuzzer_standalone_3_enterprise-rhel-62-64-bit", "variant": "enterprise-rhel-62-64-bit", "flip_count": 1, } details = {"errorLabels": []} mongo_mock.task_mappings_tasks.return_value.bulk_write.side_effect = BulkWriteError( details) pytest.raises( BulkWriteError, under_test.update_task_mappings_tasks, [task], task_mapping_id, mongo_mock, ) exception_mock.assert_called_once_with( "bulk_write error", parent=task_mapping_id, operations=[update_one_mock.return_value], details=details, )
def execute_command(self, generator, write_concern, session): """Execute using write commands. """ # nModified is only reported for write commands, not legacy ops. full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nModified": 0, "nRemoved": 0, "upserted": [], } op_id = _randint() def retryable_bulk(session, sock_info, retryable): self._execute_command( generator, write_concern, session, sock_info, op_id, retryable, full_result) client = self.collection.database.client with client._tmp_session(session) as s: client._retry_with_session( self.is_retryable, retryable_bulk, s, self) if full_result["writeErrors"] or full_result["writeConcernErrors"]: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result
def execute_command(self, sock_info, generator, write_concern, session): """Execute using write commands. """ # nModified is only reported for write commands, not legacy ops. full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nModified": 0, "nRemoved": 0, "upserted": [], } op_id = _randint() db_name = self.collection.database.name client = self.collection.database.client listeners = client._event_listeners with self.collection.database.client._tmp_session(session) as s: # sock_info.command checks auth, but we use sock_info.write_command. sock_info.check_session_auth_matches(s) for run in generator: cmd = SON([(_COMMANDS[run.op_type], self.collection.name), ('ordered', self.ordered)]) if write_concern.document: cmd['writeConcern'] = write_concern.document if self.bypass_doc_val and sock_info.max_wire_version >= 4: cmd['bypassDocumentValidation'] = True if s: cmd['lsid'] = s._use_lsid() client._send_cluster_time(cmd) bwc = _BulkWriteContext(db_name, cmd, sock_info, op_id, listeners, s) results = _do_batched_write_command( self.namespace, run.op_type, cmd, run.ops, True, self.collection.codec_options, bwc) _merge_command(run, full_result, results) last_result = results[-1][1] client._receive_cluster_time(last_result) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result['writeErrors']: break if full_result["writeErrors"] or full_result["writeConcernErrors"]: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result
def inner(*args, **kwargs) -> None: try: func(*args, **kwargs) except BulkWriteError as e: error(e.__str__()) raise BulkWriteError('Erro ao inserir os dados.') except OperationFailure as e: error(e.__str__()) raise OperationFailure('Erro ao realizar a operação.') except TypeError as e: error(e.__str__()) raise TypeError('Erro no tipo de dado.') finally: from .connectionDatabase import ConnectionDatabase ConnectionDatabase().get_connection.close()
def test_export_to_mongo_logs_error_correctly_on_bulk_write_error_without_duplicates( subject, logger, mongo_database): _, mongo_database = mongo_database bulk_write_error = BulkWriteError({ "errorLabels": [], "writeErrors": [{ "code": 999 }] }) with patch.object(Collection, "insert_many", side_effect=bulk_write_error): with pytest.raises(TransientRabbitError): subject.export_to_mongo() logger.critical.assert_called_once() logger.exception.assert_called_once_with(bulk_write_error)
def execute(self): """ Execute the bulk query. :return: :py:class:`BulkResult` instance """ try: raw_data = self._bulk_mongo.execute() except BulkWriteError as exc: self._error = True raw_data = exc.details self._result = BulkResult(self, raw_data) if self._error and self._raise_on_errors: raise BulkWriteError(raw_data) return self._result
def execute_command(self, sock_info, generator, write_concern): """Execute using write commands. """ # nModified is only reported for write commands, not legacy ops. full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nModified": 0, "nRemoved": 0, "upserted": [], } op_id = _randint() db_name = self.collection.database.name listeners = self.collection.database.client._event_listeners for run in generator: cmd = SON([(_COMMANDS[run.op_type], self.collection.name), ('ordered', self.ordered)]) if write_concern.document: cmd['writeConcern'] = write_concern.document bwc = _BulkWriteContext(db_name, cmd, sock_info, op_id, listeners) results = _do_batched_write_command(self.namespace, run.op_type, cmd, run.ops, True, self.collection.codec_options, bwc) _merge_command(run, full_result, results) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result['writeErrors']: break if full_result["writeErrors"] or full_result["writeConcernErrors"]: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result
def _execute_bulk(self, bulk): if not bulk.ops: raise InvalidOperation("No operations to execute") if bulk.executed: raise InvalidOperation("Bulk operations can only be executed once") bulk.executed = True if bulk.ordered: generator = bulk.gen_ordered() else: generator = bulk.gen_unordered() full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nModified": 0, "nRemoved": 0, "upserted": [], } for run in generator: result = yield self._execute_batch_command(run.op_type, run.ops, bulk.ordered) _merge_command(run, full_result, result) if bulk.ordered and full_result["writeErrors"]: break if self.write_concern.acknowledged: if full_result["writeErrors"] or full_result["writeConcernErrors"]: if full_result["writeErrors"]: full_result["writeErrors"].sort( key=lambda error: error["index"]) raise BulkWriteError(full_result) defer.returnValue( BulkWriteResult(full_result, self.write_concern.acknowledged))
async def execute_command(self, connection: 'aiomongo.Connection', generator: Iterable[_Run], write_concern: WriteConcern) -> dict: """Execute using write commands. """ # nModified is only reported for write commands, not legacy ops. full_result = { 'writeErrors': [], 'writeConcernErrors': [], 'nInserted': 0, 'nUpserted': 0, 'nMatched': 0, 'nModified': 0, 'nRemoved': 0, 'upserted': [], } for run in generator: cmd = SON([(_COMMANDS[run.op_type], self.collection.name), ('ordered', self.ordered)]) if write_concern.document: cmd['writeConcern'] = write_concern.document if self.bypass_doc_val and connection.max_wire_version >= 4: cmd['bypassDocumentValidation'] = True results = await self._do_batched_write_command( self.namespace, run.op_type, cmd, run.ops, True, self.collection.codec_options, connection) _merge_command(run, full_result, results) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result['writeErrors']: break if full_result['writeErrors'] or full_result['writeConcernErrors']: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result
def execute_command(self, generator, write_concern): """Execute using write commands. """ uuid_subtype = self.collection.uuid_subtype client = self.collection.database.connection # nModified is only reported for write commands, not legacy ops. full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nModified": 0, "nRemoved": 0, "upserted": [], } for run in generator: cmd = SON([(_COMMANDS[run.op_type], self.collection.name), ('ordered', self.ordered)]) if write_concern: cmd['writeConcern'] = write_concern results = _do_batched_write_command(self.namespace, run.op_type, cmd, run.ops, True, uuid_subtype, client) _merge_command(run, full_result, results) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result['writeErrors']: break if full_result["writeErrors"] or full_result["writeConcernErrors"]: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result
def test_export_to_mongo_logs_error_correctly_on_bulk_write_error_with_mix_of_errors( subject, mongo_database): _, mongo_database = mongo_database bulk_write_error = BulkWriteError({ "errorLabels": [], "writeErrors": [{ "code": 11000, "op": MagicMock() }, { "code": 999 }] }) with patch.object(Collection, "insert_many", side_effect=bulk_write_error): subject.export_to_mongo() # No documents were inserted in either collection samples_collection = get_mongo_collection(mongo_database, COLLECTION_SAMPLES) assert samples_collection.count_documents({}) == 0 source_plates_collection = get_mongo_collection(mongo_database, COLLECTION_SOURCE_PLATES) assert source_plates_collection.count_documents({}) == 0
def raiseError(*args, **kwargs): details = {"writeErrors": [{"code": 2345}]} raise BulkWriteError(details)
def execute_command(self, sock_info, generator, write_concern, session): """Execute using write commands. """ # nModified is only reported for write commands, not legacy ops. full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nModified": 0, "nRemoved": 0, "upserted": [], } op_id = _randint() db_name = self.collection.database.name client = self.collection.database.client listeners = client._event_listeners with self.collection.database.client._tmp_session(session) as s: # sock_info.command checks auth, but we use sock_info.write_command. sock_info.check_session_auth_matches(s) for run in generator: cmd = SON([(_COMMANDS[run.op_type], self.collection.name), ('ordered', self.ordered)]) if write_concern.document: cmd['writeConcern'] = write_concern.document if self.bypass_doc_val and sock_info.max_wire_version >= 4: cmd['bypassDocumentValidation'] = True if s: cmd['lsid'] = s._use_lsid() bwc = _BulkWriteContext(db_name, cmd, sock_info, op_id, listeners, s) results = [] idx_offset = 0 while idx_offset < len(run.ops): check_keys = run.op_type == _INSERT ops = islice(run.ops, idx_offset, None) # Run as many ops as possible. client._send_cluster_time(cmd, s) request_id, msg, to_send = _do_batched_write_command( self.namespace, run.op_type, cmd, ops, check_keys, self.collection.codec_options, bwc) if not to_send: raise InvalidOperation("cannot do an empty bulk write") result = bwc.write_command(request_id, msg, to_send) client._receive_cluster_time(result) if s is not None: s._advance_cluster_time(result.get("$clusterTime")) s._advance_operation_time(result.get("operationTime")) results.append((idx_offset, result)) if self.ordered and "writeErrors" in result: break idx_offset += len(to_send) _merge_command(run, full_result, results) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result['writeErrors']: break if full_result["writeErrors"] or full_result["writeConcernErrors"]: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result
def execute_legacy(self, sock_info, generator, write_concern): """Execute using legacy wire protocol ops. """ coll = self.collection full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nRemoved": 0, "upserted": [], } op_id = _randint() stop = False for run in generator: for idx, operation in enumerate(run.ops): try: # To do per-operation reporting we have to do ops one # at a time. That means the performance of bulk insert # will be slower here than calling Collection.insert() if run.op_type == _INSERT: coll._insert(sock_info, operation, self.ordered, write_concern=write_concern, op_id=op_id) result = {} elif run.op_type == _UPDATE: doc = operation['u'] check_keys = True if doc and next(iter(doc)).startswith('$'): check_keys = False result = coll._update(sock_info, operation['q'], doc, operation['upsert'], check_keys, operation['multi'], write_concern=write_concern, op_id=op_id, ordered=self.ordered) else: result = coll._delete(sock_info, operation['q'], not operation['limit'], write_concern, op_id, self.ordered) _merge_legacy(run, full_result, result, idx) except DocumentTooLarge as exc: # MongoDB 2.6 uses error code 2 for "too large". error = _make_error(run.index(idx), _BAD_VALUE, str(exc), operation) full_result['writeErrors'].append(error) if self.ordered: stop = True break except OperationFailure as exc: if not exc.details: # Some error not related to the write operation # (e.g. kerberos failure). Re-raise immediately. raise _merge_legacy(run, full_result, exc.details, idx) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result["writeErrors"]: stop = True break if stop: break if full_result["writeErrors"] or full_result['writeConcernErrors']: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result
# (e.g. kerberos failure). Re-raise immediately. raise _merge_legacy(run, full_result, exc.details, idx) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result["writeErrors"]: stop = True break if stop: break if full_result["writeErrors"] or full_result['writeConcernErrors']: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result def execute(self, write_concern): """Execute operations. """ if not self.ops: raise InvalidOperation('No operations to execute') if self.executed: raise InvalidOperation('Bulk operations can ' 'only be executed once.') self.executed = True client = self.collection.database.connection client._ensure_connected(sync=True) write_concern = write_concern or self.collection.write_concern
SAVE_EXCEPT_TEST_CASES = [ ( { "id": "opsmatters_uk.ThelatestupdateforBroadcomincludesDXNetOps202Netwo.opentracing" }, BulkWriteError( { "writeErrors": [ { "index": 0, "code": 11000, "keyPattern": {"id": 1}, "keyValue": { "id": "opsmatters_uk.ThelatestupdateforBroadcomincludesDXNetOps202Netwo.opentracing" }, } ], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nModified": 0, "nRemoved": 0, "upserted": [], }, ), 0, { "writeErrors": [ { "index": 0,
def test_pickle_BulkWriteError(self): exc = BulkWriteError({}) self.assertOperationFailureEqual(exc, pickle.loads(pickle.dumps(exc))) self.assertIn("batch op errors occurred", str(exc))
def raise_error(): error["writeErrors"].sort(key=lambda error: error["index"]) for write_error in error["writeErrors"]: write_error[u"op"] = documents[write_error["index"]] raise BulkWriteError(error)