def insert_many(self, docs, **kwargs):
docs_with_id = [add_id(fix_date(d)) for d in docs or []]
ordered = kwargs['ordered'] if 'ordered' in kwargs else True
success = True
results = {
'nInserted': 0,
'nMatched': 0,
'nModified': 0,
'nRemoved': 0,
'nUpserted': 0,
'writeConcernErrors': [],
'writeErrors': []
}
for doc in docs_with_id:
try:
self._check_duplicate(doc)
self.data.append(doc)
results['nInserted'] += 1
except DuplicateKeyError:
success = False
if ordered:
raise BulkWriteError(results)
else:
continue
if success:
return InsertManyResult([d['_id'] for d in docs_with_id])
else:
raise BulkWriteError(results)
def _raise_bulk_write_error(full_result):
"""Raise a BulkWriteError from the full bulk api result.
"""
if full_result["writeErrors"]:
full_result["writeErrors"].sort(
key=lambda error: error["index"])
raise BulkWriteError(full_result)
def test_task_mappings_exceptions(self, update_one_mock, exception_mock):
mongo_mock = MagicMock()
test_mapping_id = {"test_mapping_id": 1}
test_file = {
"name": "jstests/core/txns/commands_not_allowed_in_txn.js",
"test_file_seen_count": 1,
}
details = {"errorLabels": []}
mongo_mock.test_mappings_test_files.return_value.bulk_write.side_effect = BulkWriteError(
details)
pytest.raises(
BulkWriteError,
under_test.update_test_mappings_test_files,
[test_file],
test_mapping_id,
mongo_mock,
)
exception_mock.assert_called_once_with(
"bulk_write error",
parent=test_mapping_id,
operations=[update_one_mock.return_value],
details=details,
)
def test_task_mappings_exceptions(self, update_one_mock, exception_mock):
mongo_mock = MagicMock()
task_mapping_id = {"task_mapping_id": 1}
task = {
"name": "query_fuzzer_standalone_3_enterprise-rhel-62-64-bit",
"variant": "enterprise-rhel-62-64-bit",
"flip_count": 1,
}
details = {"errorLabels": []}
mongo_mock.task_mappings_tasks.return_value.bulk_write.side_effect = BulkWriteError(
details)
pytest.raises(
BulkWriteError,
under_test.update_task_mappings_tasks,
[task],
task_mapping_id,
mongo_mock,
)
exception_mock.assert_called_once_with(
"bulk_write error",
parent=task_mapping_id,
operations=[update_one_mock.return_value],
details=details,
)
def execute_command(self, generator, write_concern, session):
"""Execute using write commands.
"""
# nModified is only reported for write commands, not legacy ops.
full_result = {
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nRemoved": 0,
"upserted": [],
}
op_id = _randint()
def retryable_bulk(session, sock_info, retryable):
self._execute_command(
generator, write_concern, session, sock_info, op_id,
retryable, full_result)
client = self.collection.database.client
with client._tmp_session(session) as s:
client._retry_with_session(
self.is_retryable, retryable_bulk, s, self)
if full_result["writeErrors"] or full_result["writeConcernErrors"]:
if full_result['writeErrors']:
full_result['writeErrors'].sort(
key=lambda error: error['index'])
raise BulkWriteError(full_result)
return full_result
def execute_command(self, sock_info, generator, write_concern, session):
"""Execute using write commands.
"""
# nModified is only reported for write commands, not legacy ops.
full_result = {
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nRemoved": 0,
"upserted": [],
}
op_id = _randint()
db_name = self.collection.database.name
client = self.collection.database.client
listeners = client._event_listeners
with self.collection.database.client._tmp_session(session) as s:
# sock_info.command checks auth, but we use sock_info.write_command.
sock_info.check_session_auth_matches(s)
for run in generator:
cmd = SON([(_COMMANDS[run.op_type], self.collection.name),
('ordered', self.ordered)])
if write_concern.document:
cmd['writeConcern'] = write_concern.document
if self.bypass_doc_val and sock_info.max_wire_version >= 4:
cmd['bypassDocumentValidation'] = True
if s:
cmd['lsid'] = s._use_lsid()
client._send_cluster_time(cmd)
bwc = _BulkWriteContext(db_name, cmd, sock_info, op_id,
listeners, s)
results = _do_batched_write_command(
self.namespace, run.op_type, cmd,
run.ops, True, self.collection.codec_options, bwc)
_merge_command(run, full_result, results)
last_result = results[-1][1]
client._receive_cluster_time(last_result)
# We're supposed to continue if errors are
# at the write concern level (e.g. wtimeout)
if self.ordered and full_result['writeErrors']:
break
if full_result["writeErrors"] or full_result["writeConcernErrors"]:
if full_result['writeErrors']:
full_result['writeErrors'].sort(
key=lambda error: error['index'])
raise BulkWriteError(full_result)
return full_result
def inner(*args, **kwargs) -> None:
try:
func(*args, **kwargs)
except BulkWriteError as e:
error(e.__str__())
raise BulkWriteError('Erro ao inserir os dados.')
except OperationFailure as e:
error(e.__str__())
raise OperationFailure('Erro ao realizar a operação.')
except TypeError as e:
error(e.__str__())
raise TypeError('Erro no tipo de dado.')
finally:
from .connectionDatabase import ConnectionDatabase
ConnectionDatabase().get_connection.close()
def test_export_to_mongo_logs_error_correctly_on_bulk_write_error_without_duplicates(
subject, logger, mongo_database):
_, mongo_database = mongo_database
bulk_write_error = BulkWriteError({
"errorLabels": [],
"writeErrors": [{
"code": 999
}]
})
with patch.object(Collection, "insert_many", side_effect=bulk_write_error):
with pytest.raises(TransientRabbitError):
subject.export_to_mongo()
logger.critical.assert_called_once()
logger.exception.assert_called_once_with(bulk_write_error)
def execute(self):
""" Execute the bulk query.
:return: :py:class:`BulkResult` instance
"""
try:
raw_data = self._bulk_mongo.execute()
except BulkWriteError as exc:
self._error = True
raw_data = exc.details
self._result = BulkResult(self, raw_data)
if self._error and self._raise_on_errors:
raise BulkWriteError(raw_data)
return self._result
def execute_command(self, sock_info, generator, write_concern):
"""Execute using write commands.
"""
# nModified is only reported for write commands, not legacy ops.
full_result = {
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nRemoved": 0,
"upserted": [],
}
op_id = _randint()
db_name = self.collection.database.name
listeners = self.collection.database.client._event_listeners
for run in generator:
cmd = SON([(_COMMANDS[run.op_type], self.collection.name),
('ordered', self.ordered)])
if write_concern.document:
cmd['writeConcern'] = write_concern.document
bwc = _BulkWriteContext(db_name, cmd, sock_info, op_id, listeners)
results = _do_batched_write_command(self.namespace, run.op_type,
cmd, run.ops, True,
self.collection.codec_options,
bwc)
_merge_command(run, full_result, results)
# We're supposed to continue if errors are
# at the write concern level (e.g. wtimeout)
if self.ordered and full_result['writeErrors']:
break
if full_result["writeErrors"] or full_result["writeConcernErrors"]:
if full_result['writeErrors']:
full_result['writeErrors'].sort(
key=lambda error: error['index'])
raise BulkWriteError(full_result)
return full_result
def _execute_bulk(self, bulk):
if not bulk.ops:
raise InvalidOperation("No operations to execute")
if bulk.executed:
raise InvalidOperation("Bulk operations can only be executed once")
bulk.executed = True
if bulk.ordered:
generator = bulk.gen_ordered()
else:
generator = bulk.gen_unordered()
full_result = {
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nRemoved": 0,
"upserted": [],
}
for run in generator:
result = yield self._execute_batch_command(run.op_type, run.ops,
bulk.ordered)
_merge_command(run, full_result, result)
if bulk.ordered and full_result["writeErrors"]:
break
if self.write_concern.acknowledged:
if full_result["writeErrors"] or full_result["writeConcernErrors"]:
if full_result["writeErrors"]:
full_result["writeErrors"].sort(
key=lambda error: error["index"])
raise BulkWriteError(full_result)
defer.returnValue(
BulkWriteResult(full_result, self.write_concern.acknowledged))
async def execute_command(self, connection: 'aiomongo.Connection',
generator: Iterable[_Run],
write_concern: WriteConcern) -> dict:
"""Execute using write commands.
"""
# nModified is only reported for write commands, not legacy ops.
full_result = {
'writeErrors': [],
'writeConcernErrors': [],
'nInserted': 0,
'nUpserted': 0,
'nMatched': 0,
'nModified': 0,
'nRemoved': 0,
'upserted': [],
}
for run in generator:
cmd = SON([(_COMMANDS[run.op_type], self.collection.name),
('ordered', self.ordered)])
if write_concern.document:
cmd['writeConcern'] = write_concern.document
if self.bypass_doc_val and connection.max_wire_version >= 4:
cmd['bypassDocumentValidation'] = True
results = await self._do_batched_write_command(
self.namespace, run.op_type, cmd, run.ops, True,
self.collection.codec_options, connection)
_merge_command(run, full_result, results)
# We're supposed to continue if errors are
# at the write concern level (e.g. wtimeout)
if self.ordered and full_result['writeErrors']:
break
if full_result['writeErrors'] or full_result['writeConcernErrors']:
if full_result['writeErrors']:
full_result['writeErrors'].sort(
key=lambda error: error['index'])
raise BulkWriteError(full_result)
return full_result
def execute_command(self, generator, write_concern):
"""Execute using write commands.
"""
uuid_subtype = self.collection.uuid_subtype
client = self.collection.database.connection
# nModified is only reported for write commands, not legacy ops.
full_result = {
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nRemoved": 0,
"upserted": [],
}
for run in generator:
cmd = SON([(_COMMANDS[run.op_type], self.collection.name),
('ordered', self.ordered)])
if write_concern:
cmd['writeConcern'] = write_concern
results = _do_batched_write_command(self.namespace, run.op_type,
cmd, run.ops, True,
uuid_subtype, client)
_merge_command(run, full_result, results)
# We're supposed to continue if errors are
# at the write concern level (e.g. wtimeout)
if self.ordered and full_result['writeErrors']:
break
if full_result["writeErrors"] or full_result["writeConcernErrors"]:
if full_result['writeErrors']:
full_result['writeErrors'].sort(
key=lambda error: error['index'])
raise BulkWriteError(full_result)
return full_result
def test_export_to_mongo_logs_error_correctly_on_bulk_write_error_with_mix_of_errors(
subject, mongo_database):
_, mongo_database = mongo_database
bulk_write_error = BulkWriteError({
"errorLabels": [],
"writeErrors": [{
"code": 11000,
"op": MagicMock()
}, {
"code": 999
}]
})
with patch.object(Collection, "insert_many", side_effect=bulk_write_error):
subject.export_to_mongo()
# No documents were inserted in either collection
samples_collection = get_mongo_collection(mongo_database,
COLLECTION_SAMPLES)
assert samples_collection.count_documents({}) == 0
source_plates_collection = get_mongo_collection(mongo_database,
COLLECTION_SOURCE_PLATES)
assert source_plates_collection.count_documents({}) == 0
def raiseError(*args, **kwargs):
details = {"writeErrors": [{"code": 2345}]}
raise BulkWriteError(details)
def execute_command(self, sock_info, generator, write_concern, session):
"""Execute using write commands.
"""
# nModified is only reported for write commands, not legacy ops.
full_result = {
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nRemoved": 0,
"upserted": [],
}
op_id = _randint()
db_name = self.collection.database.name
client = self.collection.database.client
listeners = client._event_listeners
with self.collection.database.client._tmp_session(session) as s:
# sock_info.command checks auth, but we use sock_info.write_command.
sock_info.check_session_auth_matches(s)
for run in generator:
cmd = SON([(_COMMANDS[run.op_type], self.collection.name),
('ordered', self.ordered)])
if write_concern.document:
cmd['writeConcern'] = write_concern.document
if self.bypass_doc_val and sock_info.max_wire_version >= 4:
cmd['bypassDocumentValidation'] = True
if s:
cmd['lsid'] = s._use_lsid()
bwc = _BulkWriteContext(db_name, cmd, sock_info, op_id,
listeners, s)
results = []
idx_offset = 0
while idx_offset < len(run.ops):
check_keys = run.op_type == _INSERT
ops = islice(run.ops, idx_offset, None)
# Run as many ops as possible.
client._send_cluster_time(cmd, s)
request_id, msg, to_send = _do_batched_write_command(
self.namespace, run.op_type, cmd, ops, check_keys,
self.collection.codec_options, bwc)
if not to_send:
raise InvalidOperation("cannot do an empty bulk write")
result = bwc.write_command(request_id, msg, to_send)
client._receive_cluster_time(result)
if s is not None:
s._advance_cluster_time(result.get("$clusterTime"))
s._advance_operation_time(result.get("operationTime"))
results.append((idx_offset, result))
if self.ordered and "writeErrors" in result:
break
idx_offset += len(to_send)
_merge_command(run, full_result, results)
# We're supposed to continue if errors are
# at the write concern level (e.g. wtimeout)
if self.ordered and full_result['writeErrors']:
break
if full_result["writeErrors"] or full_result["writeConcernErrors"]:
if full_result['writeErrors']:
full_result['writeErrors'].sort(
key=lambda error: error['index'])
raise BulkWriteError(full_result)
return full_result
def execute_legacy(self, sock_info, generator, write_concern):
"""Execute using legacy wire protocol ops.
"""
coll = self.collection
full_result = {
"writeErrors": [],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nRemoved": 0,
"upserted": [],
}
op_id = _randint()
stop = False
for run in generator:
for idx, operation in enumerate(run.ops):
try:
# To do per-operation reporting we have to do ops one
# at a time. That means the performance of bulk insert
# will be slower here than calling Collection.insert()
if run.op_type == _INSERT:
coll._insert(sock_info,
operation,
self.ordered,
write_concern=write_concern,
op_id=op_id)
result = {}
elif run.op_type == _UPDATE:
doc = operation['u']
check_keys = True
if doc and next(iter(doc)).startswith('$'):
check_keys = False
result = coll._update(sock_info,
operation['q'],
doc,
operation['upsert'],
check_keys,
operation['multi'],
write_concern=write_concern,
op_id=op_id,
ordered=self.ordered)
else:
result = coll._delete(sock_info, operation['q'],
not operation['limit'],
write_concern, op_id,
self.ordered)
_merge_legacy(run, full_result, result, idx)
except DocumentTooLarge as exc:
# MongoDB 2.6 uses error code 2 for "too large".
error = _make_error(run.index(idx), _BAD_VALUE, str(exc),
operation)
full_result['writeErrors'].append(error)
if self.ordered:
stop = True
break
except OperationFailure as exc:
if not exc.details:
# Some error not related to the write operation
# (e.g. kerberos failure). Re-raise immediately.
raise
_merge_legacy(run, full_result, exc.details, idx)
# We're supposed to continue if errors are
# at the write concern level (e.g. wtimeout)
if self.ordered and full_result["writeErrors"]:
stop = True
break
if stop:
break
if full_result["writeErrors"] or full_result['writeConcernErrors']:
if full_result['writeErrors']:
full_result['writeErrors'].sort(
key=lambda error: error['index'])
raise BulkWriteError(full_result)
return full_result
# (e.g. kerberos failure). Re-raise immediately.
raise
_merge_legacy(run, full_result, exc.details, idx)
# We're supposed to continue if errors are
# at the write concern level (e.g. wtimeout)
if self.ordered and full_result["writeErrors"]:
stop = True
break
if stop:
break
if full_result["writeErrors"] or full_result['writeConcernErrors']:
if full_result['writeErrors']:
full_result['writeErrors'].sort(
key=lambda error: error['index'])
raise BulkWriteError(full_result)
return full_result
def execute(self, write_concern):
"""Execute operations.
"""
if not self.ops:
raise InvalidOperation('No operations to execute')
if self.executed:
raise InvalidOperation('Bulk operations can '
'only be executed once.')
self.executed = True
client = self.collection.database.connection
client._ensure_connected(sync=True)
write_concern = write_concern or self.collection.write_concern
SAVE_EXCEPT_TEST_CASES = [
(
{
"id": "opsmatters_uk.ThelatestupdateforBroadcomincludesDXNetOps202Netwo.opentracing"
},
BulkWriteError(
{
"writeErrors": [
{
"index": 0,
"code": 11000,
"keyPattern": {"id": 1},
"keyValue": {
"id": "opsmatters_uk.ThelatestupdateforBroadcomincludesDXNetOps202Netwo.opentracing"
},
}
],
"writeConcernErrors": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 0,
"nModified": 0,
"nRemoved": 0,
"upserted": [],
},
),
0,
{
"writeErrors": [
{
"index": 0,
def test_pickle_BulkWriteError(self):
exc = BulkWriteError({})
self.assertOperationFailureEqual(exc, pickle.loads(pickle.dumps(exc)))
self.assertIn("batch op errors occurred", str(exc))
def raise_error():
error["writeErrors"].sort(key=lambda error: error["index"])
for write_error in error["writeErrors"]:
write_error[u"op"] = documents[write_error["index"]]
raise BulkWriteError(error)
