def setUpClass(cls):
super(BulkTestBase, cls).setUpClass()
cls.coll = cls.db.test
cls.coll_w0 = cls.coll.with_options(write_concern=WriteConcern(w=0))
import threading
import time
import warnings
from collections import defaultdict
from functools import partial
from pymongo import MongoClient, monitoring
from pymongo.errors import AutoReconnect, OperationFailure
from pymongo.server_selectors import (any_server_selector,
writable_server_selector)
from pymongo.write_concern import WriteConcern
from test import (client_context, db_user, db_pwd)
from test.version import Version
IMPOSSIBLE_WRITE_CONCERN = WriteConcern(w=len(client_context.nodes) + 1)
class WhiteListEventListener(monitoring.CommandListener):
def __init__(self, *commands):
self.commands = set(commands)
self.results = defaultdict(list)
def started(self, event):
if event.command_name in self.commands:
self.results['started'].append(event)
def succeeded(self, event):
if event.command_name in self.commands:
self.results['succeeded'].append(event)
class Meta:
write_concern = WriteConcern(j=True)
connection_alias = 'Messages'
from pymongo.read_concern import ReadConcern
from pymongo.read_preferences import ReadPreference
from pymongo.server_selectors import (any_server_selector,
writable_server_selector)
from pymongo.server_type import SERVER_TYPE
from pymongo.write_concern import WriteConcern
from test import (client_context, db_user, db_pwd)
if sys.version_info[0] < 3:
# Python 2.7, use our backport.
from test.barrier import Barrier
else:
from threading import Barrier
IMPOSSIBLE_WRITE_CONCERN = WriteConcern(w=50)
class CMAPListener(ConnectionPoolListener):
def __init__(self):
self.events = []
def reset(self):
self.events = []
def add_event(self, event):
self.events.append(event)
def event_count(self, event_type):
return len([
event for event in self.events[:] if isinstance(event, event_type)
from pymongo.read_concern import ReadConcern
from pymongo.read_preferences import ReadPreference
from pymongo.write_concern import WriteConcern
from pymongo.errors import InvalidOperation
from iu_mongo.errors import TransactionError
__all__ = ['Session', 'TransactionContext']
DEFAULT_READ_CONCERN = ReadConcern('majority')
DEFAULT_WRITE_CONCERN = WriteConcern(w='majority', wtimeout=5000)
DEFAULT_READ_PREFERENCE = ReadPreference.PRIMARY
class TransactionContext(object):
def __init__(self, pymongo_transaction_context, pymongo_session):
self._pymongo_transaction_context = pymongo_transaction_context
self._pymongo_session = pymongo_session
def __enter__(self):
self._pymongo_transaction_context.__enter__()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._pymongo_transaction_context.__exit__(exc_type, exc_val, exc_tb)
@property
def _transaction(self):
return self._pymongo_session._transaction
@property
def transaction_id(self):
class Meta:
write_concern = WriteConcern(w=1)
connection_alias = "reviewer"
final = True
def create_with_schema(coll, json_schema):
"""Create and return a Collection with a jsonSchema."""
coll.with_options(write_concern=WriteConcern(w='majority')).drop()
return coll.database.create_collection(
coll.name, validator={'$jsonSchema': json_schema}, codec_options=OPTS)
def set_write_concern(collection, write_concerns):
combined_concerns = dict(list(collection.write_concern.document.items()))
combined_concerns.update(write_concerns)
yield collection.with_options(write_concern=WriteConcern(
**combined_concerns))
class Meta:
write_concern = WriteConcern(w=1)
connection_alias = 'my-app'
final = True
indexes = [IndexModel([('word', TEXT)])]
class Meta:
write_concern = WriteConcern(j=True)
connection_alias = 'my-app'
"""
def output(self):
transaction_count = self.stats_collector.transactions.get_count()
transaction_time = self.stats_collector.transaction_timer.get_total_time()
sys.stderr.write("Number of transactions executed: %s\n" % transaction_count)
sys.stderr.write("Total execution time: %s\n" % transaction_time)
sys.stderr.write("Execution throughput: %s (xact/s)\n" % (transaction_count * 1.0 / transaction_time))
if __name__ == "__main__":
ap = argparse.ArgumentParser()
# List of arguments
ap.add_argument("-n", "--number", required=False, help="Number of transactions to be executed")
ap.add_argument("-t", "--type", required=False, help="Read/Write concern type")
args = vars(ap.parse_args())
num_trans = int(args['number']) if args['number'] else 1000000000
# Consistency level: 1 / not supplied for "local" for the read_concern & "1" for write concern
# not-1 for "majority" for both read and write concerns
concern_type = int(args['type']) if args['type'] else 0
if (concern_type == 1):
read_concern = ReadConcern("majority")
write_concern = WriteConcern("majority")
else:
read_concern = ReadConcern("local")
write_concern = WriteConcern(1)
print "Executing client"
client = Client(num_trans)
client.execute()
def test_transactions(self):
# Transaction examples
client = self.client
self.addCleanup(client.drop_database, "hr")
self.addCleanup(client.drop_database, "reporting")
employees = client.hr.employees
events = client.reporting.events
employees.insert_one({"employee": 3, "status": "Active"})
events.insert_one(
{"employee": 3, "status": {"new": "Active", "old": None}})
# Start Transactions Intro Example 1
def update_employee_info(session):
employees_coll = session.client.hr.employees
events_coll = session.client.reporting.events
with session.start_transaction(
read_concern=ReadConcern("snapshot"),
write_concern=WriteConcern(w="majority")):
employees_coll.update_one(
{"employee": 3}, {"$set": {"status": "Inactive"}},
session=session)
events_coll.insert_one(
{"employee": 3, "status": {
"new": "Inactive", "old": "Active"}},
session=session)
while True:
try:
# Commit uses write concern set at transaction start.
session.commit_transaction()
print("Transaction committed.")
break
except (ConnectionFailure, OperationFailure) as exc:
# Can retry commit
if exc.has_error_label(
"UnknownTransactionCommitResult"):
print("UnknownTransactionCommitResult, retrying "
"commit operation ...")
continue
else:
print("Error during commit ...")
raise
# End Transactions Intro Example 1
with client.start_session() as session:
update_employee_info(session)
employee = employees.find_one({"employee": 3})
self.assertIsNotNone(employee)
self.assertEqual(employee['status'], 'Inactive')
# Start Transactions Retry Example 1
def run_transaction_with_retry(txn_func, session):
while True:
try:
txn_func(session) # performs transaction
break
except (ConnectionFailure, OperationFailure) as exc:
print("Transaction aborted. Caught exception during "
"transaction.")
# If transient error, retry the whole transaction
if exc.has_error_label("TransientTransactionError"):
print("TransientTransactionError, retrying"
"transaction ...")
continue
else:
raise
# End Transactions Retry Example 1
with client.start_session() as session:
run_transaction_with_retry(update_employee_info, session)
employee = employees.find_one({"employee": 3})
self.assertIsNotNone(employee)
self.assertEqual(employee['status'], 'Inactive')
# Start Transactions Retry Example 2
def commit_with_retry(session):
while True:
try:
# Commit uses write concern set at transaction start.
session.commit_transaction()
print("Transaction committed.")
break
except (ConnectionFailure, OperationFailure) as exc:
# Can retry commit
if exc.has_error_label("UnknownTransactionCommitResult"):
print("UnknownTransactionCommitResult, retrying "
"commit operation ...")
continue
else:
print("Error during commit ...")
raise
# End Transactions Retry Example 2
# Test commit_with_retry from the previous examples
def _insert_employee_retry_commit(session):
with session.start_transaction():
employees.insert_one(
{"employee": 4, "status": "Active"},
session=session)
events.insert_one(
{"employee": 4, "status": {"new": "Active", "old": None}},
session=session)
commit_with_retry(session)
with client.start_session() as session:
run_transaction_with_retry(_insert_employee_retry_commit, session)
employee = employees.find_one({"employee": 4})
self.assertIsNotNone(employee)
self.assertEqual(employee['status'], 'Active')
# Start Transactions Retry Example 3
def run_transaction_with_retry(txn_func, session):
while True:
try:
txn_func(session) # performs transaction
break
except (ConnectionFailure, OperationFailure) as exc:
# If transient error, retry the whole transaction
if exc.has_error_label("TransientTransactionError"):
print("TransientTransactionError, retrying "
"transaction ...")
continue
else:
raise
def commit_with_retry(session):
while True:
try:
# Commit uses write concern set at transaction start.
session.commit_transaction()
print("Transaction committed.")
break
except (ConnectionFailure, OperationFailure) as exc:
# Can retry commit
if exc.has_error_label("UnknownTransactionCommitResult"):
print("UnknownTransactionCommitResult, retrying "
"commit operation ...")
continue
else:
print("Error during commit ...")
raise
# Updates two collections in a transactions
def update_employee_info(session):
employees_coll = session.client.hr.employees
events_coll = session.client.reporting.events
with session.start_transaction(
read_concern=ReadConcern("snapshot"),
write_concern=WriteConcern(w="majority"),
read_preference=ReadPreference.PRIMARY):
employees_coll.update_one(
{"employee": 3}, {"$set": {"status": "Inactive"}},
session=session)
events_coll.insert_one(
{"employee": 3, "status": {
"new": "Inactive", "old": "Active"}},
session=session)
commit_with_retry(session)
# Start a session.
with client.start_session() as session:
try:
run_transaction_with_retry(update_employee_info, session)
except Exception as exc:
# Do something with error.
raise
# End Transactions Retry Example 3
employee = employees.find_one({"employee": 3})
self.assertIsNotNone(employee)
self.assertEqual(employee['status'], 'Inactive')
MongoClient = lambda _: rs_client()
uriString = None
# Start Transactions withTxn API Example 1
# For a replica set, include the replica set name and a seedlist of the members in the URI string; e.g.
# uriString = 'mongodb://mongodb0.example.com:27017,mongodb1.example.com:27017/?replicaSet=myRepl'
# For a sharded cluster, connect to the mongos instances; e.g.
# uriString = 'mongodb://mongos0.example.com:27017,mongos1.example.com:27017/'
client = MongoClient(uriString)
wc_majority = WriteConcern("majority", wtimeout=1000)
# Prereq: Create collections.
client.get_database(
"mydb1", write_concern=wc_majority).foo.insert_one({'abc': 0})
client.get_database(
"mydb2", write_concern=wc_majority).bar.insert_one({'xyz': 0})
# Step 1: Define the callback that specifies the sequence of operations to perform inside the transactions.
def callback(session):
collection_one = session.client.mydb1.foo
collection_two = session.client.mydb2.bar
# Important:: You must pass the session to the operations.
collection_one.insert_one({'abc': 1}, session=session)
collection_two.insert_one({'xyz': 999}, session=session)
# Step 2: Start a client session.
with client.start_session() as session:
# Step 3: Use with_transaction to start a transaction, execute the callback, and commit (or abort on error).
session.with_transaction(
callback, read_concern=ReadConcern('local'),
write_concern=wc_majority,
read_preference=ReadPreference.PRIMARY)
def test_write_concern_failure_unordered(self):
# Ensure we don't raise on wnote.
coll_ww = self.coll.with_options(write_concern=WriteConcern(w=self.w))
result = coll_ww.bulk_write([
DeleteOne({"something": "that does no exist"})], ordered=False)
self.assertTrue(result.acknowledged)
requests = [
InsertOne({'a': 1}),
UpdateOne({'a': 3}, {'$set': {'a': 3, 'b': 1}}, upsert=True),
InsertOne({'a': 2}),
]
# Replication wtimeout is a 'soft' error.
# It shouldn't stop batch processing.
try:
self.cause_wtimeout(requests, ordered=False)
except BulkWriteError as exc:
result = exc.details
self.assertEqual(exc.code, 65)
else:
self.fail("Error not raised")
self.assertEqual(2, result['nInserted'])
self.assertEqual(1, result['nUpserted'])
self.assertEqual(0, len(result['writeErrors']))
# When talking to legacy servers there will be a
# write concern error for each operation.
self.assertTrue(len(result['writeConcernErrors']) > 1)
self.coll.delete_many({})
self.coll.create_index('a', unique=True)
self.addCleanup(self.coll.drop_index, [('a', 1)])
# Fail due to write concern support as well
# as duplicate key error on unordered batch.
requests = [
InsertOne({'a': 1}),
UpdateOne({'a': 3}, {'$set': {'a': 3, 'b': 1}}, upsert=True),
InsertOne({'a': 1}),
InsertOne({'a': 2}),
]
try:
self.cause_wtimeout(requests, ordered=False)
except BulkWriteError as exc:
result = exc.details
self.assertEqual(exc.code, 65)
else:
self.fail("Error not raised")
self.assertEqual(2, result['nInserted'])
self.assertEqual(1, result['nUpserted'])
self.assertEqual(1, len(result['writeErrors']))
# When talking to legacy servers there will be a
# write concern error for each operation.
self.assertTrue(len(result['writeConcernErrors']) > 1)
failed = result['writeErrors'][0]
self.assertEqual(2, failed['index'])
self.assertEqual(11000, failed['code'])
self.assertTrue(isinstance(failed['errmsg'], str))
self.assertEqual(1, failed['op']['a'])
failed = result['writeConcernErrors'][0]
self.assertEqual(64, failed['code'])
self.assertTrue(isinstance(failed['errmsg'], str))
upserts = result['upserted']
self.assertEqual(1, len(upserts))
self.assertEqual(1, upserts[0]['index'])
self.assertTrue(upserts[0].get('_id'))
def test_write_concern_failure_ordered(self):
# Ensure we don't raise on wnote.
coll_ww = self.coll.with_options(write_concern=WriteConcern(w=self.w))
result = coll_ww.bulk_write([
DeleteOne({"something": "that does no exist"})])
self.assertTrue(result.acknowledged)
requests = [
InsertOne({'a': 1}),
InsertOne({'a': 2})
]
# Replication wtimeout is a 'soft' error.
# It shouldn't stop batch processing.
try:
self.cause_wtimeout(requests, ordered=True)
except BulkWriteError as exc:
result = exc.details
self.assertEqual(exc.code, 65)
else:
self.fail("Error not raised")
self.assertEqualResponse(
{'nMatched': 0,
'nModified': 0,
'nUpserted': 0,
'nInserted': 2,
'nRemoved': 0,
'upserted': [],
'writeErrors': []},
result)
# When talking to legacy servers there will be a
# write concern error for each operation.
self.assertTrue(len(result['writeConcernErrors']) > 0)
failed = result['writeConcernErrors'][0]
self.assertEqual(64, failed['code'])
self.assertTrue(isinstance(failed['errmsg'], str))
self.coll.delete_many({})
self.coll.create_index('a', unique=True)
self.addCleanup(self.coll.drop_index, [('a', 1)])
# Fail due to write concern support as well
# as duplicate key error on ordered batch.
requests = [
InsertOne({'a': 1}),
ReplaceOne({'a': 3}, {'b': 1}, upsert=True),
InsertOne({'a': 1}),
InsertOne({'a': 2}),
]
try:
self.cause_wtimeout(requests, ordered=True)
except BulkWriteError as exc:
result = exc.details
self.assertEqual(exc.code, 65)
else:
self.fail("Error not raised")
self.assertEqualResponse(
{'nMatched': 0,
'nModified': 0,
'nUpserted': 1,
'nInserted': 1,
'nRemoved': 0,
'upserted': [{'index': 1, '_id': '...'}],
'writeErrors': [
{'index': 2,
'code': 11000,
'errmsg': '...',
'op': {'_id': '...', 'a': 1}}]},
result)
self.assertTrue(len(result['writeConcernErrors']) > 1)
failed = result['writeErrors'][0]
self.assertTrue("duplicate" in failed['errmsg'])
class Meta:
write_concern = WriteConcern(j=True)
connection_alias = 'StaticChannels'
class Meta:
write_concern = WriteConcern(j=True)
class Meta:
write_concern = WriteConcern(w=1)
connection_alias = "reviewer"
final = True
indexes = [IndexModel([("name", pymongo.DESCENDING)], unique=True)]
class Meta:
write_concern = WriteConcern(j=True)
connection_alias = 'recommend-ahp'
def _test_auto_encrypt(self, opts):
client = rs_or_single_client(auto_encryption_opts=opts)
self.addCleanup(client.close)
# Create the encrypted field's data key.
key_vault = create_key_vault(
self.client.admin.datakeys,
json_data('custom', 'key-document-local.json'))
self.addCleanup(key_vault.drop)
# Collection.insert_one/insert_many auto encrypts.
docs = [{
'_id': 0,
'ssn': '000'
}, {
'_id': 1,
'ssn': '111'
}, {
'_id': 2,
'ssn': '222'
}, {
'_id': 3,
'ssn': '333'
}, {
'_id': 4,
'ssn': '444'
}, {
'_id': 5,
'ssn': '555'
}]
encrypted_coll = client.pymongo_test.test
encrypted_coll.insert_one(docs[0])
encrypted_coll.insert_many(docs[1:3])
unack = encrypted_coll.with_options(write_concern=WriteConcern(w=0))
unack.insert_one(docs[3])
unack.insert_many(docs[4:], ordered=False)
wait_until(lambda: self.db.test.count_documents({}) == len(docs),
'insert documents with w=0')
# Database.command auto decrypts.
res = client.pymongo_test.command('find',
'test',
filter={'ssn': '000'})
decrypted_docs = res['cursor']['firstBatch']
self.assertEqual(decrypted_docs, [{'_id': 0, 'ssn': '000'}])
# Collection.find auto decrypts.
decrypted_docs = list(encrypted_coll.find())
self.assertEqual(decrypted_docs, docs)
# Collection.find auto decrypts getMores.
decrypted_docs = list(encrypted_coll.find(batch_size=1))
self.assertEqual(decrypted_docs, docs)
# Collection.aggregate auto decrypts.
decrypted_docs = list(encrypted_coll.aggregate([]))
self.assertEqual(decrypted_docs, docs)
# Collection.aggregate auto decrypts getMores.
decrypted_docs = list(encrypted_coll.aggregate([], batchSize=1))
self.assertEqual(decrypted_docs, docs)
# Collection.distinct auto decrypts.
decrypted_ssns = encrypted_coll.distinct('ssn')
self.assertEqual(set(decrypted_ssns), set(d['ssn'] for d in docs))
# Make sure the field is actually encrypted.
for encrypted_doc in self.db.test.find():
self.assertIsInstance(encrypted_doc['_id'], int)
self.assertEncrypted(encrypted_doc['ssn'])
# Attempt to encrypt an unencodable object.
with self.assertRaises(BSONError):
encrypted_coll.insert_one({'unencodeable': object()})
class Meta:
write_concern = WriteConcern(j=True)
ignore_unknown_fields = True
connection_alias = 'onx-app'
def test_unacknowledged(self):
with self.client.start_session(causal_consistency=True) as s:
coll = self.client.pymongo_test.get_collection(
'test', write_concern=WriteConcern(w=0))
coll.insert_one({}, session=s)
self.assertIsNone(s.operation_time)
class Meta:
final = True
write_concern = WriteConcern(j=True)
def run_scenario(self):
dbname = scenario_def['database_name']
collname = scenario_def['collection_name']
coll = self.client[dbname][collname]
coll.drop()
coll.insert_many(scenario_def['data'])
self.listener.results.clear()
name = camel_to_snake(test['operation']['name'])
if 'read_preference' in test['operation']:
mode = read_pref_mode_from_name(
test['operation']['read_preference']['mode'])
coll = coll.with_options(
read_preference=make_read_preference(mode, None))
test_args = test['operation']['arguments']
if 'writeConcern' in test_args:
concern = test_args.pop('writeConcern')
coll = coll.with_options(
write_concern=WriteConcern(**concern))
args = {}
for arg in test_args:
args[camel_to_snake(arg)] = test_args[arg]
if name == 'bulk_write':
bulk_args = []
for request in args['requests']:
opname = next(iter(request))
klass = opname[0:1].upper() + opname[1:]
arg = getattr(pymongo, klass)(**request[opname])
bulk_args.append(arg)
try:
coll.bulk_write(bulk_args, args.get('ordered', True))
except OperationFailure:
pass
elif name == 'find':
if 'sort' in args:
args['sort'] = list(args['sort'].items())
for arg in 'skip', 'limit':
if arg in args:
args[arg] = int(args[arg])
try:
# Iterate the cursor.
tuple(coll.find(**args))
except OperationFailure:
pass
# Wait for the killCursors thread to run if necessary.
if 'limit' in args and client_context.version[:2] < (3, 1):
self.client._kill_cursors_executor.wake()
started = self.listener.results['started']
succeeded = self.listener.results['succeeded']
wait_until(
lambda: started[-1].command_name == 'killCursors',
"publish a start event for killCursors.")
wait_until(
lambda: succeeded[-1].command_name == 'killCursors',
"publish a succeeded event for killCursors.")
else:
try:
getattr(coll, name)(**args)
except OperationFailure:
pass
res = self.listener.results
for expectation in test['expectations']:
event_type = next(iter(expectation))
if event_type == "command_started_event":
event = res['started'][0] if len(res['started']) else None
if event is not None:
# The tests substitute 42 for any number other than 0.
if (event.command_name == 'getMore'
and event.command['getMore']):
event.command['getMore'] = 42
elif event.command_name == 'killCursors':
event.command['cursors'] = [42]
elif event_type == "command_succeeded_event":
event = (
res['succeeded'].pop(0) if len(res['succeeded']) else None)
if event is not None:
reply = event.reply
# The tests substitute 42 for any number other than 0,
# and "" for any error message.
if 'writeErrors' in reply:
for doc in reply['writeErrors']:
doc['code'] = 42
doc['errmsg'] = ""
elif 'cursor' in reply:
if reply['cursor']['id']:
reply['cursor']['id'] = 42
elif event.command_name == 'killCursors':
# Make the tests continue to pass when the killCursors
# command is actually in use.
if 'cursorsKilled' in reply:
reply.pop('cursorsKilled')
reply['cursorsUnknown'] = [42]
# Found succeeded event. Pop related started event.
res['started'].pop(0)
elif event_type == "command_failed_event":
event = res['failed'].pop(0) if len(res['failed']) else None
if event is not None:
# Found failed event. Pop related started event.
res['started'].pop(0)
else:
self.fail("Unknown event type")
if event is None:
event_name = event_type.split('_')[1]
self.fail(
"Expected %s event for %s command. Actual "
"results:%s" % (
event_name,
expectation[event_type]['command_name'],
format_actual_results(res)))
for attr, expected in expectation[event_type].items():
actual = getattr(event, attr)
if isinstance(expected, dict):
for key, val in expected.items():
self.assertEqual(val, actual[key])
else:
self.assertEqual(actual, expected)
def start_transaction(session):
session.start_transaction(
read_concern=ReadConcern("snapshot"),
write_concern=WriteConcern(w="majority"))
return session
def test_rollback(self):
"""Test the rollback method in a sharded environment
Cases:
1. Documents on both shards, rollback on one shard
2. Documents on both shards, rollback on both shards
"""
self.opman1.start()
self.opman2.start()
# Insert first documents while primaries are up
db_main = self.mongos_conn["test"]["mcsharded"]
db_main2 = db_main.with_options(write_concern=WriteConcern(w=2))
db_main2.insert_one({"i": 0})
db_main2.insert_one({"i": 1000})
self.assertEqual(self.shard1_conn["test"]["mcsharded"].count(), 1)
self.assertEqual(self.shard2_conn["test"]["mcsharded"].count(), 1)
# Case 1: only one primary goes down, shard1 in this case
self.cluster.shards[0].primary.stop(destroy=False)
# Wait for the secondary to be promoted
shard1_secondary_admin = self.shard1_secondary_conn["admin"]
assert_soon(
lambda: shard1_secondary_admin.command("isMaster")["ismaster"])
# Insert another document. This will be rolled back later
def insert_one(doc):
if not db_main.find_one(doc):
return db_main.insert_one(doc)
return True
assert_soon(lambda: retry_until_ok(insert_one, {"i": 1}),
"could not insert into shard1 with one node down")
db_secondary1 = self.shard1_secondary_conn["test"]["mcsharded"]
db_secondary2 = self.shard2_secondary_conn["test"]["mcsharded"]
self.assertEqual(db_secondary1.count(), 2)
# Wait for replication on the doc manager
# Note that both OplogThreads share the same doc manager
c = lambda: len(self.opman1.doc_managers[0]._search()) == 3
assert_soon(c,
"not all writes were replicated to doc manager",
max_tries=120)
# Kill the new primary
self.cluster.shards[0].secondary.stop(destroy=False)
# Start both servers back up
self.cluster.shards[0].primary.start()
primary_admin = self.shard1_conn["admin"]
c = lambda: primary_admin.command("isMaster")["ismaster"]
assert_soon(lambda: retry_until_ok(c))
self.cluster.shards[0].secondary.start()
secondary_admin = self.shard1_secondary_conn["admin"]
c = lambda: secondary_admin.command("replSetGetStatus")["myState"] == 2
assert_soon(c)
query = {"i": {"$lt": 1000}}
assert_soon(lambda: retry_until_ok(db_main.find(query).count) > 0)
# Only first document should exist in MongoDB
self.assertEqual(db_main.find(query).count(), 1)
self.assertEqual(db_main.find_one(query)["i"], 0)
def check_docman_rollback():
docman_docs = [
d for d in self.opman1.doc_managers[0]._search()
if d["i"] < 1000
]
return len(docman_docs) == 1 and docman_docs[0]["i"] == 0
assert_soon(check_docman_rollback, "doc manager did not roll back")
# Wait for previous rollback to complete.
# Insert/delete one document to jump-start replication to secondaries
# in MongoDB 3.x.
db_main.insert_one({'i': -1})
db_main.delete_one({'i': -1})
def rollback_done():
secondary1_count = retry_until_ok(db_secondary1.count)
secondary2_count = retry_until_ok(db_secondary2.count)
return (1, 1) == (secondary1_count, secondary2_count)
assert_soon(rollback_done,
"rollback never replicated to one or more secondaries")
##############################
# Case 2: Primaries on both shards go down
self.cluster.shards[0].primary.stop(destroy=False)
self.cluster.shards[1].primary.stop(destroy=False)
# Wait for the secondaries to be promoted
shard1_secondary_admin = self.shard1_secondary_conn["admin"]
shard2_secondary_admin = self.shard2_secondary_conn["admin"]
assert_soon(
lambda: shard1_secondary_admin.command("isMaster")["ismaster"])
assert_soon(
lambda: shard2_secondary_admin.command("isMaster")["ismaster"])
# Insert another document on each shard which will be rolled back later
assert_soon(lambda: retry_until_ok(insert_one, {"i": 1}),
"could not insert into shard1 with one node down")
self.assertEqual(db_secondary1.count(), 2)
assert_soon(lambda: retry_until_ok(insert_one, {"i": 1001}),
"could not insert into shard2 with one node down")
self.assertEqual(db_secondary2.count(), 2)
# Wait for replication on the doc manager
c = lambda: len(self.opman1.doc_managers[0]._search()) == 4
assert_soon(c, "not all writes were replicated to doc manager")
# Kill the new primaries
self.cluster.shards[0].secondary.stop(destroy=False)
self.cluster.shards[1].secondary.stop(destroy=False)
# Start the servers back up...
# Shard 1
self.cluster.shards[0].primary.start()
c = lambda: self.shard1_conn['admin'].command("isMaster")["ismaster"]
assert_soon(lambda: retry_until_ok(c))
self.cluster.shards[0].secondary.start()
secondary_admin = self.shard1_secondary_conn["admin"]
c = lambda: secondary_admin.command("replSetGetStatus")["myState"] == 2
assert_soon(c)
# Shard 2
self.cluster.shards[1].primary.start()
c = lambda: self.shard2_conn['admin'].command("isMaster")["ismaster"]
assert_soon(lambda: retry_until_ok(c))
self.cluster.shards[1].secondary.start()
secondary_admin = self.shard2_secondary_conn["admin"]
c = lambda: secondary_admin.command("replSetGetStatus")["myState"] == 2
assert_soon(c)
# Wait for the shards to come online
assert_soon(lambda: retry_until_ok(db_main.find(query).count) > 0)
query2 = {"i": {"$gte": 1000}}
assert_soon(lambda: retry_until_ok(db_main.find(query2).count) > 0)
# Only first documents should exist in MongoDB
self.assertEqual(db_main.find(query).count(), 1)
self.assertEqual(db_main.find_one(query)["i"], 0)
self.assertEqual(db_main.find(query2).count(), 1)
self.assertEqual(db_main.find_one(query2)["i"], 1000)
# Same should hold for the doc manager
assert_soon(lambda: len(self.opman1.doc_managers[0]._search()) == 2)
i_values = [d["i"] for d in self.opman1.doc_managers[0]._search()]
self.assertIn(0, i_values)
self.assertIn(1000, i_values)
class Meta:
write_concern = WriteConcern(j=True)
connection_alias = 'athenaDB'
def _database_default_options(self, name):
"""Get a Database instance with the default settings."""
return self.get_database(name,
codec_options=DEFAULT_CODEC_OPTIONS,
read_preference=ReadPreference.PRIMARY,
write_concern=WriteConcern())
class Meta:
write_concern = WriteConcern(j=True)
connection_alias = 'Money-Manager'
collection_name = "Accounts"
def execute_no_results(self, sock_info, generator):
"""Execute all operations, returning no results (w=0).
"""
if self.uses_collation:
raise ConfigurationError(
'Collation is unsupported for unacknowledged writes.')
if self.uses_array_filters:
raise ConfigurationError(
'arrayFilters is unsupported for unacknowledged writes.')
if self.uses_hint:
raise ConfigurationError(
'hint is unsupported for unacknowledged writes.')
# Cannot have both unacknowledged writes and bypass document validation.
if self.bypass_doc_val and sock_info.max_wire_version >= 4:
raise OperationFailure("Cannot set bypass_document_validation with"
" unacknowledged write concern")
# OP_MSG
if sock_info.max_wire_version > 5:
if self.ordered:
return self.execute_command_no_results(sock_info, generator)
return self.execute_op_msg_no_results(sock_info, generator)
coll = self.collection
# If ordered is True we have to send GLE or use write
# commands so we can abort on the first error.
write_concern = WriteConcern(w=int(self.ordered))
op_id = _randint()
next_run = next(generator)
while next_run:
# An ordered bulk write needs to send acknowledged writes to short
# circuit the next run. However, the final message on the final
# run can be unacknowledged.
run = next_run
next_run = next(generator, None)
needs_ack = self.ordered and next_run is not None
try:
if run.op_type == _INSERT:
self.execute_insert_no_results(
sock_info, run, op_id, needs_ack)
elif run.op_type == _UPDATE:
for operation in run.ops:
doc = operation['u']
check_keys = True
if doc and next(iter(doc)).startswith('$'):
check_keys = False
coll._update(
sock_info,
operation['q'],
doc,
operation['upsert'],
check_keys,
operation['multi'],
write_concern=write_concern,
op_id=op_id,
ordered=self.ordered,
bypass_doc_val=self.bypass_doc_val)
else:
for operation in run.ops:
coll._delete(sock_info,
operation['q'],
not operation['limit'],
write_concern,
op_id,
self.ordered)
except OperationFailure:
if self.ordered:
break
class Meta:
write_concern = WriteConcern(j=True)
connection_alias = 'suspend_list'
final = True
