def run_scenario(self):
listener = OvertCommandListener()
# New client, to avoid interference from pooled sessions.
# Convert test['clientOptions'] to dict to avoid a Jython bug using "**"
# with ScenarioDict.
client = rs_client(event_listeners=[listener],
**dict(test['clientOptions']))
# Close the client explicitly to avoid having too many threads open.
self.addCleanup(client.close)
# Kill all sessions before and after each test to prevent an open
# transaction (from a test failure) from blocking collection/database
# operations during test set up and tear down.
def kill_all_sessions():
try:
client.admin.command('killAllSessions', [])
except OperationFailure:
# "operation was interrupted" by killing the command's
# own session.
pass
kill_all_sessions()
self.addCleanup(kill_all_sessions)
database_name = scenario_def['database_name']
collection_name = scenario_def['collection_name']
write_concern_db = client.get_database(
database_name, write_concern=WriteConcern(w='majority'))
write_concern_coll = write_concern_db[collection_name]
write_concern_coll.drop()
write_concern_db.create_collection(collection_name)
if scenario_def['data']:
# Load data.
write_concern_coll.insert_many(scenario_def['data'])
# Create session0 and session1.
sessions = {}
session_ids = {}
for i in range(2):
session_name = 'session%d' % i
opts = camel_to_snake_args(test['sessionOptions'][session_name])
if 'default_transaction_options' in opts:
txn_opts = opts['default_transaction_options']
if 'readConcern' in txn_opts:
read_concern = ReadConcern(**dict(txn_opts['readConcern']))
else:
read_concern = None
if 'writeConcern' in txn_opts:
write_concern = WriteConcern(
**dict(txn_opts['writeConcern']))
else:
write_concern = None
if 'readPreference' in txn_opts:
read_pref = parse_read_preference(
txn_opts['readPreference'])
else:
read_pref = None
txn_opts = client_session.TransactionOptions(
read_concern=read_concern,
write_concern=write_concern,
read_preference=read_pref,
)
opts['default_transaction_options'] = txn_opts
s = client.start_session(**dict(opts))
sessions[session_name] = s
# Store lsid so we can access it after end_session, in check_events.
session_ids[session_name] = s.session_id
self.addCleanup(end_sessions, sessions)
if 'failPoint' in test:
self.set_fail_point(test['failPoint'])
self.addCleanup(self.set_fail_point, {
'configureFailPoint': 'failCommand',
'mode': 'off'
})
listener.results.clear()
collection = client[database_name][collection_name]
for op in test['operations']:
expected_result = op.get('result')
if expect_error(expected_result):
with self.assertRaises(PyMongoError,
msg=op['name']) as context:
self.run_operation(sessions, collection, op.copy())
if expect_error_message(expected_result):
self.assertIn(expected_result['errorContains'].lower(),
str(context.exception).lower())
if expect_error_code(expected_result):
self.assertEqual(expected_result['errorCodeName'],
context.exception.details.get('codeName'))
if expect_error_labels_contain(expected_result):
self.assertErrorLabelsContain(
context.exception,
expected_result['errorLabelsContain'])
if expect_error_labels_omit(expected_result):
self.assertErrorLabelsOmit(
context.exception, expected_result['errorLabelsOmit'])
else:
result = self.run_operation(sessions, collection, op.copy())
if 'result' in op:
if op['name'] == 'runCommand':
self.check_command_result(expected_result, result)
else:
self.check_result(expected_result, result)
for s in sessions.values():
s.end_session()
self.check_events(test, listener, session_ids)
# Assert final state is expected.
expected_c = test['outcome'].get('collection')
if expected_c is not None:
# Read from the primary to ensure causal consistency.
primary_coll = collection.with_options(
read_preference=ReadPreference.PRIMARY)
self.assertEqual(list(primary_coll.find()), expected_c['data'])
def run_scenario(self, scenario_def, test):
self.maybe_skip_scenario(test)
# Kill all sessions before and after each test to prevent an open
# transaction (from a test failure) from blocking collection/database
# operations during test set up and tear down.
self.kill_all_sessions()
self.addCleanup(self.kill_all_sessions)
self.setup_scenario(scenario_def)
database_name = self.get_scenario_db_name(scenario_def)
collection_name = self.get_scenario_coll_name(scenario_def)
# SPEC-1245 workaround StaleDbVersion on distinct
for c in self.mongos_clients:
c[database_name][collection_name].distinct("x")
# Configure the fail point before creating the client.
if 'failPoint' in test:
fp = test['failPoint']
self.set_fail_point(fp)
self.addCleanup(self.set_fail_point, {
'configureFailPoint': fp['configureFailPoint'],
'mode': 'off'
})
listener = OvertCommandListener()
pool_listener = CMAPListener()
server_listener = ServerAndTopologyEventListener()
# Create a new client, to avoid interference from pooled sessions.
client_options = self.parse_client_options(test['clientOptions'])
# MMAPv1 does not support retryable writes.
if (client_options.get('retryWrites') is True
and client_context.storage_engine == 'mmapv1'):
self.skipTest("MMAPv1 does not support retryWrites=True")
use_multi_mongos = test['useMultipleMongoses']
if client_context.is_mongos and use_multi_mongos:
client = rs_client(
client_context.mongos_seeds(),
event_listeners=[listener, pool_listener, server_listener],
**client_options)
else:
client = rs_client(
event_listeners=[listener, pool_listener, server_listener],
**client_options)
self.scenario_client = client
self.listener = listener
self.pool_listener = pool_listener
self.server_listener = server_listener
# Close the client explicitly to avoid having too many threads open.
self.addCleanup(client.close)
# Create session0 and session1.
sessions = {}
session_ids = {}
for i in range(2):
# Don't attempt to create sessions if they are not supported by
# the running server version.
if not client_context.sessions_enabled:
break
session_name = 'session%d' % i
opts = camel_to_snake_args(test['sessionOptions'][session_name])
if 'default_transaction_options' in opts:
txn_opts = self.parse_options(
opts['default_transaction_options'])
txn_opts = client_session.TransactionOptions(**txn_opts)
opts['default_transaction_options'] = txn_opts
s = client.start_session(**dict(opts))
sessions[session_name] = s
# Store lsid so we can access it after end_session, in check_events.
session_ids[session_name] = s.session_id
self.addCleanup(end_sessions, sessions)
collection = client[database_name][collection_name]
self.run_test_ops(sessions, collection, test)
end_sessions(sessions)
self.check_events(test, listener, session_ids)
# Disable fail points.
if 'failPoint' in test:
fp = test['failPoint']
self.set_fail_point({
'configureFailPoint': fp['configureFailPoint'],
'mode': 'off'
})
# Assert final state is expected.
outcome = test['outcome']
expected_c = outcome.get('collection')
if expected_c is not None:
outcome_coll_name = self.get_outcome_coll_name(outcome, collection)
# Read from the primary with local read concern to ensure causal
# consistency.
outcome_coll = client_context.client[
collection.database.name].get_collection(
outcome_coll_name,
read_preference=ReadPreference.PRIMARY,
read_concern=ReadConcern('local'))
actual_data = list(outcome_coll.find(sort=[('_id', 1)]))
# The expected data needs to be the left hand side here otherwise
# CompareType(Binary) doesn't work.
self.assertEqual(wrap_types(expected_c['data']), actual_data)
def run_scenario(self):
listener = OvertCommandListener()
# New client, to avoid interference from pooled sessions.
# Convert test['clientOptions'] to dict to avoid a Jython bug using "**"
# with ScenarioDict.
client = rs_client(event_listeners=[listener],
**dict(test['clientOptions']))
try:
client.admin.command('killAllSessions', [])
except OperationFailure:
# "operation was interrupted" by killing the command's own session.
pass
write_concern_db = client.get_database(
'transaction-tests', write_concern=WriteConcern(w='majority'))
write_concern_db.test.drop()
write_concern_db.create_collection('test')
if scenario_def['data']:
# Load data.
write_concern_db.test.insert_many(scenario_def['data'])
# Create session0 and session1.
sessions = {}
session_ids = {}
for i in range(2):
session_name = 'session%d' % i
opts = camel_to_snake_args(test['sessionOptions'][session_name])
if 'default_transaction_options' in opts:
txn_opts = opts['default_transaction_options']
if 'readConcern' in txn_opts:
read_concern = ReadConcern(**dict(txn_opts['readConcern']))
else:
read_concern = None
if 'writeConcern' in txn_opts:
write_concern = WriteConcern(
**dict(txn_opts['writeConcern']))
else:
write_concern = None
if 'readPreference' in txn_opts:
read_pref = parse_read_preference(
txn_opts['readPreference'])
else:
read_pref = None
txn_opts = client_session.TransactionOptions(
read_concern=read_concern,
write_concern=write_concern,
read_preference=read_pref,
)
opts['default_transaction_options'] = txn_opts
s = client.start_session(**dict(opts))
sessions[session_name] = s
# Store lsid so we can access it after end_session, in check_events.
session_ids[session_name] = s.session_id
self.addCleanup(end_sessions, sessions)
listener.results.clear()
collection = client['transaction-tests'].test
for op in test['operations']:
expected_result = op.get('result')
if expect_error_message(expected_result):
with self.assertRaises(PyMongoError,
msg=op.get('name')) as context:
self.run_operation(sessions, collection, op.copy())
self.assertIn(expected_result['errorContains'].lower(),
str(context.exception).lower())
elif expect_error_code(expected_result):
with self.assertRaises(OperationFailure) as context:
self.run_operation(sessions, collection, op.copy())
self.assertEqual(expected_result['errorCodeName'],
context.exception.details.get('codeName'))
else:
result = self.run_operation(sessions, collection, op.copy())
if 'result' in op:
self.check_result(expected_result, result)
for s in sessions.values():
s.end_session()
self.check_events(test, listener, session_ids)
# Assert final state is expected.
expected_c = test['outcome'].get('collection')
if expected_c is not None:
# Read from the primary to ensure causal consistency.
primary_coll = collection.with_options(
read_preference=ReadPreference.PRIMARY)
self.assertEqual(list(primary_coll.find()), expected_c['data'])
def run_scenario(self, scenario_def, test):
self.maybe_skip_scenario(test)
listener = OvertCommandListener()
# Create a new client, to avoid interference from pooled sessions.
# Convert test['clientOptions'] to dict to avoid a Jython bug using
# "**" with ScenarioDict.
client_options = dict(test['clientOptions'])
use_multi_mongos = test['useMultipleMongoses']
if client_context.is_mongos and use_multi_mongos:
client = rs_client(client_context.mongos_seeds(),
event_listeners=[listener],
**client_options)
else:
client = rs_client(event_listeners=[listener], **client_options)
# Close the client explicitly to avoid having too many threads open.
self.addCleanup(client.close)
# Kill all sessions before and after each test to prevent an open
# transaction (from a test failure) from blocking collection/database
# operations during test set up and tear down.
self.kill_all_sessions()
self.addCleanup(self.kill_all_sessions)
database_name = scenario_def['database_name']
write_concern_db = client_context.client.get_database(
database_name, write_concern=WriteConcern(w='majority'))
if 'bucket_name' in scenario_def:
# Create a bucket for the retryable reads GridFS tests.
collection_name = scenario_def['bucket_name']
client_context.client.drop_database(database_name)
if scenario_def['data']:
data = scenario_def['data']
# Load data.
write_concern_db['fs.chunks'].insert_many(data['fs.chunks'])
write_concern_db['fs.files'].insert_many(data['fs.files'])
else:
collection_name = scenario_def['collection_name']
write_concern_coll = write_concern_db[collection_name]
write_concern_coll.drop()
write_concern_db.create_collection(collection_name)
if scenario_def['data']:
# Load data.
write_concern_coll.insert_many(scenario_def['data'])
# SPEC-1245 workaround StaleDbVersion on distinct
for c in self.mongos_clients:
c[database_name][collection_name].distinct("x")
# Create session0 and session1.
sessions = {}
session_ids = {}
for i in range(2):
session_name = 'session%d' % i
opts = camel_to_snake_args(test['sessionOptions'][session_name])
if 'default_transaction_options' in opts:
txn_opts = opts['default_transaction_options']
if 'readConcern' in txn_opts:
read_concern = ReadConcern(**dict(txn_opts['readConcern']))
else:
read_concern = None
if 'writeConcern' in txn_opts:
write_concern = WriteConcern(
**dict(txn_opts['writeConcern']))
else:
write_concern = None
if 'readPreference' in txn_opts:
read_pref = parse_read_preference(
txn_opts['readPreference'])
else:
read_pref = None
txn_opts = client_session.TransactionOptions(
read_concern=read_concern,
write_concern=write_concern,
read_preference=read_pref,
)
opts['default_transaction_options'] = txn_opts
s = client.start_session(**dict(opts))
sessions[session_name] = s
# Store lsid so we can access it after end_session, in check_events.
session_ids[session_name] = s.session_id
self.addCleanup(end_sessions, sessions)
if 'failPoint' in test:
self.set_fail_point(test['failPoint'])
self.addCleanup(self.set_fail_point, {
'configureFailPoint': 'failCommand',
'mode': 'off'
})
listener.results.clear()
collection = client[database_name][collection_name]
self.run_operations(sessions, collection, test['operations'])
end_sessions(sessions)
self.check_events(test, listener, session_ids)
# Disable fail points.
if 'failPoint' in test:
self.set_fail_point({
'configureFailPoint': 'failCommand',
'mode': 'off'
})
# Assert final state is expected.
expected_c = test['outcome'].get('collection')
if expected_c is not None:
# Read from the primary with local read concern to ensure causal
# consistency.
primary_coll = collection.with_options(
read_preference=ReadPreference.PRIMARY,
read_concern=ReadConcern('local'))
self.assertEqual(list(primary_coll.find()), expected_c['data'])
def run_scenario(self):
listener = TestListener()
# New client, to avoid interference from pooled sessions.
client = self.motor_rsc(event_listeners=[listener],
**test['clientOptions'])
try:
yield client.admin.command('killAllSessions', [])
except OperationFailure:
# "operation was interrupted" by killing the command's own session.
pass
if test['failPoint']:
yield client.admin.command(test['failPoint'])
database_name = scenario_def['database_name']
collection_name = scenario_def['collection_name']
write_concern_db = client.get_database(
database_name, write_concern=WriteConcern(w='majority'))
write_concern_coll = write_concern_db[collection_name]
yield write_concern_coll.drop()
yield write_concern_db.create_collection(collection_name)
if scenario_def['data']:
# Load data.
yield write_concern_coll.insert_many(scenario_def['data'])
# Create session0 and session1.
sessions = {}
session_ids = {}
for i in range(2):
session_name = 'session%d' % i
opts = camel_to_snake_args(test['sessionOptions'][session_name])
if 'default_transaction_options' in opts:
txn_opts = opts['default_transaction_options']
if 'readConcern' in txn_opts:
read_concern = ReadConcern(**txn_opts['readConcern'])
else:
read_concern = None
if 'writeConcern' in txn_opts:
write_concern = WriteConcern(**txn_opts['writeConcern'])
else:
write_concern = None
if 'readPreference' in txn_opts:
read_pref = parse_read_preference(
txn_opts['readPreference'])
else:
read_pref = None
txn_opts = client_session.TransactionOptions(
read_concern=read_concern,
write_concern=write_concern,
read_preference=read_pref,
)
opts['default_transaction_options'] = txn_opts
s = yield client.start_session(**opts)
sessions[session_name] = s
# Store lsid so we can access it after end_session, in check_events.
session_ids[session_name] = s.session_id
self.addCleanup(end_sessions, sessions)
listener.results.clear()
collection = client[database_name][collection_name]
for op in test['operations']:
expected_result = op.get('result')
if expect_error(expected_result):
with self.assertRaises(PyMongoError,
msg=op.get('name')) as context:
yield self.run_operation(sessions, collection, op.copy())
err = context.exception
if expected_result['errorContains']:
self.assertIn(expected_result['errorContains'].lower(),
str(err).lower())
if expected_result['errorCodeName']:
self.assertEqual(expected_result['errorCodeName'],
err.details.get('codeName'))
for label in expected_result.get('errorLabelsContain', []):
self.assertTrue(
err.has_error_label(label),
"%r should have errorLabel %s" % (err, label))
for label in expected_result.get('errorLabelsOmit', []):
self.assertFalse(
err.has_error_label(label),
"%r should NOT have errorLabel %s" % (err, label))
else:
result = yield self.run_operation(
sessions, collection, op.copy())
if 'result' in op:
self.check_result(expected_result, result)
for s in sessions.values():
yield s.end_session()
self.check_events(test, listener, session_ids)
# Assert final state is expected.
expected = test['outcome'].get('collection')
if expected is not None:
# Read from the primary to ensure causal consistency.
primary_coll = collection.with_options(
read_preference=ReadPreference.PRIMARY)
docs = yield primary_coll.find().to_list(length=None)
self.assertEqual(expected['data'], docs)
