def run_operation(self, collection, test): # Iterate over all operations. for opdef in test['operations']: # Convert command from CamelCase to pymongo.collection method. operation = camel_to_snake(opdef['name']) # Get command handle on target entity (collection/database). target_object = opdef.get('object', 'collection') if target_object == 'database': cmd = getattr(collection.database, operation) elif target_object == 'collection': collection = collection.with_options(**dict( parse_collection_options(opdef.get('collectionOptions', {})))) cmd = getattr(collection, operation) else: self.fail("Unknown object name %s" % (target_object, )) # Convert arguments to snake_case and handle special cases. arguments = opdef['arguments'] options = arguments.pop("options", {}) for option_name in options: arguments[camel_to_snake(option_name)] = options[option_name] if operation == "bulk_write": # Parse each request into a bulk write model. requests = [] for request in arguments["requests"]: bulk_model = camel_to_upper_camel(request["name"]) bulk_class = getattr(operations, bulk_model) bulk_arguments = camel_to_snake_args(request["arguments"]) requests.append(bulk_class(**bulk_arguments)) arguments["requests"] = requests else: for arg_name in list(arguments): c2s = camel_to_snake(arg_name) # PyMongo accepts sort as list of tuples. if arg_name == "sort": sort_dict = arguments[arg_name] arguments[arg_name] = list(iteritems(sort_dict)) # Named "key" instead not fieldName. if arg_name == "fieldName": arguments["key"] = arguments.pop(arg_name) # Aggregate uses "batchSize", while find uses batch_size. elif arg_name == "batchSize" and operation == "aggregate": continue # Requires boolean returnDocument. elif arg_name == "returnDocument": arguments[c2s] = arguments[arg_name] == "After" else: arguments[c2s] = arguments.pop(arg_name) if opdef.get('error') is True: with self.assertRaises(PyMongoError): cmd(**arguments) else: result = cmd(**arguments) self.check_result(opdef.get('result'), result)
def run_operation(self, collection, test): # Iterate over all operations. for opdef in test['operations']: # Convert command from CamelCase to pymongo.collection method. operation = camel_to_snake(opdef['name']) # Get command handle on target entity (collection/database). target_object = opdef.get('object', 'collection') if target_object == 'database': cmd = getattr(collection.database, operation) elif target_object == 'collection': collection = collection.with_options(**dict( parse_collection_options(opdef.get( 'collectionOptions', {})))) cmd = getattr(collection, operation) else: self.fail("Unknown object name %s" % (target_object,)) # Convert arguments to snake_case and handle special cases. arguments = opdef['arguments'] options = arguments.pop("options", {}) for option_name in options: arguments[camel_to_snake(option_name)] = options[option_name] if operation == "bulk_write": # Parse each request into a bulk write model. requests = [] for request in arguments["requests"]: bulk_model = camel_to_upper_camel(request["name"]) bulk_class = getattr(operations, bulk_model) bulk_arguments = camel_to_snake_args(request["arguments"]) requests.append(bulk_class(**bulk_arguments)) arguments["requests"] = requests else: for arg_name in list(arguments): c2s = camel_to_snake(arg_name) # PyMongo accepts sort as list of tuples. if arg_name == "sort": sort_dict = arguments[arg_name] arguments[arg_name] = list(iteritems(sort_dict)) # Named "key" instead not fieldName. if arg_name == "fieldName": arguments["key"] = arguments.pop(arg_name) # Aggregate uses "batchSize", while find uses batch_size. elif arg_name == "batchSize" and operation == "aggregate": continue # Requires boolean returnDocument. elif arg_name == "returnDocument": arguments[c2s] = arguments[arg_name] == "After" else: arguments[c2s] = arguments.pop(arg_name) if opdef.get('error') is True: with self.assertRaises(PyMongoError): cmd(**arguments) else: result = cmd(**arguments) self.check_result(opdef.get('result'), result)
def parse_auto_encrypt_opts(self, opts): """Parse clientOptions.autoEncryptOpts.""" opts = camel_to_snake_args(opts) kms_providers = opts['kms_providers'] if 'aws' in kms_providers: kms_providers['aws'] = AWS_CREDS if not any(AWS_CREDS.values()): self.skipTest('AWS environment credentials are not set') if 'key_vault_namespace' not in opts: opts['key_vault_namespace'] = 'keyvault.datakeys' opts = dict(opts) return AutoEncryptionOpts(**opts)
def run_operation(collection, test): # Convert command from CamelCase to pymongo.collection method. operation = camel_to_snake(test['operation']['name']) cmd = getattr(collection, operation) # Convert arguments to snake_case and handle special cases. arguments = test['operation']['arguments'] options = arguments.pop("options", {}) for option_name in options: arguments[camel_to_snake(option_name)] = options[option_name] if operation == "bulk_write": # Parse each request into a bulk write model. requests = [] for request in arguments["requests"]: bulk_model = camel_to_upper_camel(request["name"]) bulk_class = getattr(operations, bulk_model) bulk_arguments = camel_to_snake_args(request["arguments"]) requests.append(bulk_class(**bulk_arguments)) arguments["requests"] = requests else: for arg_name in list(arguments): c2s = camel_to_snake(arg_name) # PyMongo accepts sort as list of tuples. if arg_name == "sort": sort_dict = arguments[arg_name] arguments[arg_name] = list(iteritems(sort_dict)) # Named "key" instead not fieldName. if arg_name == "fieldName": arguments["key"] = arguments.pop(arg_name) # Aggregate uses "batchSize", while find uses batch_size. elif arg_name == "batchSize" and operation == "aggregate": continue # Requires boolean returnDocument. elif arg_name == "returnDocument": arguments[c2s] = arguments[arg_name] == "After" else: arguments[c2s] = arguments.pop(arg_name) result = cmd(**arguments) if operation == "aggregate": if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]: out = collection.database[arguments["pipeline"][-1]["$out"]] result = out.find() if isinstance(result, Cursor) or isinstance(result, CommandCursor): return list(result) return result
def run_operation(collection, test): # Convert command from CamelCase to pymongo.collection method. operation = camel_to_snake(test['operation']['name']) cmd = getattr(collection, operation) # Convert arguments to snake_case and handle special cases. arguments = test['operation']['arguments'] options = arguments.pop("options", {}) for option_name in options: arguments[camel_to_snake(option_name)] = options[option_name] if operation == "bulk_write": # Parse each request into a bulk write model. requests = [] for request in arguments["requests"]: bulk_model = camel_to_upper_camel(request["name"]) bulk_class = getattr(operations, bulk_model) bulk_arguments = camel_to_snake_args(request["arguments"]) requests.append(bulk_class(**bulk_arguments)) arguments["requests"] = requests else: for arg_name in list(arguments): c2s = camel_to_snake(arg_name) # PyMongo accepts sort as list of tuples. if arg_name == "sort": sort_dict = arguments[arg_name] arguments[arg_name] = list(iteritems(sort_dict)) # Named "key" instead not fieldName. if arg_name == "fieldName": arguments["key"] = arguments.pop(arg_name) # Aggregate uses "batchSize", while find uses batch_size. elif arg_name == "batchSize" and operation == "aggregate": continue # Requires boolean returnDocument. elif arg_name == "returnDocument": arguments[c2s] = arguments[arg_name] == "After" else: arguments[c2s] = arguments.pop(arg_name) result = cmd(**arguments) if operation == "aggregate": if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]: out = collection.database[arguments["pipeline"][-1]["$out"]] return out.find() return result
def run_scenario(self, scenario_def, test): self.maybe_skip_scenario(test) # Kill all sessions before and after each test to prevent an open # transaction (from a test failure) from blocking collection/database # operations during test set up and tear down. self.kill_all_sessions() self.addCleanup(self.kill_all_sessions) self.setup_scenario(scenario_def) database_name = self.get_scenario_db_name(scenario_def) collection_name = self.get_scenario_coll_name(scenario_def) # SPEC-1245 workaround StaleDbVersion on distinct for c in self.mongos_clients: c[database_name][collection_name].distinct("x") # Configure the fail point before creating the client. if 'failPoint' in test: fp = test['failPoint'] self.set_fail_point(fp) self.addCleanup(self.set_fail_point, { 'configureFailPoint': fp['configureFailPoint'], 'mode': 'off' }) listener = OvertCommandListener() pool_listener = CMAPListener() server_listener = ServerAndTopologyEventListener() # Create a new client, to avoid interference from pooled sessions. client_options = self.parse_client_options(test['clientOptions']) # MMAPv1 does not support retryable writes. if (client_options.get('retryWrites') is True and client_context.storage_engine == 'mmapv1'): self.skipTest("MMAPv1 does not support retryWrites=True") use_multi_mongos = test['useMultipleMongoses'] if client_context.is_mongos and use_multi_mongos: client = rs_client( client_context.mongos_seeds(), event_listeners=[listener, pool_listener, server_listener], **client_options) else: client = rs_client( event_listeners=[listener, pool_listener, server_listener], **client_options) self.scenario_client = client self.listener = listener self.pool_listener = pool_listener self.server_listener = server_listener # Close the client explicitly to avoid having too many threads open. self.addCleanup(client.close) # Create session0 and session1. sessions = {} session_ids = {} for i in range(2): # Don't attempt to create sessions if they are not supported by # the running server version. if not client_context.sessions_enabled: break session_name = 'session%d' % i opts = camel_to_snake_args(test['sessionOptions'][session_name]) if 'default_transaction_options' in opts: txn_opts = self.parse_options( opts['default_transaction_options']) txn_opts = client_session.TransactionOptions(**txn_opts) opts['default_transaction_options'] = txn_opts s = client.start_session(**dict(opts)) sessions[session_name] = s # Store lsid so we can access it after end_session, in check_events. session_ids[session_name] = s.session_id self.addCleanup(end_sessions, sessions) collection = client[database_name][collection_name] self.run_test_ops(sessions, collection, test) end_sessions(sessions) self.check_events(test, listener, session_ids) # Disable fail points. if 'failPoint' in test: fp = test['failPoint'] self.set_fail_point({ 'configureFailPoint': fp['configureFailPoint'], 'mode': 'off' }) # Assert final state is expected. outcome = test['outcome'] expected_c = outcome.get('collection') if expected_c is not None: outcome_coll_name = self.get_outcome_coll_name(outcome, collection) # Read from the primary with local read concern to ensure causal # consistency. outcome_coll = client_context.client[ collection.database.name].get_collection( outcome_coll_name, read_preference=ReadPreference.PRIMARY, read_concern=ReadConcern('local')) actual_data = list(outcome_coll.find(sort=[('_id', 1)])) # The expected data needs to be the left hand side here otherwise # CompareType(Binary) doesn't work. self.assertEqual(wrap_types(expected_c['data']), actual_data)
def run_operation(self, sessions, collection, operation): original_collection = collection name = camel_to_snake(operation['name']) if name == 'run_command': name = 'command' elif name == 'download_by_name': name = 'open_download_stream_by_name' elif name == 'download': name = 'open_download_stream' database = collection.database collection = database.get_collection(collection.name) if 'collectionOptions' in operation: collection = collection.with_options( **self.parse_options(operation['collectionOptions'])) object_name = self.get_object_name(operation) if object_name == 'gridfsbucket': # Only create the GridFSBucket when we need it (for the gridfs # retryable reads tests). obj = GridFSBucket(database, bucket_name=collection.name, disable_md5=True) else: objects = { 'client': database.client, 'database': database, 'collection': collection, 'testRunner': self } objects.update(sessions) obj = objects[object_name] # Combine arguments with options and handle special cases. arguments = operation.get('arguments', {}) arguments.update(arguments.pop("options", {})) self.parse_options(arguments) cmd = getattr(obj, name) for arg_name in list(arguments): c2s = camel_to_snake(arg_name) # PyMongo accepts sort as list of tuples. if arg_name == "sort": sort_dict = arguments[arg_name] arguments[arg_name] = list(iteritems(sort_dict)) # Named "key" instead not fieldName. if arg_name == "fieldName": arguments["key"] = arguments.pop(arg_name) # Aggregate uses "batchSize", while find uses batch_size. elif ((arg_name == "batchSize" or arg_name == "allowDiskUse") and name == "aggregate"): continue # Requires boolean returnDocument. elif arg_name == "returnDocument": arguments[c2s] = arguments.pop(arg_name) == "After" elif c2s == "requests": # Parse each request into a bulk write model. requests = [] for request in arguments["requests"]: bulk_model = camel_to_upper_camel(request["name"]) bulk_class = getattr(operations, bulk_model) bulk_arguments = camel_to_snake_args(request["arguments"]) requests.append(bulk_class(**dict(bulk_arguments))) arguments["requests"] = requests elif arg_name == "session": arguments['session'] = sessions[arguments['session']] elif (name in ('command', 'run_admin_command') and arg_name == 'command'): # Ensure the first key is the command name. ordered_command = SON([(operation['command_name'], 1)]) ordered_command.update(arguments['command']) arguments['command'] = ordered_command elif name == 'open_download_stream' and arg_name == 'id': arguments['file_id'] = arguments.pop(arg_name) elif name != 'find' and c2s == 'max_time_ms': # find is the only method that accepts snake_case max_time_ms. # All other methods take kwargs which must use the server's # camelCase maxTimeMS. See PYTHON-1855. arguments['maxTimeMS'] = arguments.pop('max_time_ms') elif name == 'with_transaction' and arg_name == 'callback': callback_ops = arguments[arg_name]['operations'] arguments['callback'] = lambda _: self.run_operations( sessions, original_collection, copy.deepcopy(callback_ops), in_with_transaction=True) elif name == 'drop_collection' and arg_name == 'collection': arguments['name_or_collection'] = arguments.pop(arg_name) elif name == 'create_collection' and arg_name == 'collection': arguments['name'] = arguments.pop(arg_name) elif name == 'create_index' and arg_name == 'keys': arguments['keys'] = list(arguments.pop(arg_name).items()) elif name == 'drop_index' and arg_name == 'name': arguments['index_or_name'] = arguments.pop(arg_name) else: arguments[c2s] = arguments.pop(arg_name) if name == 'run_on_thread': args = {'sessions': sessions, 'collection': collection} args.update(arguments) arguments = args result = cmd(**dict(arguments)) if name == "aggregate": if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]: # Read from the primary to ensure causal consistency. out = collection.database.get_collection( arguments["pipeline"][-1]["$out"], read_preference=ReadPreference.PRIMARY) return out.find() if name == "map_reduce": if isinstance(result, dict) and 'results' in result: return result['results'] if 'download' in name: result = Binary(result.read()) if isinstance(result, Cursor) or isinstance(result, CommandCursor): return list(result) return result
def run_scenario(self, scenario_def, test): self.maybe_skip_scenario(test) listener = OvertCommandListener() # Create a new client, to avoid interference from pooled sessions. # Convert test['clientOptions'] to dict to avoid a Jython bug using # "**" with ScenarioDict. client_options = dict(test['clientOptions']) use_multi_mongos = test['useMultipleMongoses'] if client_context.is_mongos and use_multi_mongos: client = rs_client(client_context.mongos_seeds(), event_listeners=[listener], **client_options) else: client = rs_client(event_listeners=[listener], **client_options) # Close the client explicitly to avoid having too many threads open. self.addCleanup(client.close) # Kill all sessions before and after each test to prevent an open # transaction (from a test failure) from blocking collection/database # operations during test set up and tear down. self.kill_all_sessions() self.addCleanup(self.kill_all_sessions) database_name = scenario_def['database_name'] write_concern_db = client_context.client.get_database( database_name, write_concern=WriteConcern(w='majority')) if 'bucket_name' in scenario_def: # Create a bucket for the retryable reads GridFS tests. collection_name = scenario_def['bucket_name'] client_context.client.drop_database(database_name) if scenario_def['data']: data = scenario_def['data'] # Load data. write_concern_db['fs.chunks'].insert_many(data['fs.chunks']) write_concern_db['fs.files'].insert_many(data['fs.files']) else: collection_name = scenario_def['collection_name'] write_concern_coll = write_concern_db[collection_name] write_concern_coll.drop() write_concern_db.create_collection(collection_name) if scenario_def['data']: # Load data. write_concern_coll.insert_many(scenario_def['data']) # SPEC-1245 workaround StaleDbVersion on distinct for c in self.mongos_clients: c[database_name][collection_name].distinct("x") # Create session0 and session1. sessions = {} session_ids = {} for i in range(2): session_name = 'session%d' % i opts = camel_to_snake_args(test['sessionOptions'][session_name]) if 'default_transaction_options' in opts: txn_opts = opts['default_transaction_options'] if 'readConcern' in txn_opts: read_concern = ReadConcern(**dict(txn_opts['readConcern'])) else: read_concern = None if 'writeConcern' in txn_opts: write_concern = WriteConcern( **dict(txn_opts['writeConcern'])) else: write_concern = None if 'readPreference' in txn_opts: read_pref = parse_read_preference( txn_opts['readPreference']) else: read_pref = None txn_opts = client_session.TransactionOptions( read_concern=read_concern, write_concern=write_concern, read_preference=read_pref, ) opts['default_transaction_options'] = txn_opts s = client.start_session(**dict(opts)) sessions[session_name] = s # Store lsid so we can access it after end_session, in check_events. session_ids[session_name] = s.session_id self.addCleanup(end_sessions, sessions) if 'failPoint' in test: self.set_fail_point(test['failPoint']) self.addCleanup(self.set_fail_point, { 'configureFailPoint': 'failCommand', 'mode': 'off' }) listener.results.clear() collection = client[database_name][collection_name] self.run_operations(sessions, collection, test['operations']) end_sessions(sessions) self.check_events(test, listener, session_ids) # Disable fail points. if 'failPoint' in test: self.set_fail_point({ 'configureFailPoint': 'failCommand', 'mode': 'off' }) # Assert final state is expected. expected_c = test['outcome'].get('collection') if expected_c is not None: # Read from the primary with local read concern to ensure causal # consistency. primary_coll = collection.with_options( read_preference=ReadPreference.PRIMARY, read_concern=ReadConcern('local')) self.assertEqual(list(primary_coll.find()), expected_c['data'])
def run_scenario(self): if test.get('skipReason'): raise unittest.SkipTest(test.get('skipReason')) listener = OvertCommandListener() # Create a new client, to avoid interference from pooled sessions. # Convert test['clientOptions'] to dict to avoid a Jython bug using # "**" with ScenarioDict. client_options = dict(test['clientOptions']) use_multi_mongos = test['useMultipleMongoses'] if client_context.is_mongos and use_multi_mongos: client = rs_client(client_context.mongos_seeds(), event_listeners=[listener], **client_options) else: client = rs_client(event_listeners=[listener], **client_options) # Close the client explicitly to avoid having too many threads open. self.addCleanup(client.close) # Kill all sessions before and after each test to prevent an open # transaction (from a test failure) from blocking collection/database # operations during test set up and tear down. self.kill_all_sessions() self.addCleanup(self.kill_all_sessions) database_name = scenario_def['database_name'] collection_name = scenario_def['collection_name'] write_concern_db = client.get_database( database_name, write_concern=WriteConcern(w='majority')) write_concern_coll = write_concern_db[collection_name] write_concern_coll.drop() write_concern_db.create_collection(collection_name) if scenario_def['data']: # Load data. write_concern_coll.insert_many(scenario_def['data']) # Create session0 and session1. sessions = {} session_ids = {} for i in range(2): session_name = 'session%d' % i opts = camel_to_snake_args(test['sessionOptions'][session_name]) if 'default_transaction_options' in opts: txn_opts = opts['default_transaction_options'] if 'readConcern' in txn_opts: read_concern = ReadConcern(**dict(txn_opts['readConcern'])) else: read_concern = None if 'writeConcern' in txn_opts: write_concern = WriteConcern( **dict(txn_opts['writeConcern'])) else: write_concern = None if 'readPreference' in txn_opts: read_pref = parse_read_preference( txn_opts['readPreference']) else: read_pref = None txn_opts = client_session.TransactionOptions( read_concern=read_concern, write_concern=write_concern, read_preference=read_pref, ) opts['default_transaction_options'] = txn_opts s = client.start_session(**dict(opts)) sessions[session_name] = s # Store lsid so we can access it after end_session, in check_events. session_ids[session_name] = s.session_id self.addCleanup(end_sessions, sessions) if 'failPoint' in test: self.set_fail_point(test['failPoint']) self.addCleanup(self.set_fail_point, { 'configureFailPoint': 'failCommand', 'mode': 'off' }) listener.results.clear() collection = client[database_name][collection_name] for op in test['operations']: expected_result = op.get('result') if expect_error(expected_result): with self.assertRaises(PyMongoError, msg=op['name']) as context: self.run_operation(sessions, collection, op.copy()) if expect_error_message(expected_result): self.assertIn(expected_result['errorContains'].lower(), str(context.exception).lower()) if expect_error_code(expected_result): self.assertEqual(expected_result['errorCodeName'], context.exception.details.get('codeName')) if expect_error_labels_contain(expected_result): self.assertErrorLabelsContain( context.exception, expected_result['errorLabelsContain']) if expect_error_labels_omit(expected_result): self.assertErrorLabelsOmit( context.exception, expected_result['errorLabelsOmit']) else: result = self.run_operation(sessions, collection, op.copy()) if 'result' in op: if op['name'] == 'runCommand': self.check_command_result(expected_result, result) else: self.check_result(expected_result, result) for s in sessions.values(): s.end_session() self.check_events(test, listener, session_ids) # Disable fail points. self.set_fail_point({ 'configureFailPoint': 'failCommand', 'mode': 'off' }) # Assert final state is expected. expected_c = test['outcome'].get('collection') if expected_c is not None: # Read from the primary with local read concern to ensure causal # consistency. primary_coll = collection.with_options( read_preference=ReadPreference.PRIMARY, read_concern=ReadConcern('local')) self.assertEqual(list(primary_coll.find()), expected_c['data'])
def run_operation(self, sessions, collection, operation): name = camel_to_snake(operation['name']) if name == 'run_command': name = 'command' self.transaction_test_debug(name) def parse_options(opts): if 'readPreference' in opts: opts['read_preference'] = parse_read_preference( opts.pop('readPreference')) if 'writeConcern' in opts: opts['write_concern'] = WriteConcern( **dict(opts.pop('writeConcern'))) if 'readConcern' in opts: opts['read_concern'] = ReadConcern( **dict(opts.pop('readConcern'))) return opts database = collection.database collection = database.get_collection(collection.name) if 'collectionOptions' in operation: collection = collection.with_options( **dict(parse_options(operation['collectionOptions']))) objects = { 'database': database, 'collection': collection, 'testRunner': self } objects.update(sessions) obj = objects[operation['object']] # Combine arguments with options and handle special cases. arguments = operation.get('arguments', {}) arguments.update(arguments.pop("options", {})) parse_options(arguments) cmd = getattr(obj, name) for arg_name in list(arguments): c2s = camel_to_snake(arg_name) # PyMongo accepts sort as list of tuples. if arg_name == "sort": sort_dict = arguments[arg_name] arguments[arg_name] = list(iteritems(sort_dict)) # Named "key" instead not fieldName. if arg_name == "fieldName": arguments["key"] = arguments.pop(arg_name) # Aggregate uses "batchSize", while find uses batch_size. elif arg_name == "batchSize" and name == "aggregate": continue # Requires boolean returnDocument. elif arg_name == "returnDocument": arguments[c2s] = arguments[arg_name] == "After" elif c2s == "requests": # Parse each request into a bulk write model. requests = [] for request in arguments["requests"]: bulk_model = camel_to_upper_camel(request["name"]) bulk_class = getattr(operations, bulk_model) bulk_arguments = camel_to_snake_args(request["arguments"]) requests.append(bulk_class(**dict(bulk_arguments))) arguments["requests"] = requests elif arg_name == "session": arguments['session'] = sessions[arguments['session']] elif name == 'command' and arg_name == 'command': # Ensure the first key is the command name. ordered_command = SON([(operation['command_name'], 1)]) ordered_command.update(arguments['command']) arguments['command'] = ordered_command else: arguments[c2s] = arguments.pop(arg_name) result = cmd(**dict(arguments)) if name == "aggregate": if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]: # Read from the primary to ensure causal consistency. out = collection.database.get_collection( arguments["pipeline"][-1]["$out"], read_preference=ReadPreference.PRIMARY) return out.find() if isinstance(result, Cursor) or isinstance(result, CommandCursor): return list(result) return result
def run_scenario(self): if test.get('skipReason'): raise unittest.SkipTest(test.get('skipReason')) listener = OvertCommandListener() # Create a new client, to avoid interference from pooled sessions. # Convert test['clientOptions'] to dict to avoid a Jython bug using # "**" with ScenarioDict. client_options = dict(test['clientOptions']) use_multi_mongos = test['useMultipleMongoses'] if client_context.is_mongos and use_multi_mongos: client = rs_client(client_context.mongos_seeds(), event_listeners=[listener], **client_options) else: client = rs_client(event_listeners=[listener], **client_options) # Close the client explicitly to avoid having too many threads open. self.addCleanup(client.close) # Kill all sessions before and after each test to prevent an open # transaction (from a test failure) from blocking collection/database # operations during test set up and tear down. self.kill_all_sessions() self.addCleanup(self.kill_all_sessions) database_name = scenario_def['database_name'] collection_name = scenario_def['collection_name'] # Don't use the test client to load data. write_concern_db = client_context.client.get_database( database_name, write_concern=WriteConcern(w='majority')) write_concern_coll = write_concern_db[collection_name] write_concern_coll.drop() write_concern_db.create_collection(collection_name) if scenario_def['data']: # Load data. write_concern_coll.insert_many(scenario_def['data']) # SPEC-1245 workaround StaleDbVersion on distinct for c in self.mongos_clients: c[database_name][collection_name].distinct("x") # Create session0 and session1. sessions = {} session_ids = {} for i in range(2): session_name = 'session%d' % i opts = camel_to_snake_args(test['sessionOptions'][session_name]) if 'default_transaction_options' in opts: txn_opts = opts['default_transaction_options'] if 'readConcern' in txn_opts: read_concern = ReadConcern( **dict(txn_opts['readConcern'])) else: read_concern = None if 'writeConcern' in txn_opts: write_concern = WriteConcern( **dict(txn_opts['writeConcern'])) else: write_concern = None if 'readPreference' in txn_opts: read_pref = parse_read_preference( txn_opts['readPreference']) else: read_pref = None txn_opts = client_session.TransactionOptions( read_concern=read_concern, write_concern=write_concern, read_preference=read_pref, ) opts['default_transaction_options'] = txn_opts s = client.start_session(**dict(opts)) sessions[session_name] = s # Store lsid so we can access it after end_session, in check_events. session_ids[session_name] = s.session_id self.addCleanup(end_sessions, sessions) if 'failPoint' in test: self.set_fail_point(test['failPoint']) self.addCleanup(self.set_fail_point, { 'configureFailPoint': 'failCommand', 'mode': 'off'}) listener.results.clear() collection = client[database_name][collection_name] self.run_operations(sessions, collection, test['operations']) for s in sessions.values(): s.end_session() self.check_events(test, listener, session_ids) # Disable fail points. self.set_fail_point({ 'configureFailPoint': 'failCommand', 'mode': 'off'}) # Assert final state is expected. expected_c = test['outcome'].get('collection') if expected_c is not None: # Read from the primary with local read concern to ensure causal # consistency. primary_coll = collection.with_options( read_preference=ReadPreference.PRIMARY, read_concern=ReadConcern('local')) self.assertEqual(list(primary_coll.find()), expected_c['data'])
def run_operation(self, sessions, collection, operation): original_collection = collection name = camel_to_snake(operation['name']) if name == 'run_command': name = 'command' self.transaction_test_debug(name) def parse_options(opts): if 'readPreference' in opts: opts['read_preference'] = parse_read_preference( opts.pop('readPreference')) if 'writeConcern' in opts: opts['write_concern'] = WriteConcern( **dict(opts.pop('writeConcern'))) if 'readConcern' in opts: opts['read_concern'] = ReadConcern( **dict(opts.pop('readConcern'))) return opts database = collection.database collection = database.get_collection(collection.name) if 'collectionOptions' in operation: collection = collection.with_options( **dict(parse_options(operation['collectionOptions']))) objects = { 'database': database, 'collection': collection, 'testRunner': self } objects.update(sessions) obj = objects[operation['object']] # Combine arguments with options and handle special cases. arguments = operation.get('arguments', {}) arguments.update(arguments.pop("options", {})) parse_options(arguments) cmd = getattr(obj, name) for arg_name in list(arguments): c2s = camel_to_snake(arg_name) # PyMongo accepts sort as list of tuples. if arg_name == "sort": sort_dict = arguments[arg_name] arguments[arg_name] = list(iteritems(sort_dict)) # Named "key" instead not fieldName. if arg_name == "fieldName": arguments["key"] = arguments.pop(arg_name) # Aggregate uses "batchSize", while find uses batch_size. elif arg_name == "batchSize" and name == "aggregate": continue # Requires boolean returnDocument. elif arg_name == "returnDocument": arguments[c2s] = arguments[arg_name] == "After" elif c2s == "requests": # Parse each request into a bulk write model. requests = [] for request in arguments["requests"]: bulk_model = camel_to_upper_camel(request["name"]) bulk_class = getattr(operations, bulk_model) bulk_arguments = camel_to_snake_args(request["arguments"]) requests.append(bulk_class(**dict(bulk_arguments))) arguments["requests"] = requests elif arg_name == "session": arguments['session'] = sessions[arguments['session']] elif name == 'command' and arg_name == 'command': # Ensure the first key is the command name. ordered_command = SON([(operation['command_name'], 1)]) ordered_command.update(arguments['command']) arguments['command'] = ordered_command elif name == 'with_transaction' and arg_name == 'callback': callback_ops = arguments[arg_name]['operations'] arguments['callback'] = lambda _: self.run_operations( sessions, original_collection, copy.deepcopy(callback_ops), in_with_transaction=True) else: arguments[c2s] = arguments.pop(arg_name) result = cmd(**dict(arguments)) if name == "aggregate": if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]: # Read from the primary to ensure causal consistency. out = collection.database.get_collection( arguments["pipeline"][-1]["$out"], read_preference=ReadPreference.PRIMARY) return out.find() if isinstance(result, Cursor) or isinstance(result, CommandCursor): return list(result) return result
def _create_entity(self, entity_spec): if len(entity_spec) != 1: self._test_class.fail( "Entity spec %s did not contain exactly one top-level key" % (entity_spec, )) entity_type, spec = next(iteritems(entity_spec)) if entity_type == 'client': kwargs = {} observe_events = spec.get('observeEvents', []) ignore_commands = spec.get('ignoreCommandMonitoringEvents', []) if len(observe_events) or len(ignore_commands): ignore_commands = [cmd.lower() for cmd in ignore_commands] listener = EventListenerUtil(observe_events, ignore_commands) self._listeners[spec['id']] = listener kwargs['event_listeners'] = [listener] if client_context.is_mongos and spec.get('useMultipleMongoses'): kwargs['h'] = client_context.mongos_seeds() kwargs.update(spec.get('uriOptions', {})) server_api = spec.get('serverApi') if server_api: kwargs['server_api'] = ServerApi( server_api['version'], strict=server_api.get('strict'), deprecation_errors=server_api.get('deprecationErrors')) client = rs_or_single_client(**kwargs) self[spec['id']] = client self._test_class.addCleanup(client.close) return elif entity_type == 'database': client = self[spec['client']] if not isinstance(client, MongoClient): self._test_class.fail( 'Expected entity %s to be of type MongoClient, got %s' % (spec['client'], type(client))) options = parse_collection_or_database_options( spec.get('databaseOptions', {})) self[spec['id']] = client.get_database(spec['databaseName'], **options) return elif entity_type == 'collection': database = self[spec['database']] if not isinstance(database, Database): self._test_class.fail( 'Expected entity %s to be of type Database, got %s' % (spec['database'], type(database))) options = parse_collection_or_database_options( spec.get('collectionOptions', {})) self[spec['id']] = database.get_collection(spec['collectionName'], **options) return elif entity_type == 'session': client = self[spec['client']] if not isinstance(client, MongoClient): self._test_class.fail( 'Expected entity %s to be of type MongoClient, got %s' % (spec['client'], type(client))) opts = camel_to_snake_args(spec.get('sessionOptions', {})) if 'default_transaction_options' in opts: txn_opts = parse_spec_options( opts['default_transaction_options']) txn_opts = TransactionOptions(**txn_opts) opts = copy.deepcopy(opts) opts['default_transaction_options'] = txn_opts session = client.start_session(**dict(opts)) self[spec['id']] = session self._session_lsids[spec['id']] = copy.deepcopy(session.session_id) self._test_class.addCleanup(session.end_session) return elif entity_type == 'bucket': # TODO: implement the 'bucket' entity type self._test_class.skipTest( 'GridFS is not currently supported (PYTHON-2459)') self._test_class.fail('Unable to create entity of unknown type %s' % (entity_type, ))